Alexey Bataev | 9959db5 | 2014-05-06 10:08:46 +0000 | [diff] [blame] | 1 | //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===// |
| 2 | // |
Chandler Carruth | 2946cd7 | 2019-01-19 08:50:56 +0000 | [diff] [blame] | 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
Alexey Bataev | 9959db5 | 2014-05-06 10:08:46 +0000 | [diff] [blame] | 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This contains code to emit OpenMP nodes as LLVM code. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 13 | #include "CGCleanup.h" |
Alexey Bataev | 9959db5 | 2014-05-06 10:08:46 +0000 | [diff] [blame] | 14 | #include "CGOpenMPRuntime.h" |
| 15 | #include "CodeGenFunction.h" |
| 16 | #include "CodeGenModule.h" |
Chandler Carruth | 0d9593d | 2015-01-14 11:29:14 +0000 | [diff] [blame] | 17 | #include "TargetInfo.h" |
Alexey Bataev | 6120582 | 2019-12-04 09:50:21 -0500 | [diff] [blame] | 18 | #include "clang/AST/ASTContext.h" |
Reid Kleckner | 9803178 | 2019-12-09 16:11:56 -0800 | [diff] [blame] | 19 | #include "clang/AST/Attr.h" |
| 20 | #include "clang/AST/DeclOpenMP.h" |
Alexey Bataev | 9959db5 | 2014-05-06 10:08:46 +0000 | [diff] [blame] | 21 | #include "clang/AST/Stmt.h" |
| 22 | #include "clang/AST/StmtOpenMP.h" |
Alexey Bataev | 8bbf2e3 | 2019-11-04 09:59:11 -0500 | [diff] [blame] | 23 | #include "clang/Basic/PrettyStackTrace.h" |
Alexey Bataev | 9959db5 | 2014-05-06 10:08:46 +0000 | [diff] [blame] | 24 | using namespace clang; |
| 25 | using namespace CodeGen; |
| 26 | |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 27 | namespace { |
| 28 | /// Lexical scope for OpenMP executable constructs, that handles correct codegen |
| 29 | /// for captured expressions. |
Arpith Chacko Jacob | fe4890a | 2017-01-18 20:40:48 +0000 | [diff] [blame] | 30 | class OMPLexicalScope : public CodeGenFunction::LexicalScope { |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 31 | void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) { |
| 32 | for (const auto *C : S.clauses()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 33 | if (const auto *CPI = OMPClauseWithPreInit::get(C)) { |
| 34 | if (const auto *PreInit = |
| 35 | cast_or_null<DeclStmt>(CPI->getPreInitStmt())) { |
Alexey Bataev | 2bbf721 | 2016-03-03 03:52:24 +0000 | [diff] [blame] | 36 | for (const auto *I : PreInit->decls()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 37 | if (!I->hasAttr<OMPCaptureNoInitAttr>()) { |
Alexey Bataev | 2bbf721 | 2016-03-03 03:52:24 +0000 | [diff] [blame] | 38 | CGF.EmitVarDecl(cast<VarDecl>(*I)); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 39 | } else { |
Alexey Bataev | 2bbf721 | 2016-03-03 03:52:24 +0000 | [diff] [blame] | 40 | CodeGenFunction::AutoVarEmission Emission = |
| 41 | CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); |
| 42 | CGF.EmitAutoVarCleanups(Emission); |
| 43 | } |
| 44 | } |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 45 | } |
| 46 | } |
| 47 | } |
| 48 | } |
Alexey Bataev | 4ba78a4 | 2016-04-27 07:56:03 +0000 | [diff] [blame] | 49 | CodeGenFunction::OMPPrivateScope InlinedShareds; |
| 50 | |
| 51 | static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { |
| 52 | return CGF.LambdaCaptureFields.lookup(VD) || |
| 53 | (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || |
| 54 | (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl)); |
| 55 | } |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 56 | |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 57 | public: |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 58 | OMPLexicalScope( |
| 59 | CodeGenFunction &CGF, const OMPExecutableDirective &S, |
| 60 | const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None, |
| 61 | const bool EmitPreInitStmt = true) |
Alexey Bataev | 4ba78a4 | 2016-04-27 07:56:03 +0000 | [diff] [blame] | 62 | : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), |
| 63 | InlinedShareds(CGF) { |
Arpith Chacko Jacob | fe4890a | 2017-01-18 20:40:48 +0000 | [diff] [blame] | 64 | if (EmitPreInitStmt) |
| 65 | emitPreInitStmt(CGF, S); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 66 | if (!CapturedRegion.hasValue()) |
| 67 | return; |
| 68 | assert(S.hasAssociatedStmt() && |
| 69 | "Expected associated statement for inlined directive."); |
| 70 | const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 71 | for (const auto &C : CS->captures()) { |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 72 | if (C.capturesVariable() || C.capturesVariableByCopy()) { |
| 73 | auto *VD = C.getCapturedVar(); |
| 74 | assert(VD == VD->getCanonicalDecl() && |
| 75 | "Canonical decl must be captured."); |
| 76 | DeclRefExpr DRE( |
Bruno Ricci | 5fc4db7 | 2018-12-21 14:10:18 +0000 | [diff] [blame] | 77 | CGF.getContext(), const_cast<VarDecl *>(VD), |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 78 | isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && |
| 79 | InlinedShareds.isGlobalVarCaptured(VD)), |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 80 | VD->getType().getNonReferenceType(), VK_LValue, C.getLocation()); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 81 | InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 82 | return CGF.EmitLValue(&DRE).getAddress(CGF); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 83 | }); |
Alexey Bataev | 4ba78a4 | 2016-04-27 07:56:03 +0000 | [diff] [blame] | 84 | } |
| 85 | } |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 86 | (void)InlinedShareds.Privatize(); |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 87 | } |
| 88 | }; |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 89 | |
Arpith Chacko Jacob | fe4890a | 2017-01-18 20:40:48 +0000 | [diff] [blame] | 90 | /// Lexical scope for OpenMP parallel construct, that handles correct codegen |
| 91 | /// for captured expressions. |
| 92 | class OMPParallelScope final : public OMPLexicalScope { |
| 93 | bool EmitPreInitStmt(const OMPExecutableDirective &S) { |
| 94 | OpenMPDirectiveKind Kind = S.getDirectiveKind(); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 95 | return !(isOpenMPTargetExecutionDirective(Kind) || |
| 96 | isOpenMPLoopBoundSharingDirective(Kind)) && |
Arpith Chacko Jacob | fe4890a | 2017-01-18 20:40:48 +0000 | [diff] [blame] | 97 | isOpenMPParallelDirective(Kind); |
| 98 | } |
| 99 | |
| 100 | public: |
| 101 | OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 102 | : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, |
| 103 | EmitPreInitStmt(S)) {} |
Arpith Chacko Jacob | fe4890a | 2017-01-18 20:40:48 +0000 | [diff] [blame] | 104 | }; |
| 105 | |
Arpith Chacko Jacob | 99a1e0e | 2017-01-25 02:18:43 +0000 | [diff] [blame] | 106 | /// Lexical scope for OpenMP teams construct, that handles correct codegen |
| 107 | /// for captured expressions. |
| 108 | class OMPTeamsScope final : public OMPLexicalScope { |
| 109 | bool EmitPreInitStmt(const OMPExecutableDirective &S) { |
| 110 | OpenMPDirectiveKind Kind = S.getDirectiveKind(); |
| 111 | return !isOpenMPTargetExecutionDirective(Kind) && |
| 112 | isOpenMPTeamsDirective(Kind); |
| 113 | } |
| 114 | |
| 115 | public: |
| 116 | OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 117 | : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, |
| 118 | EmitPreInitStmt(S)) {} |
Arpith Chacko Jacob | 99a1e0e | 2017-01-25 02:18:43 +0000 | [diff] [blame] | 119 | }; |
| 120 | |
Alexey Bataev | 5a3af13 | 2016-03-29 08:58:54 +0000 | [diff] [blame] | 121 | /// Private scope for OpenMP loop-based directives, that supports capturing |
| 122 | /// of used expression from loop statement. |
| 123 | class OMPLoopScope : public CodeGenFunction::RunCleanupsScope { |
| 124 | void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) { |
Alexey Bataev | ab4ea22 | 2018-03-07 18:17:06 +0000 | [diff] [blame] | 125 | CodeGenFunction::OMPMapVars PreCondVars; |
Alexey Bataev | f71939c | 2019-09-18 19:24:07 +0000 | [diff] [blame] | 126 | llvm::DenseSet<const VarDecl *> EmittedAsPrivate; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 127 | for (const auto *E : S.counters()) { |
Alexey Bataev | e83b3e8 | 2017-12-08 20:18:58 +0000 | [diff] [blame] | 128 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); |
Alexey Bataev | f71939c | 2019-09-18 19:24:07 +0000 | [diff] [blame] | 129 | EmittedAsPrivate.insert(VD->getCanonicalDecl()); |
Alexey Bataev | ab4ea22 | 2018-03-07 18:17:06 +0000 | [diff] [blame] | 130 | (void)PreCondVars.setVarAddr( |
| 131 | CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); |
Alexey Bataev | e83b3e8 | 2017-12-08 20:18:58 +0000 | [diff] [blame] | 132 | } |
Alexey Bataev | f71939c | 2019-09-18 19:24:07 +0000 | [diff] [blame] | 133 | // Mark private vars as undefs. |
| 134 | for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) { |
| 135 | for (const Expr *IRef : C->varlists()) { |
| 136 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl()); |
| 137 | if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { |
| 138 | (void)PreCondVars.setVarAddr( |
| 139 | CGF, OrigVD, |
| 140 | Address(llvm::UndefValue::get( |
| 141 | CGF.ConvertTypeForMem(CGF.getContext().getPointerType( |
| 142 | OrigVD->getType().getNonReferenceType()))), |
| 143 | CGF.getContext().getDeclAlign(OrigVD))); |
| 144 | } |
| 145 | } |
| 146 | } |
Alexey Bataev | ab4ea22 | 2018-03-07 18:17:06 +0000 | [diff] [blame] | 147 | (void)PreCondVars.apply(CGF); |
Alexey Bataev | bef93a9 | 2019-10-07 18:54:57 +0000 | [diff] [blame] | 148 | // Emit init, __range and __end variables for C++ range loops. |
| 149 | const Stmt *Body = |
| 150 | S.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers(); |
| 151 | for (unsigned Cnt = 0; Cnt < S.getCollapsedNumber(); ++Cnt) { |
Alexey Bataev | 8bbf2e3 | 2019-11-04 09:59:11 -0500 | [diff] [blame] | 152 | Body = OMPLoopDirective::tryToFindNextInnerLoop( |
| 153 | Body, /*TryImperfectlyNestedLoops=*/true); |
Alexey Bataev | bef93a9 | 2019-10-07 18:54:57 +0000 | [diff] [blame] | 154 | if (auto *For = dyn_cast<ForStmt>(Body)) { |
| 155 | Body = For->getBody(); |
| 156 | } else { |
| 157 | assert(isa<CXXForRangeStmt>(Body) && |
Alexey Bataev | d457f7e | 2019-10-07 19:57:40 +0000 | [diff] [blame] | 158 | "Expected canonical for loop or range-based for loop."); |
Alexey Bataev | bef93a9 | 2019-10-07 18:54:57 +0000 | [diff] [blame] | 159 | auto *CXXFor = cast<CXXForRangeStmt>(Body); |
| 160 | if (const Stmt *Init = CXXFor->getInit()) |
| 161 | CGF.EmitStmt(Init); |
| 162 | CGF.EmitStmt(CXXFor->getRangeStmt()); |
| 163 | CGF.EmitStmt(CXXFor->getEndStmt()); |
| 164 | Body = CXXFor->getBody(); |
| 165 | } |
| 166 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 167 | if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) { |
George Burgess IV | 00f70bd | 2018-03-01 05:43:23 +0000 | [diff] [blame] | 168 | for (const auto *I : PreInits->decls()) |
| 169 | CGF.EmitVarDecl(cast<VarDecl>(*I)); |
Alexey Bataev | 5a3af13 | 2016-03-29 08:58:54 +0000 | [diff] [blame] | 170 | } |
Alexey Bataev | ab4ea22 | 2018-03-07 18:17:06 +0000 | [diff] [blame] | 171 | PreCondVars.restore(CGF); |
Alexey Bataev | 5a3af13 | 2016-03-29 08:58:54 +0000 | [diff] [blame] | 172 | } |
| 173 | |
| 174 | public: |
| 175 | OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S) |
| 176 | : CodeGenFunction::RunCleanupsScope(CGF) { |
| 177 | emitPreInitStmt(CGF, S); |
| 178 | } |
| 179 | }; |
| 180 | |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 181 | class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope { |
| 182 | CodeGenFunction::OMPPrivateScope InlinedShareds; |
| 183 | |
| 184 | static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { |
| 185 | return CGF.LambdaCaptureFields.lookup(VD) || |
| 186 | (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || |
| 187 | (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && |
| 188 | cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); |
| 189 | } |
| 190 | |
| 191 | public: |
| 192 | OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) |
| 193 | : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), |
| 194 | InlinedShareds(CGF) { |
| 195 | for (const auto *C : S.clauses()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 196 | if (const auto *CPI = OMPClauseWithPreInit::get(C)) { |
| 197 | if (const auto *PreInit = |
| 198 | cast_or_null<DeclStmt>(CPI->getPreInitStmt())) { |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 199 | for (const auto *I : PreInit->decls()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 200 | if (!I->hasAttr<OMPCaptureNoInitAttr>()) { |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 201 | CGF.EmitVarDecl(cast<VarDecl>(*I)); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 202 | } else { |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 203 | CodeGenFunction::AutoVarEmission Emission = |
| 204 | CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); |
| 205 | CGF.EmitAutoVarCleanups(Emission); |
| 206 | } |
| 207 | } |
| 208 | } |
| 209 | } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) { |
| 210 | for (const Expr *E : UDP->varlists()) { |
| 211 | const Decl *D = cast<DeclRefExpr>(E)->getDecl(); |
| 212 | if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D)) |
| 213 | CGF.EmitVarDecl(*OED); |
| 214 | } |
| 215 | } |
| 216 | } |
| 217 | if (!isOpenMPSimdDirective(S.getDirectiveKind())) |
| 218 | CGF.EmitOMPPrivateClause(S, InlinedShareds); |
| 219 | if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) { |
| 220 | if (const Expr *E = TG->getReductionRef()) |
| 221 | CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl())); |
| 222 | } |
| 223 | const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt()); |
| 224 | while (CS) { |
| 225 | for (auto &C : CS->captures()) { |
| 226 | if (C.capturesVariable() || C.capturesVariableByCopy()) { |
| 227 | auto *VD = C.getCapturedVar(); |
| 228 | assert(VD == VD->getCanonicalDecl() && |
| 229 | "Canonical decl must be captured."); |
Bruno Ricci | 5fc4db7 | 2018-12-21 14:10:18 +0000 | [diff] [blame] | 230 | DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 231 | isCapturedVar(CGF, VD) || |
| 232 | (CGF.CapturedStmtInfo && |
| 233 | InlinedShareds.isGlobalVarCaptured(VD)), |
| 234 | VD->getType().getNonReferenceType(), VK_LValue, |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 235 | C.getLocation()); |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 236 | InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 237 | return CGF.EmitLValue(&DRE).getAddress(CGF); |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 238 | }); |
| 239 | } |
| 240 | } |
| 241 | CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt()); |
| 242 | } |
| 243 | (void)InlinedShareds.Privatize(); |
| 244 | } |
| 245 | }; |
| 246 | |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 247 | } // namespace |
| 248 | |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 249 | static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, |
| 250 | const OMPExecutableDirective &S, |
| 251 | const RegionCodeGenTy &CodeGen); |
| 252 | |
Alexey Bataev | f47c4b4 | 2017-09-26 13:47:31 +0000 | [diff] [blame] | 253 | LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 254 | if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) { |
| 255 | if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) { |
Alexey Bataev | f47c4b4 | 2017-09-26 13:47:31 +0000 | [diff] [blame] | 256 | OrigVD = OrigVD->getCanonicalDecl(); |
| 257 | bool IsCaptured = |
| 258 | LambdaCaptureFields.lookup(OrigVD) || |
| 259 | (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) || |
| 260 | (CurCodeDecl && isa<BlockDecl>(CurCodeDecl)); |
Bruno Ricci | 5fc4db7 | 2018-12-21 14:10:18 +0000 | [diff] [blame] | 261 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured, |
Alexey Bataev | f47c4b4 | 2017-09-26 13:47:31 +0000 | [diff] [blame] | 262 | OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc()); |
| 263 | return EmitLValue(&DRE); |
| 264 | } |
| 265 | } |
| 266 | return EmitLValue(E); |
| 267 | } |
| 268 | |
Alexey Bataev | 1189bd0 | 2016-01-26 12:20:39 +0000 | [diff] [blame] | 269 | llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 270 | ASTContext &C = getContext(); |
Alexey Bataev | 1189bd0 | 2016-01-26 12:20:39 +0000 | [diff] [blame] | 271 | llvm::Value *Size = nullptr; |
| 272 | auto SizeInChars = C.getTypeSizeInChars(Ty); |
| 273 | if (SizeInChars.isZero()) { |
| 274 | // getTypeSizeInChars() returns 0 for a VLA. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 275 | while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) { |
| 276 | VlaSizePair VlaSize = getVLASize(VAT); |
Sander de Smalen | 891af03a | 2018-02-03 13:55:59 +0000 | [diff] [blame] | 277 | Ty = VlaSize.Type; |
| 278 | Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) |
| 279 | : VlaSize.NumElts; |
Alexey Bataev | 1189bd0 | 2016-01-26 12:20:39 +0000 | [diff] [blame] | 280 | } |
| 281 | SizeInChars = C.getTypeSizeInChars(Ty); |
| 282 | if (SizeInChars.isZero()) |
| 283 | return llvm::ConstantInt::get(SizeTy, /*V=*/0); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 284 | return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars)); |
| 285 | } |
| 286 | return CGM.getSize(SizeInChars); |
Alexey Bataev | 1189bd0 | 2016-01-26 12:20:39 +0000 | [diff] [blame] | 287 | } |
| 288 | |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 289 | void CodeGenFunction::GenerateOpenMPCapturedVars( |
Samuel Antao | 4af1b7b | 2015-12-02 17:44:43 +0000 | [diff] [blame] | 290 | const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) { |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 291 | const RecordDecl *RD = S.getCapturedRecordDecl(); |
| 292 | auto CurField = RD->field_begin(); |
| 293 | auto CurCap = S.captures().begin(); |
| 294 | for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(), |
| 295 | E = S.capture_init_end(); |
| 296 | I != E; ++I, ++CurField, ++CurCap) { |
| 297 | if (CurField->hasCapturedVLAType()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 298 | const VariableArrayType *VAT = CurField->getCapturedVLAType(); |
| 299 | llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()]; |
Samuel Antao | bed3c46 | 2015-10-02 16:14:20 +0000 | [diff] [blame] | 300 | CapturedVars.push_back(Val); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 301 | } else if (CurCap->capturesThis()) { |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 302 | CapturedVars.push_back(CXXThisValue); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 303 | } else if (CurCap->capturesVariableByCopy()) { |
Alexey Bataev | 1e49137 | 2018-01-23 18:44:14 +0000 | [diff] [blame] | 304 | llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation()); |
Samuel Antao | 6d00426 | 2016-06-16 18:39:34 +0000 | [diff] [blame] | 305 | |
| 306 | // If the field is not a pointer, we need to save the actual value |
| 307 | // and load it as a void pointer. |
| 308 | if (!CurField->getType()->isAnyPointerType()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 309 | ASTContext &Ctx = getContext(); |
| 310 | Address DstAddr = CreateMemTemp( |
Samuel Antao | 6d00426 | 2016-06-16 18:39:34 +0000 | [diff] [blame] | 311 | Ctx.getUIntPtrType(), |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 312 | Twine(CurCap->getCapturedVar()->getName(), ".casted")); |
Samuel Antao | 6d00426 | 2016-06-16 18:39:34 +0000 | [diff] [blame] | 313 | LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType()); |
| 314 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 315 | llvm::Value *SrcAddrVal = EmitScalarConversion( |
Samuel Antao | 6d00426 | 2016-06-16 18:39:34 +0000 | [diff] [blame] | 316 | DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()), |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 317 | Ctx.getPointerType(CurField->getType()), CurCap->getLocation()); |
Samuel Antao | 6d00426 | 2016-06-16 18:39:34 +0000 | [diff] [blame] | 318 | LValue SrcLV = |
| 319 | MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType()); |
| 320 | |
| 321 | // Store the value using the source type pointer. |
| 322 | EmitStoreThroughLValue(RValue::get(CV), SrcLV); |
| 323 | |
| 324 | // Load the value using the destination type pointer. |
Alexey Bataev | 1e49137 | 2018-01-23 18:44:14 +0000 | [diff] [blame] | 325 | CV = EmitLoadOfScalar(DstLV, CurCap->getLocation()); |
Samuel Antao | 6d00426 | 2016-06-16 18:39:34 +0000 | [diff] [blame] | 326 | } |
| 327 | CapturedVars.push_back(CV); |
| 328 | } else { |
Samuel Antao | 4af1b7b | 2015-12-02 17:44:43 +0000 | [diff] [blame] | 329 | assert(CurCap->capturesVariable() && "Expected capture by reference."); |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 330 | CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer()); |
Samuel Antao | 4af1b7b | 2015-12-02 17:44:43 +0000 | [diff] [blame] | 331 | } |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 332 | } |
| 333 | } |
| 334 | |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 335 | static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, |
| 336 | QualType DstType, StringRef Name, |
Alexey Bataev | 06e80f6 | 2019-05-23 18:19:54 +0000 | [diff] [blame] | 337 | LValue AddrLV) { |
Samuel Antao | 4af1b7b | 2015-12-02 17:44:43 +0000 | [diff] [blame] | 338 | ASTContext &Ctx = CGF.getContext(); |
| 339 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 340 | llvm::Value *CastedPtr = CGF.EmitScalarConversion( |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 341 | AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 342 | Ctx.getPointerType(DstType), Loc); |
| 343 | Address TmpAddr = |
Samuel Antao | 4af1b7b | 2015-12-02 17:44:43 +0000 | [diff] [blame] | 344 | CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType)) |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 345 | .getAddress(CGF); |
Samuel Antao | 4af1b7b | 2015-12-02 17:44:43 +0000 | [diff] [blame] | 346 | return TmpAddr; |
| 347 | } |
| 348 | |
Alexey Bataev | f7ce166 | 2017-04-10 19:16:45 +0000 | [diff] [blame] | 349 | static QualType getCanonicalParamType(ASTContext &C, QualType T) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 350 | if (T->isLValueReferenceType()) |
Alexey Bataev | f7ce166 | 2017-04-10 19:16:45 +0000 | [diff] [blame] | 351 | return C.getLValueReferenceType( |
| 352 | getCanonicalParamType(C, T.getNonReferenceType()), |
| 353 | /*SpelledAsLValue=*/false); |
Alexey Bataev | f7ce166 | 2017-04-10 19:16:45 +0000 | [diff] [blame] | 354 | if (T->isPointerType()) |
| 355 | return C.getPointerType(getCanonicalParamType(C, T->getPointeeType())); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 356 | if (const ArrayType *A = T->getAsArrayTypeUnsafe()) { |
| 357 | if (const auto *VLA = dyn_cast<VariableArrayType>(A)) |
Alexey Bataev | 1b48c5e | 2017-10-24 19:52:31 +0000 | [diff] [blame] | 358 | return getCanonicalParamType(C, VLA->getElementType()); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 359 | if (!A->isVariablyModifiedType()) |
Alexey Bataev | 1b48c5e | 2017-10-24 19:52:31 +0000 | [diff] [blame] | 360 | return C.getCanonicalType(T); |
| 361 | } |
Alexey Bataev | f7ce166 | 2017-04-10 19:16:45 +0000 | [diff] [blame] | 362 | return C.getCanonicalParamType(T); |
| 363 | } |
| 364 | |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 365 | namespace { |
| 366 | /// Contains required data for proper outlined function codegen. |
| 367 | struct FunctionOptions { |
| 368 | /// Captured statement for which the function is generated. |
| 369 | const CapturedStmt *S = nullptr; |
| 370 | /// true if cast to/from UIntPtr is required for variables captured by |
| 371 | /// value. |
Alexey Bataev | 3b8d558 | 2017-08-08 18:04:06 +0000 | [diff] [blame] | 372 | const bool UIntPtrCastRequired = true; |
Alexey Bataev | e754b18 | 2017-08-09 19:38:53 +0000 | [diff] [blame] | 373 | /// true if only casted arguments must be registered as local args or VLA |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 374 | /// sizes. |
Alexey Bataev | 3b8d558 | 2017-08-08 18:04:06 +0000 | [diff] [blame] | 375 | const bool RegisterCastedArgsOnly = false; |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 376 | /// Name of the generated function. |
Alexey Bataev | 3b8d558 | 2017-08-08 18:04:06 +0000 | [diff] [blame] | 377 | const StringRef FunctionName; |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 378 | explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired, |
| 379 | bool RegisterCastedArgsOnly, |
Alexey Bataev | 4aa1905 | 2017-08-08 16:45:36 +0000 | [diff] [blame] | 380 | StringRef FunctionName) |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 381 | : S(S), UIntPtrCastRequired(UIntPtrCastRequired), |
| 382 | RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly), |
Alexey Bataev | 4aa1905 | 2017-08-08 16:45:36 +0000 | [diff] [blame] | 383 | FunctionName(FunctionName) {} |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 384 | }; |
| 385 | } |
| 386 | |
Alexey Bataev | e754b18 | 2017-08-09 19:38:53 +0000 | [diff] [blame] | 387 | static llvm::Function *emitOutlinedFunctionPrologue( |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 388 | CodeGenFunction &CGF, FunctionArgList &Args, |
Alexey Bataev | 3b8d558 | 2017-08-08 18:04:06 +0000 | [diff] [blame] | 389 | llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 390 | &LocalAddrs, |
| 391 | llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> |
| 392 | &VLASizes, |
| 393 | llvm::Value *&CXXThisValue, const FunctionOptions &FO) { |
| 394 | const CapturedDecl *CD = FO.S->getCapturedDecl(); |
| 395 | const RecordDecl *RD = FO.S->getCapturedRecordDecl(); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 396 | assert(CD->hasBody() && "missing CapturedDecl body"); |
| 397 | |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 398 | CXXThisValue = nullptr; |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 399 | // Build the argument list. |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 400 | CodeGenModule &CGM = CGF.CGM; |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 401 | ASTContext &Ctx = CGM.getContext(); |
Alexey Bataev | 3b8d558 | 2017-08-08 18:04:06 +0000 | [diff] [blame] | 402 | FunctionArgList TargetArgs; |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 403 | Args.append(CD->param_begin(), |
| 404 | std::next(CD->param_begin(), CD->getContextParamPosition())); |
Alexey Bataev | 3b8d558 | 2017-08-08 18:04:06 +0000 | [diff] [blame] | 405 | TargetArgs.append( |
| 406 | CD->param_begin(), |
| 407 | std::next(CD->param_begin(), CD->getContextParamPosition())); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 408 | auto I = FO.S->captures().begin(); |
Alexey Bataev | b45d43c | 2017-11-22 16:02:03 +0000 | [diff] [blame] | 409 | FunctionDecl *DebugFunctionDecl = nullptr; |
| 410 | if (!FO.UIntPtrCastRequired) { |
| 411 | FunctionProtoType::ExtProtoInfo EPI; |
Jonas Devlieghere | 64a2630 | 2018-11-11 00:56:15 +0000 | [diff] [blame] | 412 | QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI); |
Alexey Bataev | b45d43c | 2017-11-22 16:02:03 +0000 | [diff] [blame] | 413 | DebugFunctionDecl = FunctionDecl::Create( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 414 | Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(), |
Jonas Devlieghere | 64a2630 | 2018-11-11 00:56:15 +0000 | [diff] [blame] | 415 | SourceLocation(), DeclarationName(), FunctionTy, |
| 416 | Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static, |
| 417 | /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false); |
Alexey Bataev | b45d43c | 2017-11-22 16:02:03 +0000 | [diff] [blame] | 418 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 419 | for (const FieldDecl *FD : RD->fields()) { |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 420 | QualType ArgType = FD->getType(); |
| 421 | IdentifierInfo *II = nullptr; |
| 422 | VarDecl *CapVar = nullptr; |
Samuel Antao | 4af1b7b | 2015-12-02 17:44:43 +0000 | [diff] [blame] | 423 | |
| 424 | // If this is a capture by copy and the type is not a pointer, the outlined |
| 425 | // function argument type should be uintptr and the value properly casted to |
| 426 | // uintptr. This is necessary given that the runtime library is only able to |
| 427 | // deal with pointers. We can pass in the same way the VLA type sizes to the |
| 428 | // outlined function. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 429 | if (FO.UIntPtrCastRequired && |
| 430 | ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) || |
| 431 | I->capturesVariableArrayType())) |
| 432 | ArgType = Ctx.getUIntPtrType(); |
Samuel Antao | 4af1b7b | 2015-12-02 17:44:43 +0000 | [diff] [blame] | 433 | |
| 434 | if (I->capturesVariable() || I->capturesVariableByCopy()) { |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 435 | CapVar = I->getCapturedVar(); |
| 436 | II = CapVar->getIdentifier(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 437 | } else if (I->capturesThis()) { |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 438 | II = &Ctx.Idents.get("this"); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 439 | } else { |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 440 | assert(I->capturesVariableArrayType()); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 441 | II = &Ctx.Idents.get("vla"); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 442 | } |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 443 | if (ArgType->isVariablyModifiedType()) |
Alexey Bataev | 1b48c5e | 2017-10-24 19:52:31 +0000 | [diff] [blame] | 444 | ArgType = getCanonicalParamType(Ctx, ArgType); |
Alexey Bataev | b45d43c | 2017-11-22 16:02:03 +0000 | [diff] [blame] | 445 | VarDecl *Arg; |
| 446 | if (DebugFunctionDecl && (CapVar || I->capturesThis())) { |
| 447 | Arg = ParmVarDecl::Create( |
| 448 | Ctx, DebugFunctionDecl, |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 449 | CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(), |
Alexey Bataev | b45d43c | 2017-11-22 16:02:03 +0000 | [diff] [blame] | 450 | CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType, |
| 451 | /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr); |
| 452 | } else { |
| 453 | Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(), |
| 454 | II, ArgType, ImplicitParamDecl::Other); |
| 455 | } |
Alexey Bataev | 3b8d558 | 2017-08-08 18:04:06 +0000 | [diff] [blame] | 456 | Args.emplace_back(Arg); |
| 457 | // Do not cast arguments if we emit function with non-original types. |
| 458 | TargetArgs.emplace_back( |
| 459 | FO.UIntPtrCastRequired |
| 460 | ? Arg |
| 461 | : CGM.getOpenMPRuntime().translateParameter(FD, Arg)); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 462 | ++I; |
| 463 | } |
| 464 | Args.append( |
| 465 | std::next(CD->param_begin(), CD->getContextParamPosition() + 1), |
| 466 | CD->param_end()); |
Alexey Bataev | 3b8d558 | 2017-08-08 18:04:06 +0000 | [diff] [blame] | 467 | TargetArgs.append( |
| 468 | std::next(CD->param_begin(), CD->getContextParamPosition() + 1), |
| 469 | CD->param_end()); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 470 | |
| 471 | // Create the function declaration. |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 472 | const CGFunctionInfo &FuncInfo = |
Alexey Bataev | 3b8d558 | 2017-08-08 18:04:06 +0000 | [diff] [blame] | 473 | CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 474 | llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo); |
| 475 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 476 | auto *F = |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 477 | llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage, |
| 478 | FO.FunctionName, &CGM.getModule()); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 479 | CGM.SetInternalFunctionAttributes(CD, F, FuncInfo); |
| 480 | if (CD->isNothrow()) |
Alexey Bataev | 2c7eee5 | 2017-08-04 19:10:54 +0000 | [diff] [blame] | 481 | F->setDoesNotThrow(); |
Alexey Bataev | c0f879b | 2018-04-10 20:10:53 +0000 | [diff] [blame] | 482 | F->setDoesNotRecurse(); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 483 | |
| 484 | // Generate the function. |
Alexey Bataev | 6e01dc1 | 2017-08-14 16:03:47 +0000 | [diff] [blame] | 485 | CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 486 | FO.S->getBeginLoc(), CD->getBody()->getBeginLoc()); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 487 | unsigned Cnt = CD->getContextParamPosition(); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 488 | I = FO.S->captures().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 489 | for (const FieldDecl *FD : RD->fields()) { |
Alexey Bataev | 3b8d558 | 2017-08-08 18:04:06 +0000 | [diff] [blame] | 490 | // Do not map arguments if we emit function with non-original types. |
| 491 | Address LocalAddr(Address::invalid()); |
| 492 | if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) { |
| 493 | LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], |
| 494 | TargetArgs[Cnt]); |
| 495 | } else { |
| 496 | LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); |
| 497 | } |
Samuel Antao | 4af1b7b | 2015-12-02 17:44:43 +0000 | [diff] [blame] | 498 | // If we are capturing a pointer by copy we don't need to do anything, just |
| 499 | // use the value that we get from the arguments. |
| 500 | if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) { |
Samuel Antao | 403ffd4 | 2016-07-27 22:49:49 +0000 | [diff] [blame] | 501 | const VarDecl *CurVD = I->getCapturedVar(); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 502 | if (!FO.RegisterCastedArgsOnly) |
| 503 | LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}}); |
Richard Trieu | cc3949d | 2016-02-18 22:34:54 +0000 | [diff] [blame] | 504 | ++Cnt; |
| 505 | ++I; |
Samuel Antao | 4af1b7b | 2015-12-02 17:44:43 +0000 | [diff] [blame] | 506 | continue; |
| 507 | } |
| 508 | |
Ivan A. Kosarev | 5f8c0ca | 2017-10-10 09:39:32 +0000 | [diff] [blame] | 509 | LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), |
| 510 | AlignmentSource::Decl); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 511 | if (FD->hasCapturedVLAType()) { |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 512 | if (FO.UIntPtrCastRequired) { |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 513 | ArgLVal = CGF.MakeAddrLValue( |
| 514 | castValueFromUintptr(CGF, I->getLocation(), FD->getType(), |
| 515 | Args[Cnt]->getName(), ArgLVal), |
| 516 | FD->getType(), AlignmentSource::Decl); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 517 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 518 | llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); |
| 519 | const VariableArrayType *VAT = FD->getCapturedVLAType(); |
| 520 | VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 521 | } else if (I->capturesVariable()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 522 | const VarDecl *Var = I->getCapturedVar(); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 523 | QualType VarTy = Var->getType(); |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 524 | Address ArgAddr = ArgLVal.getAddress(CGF); |
Alexey Bataev | 06e80f6 | 2019-05-23 18:19:54 +0000 | [diff] [blame] | 525 | if (ArgLVal.getType()->isLValueReferenceType()) { |
| 526 | ArgAddr = CGF.EmitLoadOfReference(ArgLVal); |
| 527 | } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) { |
| 528 | assert(ArgLVal.getType()->isPointerType()); |
| 529 | ArgAddr = CGF.EmitLoadOfPointer( |
| 530 | ArgAddr, ArgLVal.getType()->castAs<PointerType>()); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 531 | } |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 532 | if (!FO.RegisterCastedArgsOnly) { |
| 533 | LocalAddrs.insert( |
| 534 | {Args[Cnt], |
| 535 | {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}}); |
| 536 | } |
Samuel Antao | 4af1b7b | 2015-12-02 17:44:43 +0000 | [diff] [blame] | 537 | } else if (I->capturesVariableByCopy()) { |
| 538 | assert(!FD->getType()->isAnyPointerType() && |
| 539 | "Not expecting a captured pointer."); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 540 | const VarDecl *Var = I->getCapturedVar(); |
Alexey Bataev | 06e80f6 | 2019-05-23 18:19:54 +0000 | [diff] [blame] | 541 | LocalAddrs.insert({Args[Cnt], |
| 542 | {Var, FO.UIntPtrCastRequired |
| 543 | ? castValueFromUintptr( |
| 544 | CGF, I->getLocation(), FD->getType(), |
| 545 | Args[Cnt]->getName(), ArgLVal) |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 546 | : ArgLVal.getAddress(CGF)}}); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 547 | } else { |
| 548 | // If 'this' is captured, load it into CXXThisValue. |
| 549 | assert(I->capturesThis()); |
Alexey Bataev | 1e49137 | 2018-01-23 18:44:14 +0000 | [diff] [blame] | 550 | CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 551 | LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}}); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 552 | } |
Richard Trieu | cc3949d | 2016-02-18 22:34:54 +0000 | [diff] [blame] | 553 | ++Cnt; |
| 554 | ++I; |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 555 | } |
| 556 | |
Alexey Bataev | e754b18 | 2017-08-09 19:38:53 +0000 | [diff] [blame] | 557 | return F; |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 558 | } |
| 559 | |
| 560 | llvm::Function * |
| 561 | CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) { |
| 562 | assert( |
| 563 | CapturedStmtInfo && |
| 564 | "CapturedStmtInfo should be set when generating the captured function"); |
| 565 | const CapturedDecl *CD = S.getCapturedDecl(); |
| 566 | // Build the argument list. |
| 567 | bool NeedWrapperFunction = |
| 568 | getDebugInfo() && |
| 569 | CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo; |
| 570 | FunctionArgList Args; |
Alexey Bataev | 3b8d558 | 2017-08-08 18:04:06 +0000 | [diff] [blame] | 571 | llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs; |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 572 | llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes; |
Alexey Bataev | e754b18 | 2017-08-09 19:38:53 +0000 | [diff] [blame] | 573 | SmallString<256> Buffer; |
| 574 | llvm::raw_svector_ostream Out(Buffer); |
| 575 | Out << CapturedStmtInfo->getHelperName(); |
| 576 | if (NeedWrapperFunction) |
| 577 | Out << "_debug__"; |
Alexey Bataev | 4aa1905 | 2017-08-08 16:45:36 +0000 | [diff] [blame] | 578 | FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false, |
Alexey Bataev | e754b18 | 2017-08-09 19:38:53 +0000 | [diff] [blame] | 579 | Out.str()); |
| 580 | llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs, |
| 581 | VLASizes, CXXThisValue, FO); |
Alexey Bataev | 06e80f6 | 2019-05-23 18:19:54 +0000 | [diff] [blame] | 582 | CodeGenFunction::OMPPrivateScope LocalScope(*this); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 583 | for (const auto &LocalAddrPair : LocalAddrs) { |
| 584 | if (LocalAddrPair.second.first) { |
Alexey Bataev | 06e80f6 | 2019-05-23 18:19:54 +0000 | [diff] [blame] | 585 | LocalScope.addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() { |
| 586 | return LocalAddrPair.second.second; |
| 587 | }); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 588 | } |
| 589 | } |
Alexey Bataev | 06e80f6 | 2019-05-23 18:19:54 +0000 | [diff] [blame] | 590 | (void)LocalScope.Privatize(); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 591 | for (const auto &VLASizePair : VLASizes) |
| 592 | VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second; |
Serge Pavlov | 3a56145 | 2015-12-06 14:32:39 +0000 | [diff] [blame] | 593 | PGO.assignRegionCounters(GlobalDecl(CD), F); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 594 | CapturedStmtInfo->EmitBody(*this, CD->getBody()); |
Alexey Bataev | 06e80f6 | 2019-05-23 18:19:54 +0000 | [diff] [blame] | 595 | (void)LocalScope.ForceCleanup(); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 596 | FinishFunction(CD->getBodyRBrace()); |
Alexey Bataev | e754b18 | 2017-08-09 19:38:53 +0000 | [diff] [blame] | 597 | if (!NeedWrapperFunction) |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 598 | return F; |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 599 | |
Alexey Bataev | efd884d | 2017-08-04 21:26:25 +0000 | [diff] [blame] | 600 | FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true, |
Alexey Bataev | e754b18 | 2017-08-09 19:38:53 +0000 | [diff] [blame] | 601 | /*RegisterCastedArgsOnly=*/true, |
| 602 | CapturedStmtInfo->getHelperName()); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 603 | CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true); |
Gheorghe-Teodor Bercea | d3dcf2f | 2018-03-14 14:17:45 +0000 | [diff] [blame] | 604 | WrapperCGF.CapturedStmtInfo = CapturedStmtInfo; |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 605 | Args.clear(); |
| 606 | LocalAddrs.clear(); |
| 607 | VLASizes.clear(); |
| 608 | llvm::Function *WrapperF = |
| 609 | emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes, |
Alexey Bataev | e754b18 | 2017-08-09 19:38:53 +0000 | [diff] [blame] | 610 | WrapperCGF.CXXThisValue, WrapperFO); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 611 | llvm::SmallVector<llvm::Value *, 4> CallArgs; |
| 612 | for (const auto *Arg : Args) { |
| 613 | llvm::Value *CallArg; |
| 614 | auto I = LocalAddrs.find(Arg); |
| 615 | if (I != LocalAddrs.end()) { |
Alexey Bataev | 7ba57af | 2017-10-17 16:47:34 +0000 | [diff] [blame] | 616 | LValue LV = WrapperCGF.MakeAddrLValue( |
| 617 | I->second.second, |
| 618 | I->second.first ? I->second.first->getType() : Arg->getType(), |
| 619 | AlignmentSource::Decl); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 620 | CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc()); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 621 | } else { |
| 622 | auto EI = VLASizes.find(Arg); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 623 | if (EI != VLASizes.end()) { |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 624 | CallArg = EI->second.second; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 625 | } else { |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 626 | LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg), |
Ivan A. Kosarev | 5f8c0ca | 2017-10-10 09:39:32 +0000 | [diff] [blame] | 627 | Arg->getType(), |
| 628 | AlignmentSource::Decl); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 629 | CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc()); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 630 | } |
| 631 | } |
Alexey Bataev | 7ba57af | 2017-10-17 16:47:34 +0000 | [diff] [blame] | 632 | CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType())); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 633 | } |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 634 | CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(), |
Alexey Bataev | 3c595a6 | 2017-08-14 15:01:03 +0000 | [diff] [blame] | 635 | F, CallArgs); |
Alexey Bataev | 1fdfdf7 | 2017-06-29 16:43:05 +0000 | [diff] [blame] | 636 | WrapperCGF.FinishFunction(); |
| 637 | return WrapperF; |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 638 | } |
| 639 | |
Alexey Bataev | 9959db5 | 2014-05-06 10:08:46 +0000 | [diff] [blame] | 640 | //===----------------------------------------------------------------------===// |
| 641 | // OpenMP Directive Emission |
| 642 | //===----------------------------------------------------------------------===// |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 643 | void CodeGenFunction::EmitOMPAggregateAssign( |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 644 | Address DestAddr, Address SrcAddr, QualType OriginalType, |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 645 | const llvm::function_ref<void(Address, Address)> CopyGen) { |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 646 | // Perform element-by-element initialization. |
| 647 | QualType ElementTy; |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 648 | |
| 649 | // Drill down to the base element type on both arrays. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 650 | const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe(); |
| 651 | llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr); |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 652 | SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); |
| 653 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 654 | llvm::Value *SrcBegin = SrcAddr.getPointer(); |
| 655 | llvm::Value *DestBegin = DestAddr.getPointer(); |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 656 | // Cast from pointer to array type to pointer to single element. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 657 | llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements); |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 658 | // The basic structure here is a while-do loop. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 659 | llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body"); |
| 660 | llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done"); |
| 661 | llvm::Value *IsEmpty = |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 662 | Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty"); |
| 663 | Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); |
Alexey Bataev | 4a5bb77 | 2014-10-08 14:01:46 +0000 | [diff] [blame] | 664 | |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 665 | // Enter the loop body, making that address the current address. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 666 | llvm::BasicBlock *EntryBB = Builder.GetInsertBlock(); |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 667 | EmitBlock(BodyBB); |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 668 | |
| 669 | CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy); |
| 670 | |
| 671 | llvm::PHINode *SrcElementPHI = |
| 672 | Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); |
| 673 | SrcElementPHI->addIncoming(SrcBegin, EntryBB); |
| 674 | Address SrcElementCurrent = |
| 675 | Address(SrcElementPHI, |
| 676 | SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize)); |
| 677 | |
| 678 | llvm::PHINode *DestElementPHI = |
| 679 | Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast"); |
| 680 | DestElementPHI->addIncoming(DestBegin, EntryBB); |
| 681 | Address DestElementCurrent = |
| 682 | Address(DestElementPHI, |
| 683 | DestAddr.getAlignment().alignmentOfArrayElement(ElementSize)); |
Alexey Bataev | 4a5bb77 | 2014-10-08 14:01:46 +0000 | [diff] [blame] | 684 | |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 685 | // Emit copy. |
| 686 | CopyGen(DestElementCurrent, SrcElementCurrent); |
| 687 | |
| 688 | // Shift the address forward by one element. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 689 | llvm::Value *DestElementNext = Builder.CreateConstGEP1_32( |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 690 | DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 691 | llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32( |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 692 | SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element"); |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 693 | // Check whether we've reached the end. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 694 | llvm::Value *Done = |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 695 | Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); |
| 696 | Builder.CreateCondBr(Done, DoneBB, BodyBB); |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 697 | DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock()); |
| 698 | SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock()); |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 699 | |
| 700 | // Done. |
| 701 | EmitBlock(DoneBB, /*IsFinished=*/true); |
| 702 | } |
| 703 | |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 704 | void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr, |
| 705 | Address SrcAddr, const VarDecl *DestVD, |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 706 | const VarDecl *SrcVD, const Expr *Copy) { |
| 707 | if (OriginalType->isArrayType()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 708 | const auto *BO = dyn_cast<BinaryOperator>(Copy); |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 709 | if (BO && BO->getOpcode() == BO_Assign) { |
| 710 | // Perform simple memcpy for simple copying. |
Ivan A. Kosarev | 1860b52 | 2018-01-25 14:21:55 +0000 | [diff] [blame] | 711 | LValue Dest = MakeAddrLValue(DestAddr, OriginalType); |
| 712 | LValue Src = MakeAddrLValue(SrcAddr, OriginalType); |
| 713 | EmitAggregateAssign(Dest, Src, OriginalType); |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 714 | } else { |
| 715 | // For arrays with complex element types perform element by element |
| 716 | // copying. |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 717 | EmitOMPAggregateAssign( |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 718 | DestAddr, SrcAddr, OriginalType, |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 719 | [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) { |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 720 | // Working with the single array element, so have to remap |
| 721 | // destination and source variables to corresponding array |
| 722 | // elements. |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 723 | CodeGenFunction::OMPPrivateScope Remap(*this); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 724 | Remap.addPrivate(DestVD, [DestElement]() { return DestElement; }); |
| 725 | Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; }); |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 726 | (void)Remap.Privatize(); |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 727 | EmitIgnoredExpr(Copy); |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 728 | }); |
Alexey Bataev | 4a5bb77 | 2014-10-08 14:01:46 +0000 | [diff] [blame] | 729 | } |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 730 | } else { |
| 731 | // Remap pseudo source variable to private copy. |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 732 | CodeGenFunction::OMPPrivateScope Remap(*this); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 733 | Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; }); |
| 734 | Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; }); |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 735 | (void)Remap.Privatize(); |
| 736 | // Emit copying of the whole variable. |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 737 | EmitIgnoredExpr(Copy); |
Alexey Bataev | 4a5bb77 | 2014-10-08 14:01:46 +0000 | [diff] [blame] | 738 | } |
Alexey Bataev | 4a5bb77 | 2014-10-08 14:01:46 +0000 | [diff] [blame] | 739 | } |
| 740 | |
Alexey Bataev | 69c62a9 | 2015-04-15 04:52:20 +0000 | [diff] [blame] | 741 | bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, |
| 742 | OMPPrivateScope &PrivateScope) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 743 | if (!HaveInsertPoint()) |
| 744 | return false; |
Alexey Bataev | 1af5bd5 | 2019-03-05 17:47:18 +0000 | [diff] [blame] | 745 | bool DeviceConstTarget = |
| 746 | getLangOpts().OpenMPIsDevice && |
| 747 | isOpenMPTargetExecutionDirective(D.getDirectiveKind()); |
Alexey Bataev | cd8b6a2 | 2016-02-15 08:07:17 +0000 | [diff] [blame] | 748 | bool FirstprivateIsLastprivate = false; |
| 749 | llvm::DenseSet<const VarDecl *> Lastprivates; |
| 750 | for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) { |
| 751 | for (const auto *D : C->varlists()) |
| 752 | Lastprivates.insert( |
| 753 | cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl()); |
| 754 | } |
Alexey Bataev | 69c62a9 | 2015-04-15 04:52:20 +0000 | [diff] [blame] | 755 | llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 756 | llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions; |
| 757 | getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind()); |
| 758 | // Force emission of the firstprivate copy if the directive does not emit |
| 759 | // outlined function, like omp for, omp simd, omp distribute etc. |
| 760 | bool MustEmitFirstprivateCopy = |
| 761 | CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown; |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 762 | for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) { |
Alexey Bataev | 4a5bb77 | 2014-10-08 14:01:46 +0000 | [diff] [blame] | 763 | auto IRef = C->varlist_begin(); |
| 764 | auto InitsRef = C->inits().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 765 | for (const Expr *IInit : C->private_copies()) { |
| 766 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); |
Alexey Bataev | 7ace49d | 2016-05-17 08:55:33 +0000 | [diff] [blame] | 767 | bool ThisFirstprivateIsLastprivate = |
| 768 | Lastprivates.count(OrigVD->getCanonicalDecl()) > 0; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 769 | const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD); |
Alexey Bataev | 9c39781 | 2019-04-03 17:57:06 +0000 | [diff] [blame] | 770 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 771 | if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD && |
Alexey Bataev | 9c39781 | 2019-04-03 17:57:06 +0000 | [diff] [blame] | 772 | !FD->getType()->isReferenceType() && |
| 773 | (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) { |
Alexey Bataev | 7ace49d | 2016-05-17 08:55:33 +0000 | [diff] [blame] | 774 | EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()); |
| 775 | ++IRef; |
| 776 | ++InitsRef; |
| 777 | continue; |
| 778 | } |
Alexey Bataev | 1af5bd5 | 2019-03-05 17:47:18 +0000 | [diff] [blame] | 779 | // Do not emit copy for firstprivate constant variables in target regions, |
| 780 | // captured by reference. |
| 781 | if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) && |
Alexey Bataev | 9c39781 | 2019-04-03 17:57:06 +0000 | [diff] [blame] | 782 | FD && FD->getType()->isReferenceType() && |
| 783 | (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) { |
Alexey Bataev | 1af5bd5 | 2019-03-05 17:47:18 +0000 | [diff] [blame] | 784 | (void)CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(*this, |
| 785 | OrigVD); |
| 786 | ++IRef; |
| 787 | ++InitsRef; |
| 788 | continue; |
| 789 | } |
Alexey Bataev | cd8b6a2 | 2016-02-15 08:07:17 +0000 | [diff] [blame] | 790 | FirstprivateIsLastprivate = |
Alexey Bataev | 7ace49d | 2016-05-17 08:55:33 +0000 | [diff] [blame] | 791 | FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate; |
Alexey Bataev | cd8b6a2 | 2016-02-15 08:07:17 +0000 | [diff] [blame] | 792 | if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 793 | const auto *VDInit = |
| 794 | cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl()); |
Alexey Bataev | 69c62a9 | 2015-04-15 04:52:20 +0000 | [diff] [blame] | 795 | bool IsRegistered; |
Bruno Ricci | 5fc4db7 | 2018-12-21 14:10:18 +0000 | [diff] [blame] | 796 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), |
Alexey Bataev | 7ace49d | 2016-05-17 08:55:33 +0000 | [diff] [blame] | 797 | /*RefersToEnclosingVariableOrCapture=*/FD != nullptr, |
| 798 | (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); |
Alexey Bataev | e0ef04f | 2019-05-23 22:30:43 +0000 | [diff] [blame] | 799 | LValue OriginalLVal; |
| 800 | if (!FD) { |
| 801 | // Check if the firstprivate variable is just a constant value. |
| 802 | ConstantEmission CE = tryEmitAsConstant(&DRE); |
| 803 | if (CE && !CE.isReference()) { |
| 804 | // Constant value, no need to create a copy. |
| 805 | ++IRef; |
| 806 | ++InitsRef; |
| 807 | continue; |
| 808 | } |
| 809 | if (CE && CE.isReference()) { |
| 810 | OriginalLVal = CE.getReferenceLValue(*this, &DRE); |
| 811 | } else { |
| 812 | assert(!CE && "Expected non-constant firstprivate."); |
| 813 | OriginalLVal = EmitLValue(&DRE); |
| 814 | } |
| 815 | } else { |
| 816 | OriginalLVal = EmitLValue(&DRE); |
| 817 | } |
Alexey Bataev | feddd64 | 2016-04-22 09:05:03 +0000 | [diff] [blame] | 818 | QualType Type = VD->getType(); |
Alexey Bataev | 1d9c15c | 2015-05-19 12:31:28 +0000 | [diff] [blame] | 819 | if (Type->isArrayType()) { |
Alexey Bataev | 69c62a9 | 2015-04-15 04:52:20 +0000 | [diff] [blame] | 820 | // Emit VarDecl with copy init for arrays. |
| 821 | // Get the address of the original variable captured in current |
| 822 | // captured region. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 823 | IsRegistered = PrivateScope.addPrivate( |
| 824 | OrigVD, [this, VD, Type, OriginalLVal, VDInit]() { |
| 825 | AutoVarEmission Emission = EmitAutoVarAlloca(*VD); |
| 826 | const Expr *Init = VD->getInit(); |
| 827 | if (!isa<CXXConstructExpr>(Init) || |
| 828 | isTrivialInitializer(Init)) { |
| 829 | // Perform simple memcpy. |
| 830 | LValue Dest = |
| 831 | MakeAddrLValue(Emission.getAllocatedAddress(), Type); |
| 832 | EmitAggregateAssign(Dest, OriginalLVal, Type); |
| 833 | } else { |
| 834 | EmitOMPAggregateAssign( |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 835 | Emission.getAllocatedAddress(), |
| 836 | OriginalLVal.getAddress(*this), Type, |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 837 | [this, VDInit, Init](Address DestElement, |
| 838 | Address SrcElement) { |
| 839 | // Clean up any temporaries needed by the |
| 840 | // initialization. |
| 841 | RunCleanupsScope InitScope(*this); |
| 842 | // Emit initialization for single element. |
| 843 | setAddrOfLocalVar(VDInit, SrcElement); |
| 844 | EmitAnyExprToMem(Init, DestElement, |
| 845 | Init->getType().getQualifiers(), |
| 846 | /*IsInitializer*/ false); |
| 847 | LocalDeclMap.erase(VDInit); |
| 848 | }); |
| 849 | } |
| 850 | EmitAutoVarCleanups(Emission); |
| 851 | return Emission.getAllocatedAddress(); |
| 852 | }); |
Alexey Bataev | 69c62a9 | 2015-04-15 04:52:20 +0000 | [diff] [blame] | 853 | } else { |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 854 | Address OriginalAddr = OriginalLVal.getAddress(*this); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 855 | IsRegistered = PrivateScope.addPrivate( |
| 856 | OrigVD, [this, VDInit, OriginalAddr, VD]() { |
| 857 | // Emit private VarDecl with copy init. |
| 858 | // Remap temp VDInit variable to the address of the original |
| 859 | // variable (for proper handling of captured global variables). |
| 860 | setAddrOfLocalVar(VDInit, OriginalAddr); |
| 861 | EmitDecl(*VD); |
| 862 | LocalDeclMap.erase(VDInit); |
| 863 | return GetAddrOfLocalVar(VD); |
| 864 | }); |
Alexey Bataev | 69c62a9 | 2015-04-15 04:52:20 +0000 | [diff] [blame] | 865 | } |
| 866 | assert(IsRegistered && |
| 867 | "firstprivate var already registered as private"); |
| 868 | // Silence the warning about unused variable. |
| 869 | (void)IsRegistered; |
| 870 | } |
Richard Trieu | cc3949d | 2016-02-18 22:34:54 +0000 | [diff] [blame] | 871 | ++IRef; |
| 872 | ++InitsRef; |
Alexey Bataev | 4a5bb77 | 2014-10-08 14:01:46 +0000 | [diff] [blame] | 873 | } |
| 874 | } |
Alexey Bataev | cd8b6a2 | 2016-02-15 08:07:17 +0000 | [diff] [blame] | 875 | return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty(); |
Alexey Bataev | 4a5bb77 | 2014-10-08 14:01:46 +0000 | [diff] [blame] | 876 | } |
| 877 | |
Alexey Bataev | 03b340a | 2014-10-21 03:16:40 +0000 | [diff] [blame] | 878 | void CodeGenFunction::EmitOMPPrivateClause( |
| 879 | const OMPExecutableDirective &D, |
| 880 | CodeGenFunction::OMPPrivateScope &PrivateScope) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 881 | if (!HaveInsertPoint()) |
| 882 | return; |
Alexey Bataev | 50a6458 | 2015-04-22 12:24:45 +0000 | [diff] [blame] | 883 | llvm::DenseSet<const VarDecl *> EmittedAsPrivate; |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 884 | for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) { |
Alexey Bataev | 03b340a | 2014-10-21 03:16:40 +0000 | [diff] [blame] | 885 | auto IRef = C->varlist_begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 886 | for (const Expr *IInit : C->private_copies()) { |
| 887 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); |
Alexey Bataev | 50a6458 | 2015-04-22 12:24:45 +0000 | [diff] [blame] | 888 | if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 889 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); |
| 890 | bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() { |
| 891 | // Emit private VarDecl with copy init. |
| 892 | EmitDecl(*VD); |
| 893 | return GetAddrOfLocalVar(VD); |
| 894 | }); |
Alexey Bataev | 50a6458 | 2015-04-22 12:24:45 +0000 | [diff] [blame] | 895 | assert(IsRegistered && "private var already registered as private"); |
| 896 | // Silence the warning about unused variable. |
| 897 | (void)IsRegistered; |
| 898 | } |
Alexey Bataev | 03b340a | 2014-10-21 03:16:40 +0000 | [diff] [blame] | 899 | ++IRef; |
| 900 | } |
| 901 | } |
| 902 | } |
| 903 | |
Alexey Bataev | f56f98c | 2015-04-16 05:39:01 +0000 | [diff] [blame] | 904 | bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 905 | if (!HaveInsertPoint()) |
| 906 | return false; |
Alexey Bataev | f56f98c | 2015-04-16 05:39:01 +0000 | [diff] [blame] | 907 | // threadprivate_var1 = master_threadprivate_var1; |
| 908 | // operator=(threadprivate_var2, master_threadprivate_var2); |
| 909 | // ... |
| 910 | // __kmpc_barrier(&loc, global_tid); |
Alexey Bataev | f56f98c | 2015-04-16 05:39:01 +0000 | [diff] [blame] | 911 | llvm::DenseSet<const VarDecl *> CopiedVars; |
| 912 | llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr; |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 913 | for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) { |
Alexey Bataev | f56f98c | 2015-04-16 05:39:01 +0000 | [diff] [blame] | 914 | auto IRef = C->varlist_begin(); |
| 915 | auto ISrcRef = C->source_exprs().begin(); |
| 916 | auto IDestRef = C->destination_exprs().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 917 | for (const Expr *AssignOp : C->assignment_ops()) { |
| 918 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); |
Alexey Bataev | 1d9c15c | 2015-05-19 12:31:28 +0000 | [diff] [blame] | 919 | QualType Type = VD->getType(); |
Alexey Bataev | f56f98c | 2015-04-16 05:39:01 +0000 | [diff] [blame] | 920 | if (CopiedVars.insert(VD->getCanonicalDecl()).second) { |
Samuel Antao | 9c75cfe | 2015-07-27 16:38:06 +0000 | [diff] [blame] | 921 | // Get the address of the master variable. If we are emitting code with |
| 922 | // TLS support, the address is passed from the master as field in the |
| 923 | // captured declaration. |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 924 | Address MasterAddr = Address::invalid(); |
Samuel Antao | 9c75cfe | 2015-07-27 16:38:06 +0000 | [diff] [blame] | 925 | if (getLangOpts().OpenMPUseTLS && |
| 926 | getContext().getTargetInfo().isTLSSupported()) { |
| 927 | assert(CapturedStmtInfo->lookup(VD) && |
| 928 | "Copyin threadprivates should have been captured!"); |
Bruno Ricci | 5fc4db7 | 2018-12-21 14:10:18 +0000 | [diff] [blame] | 929 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true, |
| 930 | (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 931 | MasterAddr = EmitLValue(&DRE).getAddress(*this); |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 932 | LocalDeclMap.erase(VD); |
Samuel Antao | 9c75cfe | 2015-07-27 16:38:06 +0000 | [diff] [blame] | 933 | } else { |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 934 | MasterAddr = |
| 935 | Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD) |
| 936 | : CGM.GetAddrOfGlobal(VD), |
| 937 | getContext().getDeclAlign(VD)); |
Samuel Antao | 9c75cfe | 2015-07-27 16:38:06 +0000 | [diff] [blame] | 938 | } |
Alexey Bataev | f56f98c | 2015-04-16 05:39:01 +0000 | [diff] [blame] | 939 | // Get the address of the threadprivate variable. |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 940 | Address PrivateAddr = EmitLValue(*IRef).getAddress(*this); |
Alexey Bataev | f56f98c | 2015-04-16 05:39:01 +0000 | [diff] [blame] | 941 | if (CopiedVars.size() == 1) { |
| 942 | // At first check if current thread is a master thread. If it is, no |
| 943 | // need to copy data. |
| 944 | CopyBegin = createBasicBlock("copyin.not.master"); |
| 945 | CopyEnd = createBasicBlock("copyin.not.master.end"); |
| 946 | Builder.CreateCondBr( |
| 947 | Builder.CreateICmpNE( |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 948 | Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy), |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 949 | Builder.CreatePtrToInt(PrivateAddr.getPointer(), |
| 950 | CGM.IntPtrTy)), |
Alexey Bataev | f56f98c | 2015-04-16 05:39:01 +0000 | [diff] [blame] | 951 | CopyBegin, CopyEnd); |
| 952 | EmitBlock(CopyBegin); |
| 953 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 954 | const auto *SrcVD = |
| 955 | cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); |
| 956 | const auto *DestVD = |
| 957 | cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 958 | EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp); |
Alexey Bataev | f56f98c | 2015-04-16 05:39:01 +0000 | [diff] [blame] | 959 | } |
| 960 | ++IRef; |
| 961 | ++ISrcRef; |
| 962 | ++IDestRef; |
| 963 | } |
| 964 | } |
| 965 | if (CopyEnd) { |
| 966 | // Exit out of copying procedure for non-master thread. |
| 967 | EmitBlock(CopyEnd, /*IsFinished=*/true); |
| 968 | return true; |
| 969 | } |
| 970 | return false; |
| 971 | } |
| 972 | |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 973 | bool CodeGenFunction::EmitOMPLastprivateClauseInit( |
| 974 | const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 975 | if (!HaveInsertPoint()) |
| 976 | return false; |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 977 | bool HasAtLeastOneLastprivate = false; |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 978 | llvm::DenseSet<const VarDecl *> SIMDLCVs; |
| 979 | if (isOpenMPSimdDirective(D.getDirectiveKind())) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 980 | const auto *LoopDirective = cast<OMPLoopDirective>(&D); |
| 981 | for (const Expr *C : LoopDirective->counters()) { |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 982 | SIMDLCVs.insert( |
| 983 | cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl()); |
| 984 | } |
| 985 | } |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 986 | llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 987 | for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) { |
Alexey Bataev | d130fd1 | 2015-05-13 10:23:02 +0000 | [diff] [blame] | 988 | HasAtLeastOneLastprivate = true; |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 989 | if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) && |
| 990 | !getLangOpts().OpenMPSimd) |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 991 | break; |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 992 | auto IRef = C->varlist_begin(); |
| 993 | auto IDestRef = C->destination_exprs().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 994 | for (const Expr *IInit : C->private_copies()) { |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 995 | // Keep the address of the original variable for future update at the end |
| 996 | // of the loop. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 997 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 998 | // Taskloops do not require additional initialization, it is done in |
| 999 | // runtime support library. |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 1000 | if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1001 | const auto *DestVD = |
| 1002 | cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); |
| 1003 | PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() { |
Bruno Ricci | 5fc4db7 | 2018-12-21 14:10:18 +0000 | [diff] [blame] | 1004 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), |
| 1005 | /*RefersToEnclosingVariableOrCapture=*/ |
| 1006 | CapturedStmtInfo->lookup(OrigVD) != nullptr, |
| 1007 | (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 1008 | return EmitLValue(&DRE).getAddress(*this); |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 1009 | }); |
| 1010 | // Check if the variable is also a firstprivate: in this case IInit is |
| 1011 | // not generated. Initialization of this variable will happen in codegen |
| 1012 | // for 'firstprivate' clause. |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1013 | if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1014 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); |
| 1015 | bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() { |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 1016 | // Emit private VarDecl with copy init. |
| 1017 | EmitDecl(*VD); |
| 1018 | return GetAddrOfLocalVar(VD); |
| 1019 | }); |
Alexey Bataev | d130fd1 | 2015-05-13 10:23:02 +0000 | [diff] [blame] | 1020 | assert(IsRegistered && |
| 1021 | "lastprivate var already registered as private"); |
| 1022 | (void)IsRegistered; |
| 1023 | } |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 1024 | } |
Richard Trieu | cc3949d | 2016-02-18 22:34:54 +0000 | [diff] [blame] | 1025 | ++IRef; |
| 1026 | ++IDestRef; |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 1027 | } |
| 1028 | } |
| 1029 | return HasAtLeastOneLastprivate; |
| 1030 | } |
| 1031 | |
| 1032 | void CodeGenFunction::EmitOMPLastprivateClauseFinal( |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1033 | const OMPExecutableDirective &D, bool NoFinals, |
| 1034 | llvm::Value *IsLastIterCond) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 1035 | if (!HaveInsertPoint()) |
| 1036 | return; |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 1037 | // Emit following code: |
| 1038 | // if (<IsLastIterCond>) { |
| 1039 | // orig_var1 = private_orig_var1; |
| 1040 | // ... |
| 1041 | // orig_varn = private_orig_varn; |
| 1042 | // } |
Alexey Bataev | fc087ec | 2015-06-16 13:14:42 +0000 | [diff] [blame] | 1043 | llvm::BasicBlock *ThenBB = nullptr; |
| 1044 | llvm::BasicBlock *DoneBB = nullptr; |
| 1045 | if (IsLastIterCond) { |
| 1046 | ThenBB = createBasicBlock(".omp.lastprivate.then"); |
| 1047 | DoneBB = createBasicBlock(".omp.lastprivate.done"); |
| 1048 | Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB); |
| 1049 | EmitBlock(ThenBB); |
| 1050 | } |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1051 | llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; |
| 1052 | llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1053 | if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) { |
Alexey Bataev | 8ffcc94 | 2016-02-18 13:48:15 +0000 | [diff] [blame] | 1054 | auto IC = LoopDirective->counters().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1055 | for (const Expr *F : LoopDirective->finals()) { |
| 1056 | const auto *D = |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1057 | cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl(); |
| 1058 | if (NoFinals) |
| 1059 | AlreadyEmittedVars.insert(D); |
| 1060 | else |
| 1061 | LoopCountersAndUpdates[D] = F; |
Alexey Bataev | 8ffcc94 | 2016-02-18 13:48:15 +0000 | [diff] [blame] | 1062 | ++IC; |
Alexey Bataev | 7a228ff | 2015-05-21 07:59:51 +0000 | [diff] [blame] | 1063 | } |
| 1064 | } |
Alexey Bataev | 8ffcc94 | 2016-02-18 13:48:15 +0000 | [diff] [blame] | 1065 | for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) { |
| 1066 | auto IRef = C->varlist_begin(); |
| 1067 | auto ISrcRef = C->source_exprs().begin(); |
| 1068 | auto IDestRef = C->destination_exprs().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1069 | for (const Expr *AssignOp : C->assignment_ops()) { |
| 1070 | const auto *PrivateVD = |
| 1071 | cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); |
Alexey Bataev | 8ffcc94 | 2016-02-18 13:48:15 +0000 | [diff] [blame] | 1072 | QualType Type = PrivateVD->getType(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1073 | const auto *CanonicalVD = PrivateVD->getCanonicalDecl(); |
Alexey Bataev | 8ffcc94 | 2016-02-18 13:48:15 +0000 | [diff] [blame] | 1074 | if (AlreadyEmittedVars.insert(CanonicalVD).second) { |
| 1075 | // If lastprivate variable is a loop control variable for loop-based |
| 1076 | // directive, update its value before copyin back to original |
| 1077 | // variable. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1078 | if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1079 | EmitIgnoredExpr(FinalExpr); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1080 | const auto *SrcVD = |
| 1081 | cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); |
| 1082 | const auto *DestVD = |
| 1083 | cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); |
Alexey Bataev | 8ffcc94 | 2016-02-18 13:48:15 +0000 | [diff] [blame] | 1084 | // Get the address of the original variable. |
| 1085 | Address OriginalAddr = GetAddrOfLocalVar(DestVD); |
| 1086 | // Get the address of the private variable. |
| 1087 | Address PrivateAddr = GetAddrOfLocalVar(PrivateVD); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1088 | if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>()) |
Alexey Bataev | 8ffcc94 | 2016-02-18 13:48:15 +0000 | [diff] [blame] | 1089 | PrivateAddr = |
John McCall | 7f416cc | 2015-09-08 08:05:57 +0000 | [diff] [blame] | 1090 | Address(Builder.CreateLoad(PrivateAddr), |
| 1091 | getNaturalTypeAlignment(RefTy->getPointeeType())); |
Alexey Bataev | 8ffcc94 | 2016-02-18 13:48:15 +0000 | [diff] [blame] | 1092 | EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp); |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 1093 | } |
Alexey Bataev | 8ffcc94 | 2016-02-18 13:48:15 +0000 | [diff] [blame] | 1094 | ++IRef; |
| 1095 | ++ISrcRef; |
| 1096 | ++IDestRef; |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 1097 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1098 | if (const Expr *PostUpdate = C->getPostUpdateExpr()) |
Alexey Bataev | 005248a | 2016-02-25 05:25:57 +0000 | [diff] [blame] | 1099 | EmitIgnoredExpr(PostUpdate); |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 1100 | } |
Alexey Bataev | 8ffcc94 | 2016-02-18 13:48:15 +0000 | [diff] [blame] | 1101 | if (IsLastIterCond) |
Alexey Bataev | fc087ec | 2015-06-16 13:14:42 +0000 | [diff] [blame] | 1102 | EmitBlock(DoneBB, /*IsFinished=*/true); |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 1103 | } |
| 1104 | |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 1105 | void CodeGenFunction::EmitOMPReductionClauseInit( |
| 1106 | const OMPExecutableDirective &D, |
| 1107 | CodeGenFunction::OMPPrivateScope &PrivateScope) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 1108 | if (!HaveInsertPoint()) |
| 1109 | return; |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1110 | SmallVector<const Expr *, 4> Shareds; |
| 1111 | SmallVector<const Expr *, 4> Privates; |
| 1112 | SmallVector<const Expr *, 4> ReductionOps; |
| 1113 | SmallVector<const Expr *, 4> LHSs; |
| 1114 | SmallVector<const Expr *, 4> RHSs; |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 1115 | for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) { |
Alexey Bataev | f24e7b1 | 2015-10-08 09:10:53 +0000 | [diff] [blame] | 1116 | auto IPriv = C->privates().begin(); |
Alexey Bataev | a839ddd | 2016-03-17 10:19:46 +0000 | [diff] [blame] | 1117 | auto IRed = C->reduction_ops().begin(); |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1118 | auto ILHS = C->lhs_exprs().begin(); |
| 1119 | auto IRHS = C->rhs_exprs().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1120 | for (const Expr *Ref : C->varlists()) { |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1121 | Shareds.emplace_back(Ref); |
| 1122 | Privates.emplace_back(*IPriv); |
| 1123 | ReductionOps.emplace_back(*IRed); |
| 1124 | LHSs.emplace_back(*ILHS); |
| 1125 | RHSs.emplace_back(*IRHS); |
| 1126 | std::advance(IPriv, 1); |
| 1127 | std::advance(IRed, 1); |
| 1128 | std::advance(ILHS, 1); |
| 1129 | std::advance(IRHS, 1); |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 1130 | } |
| 1131 | } |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1132 | ReductionCodeGen RedCG(Shareds, Privates, ReductionOps); |
| 1133 | unsigned Count = 0; |
| 1134 | auto ILHS = LHSs.begin(); |
| 1135 | auto IRHS = RHSs.begin(); |
| 1136 | auto IPriv = Privates.begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1137 | for (const Expr *IRef : Shareds) { |
| 1138 | const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl()); |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1139 | // Emit private VarDecl with reduction init. |
| 1140 | RedCG.emitSharedLValue(*this, Count); |
| 1141 | RedCG.emitAggregateType(*this, Count); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1142 | AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD); |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1143 | RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(), |
| 1144 | RedCG.getSharedLValue(Count), |
| 1145 | [&Emission](CodeGenFunction &CGF) { |
| 1146 | CGF.EmitAutoVarInit(Emission); |
| 1147 | return true; |
| 1148 | }); |
| 1149 | EmitAutoVarCleanups(Emission); |
| 1150 | Address BaseAddr = RedCG.adjustPrivateAddress( |
| 1151 | *this, Count, Emission.getAllocatedAddress()); |
| 1152 | bool IsRegistered = PrivateScope.addPrivate( |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1153 | RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; }); |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1154 | assert(IsRegistered && "private var already registered as private"); |
| 1155 | // Silence the warning about unused variable. |
| 1156 | (void)IsRegistered; |
| 1157 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1158 | const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); |
| 1159 | const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); |
Jonas Hahnfeld | 4525c82 | 2017-10-23 19:01:35 +0000 | [diff] [blame] | 1160 | QualType Type = PrivateVD->getType(); |
| 1161 | bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef); |
| 1162 | if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) { |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1163 | // Store the address of the original variable associated with the LHS |
| 1164 | // implicit variable. |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 1165 | PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() { |
| 1166 | return RedCG.getSharedLValue(Count).getAddress(*this); |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1167 | }); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1168 | PrivateScope.addPrivate( |
| 1169 | RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); }); |
Jonas Hahnfeld | 4525c82 | 2017-10-23 19:01:35 +0000 | [diff] [blame] | 1170 | } else if ((isaOMPArraySectionExpr && Type->isScalarType()) || |
| 1171 | isa<ArraySubscriptExpr>(IRef)) { |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1172 | // Store the address of the original variable associated with the LHS |
| 1173 | // implicit variable. |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 1174 | PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() { |
| 1175 | return RedCG.getSharedLValue(Count).getAddress(*this); |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1176 | }); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1177 | PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() { |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1178 | return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD), |
| 1179 | ConvertTypeForMem(RHSVD->getType()), |
| 1180 | "rhs.begin"); |
| 1181 | }); |
| 1182 | } else { |
| 1183 | QualType Type = PrivateVD->getType(); |
| 1184 | bool IsArray = getContext().getAsArrayType(Type) != nullptr; |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 1185 | Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*this); |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1186 | // Store the address of the original variable associated with the LHS |
| 1187 | // implicit variable. |
| 1188 | if (IsArray) { |
| 1189 | OriginalAddr = Builder.CreateElementBitCast( |
| 1190 | OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin"); |
| 1191 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1192 | PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; }); |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1193 | PrivateScope.addPrivate( |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1194 | RHSVD, [this, PrivateVD, RHSVD, IsArray]() { |
Alexey Bataev | 5c40bec | 2017-07-13 13:36:14 +0000 | [diff] [blame] | 1195 | return IsArray |
| 1196 | ? Builder.CreateElementBitCast( |
| 1197 | GetAddrOfLocalVar(PrivateVD), |
| 1198 | ConvertTypeForMem(RHSVD->getType()), "rhs.begin") |
| 1199 | : GetAddrOfLocalVar(PrivateVD); |
| 1200 | }); |
| 1201 | } |
| 1202 | ++ILHS; |
| 1203 | ++IRHS; |
| 1204 | ++IPriv; |
| 1205 | ++Count; |
| 1206 | } |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 1207 | } |
| 1208 | |
| 1209 | void CodeGenFunction::EmitOMPReductionClauseFinal( |
Arpith Chacko Jacob | 101e8fb | 2017-02-16 16:20:16 +0000 | [diff] [blame] | 1210 | const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 1211 | if (!HaveInsertPoint()) |
| 1212 | return; |
Alexey Bataev | f24e7b1 | 2015-10-08 09:10:53 +0000 | [diff] [blame] | 1213 | llvm::SmallVector<const Expr *, 8> Privates; |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 1214 | llvm::SmallVector<const Expr *, 8> LHSExprs; |
| 1215 | llvm::SmallVector<const Expr *, 8> RHSExprs; |
| 1216 | llvm::SmallVector<const Expr *, 8> ReductionOps; |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 1217 | bool HasAtLeastOneReduction = false; |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 1218 | for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) { |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 1219 | HasAtLeastOneReduction = true; |
Alexey Bataev | f24e7b1 | 2015-10-08 09:10:53 +0000 | [diff] [blame] | 1220 | Privates.append(C->privates().begin(), C->privates().end()); |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 1221 | LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end()); |
| 1222 | RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end()); |
| 1223 | ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end()); |
| 1224 | } |
| 1225 | if (HasAtLeastOneReduction) { |
Arpith Chacko Jacob | 101e8fb | 2017-02-16 16:20:16 +0000 | [diff] [blame] | 1226 | bool WithNowait = D.getSingleClause<OMPNowaitClause>() || |
| 1227 | isOpenMPParallelDirective(D.getDirectiveKind()) || |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 1228 | ReductionKind == OMPD_simd; |
| 1229 | bool SimpleReduction = ReductionKind == OMPD_simd; |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 1230 | // Emit nowait reduction if nowait clause is present or directive is a |
| 1231 | // parallel directive (it always has implicit barrier). |
| 1232 | CGM.getOpenMPRuntime().emitReduction( |
Stephen Kelly | 1c301dc | 2018-08-09 21:09:38 +0000 | [diff] [blame] | 1233 | *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps, |
Arpith Chacko Jacob | 101e8fb | 2017-02-16 16:20:16 +0000 | [diff] [blame] | 1234 | {WithNowait, SimpleReduction, ReductionKind}); |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 1235 | } |
| 1236 | } |
| 1237 | |
Alexey Bataev | 6120507 | 2016-03-02 04:57:40 +0000 | [diff] [blame] | 1238 | static void emitPostUpdateForReductionClause( |
| 1239 | CodeGenFunction &CGF, const OMPExecutableDirective &D, |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1240 | const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) { |
Alexey Bataev | 6120507 | 2016-03-02 04:57:40 +0000 | [diff] [blame] | 1241 | if (!CGF.HaveInsertPoint()) |
| 1242 | return; |
| 1243 | llvm::BasicBlock *DoneBB = nullptr; |
| 1244 | for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1245 | if (const Expr *PostUpdate = C->getPostUpdateExpr()) { |
Alexey Bataev | 6120507 | 2016-03-02 04:57:40 +0000 | [diff] [blame] | 1246 | if (!DoneBB) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1247 | if (llvm::Value *Cond = CondGen(CGF)) { |
Alexey Bataev | 6120507 | 2016-03-02 04:57:40 +0000 | [diff] [blame] | 1248 | // If the first post-update expression is found, emit conditional |
| 1249 | // block if it was requested. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1250 | llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); |
Alexey Bataev | 6120507 | 2016-03-02 04:57:40 +0000 | [diff] [blame] | 1251 | DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done"); |
| 1252 | CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB); |
| 1253 | CGF.EmitBlock(ThenBB); |
| 1254 | } |
| 1255 | } |
| 1256 | CGF.EmitIgnoredExpr(PostUpdate); |
| 1257 | } |
| 1258 | } |
| 1259 | if (DoneBB) |
| 1260 | CGF.EmitBlock(DoneBB, /*IsFinished=*/true); |
| 1261 | } |
| 1262 | |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1263 | namespace { |
| 1264 | /// Codegen lambda for appending distribute lower and upper bounds to outlined |
| 1265 | /// parallel function. This is necessary for combined constructs such as |
| 1266 | /// 'distribute parallel for' |
| 1267 | typedef llvm::function_ref<void(CodeGenFunction &, |
| 1268 | const OMPExecutableDirective &, |
| 1269 | llvm::SmallVectorImpl<llvm::Value *> &)> |
| 1270 | CodeGenBoundParametersTy; |
| 1271 | } // anonymous namespace |
| 1272 | |
| 1273 | static void emitCommonOMPParallelDirective( |
| 1274 | CodeGenFunction &CGF, const OMPExecutableDirective &S, |
| 1275 | OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, |
| 1276 | const CodeGenBoundParametersTy &CodeGenBoundParameters) { |
Arpith Chacko Jacob | 19b911c | 2017-01-18 18:18:53 +0000 | [diff] [blame] | 1277 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel); |
James Y Knight | 9871db0 | 2019-02-05 16:42:33 +0000 | [diff] [blame] | 1278 | llvm::Function *OutlinedFn = |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1279 | CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( |
| 1280 | S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 1281 | if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) { |
Alexey Bataev | 1d67713 | 2015-04-22 13:57:31 +0000 | [diff] [blame] | 1282 | CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1283 | llvm::Value *NumThreads = |
| 1284 | CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), |
| 1285 | /*IgnoreResultAssign=*/true); |
Alexey Bataev | 1d67713 | 2015-04-22 13:57:31 +0000 | [diff] [blame] | 1286 | CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 1287 | CGF, NumThreads, NumThreadsClause->getBeginLoc()); |
Alexey Bataev | 1d67713 | 2015-04-22 13:57:31 +0000 | [diff] [blame] | 1288 | } |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 1289 | if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) { |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 1290 | CodeGenFunction::RunCleanupsScope ProcBindScope(CGF); |
Alexey Bataev | 7f210c6 | 2015-06-18 13:40:03 +0000 | [diff] [blame] | 1291 | CGF.CGM.getOpenMPRuntime().emitProcBindClause( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 1292 | CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc()); |
Alexey Bataev | 7f210c6 | 2015-06-18 13:40:03 +0000 | [diff] [blame] | 1293 | } |
Alexey Bataev | 1d67713 | 2015-04-22 13:57:31 +0000 | [diff] [blame] | 1294 | const Expr *IfCond = nullptr; |
Alexey Bataev | 7371aa3 | 2015-09-03 08:45:56 +0000 | [diff] [blame] | 1295 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { |
| 1296 | if (C->getNameModifier() == OMPD_unknown || |
| 1297 | C->getNameModifier() == OMPD_parallel) { |
| 1298 | IfCond = C->getCondition(); |
| 1299 | break; |
| 1300 | } |
Alexey Bataev | 1d67713 | 2015-04-22 13:57:31 +0000 | [diff] [blame] | 1301 | } |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 1302 | |
Arpith Chacko Jacob | fe4890a | 2017-01-18 20:40:48 +0000 | [diff] [blame] | 1303 | OMPParallelScope Scope(CGF, S); |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 1304 | llvm::SmallVector<llvm::Value *, 16> CapturedVars; |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1305 | // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk |
| 1306 | // lower and upper bounds with the pragma 'for' chunking mechanism. |
| 1307 | // The following lambda takes care of appending the lower and upper bound |
| 1308 | // parameters when necessary |
| 1309 | CodeGenBoundParameters(CGF, S, CapturedVars); |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 1310 | CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 1311 | CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn, |
Alexey Bataev | 2377fe9 | 2015-09-10 08:12:02 +0000 | [diff] [blame] | 1312 | CapturedVars, IfCond); |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 1313 | } |
| 1314 | |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1315 | static void emitEmptyBoundParameters(CodeGenFunction &, |
| 1316 | const OMPExecutableDirective &, |
| 1317 | llvm::SmallVectorImpl<llvm::Value *> &) {} |
| 1318 | |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 1319 | void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 1320 | // Emit parallel region as a standalone region. |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 1321 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 1322 | Action.Enter(CGF); |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 1323 | OMPPrivateScope PrivateScope(CGF); |
Alexey Bataev | f56f98c | 2015-04-16 05:39:01 +0000 | [diff] [blame] | 1324 | bool Copyins = CGF.EmitOMPCopyinClause(S); |
Alexey Bataev | cd8b6a2 | 2016-02-15 08:07:17 +0000 | [diff] [blame] | 1325 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); |
| 1326 | if (Copyins) { |
Alexey Bataev | 69c62a9 | 2015-04-15 04:52:20 +0000 | [diff] [blame] | 1327 | // Emit implicit barrier to synchronize threads and avoid data races on |
Alexey Bataev | cd8b6a2 | 2016-02-15 08:07:17 +0000 | [diff] [blame] | 1328 | // propagation master's thread values of threadprivate variables to local |
| 1329 | // instances of that variables of all other implicit threads. |
Alexey Bataev | 25e5b44 | 2015-09-15 12:52:43 +0000 | [diff] [blame] | 1330 | CGF.CGM.getOpenMPRuntime().emitBarrierCall( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 1331 | CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, |
Alexey Bataev | 25e5b44 | 2015-09-15 12:52:43 +0000 | [diff] [blame] | 1332 | /*ForceSimpleCall=*/true); |
Alexey Bataev | 69c62a9 | 2015-04-15 04:52:20 +0000 | [diff] [blame] | 1333 | } |
| 1334 | CGF.EmitOMPPrivateClause(S, PrivateScope); |
| 1335 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 1336 | (void)PrivateScope.Privatize(); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 1337 | CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); |
Arpith Chacko Jacob | 101e8fb | 2017-02-16 16:20:16 +0000 | [diff] [blame] | 1338 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 1339 | }; |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1340 | emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen, |
| 1341 | emitEmptyBoundParameters); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1342 | emitPostUpdateForReductionClause(*this, S, |
| 1343 | [](CodeGenFunction &) { return nullptr; }); |
Alexey Bataev | 9959db5 | 2014-05-06 10:08:46 +0000 | [diff] [blame] | 1344 | } |
Alexander Musman | 515ad8c | 2014-05-22 08:54:05 +0000 | [diff] [blame] | 1345 | |
Alexey Bataev | 8bbf2e3 | 2019-11-04 09:59:11 -0500 | [diff] [blame] | 1346 | static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, |
| 1347 | int MaxLevel, int Level = 0) { |
| 1348 | assert(Level < MaxLevel && "Too deep lookup during loop body codegen."); |
| 1349 | const Stmt *SimplifiedS = S->IgnoreContainers(); |
| 1350 | if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) { |
| 1351 | PrettyStackTraceLoc CrashInfo( |
| 1352 | CGF.getContext().getSourceManager(), CS->getLBracLoc(), |
| 1353 | "LLVM IR generation of compound statement ('{}')"); |
| 1354 | |
| 1355 | // Keep track of the current cleanup stack depth, including debug scopes. |
| 1356 | CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange()); |
| 1357 | for (const Stmt *CurStmt : CS->body()) |
| 1358 | emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level); |
| 1359 | return; |
| 1360 | } |
| 1361 | if (SimplifiedS == NextLoop) { |
| 1362 | if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) { |
| 1363 | S = For->getBody(); |
| 1364 | } else { |
| 1365 | assert(isa<CXXForRangeStmt>(SimplifiedS) && |
| 1366 | "Expected canonical for loop or range-based for loop."); |
| 1367 | const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS); |
| 1368 | CGF.EmitStmt(CXXFor->getLoopVarStmt()); |
| 1369 | S = CXXFor->getBody(); |
| 1370 | } |
| 1371 | if (Level + 1 < MaxLevel) { |
| 1372 | NextLoop = OMPLoopDirective::tryToFindNextInnerLoop( |
| 1373 | S, /*TryImperfectlyNestedLoops=*/true); |
| 1374 | emitBody(CGF, S, NextLoop, MaxLevel, Level + 1); |
| 1375 | return; |
| 1376 | } |
| 1377 | } |
| 1378 | CGF.EmitStmt(S); |
| 1379 | } |
| 1380 | |
Alexey Bataev | 0f34da1 | 2015-07-02 04:17:07 +0000 | [diff] [blame] | 1381 | void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D, |
| 1382 | JumpDest LoopExit) { |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1383 | RunCleanupsScope BodyScope(*this); |
| 1384 | // Update counters values on current iteration. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1385 | for (const Expr *UE : D.updates()) |
| 1386 | EmitIgnoredExpr(UE); |
Alexander Musman | 3276a27 | 2015-03-21 10:12:56 +0000 | [diff] [blame] | 1387 | // Update the linear variables. |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 1388 | // In distribute directives only loop counters may be marked as linear, no |
| 1389 | // need to generate the code for them. |
| 1390 | if (!isOpenMPDistributeDirective(D.getDirectiveKind())) { |
| 1391 | for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1392 | for (const Expr *UE : C->updates()) |
| 1393 | EmitIgnoredExpr(UE); |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 1394 | } |
Alexander Musman | 3276a27 | 2015-03-21 10:12:56 +0000 | [diff] [blame] | 1395 | } |
| 1396 | |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1397 | // On a continue in the body, jump to the end. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1398 | JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue"); |
Alexey Bataev | 0f34da1 | 2015-07-02 04:17:07 +0000 | [diff] [blame] | 1399 | BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); |
Alexey Bataev | f8be476 | 2019-08-14 19:30:06 +0000 | [diff] [blame] | 1400 | for (const Expr *E : D.finals_conditions()) { |
| 1401 | if (!E) |
| 1402 | continue; |
| 1403 | // Check that loop counter in non-rectangular nest fits into the iteration |
| 1404 | // space. |
| 1405 | llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next"); |
| 1406 | EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(), |
| 1407 | getProfileCount(D.getBody())); |
| 1408 | EmitBlock(NextBB); |
| 1409 | } |
Alexey Bataev | bef93a9 | 2019-10-07 18:54:57 +0000 | [diff] [blame] | 1410 | // Emit loop variables for C++ range loops. |
| 1411 | const Stmt *Body = |
| 1412 | D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers(); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1413 | // Emit loop body. |
Alexey Bataev | 8bbf2e3 | 2019-11-04 09:59:11 -0500 | [diff] [blame] | 1414 | emitBody(*this, Body, |
| 1415 | OMPLoopDirective::tryToFindNextInnerLoop( |
| 1416 | Body, /*TryImperfectlyNestedLoops=*/true), |
| 1417 | D.getCollapsedNumber()); |
| 1418 | |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1419 | // The end (updates/cleanups). |
| 1420 | EmitBlock(Continue.getBlock()); |
| 1421 | BreakContinueStack.pop_back(); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1422 | } |
| 1423 | |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 1424 | void CodeGenFunction::EmitOMPInnerLoop( |
| 1425 | const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, |
| 1426 | const Expr *IncExpr, |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1427 | const llvm::function_ref<void(CodeGenFunction &)> BodyGen, |
| 1428 | const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) { |
Alexander Musman | d196ef2 | 2014-10-07 08:57:09 +0000 | [diff] [blame] | 1429 | auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end"); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1430 | |
| 1431 | // Start the loop with a block that tests the condition. |
Alexander Musman | d196ef2 | 2014-10-07 08:57:09 +0000 | [diff] [blame] | 1432 | auto CondBlock = createBasicBlock("omp.inner.for.cond"); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1433 | EmitBlock(CondBlock); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1434 | const SourceRange R = S.getSourceRange(); |
Amara Emerson | 652795d | 2016-11-10 14:44:30 +0000 | [diff] [blame] | 1435 | LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()), |
| 1436 | SourceLocToDebugLoc(R.getEnd())); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1437 | |
| 1438 | // If there are any cleanups between here and the loop-exit scope, |
| 1439 | // create a block to stage a loop exit along. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1440 | llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); |
Alexey Bataev | 2df54a0 | 2015-03-12 08:53:29 +0000 | [diff] [blame] | 1441 | if (RequiresCleanup) |
Alexander Musman | d196ef2 | 2014-10-07 08:57:09 +0000 | [diff] [blame] | 1442 | ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup"); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1443 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1444 | llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body"); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1445 | |
Alexey Bataev | 2df54a0 | 2015-03-12 08:53:29 +0000 | [diff] [blame] | 1446 | // Emit condition. |
Justin Bogner | 66242d6 | 2015-04-23 23:06:47 +0000 | [diff] [blame] | 1447 | EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S)); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1448 | if (ExitBlock != LoopExit.getBlock()) { |
| 1449 | EmitBlock(ExitBlock); |
| 1450 | EmitBranchThroughCleanup(LoopExit); |
| 1451 | } |
| 1452 | |
| 1453 | EmitBlock(LoopBody); |
Justin Bogner | 66242d6 | 2015-04-23 23:06:47 +0000 | [diff] [blame] | 1454 | incrementProfileCounter(&S); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1455 | |
| 1456 | // Create a block for the increment. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1457 | JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc"); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1458 | BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); |
| 1459 | |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 1460 | BodyGen(*this); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1461 | |
| 1462 | // Emit "IV = IV + 1" and a back-edge to the condition block. |
| 1463 | EmitBlock(Continue.getBlock()); |
Alexey Bataev | 2df54a0 | 2015-03-12 08:53:29 +0000 | [diff] [blame] | 1464 | EmitIgnoredExpr(IncExpr); |
Alexey Bataev | 98eb6e3 | 2015-04-22 11:15:40 +0000 | [diff] [blame] | 1465 | PostIncGen(*this); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1466 | BreakContinueStack.pop_back(); |
| 1467 | EmitBranch(CondBlock); |
| 1468 | LoopStack.pop(); |
| 1469 | // Emit the fall-through block. |
| 1470 | EmitBlock(LoopExit.getBlock()); |
| 1471 | } |
| 1472 | |
Alexey Bataev | 8c3edfe | 2017-08-16 15:58:46 +0000 | [diff] [blame] | 1473 | bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 1474 | if (!HaveInsertPoint()) |
Alexey Bataev | 8c3edfe | 2017-08-16 15:58:46 +0000 | [diff] [blame] | 1475 | return false; |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1476 | // Emit inits for the linear variables. |
Alexey Bataev | 8c3edfe | 2017-08-16 15:58:46 +0000 | [diff] [blame] | 1477 | bool HasLinears = false; |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 1478 | for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1479 | for (const Expr *Init : C->inits()) { |
Alexey Bataev | 8c3edfe | 2017-08-16 15:58:46 +0000 | [diff] [blame] | 1480 | HasLinears = true; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1481 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl()); |
| 1482 | if (const auto *Ref = |
| 1483 | dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) { |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1484 | AutoVarEmission Emission = EmitAutoVarAlloca(*VD); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1485 | const auto *OrigVD = cast<VarDecl>(Ref->getDecl()); |
Bruno Ricci | 5fc4db7 | 2018-12-21 14:10:18 +0000 | [diff] [blame] | 1486 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1487 | CapturedStmtInfo->lookup(OrigVD) != nullptr, |
| 1488 | VD->getInit()->getType(), VK_LValue, |
| 1489 | VD->getInit()->getExprLoc()); |
| 1490 | EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(), |
| 1491 | VD->getType()), |
| 1492 | /*capturedByInit=*/false); |
| 1493 | EmitAutoVarCleanups(Emission); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1494 | } else { |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1495 | EmitVarDecl(*VD); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1496 | } |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1497 | } |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1498 | // Emit the linear steps for the linear clauses. |
| 1499 | // If a step is not constant, it is pre-calculated before the loop. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1500 | if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep())) |
| 1501 | if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) { |
Alexey Bataev | 3b5b5c4 | 2015-06-18 10:10:12 +0000 | [diff] [blame] | 1502 | EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl())); |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1503 | // Emit calculation of the linear step. |
Alexey Bataev | 3b5b5c4 | 2015-06-18 10:10:12 +0000 | [diff] [blame] | 1504 | EmitIgnoredExpr(CS); |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1505 | } |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1506 | } |
Alexey Bataev | 8c3edfe | 2017-08-16 15:58:46 +0000 | [diff] [blame] | 1507 | return HasLinears; |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1508 | } |
| 1509 | |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1510 | void CodeGenFunction::EmitOMPLinearClauseFinal( |
| 1511 | const OMPLoopDirective &D, |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1512 | const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) { |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1513 | if (!HaveInsertPoint()) |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 1514 | return; |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1515 | llvm::BasicBlock *DoneBB = nullptr; |
Alexander Musman | 3276a27 | 2015-03-21 10:12:56 +0000 | [diff] [blame] | 1516 | // Emit the final values of the linear variables. |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 1517 | for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { |
Alexey Bataev | 39f915b8 | 2015-05-08 10:41:21 +0000 | [diff] [blame] | 1518 | auto IC = C->varlist_begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1519 | for (const Expr *F : C->finals()) { |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1520 | if (!DoneBB) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1521 | if (llvm::Value *Cond = CondGen(*this)) { |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1522 | // If the first post-update expression is found, emit conditional |
| 1523 | // block if it was requested. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1524 | llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu"); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1525 | DoneBB = createBasicBlock(".omp.linear.pu.done"); |
| 1526 | Builder.CreateCondBr(Cond, ThenBB, DoneBB); |
| 1527 | EmitBlock(ThenBB); |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1528 | } |
| 1529 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1530 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl()); |
Bruno Ricci | 5fc4db7 | 2018-12-21 14:10:18 +0000 | [diff] [blame] | 1531 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1532 | CapturedStmtInfo->lookup(OrigVD) != nullptr, |
Alexey Bataev | 39f915b8 | 2015-05-08 10:41:21 +0000 | [diff] [blame] | 1533 | (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 1534 | Address OrigAddr = EmitLValue(&DRE).getAddress(*this); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1535 | CodeGenFunction::OMPPrivateScope VarScope(*this); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1536 | VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; }); |
Alexey Bataev | 39f915b8 | 2015-05-08 10:41:21 +0000 | [diff] [blame] | 1537 | (void)VarScope.Privatize(); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1538 | EmitIgnoredExpr(F); |
Alexey Bataev | 39f915b8 | 2015-05-08 10:41:21 +0000 | [diff] [blame] | 1539 | ++IC; |
Alexander Musman | 3276a27 | 2015-03-21 10:12:56 +0000 | [diff] [blame] | 1540 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1541 | if (const Expr *PostUpdate = C->getPostUpdateExpr()) |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1542 | EmitIgnoredExpr(PostUpdate); |
Alexander Musman | 3276a27 | 2015-03-21 10:12:56 +0000 | [diff] [blame] | 1543 | } |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1544 | if (DoneBB) |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1545 | EmitBlock(DoneBB, /*IsFinished=*/true); |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1546 | } |
| 1547 | |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1548 | static void emitAlignedClause(CodeGenFunction &CGF, |
| 1549 | const OMPExecutableDirective &D) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 1550 | if (!CGF.HaveInsertPoint()) |
| 1551 | return; |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 1552 | for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) { |
Erich Keane | f759395 | 2019-10-11 14:59:44 +0000 | [diff] [blame] | 1553 | llvm::APInt ClauseAlignment(64, 0); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1554 | if (const Expr *AlignmentExpr = Clause->getAlignment()) { |
| 1555 | auto *AlignmentCI = |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1556 | cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); |
Erich Keane | f759395 | 2019-10-11 14:59:44 +0000 | [diff] [blame] | 1557 | ClauseAlignment = AlignmentCI->getValue(); |
Alexander Musman | 09184fe | 2014-09-30 05:29:28 +0000 | [diff] [blame] | 1558 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1559 | for (const Expr *E : Clause->varlists()) { |
Erich Keane | f759395 | 2019-10-11 14:59:44 +0000 | [diff] [blame] | 1560 | llvm::APInt Alignment(ClauseAlignment); |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1561 | if (Alignment == 0) { |
| 1562 | // OpenMP [2.8.1, Description] |
| 1563 | // If no optional parameter is specified, implementation-defined default |
| 1564 | // alignments for SIMD instructions on the target platforms are assumed. |
| 1565 | Alignment = |
Alexey Bataev | 0039651 | 2015-07-02 03:40:19 +0000 | [diff] [blame] | 1566 | CGF.getContext() |
| 1567 | .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( |
| 1568 | E->getType()->getPointeeType())) |
| 1569 | .getQuantity(); |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1570 | } |
Erich Keane | f759395 | 2019-10-11 14:59:44 +0000 | [diff] [blame] | 1571 | assert((Alignment == 0 || Alignment.isPowerOf2()) && |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1572 | "alignment is not power of 2"); |
| 1573 | if (Alignment != 0) { |
| 1574 | llvm::Value *PtrValue = CGF.EmitScalarExpr(E); |
Roman Lebedev | bd1c087 | 2019-01-15 09:44:25 +0000 | [diff] [blame] | 1575 | CGF.EmitAlignmentAssumption( |
Erich Keane | f759395 | 2019-10-11 14:59:44 +0000 | [diff] [blame] | 1576 | PtrValue, E, /*No second loc needed*/ SourceLocation(), |
| 1577 | llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment)); |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1578 | } |
Alexander Musman | 09184fe | 2014-09-30 05:29:28 +0000 | [diff] [blame] | 1579 | } |
| 1580 | } |
| 1581 | } |
| 1582 | |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1583 | void CodeGenFunction::EmitOMPPrivateLoopCounters( |
| 1584 | const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) { |
| 1585 | if (!HaveInsertPoint()) |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 1586 | return; |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1587 | auto I = S.private_counters().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1588 | for (const Expr *E : S.counters()) { |
| 1589 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); |
| 1590 | const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()); |
Alexey Bataev | ab4ea22 | 2018-03-07 18:17:06 +0000 | [diff] [blame] | 1591 | // Emit var without initialization. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1592 | AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD); |
Alexey Bataev | ab4ea22 | 2018-03-07 18:17:06 +0000 | [diff] [blame] | 1593 | EmitAutoVarCleanups(VarEmission); |
| 1594 | LocalDeclMap.erase(PrivateVD); |
| 1595 | (void)LoopScope.addPrivate(VD, [&VarEmission]() { |
| 1596 | return VarEmission.getAllocatedAddress(); |
Alexey Bataev | 435ad7b | 2014-10-10 09:48:26 +0000 | [diff] [blame] | 1597 | }); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1598 | if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) || |
| 1599 | VD->hasGlobalStorage()) { |
Alexey Bataev | ab4ea22 | 2018-03-07 18:17:06 +0000 | [diff] [blame] | 1600 | (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() { |
Bruno Ricci | 5fc4db7 | 2018-12-21 14:10:18 +0000 | [diff] [blame] | 1601 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1602 | LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD), |
| 1603 | E->getType(), VK_LValue, E->getExprLoc()); |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 1604 | return EmitLValue(&DRE).getAddress(*this); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1605 | }); |
Alexey Bataev | ab4ea22 | 2018-03-07 18:17:06 +0000 | [diff] [blame] | 1606 | } else { |
| 1607 | (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() { |
| 1608 | return VarEmission.getAllocatedAddress(); |
| 1609 | }); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1610 | } |
Alexey Bataev | a889917 | 2015-08-06 12:30:57 +0000 | [diff] [blame] | 1611 | ++I; |
Alexey Bataev | 435ad7b | 2014-10-10 09:48:26 +0000 | [diff] [blame] | 1612 | } |
Alexey Bataev | f138fda | 2018-08-13 19:04:24 +0000 | [diff] [blame] | 1613 | // Privatize extra loop counters used in loops for ordered(n) clauses. |
| 1614 | for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) { |
| 1615 | if (!C->getNumForLoops()) |
| 1616 | continue; |
| 1617 | for (unsigned I = S.getCollapsedNumber(), |
| 1618 | E = C->getLoopNumIterations().size(); |
| 1619 | I < E; ++I) { |
Mike Rice | 0ed4666 | 2018-09-20 17:19:41 +0000 | [diff] [blame] | 1620 | const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I)); |
Alexey Bataev | f138fda | 2018-08-13 19:04:24 +0000 | [diff] [blame] | 1621 | const auto *VD = cast<VarDecl>(DRE->getDecl()); |
Alexey Bataev | 0d8fcdf | 2019-03-14 20:36:00 +0000 | [diff] [blame] | 1622 | // Override only those variables that can be captured to avoid re-emission |
| 1623 | // of the variables declared within the loops. |
| 1624 | if (DRE->refersToEnclosingVariableOrCapture()) { |
Alexey Bataev | f138fda | 2018-08-13 19:04:24 +0000 | [diff] [blame] | 1625 | (void)LoopScope.addPrivate(VD, [this, DRE, VD]() { |
| 1626 | return CreateMemTemp(DRE->getType(), VD->getName()); |
| 1627 | }); |
| 1628 | } |
| 1629 | } |
| 1630 | } |
Alexey Bataev | 435ad7b | 2014-10-10 09:48:26 +0000 | [diff] [blame] | 1631 | } |
| 1632 | |
Alexey Bataev | 62dbb97 | 2015-04-22 11:59:37 +0000 | [diff] [blame] | 1633 | static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, |
| 1634 | const Expr *Cond, llvm::BasicBlock *TrueBlock, |
| 1635 | llvm::BasicBlock *FalseBlock, uint64_t TrueCount) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 1636 | if (!CGF.HaveInsertPoint()) |
| 1637 | return; |
Alexey Bataev | 6e8248f | 2015-06-11 10:53:56 +0000 | [diff] [blame] | 1638 | { |
| 1639 | CodeGenFunction::OMPPrivateScope PreCondScope(CGF); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1640 | CGF.EmitOMPPrivateLoopCounters(S, PreCondScope); |
Alexey Bataev | 6e8248f | 2015-06-11 10:53:56 +0000 | [diff] [blame] | 1641 | (void)PreCondScope.Privatize(); |
Alexey Bataev | 6e8248f | 2015-06-11 10:53:56 +0000 | [diff] [blame] | 1642 | // Get initial values of real counters. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1643 | for (const Expr *I : S.inits()) { |
Alexey Bataev | 6e8248f | 2015-06-11 10:53:56 +0000 | [diff] [blame] | 1644 | CGF.EmitIgnoredExpr(I); |
| 1645 | } |
Alexey Bataev | 62dbb97 | 2015-04-22 11:59:37 +0000 | [diff] [blame] | 1646 | } |
Alexey Bataev | f8be476 | 2019-08-14 19:30:06 +0000 | [diff] [blame] | 1647 | // Create temp loop control variables with their init values to support |
| 1648 | // non-rectangular loops. |
| 1649 | CodeGenFunction::OMPMapVars PreCondVars; |
| 1650 | for (const Expr * E: S.dependent_counters()) { |
| 1651 | if (!E) |
| 1652 | continue; |
| 1653 | assert(!E->getType().getNonReferenceType()->isRecordType() && |
| 1654 | "dependent counter must not be an iterator."); |
| 1655 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); |
| 1656 | Address CounterAddr = |
| 1657 | CGF.CreateMemTemp(VD->getType().getNonReferenceType()); |
| 1658 | (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr); |
| 1659 | } |
| 1660 | (void)PreCondVars.apply(CGF); |
| 1661 | for (const Expr *E : S.dependent_inits()) { |
| 1662 | if (!E) |
| 1663 | continue; |
| 1664 | CGF.EmitIgnoredExpr(E); |
| 1665 | } |
Alexey Bataev | 62dbb97 | 2015-04-22 11:59:37 +0000 | [diff] [blame] | 1666 | // Check that loop is executed at least one time. |
| 1667 | CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); |
Alexey Bataev | f8be476 | 2019-08-14 19:30:06 +0000 | [diff] [blame] | 1668 | PreCondVars.restore(CGF); |
Alexey Bataev | 62dbb97 | 2015-04-22 11:59:37 +0000 | [diff] [blame] | 1669 | } |
| 1670 | |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1671 | void CodeGenFunction::EmitOMPLinearClause( |
| 1672 | const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { |
| 1673 | if (!HaveInsertPoint()) |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 1674 | return; |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1675 | llvm::DenseSet<const VarDecl *> SIMDLCVs; |
| 1676 | if (isOpenMPSimdDirective(D.getDirectiveKind())) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1677 | const auto *LoopDirective = cast<OMPLoopDirective>(&D); |
| 1678 | for (const Expr *C : LoopDirective->counters()) { |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1679 | SIMDLCVs.insert( |
| 1680 | cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl()); |
| 1681 | } |
| 1682 | } |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 1683 | for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) { |
Alexey Bataev | bd9fec1 | 2015-08-18 06:47:21 +0000 | [diff] [blame] | 1684 | auto CurPrivate = C->privates().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1685 | for (const Expr *E : C->varlists()) { |
| 1686 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); |
| 1687 | const auto *PrivateVD = |
Alexey Bataev | bd9fec1 | 2015-08-18 06:47:21 +0000 | [diff] [blame] | 1688 | cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl()); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1689 | if (!SIMDLCVs.count(VD->getCanonicalDecl())) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1690 | bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() { |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1691 | // Emit private VarDecl with copy init. |
| 1692 | EmitVarDecl(*PrivateVD); |
| 1693 | return GetAddrOfLocalVar(PrivateVD); |
| 1694 | }); |
| 1695 | assert(IsRegistered && "linear var already registered as private"); |
| 1696 | // Silence the warning about unused variable. |
| 1697 | (void)IsRegistered; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1698 | } else { |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1699 | EmitVarDecl(*PrivateVD); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1700 | } |
Alexey Bataev | bd9fec1 | 2015-08-18 06:47:21 +0000 | [diff] [blame] | 1701 | ++CurPrivate; |
Alexander Musman | 3276a27 | 2015-03-21 10:12:56 +0000 | [diff] [blame] | 1702 | } |
| 1703 | } |
| 1704 | } |
| 1705 | |
Alexey Bataev | 45bfad5 | 2015-08-21 12:19:04 +0000 | [diff] [blame] | 1706 | static void emitSimdlenSafelenClause(CodeGenFunction &CGF, |
Alexey Bataev | a6f2a14 | 2015-12-31 06:52:34 +0000 | [diff] [blame] | 1707 | const OMPExecutableDirective &D, |
| 1708 | bool IsMonotonic) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 1709 | if (!CGF.HaveInsertPoint()) |
| 1710 | return; |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 1711 | if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) { |
Alexey Bataev | 45bfad5 | 2015-08-21 12:19:04 +0000 | [diff] [blame] | 1712 | RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(), |
| 1713 | /*ignoreResult=*/true); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1714 | auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); |
Alexey Bataev | 45bfad5 | 2015-08-21 12:19:04 +0000 | [diff] [blame] | 1715 | CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); |
| 1716 | // In presence of finite 'safelen', it may be unsafe to mark all |
| 1717 | // the memory instructions parallel, because loop-carried |
| 1718 | // dependences of 'safelen' iterations are possible. |
Alexey Bataev | a6f2a14 | 2015-12-31 06:52:34 +0000 | [diff] [blame] | 1719 | if (!IsMonotonic) |
| 1720 | CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>()); |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 1721 | } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) { |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1722 | RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), |
| 1723 | /*ignoreResult=*/true); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1724 | auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); |
Tyler Nowicki | da46d0e | 2015-07-14 23:03:09 +0000 | [diff] [blame] | 1725 | CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1726 | // In presence of finite 'safelen', it may be unsafe to mark all |
| 1727 | // the memory instructions parallel, because loop-carried |
| 1728 | // dependences of 'safelen' iterations are possible. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1729 | CGF.LoopStack.setParallel(/*Enable=*/false); |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 1730 | } |
| 1731 | } |
| 1732 | |
Alexey Bataev | a6f2a14 | 2015-12-31 06:52:34 +0000 | [diff] [blame] | 1733 | void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D, |
| 1734 | bool IsMonotonic) { |
Alexey Bataev | 58e5bdb | 2015-06-18 04:45:29 +0000 | [diff] [blame] | 1735 | // Walk clauses and process safelen/lastprivate. |
Alexey Bataev | a6f2a14 | 2015-12-31 06:52:34 +0000 | [diff] [blame] | 1736 | LoopStack.setParallel(!IsMonotonic); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1737 | LoopStack.setVectorizeEnable(); |
Alexey Bataev | a6f2a14 | 2015-12-31 06:52:34 +0000 | [diff] [blame] | 1738 | emitSimdlenSafelenClause(*this, D, IsMonotonic); |
Alexey Bataev | 58e5bdb | 2015-06-18 04:45:29 +0000 | [diff] [blame] | 1739 | } |
| 1740 | |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1741 | void CodeGenFunction::EmitOMPSimdFinal( |
| 1742 | const OMPLoopDirective &D, |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1743 | const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) { |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 1744 | if (!HaveInsertPoint()) |
| 1745 | return; |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1746 | llvm::BasicBlock *DoneBB = nullptr; |
Alexey Bataev | 58e5bdb | 2015-06-18 04:45:29 +0000 | [diff] [blame] | 1747 | auto IC = D.counters().begin(); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1748 | auto IPC = D.private_counters().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1749 | for (const Expr *F : D.finals()) { |
| 1750 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl()); |
| 1751 | const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl()); |
| 1752 | const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1753 | if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) || |
| 1754 | OrigVD->hasGlobalStorage() || CED) { |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1755 | if (!DoneBB) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1756 | if (llvm::Value *Cond = CondGen(*this)) { |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1757 | // If the first post-update expression is found, emit conditional |
| 1758 | // block if it was requested. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1759 | llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then"); |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1760 | DoneBB = createBasicBlock(".omp.final.done"); |
| 1761 | Builder.CreateCondBr(Cond, ThenBB, DoneBB); |
| 1762 | EmitBlock(ThenBB); |
| 1763 | } |
| 1764 | } |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1765 | Address OrigAddr = Address::invalid(); |
Alexey Bataev | ab4ea22 | 2018-03-07 18:17:06 +0000 | [diff] [blame] | 1766 | if (CED) { |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 1767 | OrigAddr = |
| 1768 | EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(*this); |
Alexey Bataev | ab4ea22 | 2018-03-07 18:17:06 +0000 | [diff] [blame] | 1769 | } else { |
Bruno Ricci | 5fc4db7 | 2018-12-21 14:10:18 +0000 | [diff] [blame] | 1770 | DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD), |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1771 | /*RefersToEnclosingVariableOrCapture=*/false, |
| 1772 | (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc()); |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 1773 | OrigAddr = EmitLValue(&DRE).getAddress(*this); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1774 | } |
Alexey Bataev | 58e5bdb | 2015-06-18 04:45:29 +0000 | [diff] [blame] | 1775 | OMPPrivateScope VarScope(*this); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1776 | VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; }); |
Alexey Bataev | 58e5bdb | 2015-06-18 04:45:29 +0000 | [diff] [blame] | 1777 | (void)VarScope.Privatize(); |
| 1778 | EmitIgnoredExpr(F); |
| 1779 | } |
| 1780 | ++IC; |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 1781 | ++IPC; |
Alexey Bataev | 58e5bdb | 2015-06-18 04:45:29 +0000 | [diff] [blame] | 1782 | } |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 1783 | if (DoneBB) |
| 1784 | EmitBlock(DoneBB, /*IsFinished=*/true); |
Alexey Bataev | 58e5bdb | 2015-06-18 04:45:29 +0000 | [diff] [blame] | 1785 | } |
| 1786 | |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1787 | static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, |
| 1788 | const OMPLoopDirective &S, |
| 1789 | CodeGenFunction::JumpDest LoopExit) { |
| 1790 | CGF.EmitOMPLoopBody(S, LoopExit); |
| 1791 | CGF.EmitStopPoint(&S); |
Hans Wennborg | ed129ae | 2017-04-27 17:02:25 +0000 | [diff] [blame] | 1792 | } |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1793 | |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 1794 | /// Emit a helper variable and return corresponding lvalue. |
| 1795 | static LValue EmitOMPHelperVar(CodeGenFunction &CGF, |
| 1796 | const DeclRefExpr *Helper) { |
| 1797 | auto VDecl = cast<VarDecl>(Helper->getDecl()); |
| 1798 | CGF.EmitVarDecl(*VDecl); |
| 1799 | return CGF.EmitLValue(Helper); |
| 1800 | } |
| 1801 | |
Alexey Bataev | 103f3c9e | 2019-11-20 15:59:03 -0500 | [diff] [blame] | 1802 | static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, |
| 1803 | const RegionCodeGenTy &SimdInitGen, |
| 1804 | const RegionCodeGenTy &BodyCodeGen) { |
| 1805 | auto &&ThenGen = [&SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF, |
| 1806 | PrePostActionTy &) { |
| 1807 | CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); |
| 1808 | SimdInitGen(CGF); |
| 1809 | |
| 1810 | BodyCodeGen(CGF); |
| 1811 | }; |
| 1812 | auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) { |
| 1813 | CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); |
| 1814 | CGF.LoopStack.setVectorizeEnable(/*Enable=*/false); |
| 1815 | |
| 1816 | BodyCodeGen(CGF); |
| 1817 | }; |
| 1818 | const Expr *IfCond = nullptr; |
| 1819 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { |
| 1820 | if (CGF.getLangOpts().OpenMP >= 50 && |
| 1821 | (C->getNameModifier() == OMPD_unknown || |
| 1822 | C->getNameModifier() == OMPD_simd)) { |
| 1823 | IfCond = C->getCondition(); |
| 1824 | break; |
| 1825 | } |
| 1826 | } |
| 1827 | if (IfCond) { |
| 1828 | CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen); |
| 1829 | } else { |
| 1830 | RegionCodeGenTy ThenRCG(ThenGen); |
| 1831 | ThenRCG(CGF); |
| 1832 | } |
| 1833 | } |
| 1834 | |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 1835 | static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, |
| 1836 | PrePostActionTy &Action) { |
| 1837 | Action.Enter(CGF); |
| 1838 | assert(isOpenMPSimdDirective(S.getDirectiveKind()) && |
| 1839 | "Expected simd directive"); |
| 1840 | OMPLoopScope PreInitScope(CGF, S); |
| 1841 | // if (PreCond) { |
| 1842 | // for (IV in 0..LastIteration) BODY; |
| 1843 | // <Final counter/linear vars updates>; |
| 1844 | // } |
| 1845 | // |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 1846 | if (isOpenMPDistributeDirective(S.getDirectiveKind()) || |
| 1847 | isOpenMPWorksharingDirective(S.getDirectiveKind()) || |
| 1848 | isOpenMPTaskLoopDirective(S.getDirectiveKind())) { |
| 1849 | (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable())); |
| 1850 | (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable())); |
| 1851 | } |
Alexander Musman | a5f070a | 2014-10-01 06:03:56 +0000 | [diff] [blame] | 1852 | |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 1853 | // Emit: if (PreCond) - begin. |
| 1854 | // If the condition constant folds and can be elided, avoid emitting the |
| 1855 | // whole loop. |
| 1856 | bool CondConstant; |
| 1857 | llvm::BasicBlock *ContBlock = nullptr; |
| 1858 | if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { |
| 1859 | if (!CondConstant) |
| 1860 | return; |
| 1861 | } else { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1862 | llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 1863 | ContBlock = CGF.createBasicBlock("simd.if.end"); |
| 1864 | emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, |
| 1865 | CGF.getProfileCount(&S)); |
| 1866 | CGF.EmitBlock(ThenBlock); |
| 1867 | CGF.incrementProfileCounter(&S); |
| 1868 | } |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 1869 | |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 1870 | // Emit the loop iteration variable. |
| 1871 | const Expr *IVExpr = S.getIterationVariable(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1872 | const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl()); |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 1873 | CGF.EmitVarDecl(*IVDecl); |
| 1874 | CGF.EmitIgnoredExpr(S.getInit()); |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 1875 | |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 1876 | // Emit the iterations count variable. |
| 1877 | // If it is not a variable, Sema decided to calculate iterations count on |
| 1878 | // each iteration (e.g., it is foldable into a constant). |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1879 | if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 1880 | CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); |
| 1881 | // Emit calculation of the iterations count. |
| 1882 | CGF.EmitIgnoredExpr(S.getCalcLastIteration()); |
| 1883 | } |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 1884 | |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 1885 | emitAlignedClause(CGF, S); |
| 1886 | (void)CGF.EmitOMPLinearClauseInit(S); |
| 1887 | { |
| 1888 | CodeGenFunction::OMPPrivateScope LoopScope(CGF); |
| 1889 | CGF.EmitOMPPrivateLoopCounters(S, LoopScope); |
| 1890 | CGF.EmitOMPLinearClause(S, LoopScope); |
| 1891 | CGF.EmitOMPPrivateClause(S, LoopScope); |
| 1892 | CGF.EmitOMPReductionClauseInit(S, LoopScope); |
| 1893 | bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); |
| 1894 | (void)LoopScope.Privatize(); |
Alexey Bataev | 6070542 | 2018-10-30 15:50:12 +0000 | [diff] [blame] | 1895 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) |
| 1896 | CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); |
Alexey Bataev | d08c056 | 2019-11-19 12:07:54 -0500 | [diff] [blame] | 1897 | |
Alexey Bataev | 103f3c9e | 2019-11-20 15:59:03 -0500 | [diff] [blame] | 1898 | emitCommonSimdLoop( |
| 1899 | CGF, S, |
| 1900 | [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 1901 | CGF.EmitOMPSimdInit(S); |
| 1902 | }, |
| 1903 | [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { |
| 1904 | CGF.EmitOMPInnerLoop( |
| 1905 | S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), |
| 1906 | [&S](CodeGenFunction &CGF) { |
| 1907 | CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest()); |
| 1908 | CGF.EmitStopPoint(&S); |
| 1909 | }, |
| 1910 | [](CodeGenFunction &) {}); |
| 1911 | }); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1912 | CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 1913 | // Emit final copy of the lastprivate variables at the end of loops. |
| 1914 | if (HasLastprivateClause) |
| 1915 | CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); |
| 1916 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1917 | emitPostUpdateForReductionClause(CGF, S, |
| 1918 | [](CodeGenFunction &) { return nullptr; }); |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 1919 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1920 | CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 1921 | // Emit: if (PreCond) - end. |
| 1922 | if (ContBlock) { |
| 1923 | CGF.EmitBranch(ContBlock); |
| 1924 | CGF.EmitBlock(ContBlock, true); |
| 1925 | } |
| 1926 | } |
| 1927 | |
| 1928 | void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { |
| 1929 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 1930 | emitOMPSimdRegion(CGF, S, Action); |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 1931 | }; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 1932 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Alexey Bataev | 81c7ea0 | 2015-07-03 09:56:58 +0000 | [diff] [blame] | 1933 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); |
Alexander Musman | 515ad8c | 2014-05-22 08:54:05 +0000 | [diff] [blame] | 1934 | } |
| 1935 | |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1936 | void CodeGenFunction::EmitOMPOuterLoop( |
| 1937 | bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S, |
| 1938 | CodeGenFunction::OMPPrivateScope &LoopScope, |
| 1939 | const CodeGenFunction::OMPLoopArguments &LoopArgs, |
| 1940 | const CodeGenFunction::CodeGenLoopTy &CodeGenLoop, |
| 1941 | const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1942 | CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); |
Alexander Musman | 92bdaab | 2015-03-12 13:37:50 +0000 | [diff] [blame] | 1943 | |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 1944 | const Expr *IVExpr = S.getIterationVariable(); |
| 1945 | const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); |
| 1946 | const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); |
| 1947 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1948 | JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end"); |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 1949 | |
| 1950 | // Start the loop with a block that tests the condition. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1951 | llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond"); |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 1952 | EmitBlock(CondBlock); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1953 | const SourceRange R = S.getSourceRange(); |
Amara Emerson | 652795d | 2016-11-10 14:44:30 +0000 | [diff] [blame] | 1954 | LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()), |
| 1955 | SourceLocToDebugLoc(R.getEnd())); |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 1956 | |
| 1957 | llvm::Value *BoolCondVal = nullptr; |
Alexey Bataev | d7589ffe | 2015-05-20 13:12:48 +0000 | [diff] [blame] | 1958 | if (!DynamicOrOrdered) { |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1959 | // UB = min(UB, GlobalUB) or |
| 1960 | // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g. |
| 1961 | // 'distribute parallel for') |
| 1962 | EmitIgnoredExpr(LoopArgs.EUB); |
Alexander Musman | 92bdaab | 2015-03-12 13:37:50 +0000 | [diff] [blame] | 1963 | // IV = LB |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1964 | EmitIgnoredExpr(LoopArgs.Init); |
Alexander Musman | 92bdaab | 2015-03-12 13:37:50 +0000 | [diff] [blame] | 1965 | // IV < UB |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1966 | BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond); |
Alexander Musman | 92bdaab | 2015-03-12 13:37:50 +0000 | [diff] [blame] | 1967 | } else { |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1968 | BoolCondVal = |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 1969 | RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL, |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1970 | LoopArgs.LB, LoopArgs.UB, LoopArgs.ST); |
Alexander Musman | 92bdaab | 2015-03-12 13:37:50 +0000 | [diff] [blame] | 1971 | } |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 1972 | |
| 1973 | // If there are any cleanups between here and the loop-exit scope, |
| 1974 | // create a block to stage a loop exit along. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1975 | llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 1976 | if (LoopScope.requiresCleanups()) |
| 1977 | ExitBlock = createBasicBlock("omp.dispatch.cleanup"); |
| 1978 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1979 | llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body"); |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 1980 | Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock); |
| 1981 | if (ExitBlock != LoopExit.getBlock()) { |
| 1982 | EmitBlock(ExitBlock); |
| 1983 | EmitBranchThroughCleanup(LoopExit); |
| 1984 | } |
| 1985 | EmitBlock(LoopBody); |
| 1986 | |
Alexander Musman | 92bdaab | 2015-03-12 13:37:50 +0000 | [diff] [blame] | 1987 | // Emit "IV = LB" (in case of static schedule, we have already calculated new |
| 1988 | // LB for loop condition and emitted it above). |
Alexey Bataev | d7589ffe | 2015-05-20 13:12:48 +0000 | [diff] [blame] | 1989 | if (DynamicOrOrdered) |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 1990 | EmitIgnoredExpr(LoopArgs.Init); |
Alexander Musman | 92bdaab | 2015-03-12 13:37:50 +0000 | [diff] [blame] | 1991 | |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 1992 | // Create a block for the increment. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 1993 | JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc"); |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 1994 | BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); |
| 1995 | |
Alexey Bataev | 103f3c9e | 2019-11-20 15:59:03 -0500 | [diff] [blame] | 1996 | emitCommonSimdLoop( |
| 1997 | *this, S, |
| 1998 | [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) { |
| 1999 | // Generate !llvm.loop.parallel metadata for loads and stores for loops |
| 2000 | // with dynamic/guided scheduling and without ordered clause. |
| 2001 | if (!isOpenMPSimdDirective(S.getDirectiveKind())) |
| 2002 | CGF.LoopStack.setParallel(!IsMonotonic); |
| 2003 | else |
| 2004 | CGF.EmitOMPSimdInit(S, IsMonotonic); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2005 | }, |
Alexey Bataev | 103f3c9e | 2019-11-20 15:59:03 -0500 | [diff] [blame] | 2006 | [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered, |
| 2007 | &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { |
| 2008 | SourceLocation Loc = S.getBeginLoc(); |
| 2009 | // when 'distribute' is not combined with a 'for': |
| 2010 | // while (idx <= UB) { BODY; ++idx; } |
| 2011 | // when 'distribute' is combined with a 'for' |
| 2012 | // (e.g. 'distribute parallel for') |
| 2013 | // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; } |
| 2014 | CGF.EmitOMPInnerLoop( |
| 2015 | S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr, |
| 2016 | [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { |
| 2017 | CodeGenLoop(CGF, S, LoopExit); |
| 2018 | }, |
| 2019 | [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) { |
| 2020 | CodeGenOrdered(CGF, Loc, IVSize, IVSigned); |
| 2021 | }); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2022 | }); |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 2023 | |
| 2024 | EmitBlock(Continue.getBlock()); |
| 2025 | BreakContinueStack.pop_back(); |
Alexey Bataev | d7589ffe | 2015-05-20 13:12:48 +0000 | [diff] [blame] | 2026 | if (!DynamicOrOrdered) { |
Alexander Musman | 92bdaab | 2015-03-12 13:37:50 +0000 | [diff] [blame] | 2027 | // Emit "LB = LB + Stride", "UB = UB + Stride". |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2028 | EmitIgnoredExpr(LoopArgs.NextLB); |
| 2029 | EmitIgnoredExpr(LoopArgs.NextUB); |
Alexander Musman | 92bdaab | 2015-03-12 13:37:50 +0000 | [diff] [blame] | 2030 | } |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 2031 | |
| 2032 | EmitBranch(CondBlock); |
| 2033 | LoopStack.pop(); |
| 2034 | // Emit the fall-through block. |
| 2035 | EmitBlock(LoopExit.getBlock()); |
| 2036 | |
| 2037 | // Tell the runtime we are done. |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 2038 | auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) { |
| 2039 | if (!DynamicOrOrdered) |
Stephen Kelly | 1c301dc | 2018-08-09 21:09:38 +0000 | [diff] [blame] | 2040 | CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), |
Alexey Bataev | f43f714 | 2017-09-06 16:17:35 +0000 | [diff] [blame] | 2041 | S.getDirectiveKind()); |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 2042 | }; |
| 2043 | OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2044 | } |
| 2045 | |
| 2046 | void CodeGenFunction::EmitOMPForOuterLoop( |
Alexey Bataev | 9ebd742 | 2016-05-10 09:57:36 +0000 | [diff] [blame] | 2047 | const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic, |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2048 | const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered, |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2049 | const OMPLoopArguments &LoopArgs, |
| 2050 | const CodeGenDispatchBoundsTy &CGDispatchBounds) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2051 | CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2052 | |
| 2053 | // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime). |
Alexey Bataev | 9ebd742 | 2016-05-10 09:57:36 +0000 | [diff] [blame] | 2054 | const bool DynamicOrOrdered = |
| 2055 | Ordered || RT.isDynamic(ScheduleKind.Schedule); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2056 | |
| 2057 | assert((Ordered || |
Alexey Bataev | 9ebd742 | 2016-05-10 09:57:36 +0000 | [diff] [blame] | 2058 | !RT.isStaticNonchunked(ScheduleKind.Schedule, |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2059 | LoopArgs.Chunk != nullptr)) && |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2060 | "static non-chunked schedule does not need outer loop"); |
| 2061 | |
| 2062 | // Emit outer loop. |
| 2063 | // |
| 2064 | // OpenMP [2.7.1, Loop Construct, Description, table 2-1] |
| 2065 | // When schedule(dynamic,chunk_size) is specified, the iterations are |
| 2066 | // distributed to threads in the team in chunks as the threads request them. |
| 2067 | // Each thread executes a chunk of iterations, then requests another chunk, |
| 2068 | // until no chunks remain to be distributed. Each chunk contains chunk_size |
| 2069 | // iterations, except for the last chunk to be distributed, which may have |
| 2070 | // fewer iterations. When no chunk_size is specified, it defaults to 1. |
| 2071 | // |
| 2072 | // When schedule(guided,chunk_size) is specified, the iterations are assigned |
| 2073 | // to threads in the team in chunks as the executing threads request them. |
| 2074 | // Each thread executes a chunk of iterations, then requests another chunk, |
| 2075 | // until no chunks remain to be assigned. For a chunk_size of 1, the size of |
| 2076 | // each chunk is proportional to the number of unassigned iterations divided |
| 2077 | // by the number of threads in the team, decreasing to 1. For a chunk_size |
| 2078 | // with value k (greater than 1), the size of each chunk is determined in the |
| 2079 | // same way, with the restriction that the chunks do not contain fewer than k |
| 2080 | // iterations (except for the last chunk to be assigned, which may have fewer |
| 2081 | // than k iterations). |
| 2082 | // |
| 2083 | // When schedule(auto) is specified, the decision regarding scheduling is |
| 2084 | // delegated to the compiler and/or runtime system. The programmer gives the |
| 2085 | // implementation the freedom to choose any possible mapping of iterations to |
| 2086 | // threads in the team. |
| 2087 | // |
| 2088 | // When schedule(runtime) is specified, the decision regarding scheduling is |
| 2089 | // deferred until run time, and the schedule and chunk size are taken from the |
| 2090 | // run-sched-var ICV. If the ICV is set to auto, the schedule is |
| 2091 | // implementation defined |
| 2092 | // |
| 2093 | // while(__kmpc_dispatch_next(&LB, &UB)) { |
| 2094 | // idx = LB; |
| 2095 | // while (idx <= UB) { BODY; ++idx; |
| 2096 | // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only. |
| 2097 | // } // inner loop |
| 2098 | // } |
| 2099 | // |
| 2100 | // OpenMP [2.7.1, Loop Construct, Description, table 2-1] |
| 2101 | // When schedule(static, chunk_size) is specified, iterations are divided into |
| 2102 | // chunks of size chunk_size, and the chunks are assigned to the threads in |
| 2103 | // the team in a round-robin fashion in the order of the thread number. |
| 2104 | // |
| 2105 | // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) { |
| 2106 | // while (idx <= UB) { BODY; ++idx; } // inner loop |
| 2107 | // LB = LB + ST; |
| 2108 | // UB = UB + ST; |
| 2109 | // } |
| 2110 | // |
| 2111 | |
| 2112 | const Expr *IVExpr = S.getIterationVariable(); |
| 2113 | const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); |
| 2114 | const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); |
| 2115 | |
| 2116 | if (DynamicOrOrdered) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2117 | const std::pair<llvm::Value *, llvm::Value *> DispatchBounds = |
| 2118 | CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2119 | llvm::Value *LBVal = DispatchBounds.first; |
| 2120 | llvm::Value *UBVal = DispatchBounds.second; |
| 2121 | CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal, |
| 2122 | LoopArgs.Chunk}; |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2123 | RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize, |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2124 | IVSigned, Ordered, DipatchRTInputValues); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2125 | } else { |
Alexey Bataev | 0f87dbe | 2017-08-14 17:56:13 +0000 | [diff] [blame] | 2126 | CGOpenMPRuntime::StaticRTInput StaticInit( |
| 2127 | IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB, |
| 2128 | LoopArgs.ST, LoopArgs.Chunk); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2129 | RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(), |
Alexey Bataev | 0f87dbe | 2017-08-14 17:56:13 +0000 | [diff] [blame] | 2130 | ScheduleKind, StaticInit); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2131 | } |
| 2132 | |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2133 | auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc, |
| 2134 | const unsigned IVSize, |
| 2135 | const bool IVSigned) { |
| 2136 | if (Ordered) { |
| 2137 | CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize, |
| 2138 | IVSigned); |
| 2139 | } |
| 2140 | }; |
| 2141 | |
| 2142 | OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST, |
| 2143 | LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB); |
| 2144 | OuterLoopArgs.IncExpr = S.getInc(); |
| 2145 | OuterLoopArgs.Init = S.getInit(); |
| 2146 | OuterLoopArgs.Cond = S.getCond(); |
| 2147 | OuterLoopArgs.NextLB = S.getNextLowerBound(); |
| 2148 | OuterLoopArgs.NextUB = S.getNextUpperBound(); |
| 2149 | EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs, |
| 2150 | emitOMPLoopBodyWithStopPoint, CodeGenOrdered); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2151 | } |
| 2152 | |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2153 | static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, |
| 2154 | const unsigned IVSize, const bool IVSigned) {} |
| 2155 | |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2156 | void CodeGenFunction::EmitOMPDistributeOuterLoop( |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2157 | OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S, |
| 2158 | OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs, |
| 2159 | const CodeGenLoopTy &CodeGenLoopContent) { |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2160 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2161 | CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2162 | |
| 2163 | // Emit outer loop. |
| 2164 | // Same behavior as a OMPForOuterLoop, except that schedule cannot be |
| 2165 | // dynamic |
| 2166 | // |
| 2167 | |
| 2168 | const Expr *IVExpr = S.getIterationVariable(); |
| 2169 | const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); |
| 2170 | const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); |
| 2171 | |
Alexey Bataev | 0f87dbe | 2017-08-14 17:56:13 +0000 | [diff] [blame] | 2172 | CGOpenMPRuntime::StaticRTInput StaticInit( |
| 2173 | IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB, |
| 2174 | LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2175 | RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 2176 | |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2177 | // for combined 'distribute' and 'for' the increment expression of distribute |
Gheorghe-Teodor Bercea | e925676 | 2018-10-29 15:45:47 +0000 | [diff] [blame] | 2178 | // is stored in DistInc. For 'distribute' alone, it is in Inc. |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2179 | Expr *IncExpr; |
| 2180 | if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())) |
| 2181 | IncExpr = S.getDistInc(); |
| 2182 | else |
| 2183 | IncExpr = S.getInc(); |
| 2184 | |
| 2185 | // this routine is shared by 'omp distribute parallel for' and |
| 2186 | // 'omp distribute': select the right EUB expression depending on the |
| 2187 | // directive |
| 2188 | OMPLoopArguments OuterLoopArgs; |
| 2189 | OuterLoopArgs.LB = LoopArgs.LB; |
| 2190 | OuterLoopArgs.UB = LoopArgs.UB; |
| 2191 | OuterLoopArgs.ST = LoopArgs.ST; |
| 2192 | OuterLoopArgs.IL = LoopArgs.IL; |
| 2193 | OuterLoopArgs.Chunk = LoopArgs.Chunk; |
| 2194 | OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) |
| 2195 | ? S.getCombinedEnsureUpperBound() |
| 2196 | : S.getEnsureUpperBound(); |
| 2197 | OuterLoopArgs.IncExpr = IncExpr; |
| 2198 | OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) |
| 2199 | ? S.getCombinedInit() |
| 2200 | : S.getInit(); |
| 2201 | OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) |
| 2202 | ? S.getCombinedCond() |
| 2203 | : S.getCond(); |
| 2204 | OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) |
| 2205 | ? S.getCombinedNextLowerBound() |
| 2206 | : S.getNextLowerBound(); |
| 2207 | OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) |
| 2208 | ? S.getCombinedNextUpperBound() |
| 2209 | : S.getNextUpperBound(); |
| 2210 | |
| 2211 | EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S, |
| 2212 | LoopScope, OuterLoopArgs, CodeGenLoopContent, |
| 2213 | emitEmptyOrdered); |
| 2214 | } |
| 2215 | |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2216 | static std::pair<LValue, LValue> |
| 2217 | emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, |
| 2218 | const OMPExecutableDirective &S) { |
| 2219 | const OMPLoopDirective &LS = cast<OMPLoopDirective>(S); |
| 2220 | LValue LB = |
| 2221 | EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); |
| 2222 | LValue UB = |
| 2223 | EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); |
| 2224 | |
| 2225 | // When composing 'distribute' with 'for' (e.g. as in 'distribute |
| 2226 | // parallel for') we need to use the 'distribute' |
| 2227 | // chunk lower and upper bounds rather than the whole loop iteration |
| 2228 | // space. These are parameters to the outlined function for 'parallel' |
| 2229 | // and we copy the bounds of the previous schedule into the |
| 2230 | // the current ones. |
| 2231 | LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable()); |
| 2232 | LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable()); |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 2233 | llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( |
| 2234 | PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc()); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2235 | PrevLBVal = CGF.EmitScalarConversion( |
| 2236 | PrevLBVal, LS.getPrevLowerBoundVariable()->getType(), |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 2237 | LS.getIterationVariable()->getType(), |
| 2238 | LS.getPrevLowerBoundVariable()->getExprLoc()); |
| 2239 | llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( |
| 2240 | PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc()); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2241 | PrevUBVal = CGF.EmitScalarConversion( |
| 2242 | PrevUBVal, LS.getPrevUpperBoundVariable()->getType(), |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 2243 | LS.getIterationVariable()->getType(), |
| 2244 | LS.getPrevUpperBoundVariable()->getExprLoc()); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2245 | |
| 2246 | CGF.EmitStoreOfScalar(PrevLBVal, LB); |
| 2247 | CGF.EmitStoreOfScalar(PrevUBVal, UB); |
| 2248 | |
| 2249 | return {LB, UB}; |
| 2250 | } |
| 2251 | |
| 2252 | /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then |
| 2253 | /// we need to use the LB and UB expressions generated by the worksharing |
| 2254 | /// code generation support, whereas in non combined situations we would |
| 2255 | /// just emit 0 and the LastIteration expression |
| 2256 | /// This function is necessary due to the difference of the LB and UB |
| 2257 | /// types for the RT emission routines for 'for_static_init' and |
| 2258 | /// 'for_dispatch_init' |
| 2259 | static std::pair<llvm::Value *, llvm::Value *> |
| 2260 | emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, |
| 2261 | const OMPExecutableDirective &S, |
| 2262 | Address LB, Address UB) { |
| 2263 | const OMPLoopDirective &LS = cast<OMPLoopDirective>(S); |
| 2264 | const Expr *IVExpr = LS.getIterationVariable(); |
| 2265 | // when implementing a dynamic schedule for a 'for' combined with a |
| 2266 | // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop |
| 2267 | // is not normalized as each team only executes its own assigned |
| 2268 | // distribute chunk |
| 2269 | QualType IteratorTy = IVExpr->getType(); |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 2270 | llvm::Value *LBVal = |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2271 | CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 2272 | llvm::Value *UBVal = |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2273 | CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2274 | return {LBVal, UBVal}; |
Hans Wennborg | ed129ae | 2017-04-27 17:02:25 +0000 | [diff] [blame] | 2275 | } |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2276 | |
| 2277 | static void emitDistributeParallelForDistributeInnerBoundParams( |
| 2278 | CodeGenFunction &CGF, const OMPExecutableDirective &S, |
| 2279 | llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) { |
| 2280 | const auto &Dir = cast<OMPLoopDirective>(S); |
| 2281 | LValue LB = |
| 2282 | CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable())); |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 2283 | llvm::Value *LBCast = |
| 2284 | CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)), |
| 2285 | CGF.SizeTy, /*isSigned=*/false); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2286 | CapturedVars.push_back(LBCast); |
| 2287 | LValue UB = |
| 2288 | CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable())); |
| 2289 | |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 2290 | llvm::Value *UBCast = |
| 2291 | CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)), |
| 2292 | CGF.SizeTy, /*isSigned=*/false); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2293 | CapturedVars.push_back(UBCast); |
Hans Wennborg | ed129ae | 2017-04-27 17:02:25 +0000 | [diff] [blame] | 2294 | } |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2295 | |
| 2296 | static void |
| 2297 | emitInnerParallelForWhenCombined(CodeGenFunction &CGF, |
| 2298 | const OMPLoopDirective &S, |
| 2299 | CodeGenFunction::JumpDest LoopExit) { |
| 2300 | auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF, |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 2301 | PrePostActionTy &Action) { |
| 2302 | Action.Enter(CGF); |
Alexey Bataev | dcb4b8fb | 2017-11-22 20:19:50 +0000 | [diff] [blame] | 2303 | bool HasCancel = false; |
| 2304 | if (!isOpenMPSimdDirective(S.getDirectiveKind())) { |
| 2305 | if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S)) |
| 2306 | HasCancel = D->hasCancel(); |
| 2307 | else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S)) |
| 2308 | HasCancel = D->hasCancel(); |
Alexey Bataev | 16e7988 | 2017-11-22 21:12:03 +0000 | [diff] [blame] | 2309 | else if (const auto *D = |
| 2310 | dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S)) |
| 2311 | HasCancel = D->hasCancel(); |
Alexey Bataev | dcb4b8fb | 2017-11-22 20:19:50 +0000 | [diff] [blame] | 2312 | } |
| 2313 | CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), |
| 2314 | HasCancel); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2315 | CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(), |
| 2316 | emitDistributeParallelForInnerBounds, |
| 2317 | emitDistributeParallelForDispatchBounds); |
| 2318 | }; |
| 2319 | |
| 2320 | emitCommonOMPParallelDirective( |
Alexey Bataev | dcb4b8fb | 2017-11-22 20:19:50 +0000 | [diff] [blame] | 2321 | CGF, S, |
| 2322 | isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for, |
| 2323 | CGInlinedWorksharingLoop, |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2324 | emitDistributeParallelForDistributeInnerBoundParams); |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 2325 | } |
| 2326 | |
Carlo Bertolli | 9925f15 | 2016-06-27 14:55:37 +0000 | [diff] [blame] | 2327 | void CodeGenFunction::EmitOMPDistributeParallelForDirective( |
| 2328 | const OMPDistributeParallelForDirective &S) { |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2329 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 2330 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, |
| 2331 | S.getDistInc()); |
| 2332 | }; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2333 | OMPLexicalScope Scope(*this, S, OMPD_parallel); |
Alexey Bataev | 10a5431 | 2017-11-27 16:54:08 +0000 | [diff] [blame] | 2334 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); |
Carlo Bertolli | 9925f15 | 2016-06-27 14:55:37 +0000 | [diff] [blame] | 2335 | } |
| 2336 | |
Kelvin Li | 4a39add | 2016-07-05 05:00:15 +0000 | [diff] [blame] | 2337 | void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective( |
| 2338 | const OMPDistributeParallelForSimdDirective &S) { |
Alexey Bataev | 0b49f9e | 2017-11-27 19:38:58 +0000 | [diff] [blame] | 2339 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 2340 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, |
| 2341 | S.getDistInc()); |
| 2342 | }; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2343 | OMPLexicalScope Scope(*this, S, OMPD_parallel); |
Alexey Bataev | 0b49f9e | 2017-11-27 19:38:58 +0000 | [diff] [blame] | 2344 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); |
Kelvin Li | 4a39add | 2016-07-05 05:00:15 +0000 | [diff] [blame] | 2345 | } |
Kelvin Li | 787f3fc | 2016-07-06 04:45:38 +0000 | [diff] [blame] | 2346 | |
| 2347 | void CodeGenFunction::EmitOMPDistributeSimdDirective( |
| 2348 | const OMPDistributeSimdDirective &S) { |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 2349 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 2350 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); |
| 2351 | }; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2352 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 2353 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); |
Kelvin Li | 787f3fc | 2016-07-06 04:45:38 +0000 | [diff] [blame] | 2354 | } |
| 2355 | |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 2356 | void CodeGenFunction::EmitOMPTargetSimdDeviceFunction( |
| 2357 | CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) { |
| 2358 | // Emit SPMD target parallel for region as a standalone region. |
| 2359 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 2360 | emitOMPSimdRegion(CGF, S, Action); |
| 2361 | }; |
| 2362 | llvm::Function *Fn; |
| 2363 | llvm::Constant *Addr; |
| 2364 | // Emit target region as a standalone region. |
| 2365 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( |
| 2366 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); |
| 2367 | assert(Fn && Addr && "Target device function emission failed."); |
| 2368 | } |
| 2369 | |
Kelvin Li | 986330c | 2016-07-20 22:57:10 +0000 | [diff] [blame] | 2370 | void CodeGenFunction::EmitOMPTargetSimdDirective( |
| 2371 | const OMPTargetSimdDirective &S) { |
Alexey Bataev | f836537 | 2017-11-17 17:57:25 +0000 | [diff] [blame] | 2372 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 2373 | emitOMPSimdRegion(CGF, S, Action); |
| 2374 | }; |
| 2375 | emitCommonOMPTargetDirective(*this, S, CodeGen); |
Kelvin Li | 986330c | 2016-07-20 22:57:10 +0000 | [diff] [blame] | 2376 | } |
| 2377 | |
Alexey Bataev | a6f2a14 | 2015-12-31 06:52:34 +0000 | [diff] [blame] | 2378 | namespace { |
| 2379 | struct ScheduleKindModifiersTy { |
| 2380 | OpenMPScheduleClauseKind Kind; |
| 2381 | OpenMPScheduleClauseModifier M1; |
| 2382 | OpenMPScheduleClauseModifier M2; |
| 2383 | ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind, |
| 2384 | OpenMPScheduleClauseModifier M1, |
| 2385 | OpenMPScheduleClauseModifier M2) |
| 2386 | : Kind(Kind), M1(M1), M2(M2) {} |
| 2387 | }; |
| 2388 | } // namespace |
| 2389 | |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2390 | bool CodeGenFunction::EmitOMPWorksharingLoop( |
| 2391 | const OMPLoopDirective &S, Expr *EUB, |
| 2392 | const CodeGenLoopBoundsTy &CodeGenLoopBounds, |
| 2393 | const CodeGenDispatchBoundsTy &CGDispatchBounds) { |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2394 | // Emit the loop iteration variable. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2395 | const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable()); |
| 2396 | const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl()); |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2397 | EmitVarDecl(*IVDecl); |
| 2398 | |
| 2399 | // Emit the iterations count variable. |
| 2400 | // If it is not a variable, Sema decided to calculate iterations count on each |
| 2401 | // iteration (e.g., it is foldable into a constant). |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2402 | if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2403 | EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); |
| 2404 | // Emit calculation of the iterations count. |
| 2405 | EmitIgnoredExpr(S.getCalcLastIteration()); |
| 2406 | } |
| 2407 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2408 | CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2409 | |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 2410 | bool HasLastprivateClause; |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2411 | // Check pre-condition. |
| 2412 | { |
Alexey Bataev | 5a3af13 | 2016-03-29 08:58:54 +0000 | [diff] [blame] | 2413 | OMPLoopScope PreInitScope(*this, S); |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2414 | // Skip the entire loop if we don't meet the precondition. |
Alexey Bataev | 62dbb97 | 2015-04-22 11:59:37 +0000 | [diff] [blame] | 2415 | // If the condition constant folds and can be elided, avoid emitting the |
| 2416 | // whole loop. |
| 2417 | bool CondConstant; |
| 2418 | llvm::BasicBlock *ContBlock = nullptr; |
| 2419 | if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { |
| 2420 | if (!CondConstant) |
| 2421 | return false; |
| 2422 | } else { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2423 | llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then"); |
Alexey Bataev | 62dbb97 | 2015-04-22 11:59:37 +0000 | [diff] [blame] | 2424 | ContBlock = createBasicBlock("omp.precond.end"); |
| 2425 | emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock, |
Justin Bogner | 66242d6 | 2015-04-23 23:06:47 +0000 | [diff] [blame] | 2426 | getProfileCount(&S)); |
Alexey Bataev | 62dbb97 | 2015-04-22 11:59:37 +0000 | [diff] [blame] | 2427 | EmitBlock(ThenBlock); |
Justin Bogner | 66242d6 | 2015-04-23 23:06:47 +0000 | [diff] [blame] | 2428 | incrementProfileCounter(&S); |
Alexey Bataev | 62dbb97 | 2015-04-22 11:59:37 +0000 | [diff] [blame] | 2429 | } |
Alexey Bataev | 58e5bdb | 2015-06-18 04:45:29 +0000 | [diff] [blame] | 2430 | |
Alexey Bataev | ea33dee | 2018-02-15 23:39:43 +0000 | [diff] [blame] | 2431 | RunCleanupsScope DoacrossCleanupScope(*this); |
Alexey Bataev | 8b42706 | 2016-05-25 12:36:08 +0000 | [diff] [blame] | 2432 | bool Ordered = false; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2433 | if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) { |
Alexey Bataev | 8b42706 | 2016-05-25 12:36:08 +0000 | [diff] [blame] | 2434 | if (OrderedClause->getNumForLoops()) |
Alexey Bataev | f138fda | 2018-08-13 19:04:24 +0000 | [diff] [blame] | 2435 | RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations()); |
Alexey Bataev | 8b42706 | 2016-05-25 12:36:08 +0000 | [diff] [blame] | 2436 | else |
| 2437 | Ordered = true; |
| 2438 | } |
| 2439 | |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 2440 | llvm::DenseSet<const Expr *> EmittedFinals; |
Alexey Bataev | 58e5bdb | 2015-06-18 04:45:29 +0000 | [diff] [blame] | 2441 | emitAlignedClause(*this, S); |
Alexey Bataev | 8c3edfe | 2017-08-16 15:58:46 +0000 | [diff] [blame] | 2442 | bool HasLinears = EmitOMPLinearClauseInit(S); |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 2443 | // Emit helper vars inits. |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2444 | |
| 2445 | std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S); |
| 2446 | LValue LB = Bounds.first; |
| 2447 | LValue UB = Bounds.second; |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 2448 | LValue ST = |
| 2449 | EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable())); |
| 2450 | LValue IL = |
| 2451 | EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable())); |
| 2452 | |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2453 | // Emit 'then' code. |
| 2454 | { |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2455 | OMPPrivateScope LoopScope(*this); |
Alexey Bataev | 8c3edfe | 2017-08-16 15:58:46 +0000 | [diff] [blame] | 2456 | if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) { |
Alexey Bataev | 69c62a9 | 2015-04-15 04:52:20 +0000 | [diff] [blame] | 2457 | // Emit implicit barrier to synchronize threads and avoid data races on |
Alexey Bataev | cd8b6a2 | 2016-02-15 08:07:17 +0000 | [diff] [blame] | 2458 | // initialization of firstprivate variables and post-update of |
| 2459 | // lastprivate variables. |
Alexey Bataev | 25e5b44 | 2015-09-15 12:52:43 +0000 | [diff] [blame] | 2460 | CGM.getOpenMPRuntime().emitBarrierCall( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2461 | *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, |
Alexey Bataev | 25e5b44 | 2015-09-15 12:52:43 +0000 | [diff] [blame] | 2462 | /*ForceSimpleCall=*/true); |
Alexey Bataev | 69c62a9 | 2015-04-15 04:52:20 +0000 | [diff] [blame] | 2463 | } |
Alexey Bataev | 50a6458 | 2015-04-22 12:24:45 +0000 | [diff] [blame] | 2464 | EmitOMPPrivateClause(S, LoopScope); |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 2465 | HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); |
Alexey Bataev | 7ebe5fd | 2015-04-22 13:43:03 +0000 | [diff] [blame] | 2466 | EmitOMPReductionClauseInit(S, LoopScope); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 2467 | EmitOMPPrivateLoopCounters(S, LoopScope); |
| 2468 | EmitOMPLinearClause(S, LoopScope); |
Alexander Musman | 7931b98 | 2015-03-16 07:14:41 +0000 | [diff] [blame] | 2469 | (void)LoopScope.Privatize(); |
Alexey Bataev | 6070542 | 2018-10-30 15:50:12 +0000 | [diff] [blame] | 2470 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) |
| 2471 | CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S); |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2472 | |
| 2473 | // Detect the loop schedule kind and chunk. |
Gheorghe-Teodor Bercea | e925676 | 2018-10-29 15:45:47 +0000 | [diff] [blame] | 2474 | const Expr *ChunkExpr = nullptr; |
Alexey Bataev | 9ebd742 | 2016-05-10 09:57:36 +0000 | [diff] [blame] | 2475 | OpenMPScheduleTy ScheduleKind; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2476 | if (const auto *C = S.getSingleClause<OMPScheduleClause>()) { |
Alexey Bataev | 9ebd742 | 2016-05-10 09:57:36 +0000 | [diff] [blame] | 2477 | ScheduleKind.Schedule = C->getScheduleKind(); |
| 2478 | ScheduleKind.M1 = C->getFirstScheduleModifier(); |
| 2479 | ScheduleKind.M2 = C->getSecondScheduleModifier(); |
Gheorghe-Teodor Bercea | e925676 | 2018-10-29 15:45:47 +0000 | [diff] [blame] | 2480 | ChunkExpr = C->getChunkSize(); |
Gheorghe-Teodor Bercea | 8233af9 | 2018-09-27 20:29:00 +0000 | [diff] [blame] | 2481 | } else { |
| 2482 | // Default behaviour for schedule clause. |
| 2483 | CGM.getOpenMPRuntime().getDefaultScheduleAndChunk( |
Gheorghe-Teodor Bercea | e925676 | 2018-10-29 15:45:47 +0000 | [diff] [blame] | 2484 | *this, S, ScheduleKind.Schedule, ChunkExpr); |
| 2485 | } |
| 2486 | bool HasChunkSizeOne = false; |
| 2487 | llvm::Value *Chunk = nullptr; |
| 2488 | if (ChunkExpr) { |
| 2489 | Chunk = EmitScalarExpr(ChunkExpr); |
| 2490 | Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(), |
| 2491 | S.getIterationVariable()->getType(), |
| 2492 | S.getBeginLoc()); |
Fangrui Song | 407659a | 2018-11-30 23:41:18 +0000 | [diff] [blame] | 2493 | Expr::EvalResult Result; |
| 2494 | if (ChunkExpr->EvaluateAsInt(Result, getContext())) { |
| 2495 | llvm::APSInt EvaluatedChunk = Result.Val.getInt(); |
Gheorghe-Teodor Bercea | e925676 | 2018-10-29 15:45:47 +0000 | [diff] [blame] | 2496 | HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1); |
Fangrui Song | 407659a | 2018-11-30 23:41:18 +0000 | [diff] [blame] | 2497 | } |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 2498 | } |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2499 | const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); |
| 2500 | const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); |
Alexey Bataev | a6f2a14 | 2015-12-31 06:52:34 +0000 | [diff] [blame] | 2501 | // OpenMP 4.5, 2.7.1 Loop Construct, Description. |
| 2502 | // If the static schedule kind is specified or if the ordered clause is |
| 2503 | // specified, and if no monotonic modifier is specified, the effect will |
| 2504 | // be as if the monotonic modifier was specified. |
Gheorghe-Teodor Bercea | e925676 | 2018-10-29 15:45:47 +0000 | [diff] [blame] | 2505 | bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule, |
| 2506 | /* Chunked */ Chunk != nullptr) && HasChunkSizeOne && |
| 2507 | isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()); |
| 2508 | if ((RT.isStaticNonchunked(ScheduleKind.Schedule, |
| 2509 | /* Chunked */ Chunk != nullptr) || |
| 2510 | StaticChunkedOne) && |
Alexey Bataev | d7589ffe | 2015-05-20 13:12:48 +0000 | [diff] [blame] | 2511 | !Ordered) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2512 | JumpDest LoopExit = |
Alexey Bataev | a6f2a14 | 2015-12-31 06:52:34 +0000 | [diff] [blame] | 2513 | getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit")); |
Alexey Bataev | 103f3c9e | 2019-11-20 15:59:03 -0500 | [diff] [blame] | 2514 | emitCommonSimdLoop( |
| 2515 | *this, S, |
| 2516 | [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 2517 | if (isOpenMPSimdDirective(S.getDirectiveKind())) |
| 2518 | CGF.EmitOMPSimdInit(S, /*IsMonotonic=*/true); |
Gheorghe-Teodor Bercea | e925676 | 2018-10-29 15:45:47 +0000 | [diff] [blame] | 2519 | }, |
Alexey Bataev | 103f3c9e | 2019-11-20 15:59:03 -0500 | [diff] [blame] | 2520 | [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk, |
| 2521 | &S, ScheduleKind, LoopExit, |
| 2522 | &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { |
| 2523 | // OpenMP [2.7.1, Loop Construct, Description, table 2-1] |
| 2524 | // When no chunk_size is specified, the iteration space is divided |
| 2525 | // into chunks that are approximately equal in size, and at most |
| 2526 | // one chunk is distributed to each thread. Note that the size of |
| 2527 | // the chunks is unspecified in this case. |
| 2528 | CGOpenMPRuntime::StaticRTInput StaticInit( |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 2529 | IVSize, IVSigned, Ordered, IL.getAddress(CGF), |
| 2530 | LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF), |
Alexey Bataev | 103f3c9e | 2019-11-20 15:59:03 -0500 | [diff] [blame] | 2531 | StaticChunkedOne ? Chunk : nullptr); |
| 2532 | CGF.CGM.getOpenMPRuntime().emitForStaticInit( |
| 2533 | CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, |
| 2534 | StaticInit); |
| 2535 | // UB = min(UB, GlobalUB); |
| 2536 | if (!StaticChunkedOne) |
| 2537 | CGF.EmitIgnoredExpr(S.getEnsureUpperBound()); |
| 2538 | // IV = LB; |
| 2539 | CGF.EmitIgnoredExpr(S.getInit()); |
| 2540 | // For unchunked static schedule generate: |
| 2541 | // |
| 2542 | // while (idx <= UB) { |
| 2543 | // BODY; |
| 2544 | // ++idx; |
| 2545 | // } |
| 2546 | // |
| 2547 | // For static schedule with chunk one: |
| 2548 | // |
| 2549 | // while (IV <= PrevUB) { |
| 2550 | // BODY; |
| 2551 | // IV += ST; |
| 2552 | // } |
| 2553 | CGF.EmitOMPInnerLoop( |
| 2554 | S, LoopScope.requiresCleanups(), |
| 2555 | StaticChunkedOne ? S.getCombinedParForInDistCond() |
| 2556 | : S.getCond(), |
| 2557 | StaticChunkedOne ? S.getDistInc() : S.getInc(), |
| 2558 | [&S, LoopExit](CodeGenFunction &CGF) { |
| 2559 | CGF.EmitOMPLoopBody(S, LoopExit); |
| 2560 | CGF.EmitStopPoint(&S); |
| 2561 | }, |
| 2562 | [](CodeGenFunction &) {}); |
| 2563 | }); |
Alexey Bataev | 0f34da1 | 2015-07-02 04:17:07 +0000 | [diff] [blame] | 2564 | EmitBlock(LoopExit.getBlock()); |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2565 | // Tell the runtime we are done. |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 2566 | auto &&CodeGen = [&S](CodeGenFunction &CGF) { |
Stephen Kelly | 1c301dc | 2018-08-09 21:09:38 +0000 | [diff] [blame] | 2567 | CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), |
Alexey Bataev | f43f714 | 2017-09-06 16:17:35 +0000 | [diff] [blame] | 2568 | S.getDirectiveKind()); |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 2569 | }; |
| 2570 | OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen); |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 2571 | } else { |
Alexey Bataev | 9ebd742 | 2016-05-10 09:57:36 +0000 | [diff] [blame] | 2572 | const bool IsMonotonic = |
| 2573 | Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static || |
| 2574 | ScheduleKind.Schedule == OMPC_SCHEDULE_unknown || |
| 2575 | ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic || |
| 2576 | ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic; |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 2577 | // Emit the outer loop, which requests its work chunk [LB..UB] from |
| 2578 | // runtime and runs the inner loop to process it. |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 2579 | const OMPLoopArguments LoopArguments( |
| 2580 | LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this), |
| 2581 | IL.getAddress(*this), Chunk, EUB); |
Alexey Bataev | a6f2a14 | 2015-12-31 06:52:34 +0000 | [diff] [blame] | 2582 | EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered, |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2583 | LoopArguments, CGDispatchBounds); |
Alexander Musman | df7a8e2 | 2015-01-22 08:49:35 +0000 | [diff] [blame] | 2584 | } |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 2585 | if (isOpenMPSimdDirective(S.getDirectiveKind())) { |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2586 | EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { |
| 2587 | return CGF.Builder.CreateIsNotNull( |
| 2588 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); |
| 2589 | }); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 2590 | } |
Arpith Chacko Jacob | 101e8fb | 2017-02-16 16:20:16 +0000 | [diff] [blame] | 2591 | EmitOMPReductionClauseFinal( |
| 2592 | S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind()) |
| 2593 | ? /*Parallel and Simd*/ OMPD_parallel_for_simd |
| 2594 | : /*Parallel only*/ OMPD_parallel); |
Alexey Bataev | 6120507 | 2016-03-02 04:57:40 +0000 | [diff] [blame] | 2595 | // Emit post-update of the reduction variables if IsLastIter != 0. |
| 2596 | emitPostUpdateForReductionClause( |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2597 | *this, S, [IL, &S](CodeGenFunction &CGF) { |
Alexey Bataev | 6120507 | 2016-03-02 04:57:40 +0000 | [diff] [blame] | 2598 | return CGF.Builder.CreateIsNotNull( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2599 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); |
Alexey Bataev | 6120507 | 2016-03-02 04:57:40 +0000 | [diff] [blame] | 2600 | }); |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 2601 | // Emit final copy of the lastprivate variables if IsLastIter != 0. |
| 2602 | if (HasLastprivateClause) |
| 2603 | EmitOMPLastprivateClauseFinal( |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 2604 | S, isOpenMPSimdDirective(S.getDirectiveKind()), |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2605 | Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc()))); |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2606 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2607 | EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 2608 | return CGF.Builder.CreateIsNotNull( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2609 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); |
Alexey Bataev | ef549a8 | 2016-03-09 09:49:09 +0000 | [diff] [blame] | 2610 | }); |
Alexey Bataev | ea33dee | 2018-02-15 23:39:43 +0000 | [diff] [blame] | 2611 | DoacrossCleanupScope.ForceCleanup(); |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2612 | // We're now done with the loop, so jump to the continuation block. |
Alexey Bataev | 62dbb97 | 2015-04-22 11:59:37 +0000 | [diff] [blame] | 2613 | if (ContBlock) { |
| 2614 | EmitBranch(ContBlock); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2615 | EmitBlock(ContBlock, /*IsFinished=*/true); |
Alexey Bataev | 62dbb97 | 2015-04-22 11:59:37 +0000 | [diff] [blame] | 2616 | } |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2617 | } |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 2618 | return HasLastprivateClause; |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2619 | } |
| 2620 | |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2621 | /// The following two functions generate expressions for the loop lower |
| 2622 | /// and upper bounds in case of static and dynamic (dispatch) schedule |
| 2623 | /// of the associated 'for' or 'distribute' loop. |
| 2624 | static std::pair<LValue, LValue> |
| 2625 | emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2626 | const auto &LS = cast<OMPLoopDirective>(S); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2627 | LValue LB = |
| 2628 | EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); |
| 2629 | LValue UB = |
| 2630 | EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); |
| 2631 | return {LB, UB}; |
| 2632 | } |
| 2633 | |
| 2634 | /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not |
| 2635 | /// consider the lower and upper bound expressions generated by the |
| 2636 | /// worksharing loop support, but we use 0 and the iteration space size as |
| 2637 | /// constants |
| 2638 | static std::pair<llvm::Value *, llvm::Value *> |
| 2639 | emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, |
| 2640 | Address LB, Address UB) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2641 | const auto &LS = cast<OMPLoopDirective>(S); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2642 | const Expr *IVExpr = LS.getIterationVariable(); |
| 2643 | const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType()); |
| 2644 | llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0); |
| 2645 | llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration()); |
| 2646 | return {LBVal, UBVal}; |
| 2647 | } |
| 2648 | |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2649 | void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) { |
Alexey Bataev | 38e8953 | 2015-04-16 04:54:05 +0000 | [diff] [blame] | 2650 | bool HasLastprivates = false; |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 2651 | auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, |
| 2652 | PrePostActionTy &) { |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 2653 | OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel()); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2654 | HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), |
| 2655 | emitForLoopBounds, |
| 2656 | emitDispatchForLoopBounds); |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 2657 | }; |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 2658 | { |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2659 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 2660 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen, |
| 2661 | S.hasCancel()); |
| 2662 | } |
Alexander Musman | c638868 | 2014-12-15 07:07:06 +0000 | [diff] [blame] | 2663 | |
| 2664 | // Emit an implicit barrier at the end. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2665 | if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2666 | CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for); |
Alexey Bataev | f29276e | 2014-06-18 04:14:57 +0000 | [diff] [blame] | 2667 | } |
Alexey Bataev | d3f8dd2 | 2014-06-25 11:44:49 +0000 | [diff] [blame] | 2668 | |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 2669 | void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) { |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 2670 | bool HasLastprivates = false; |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 2671 | auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, |
| 2672 | PrePostActionTy &) { |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2673 | HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), |
| 2674 | emitForLoopBounds, |
| 2675 | emitDispatchForLoopBounds); |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 2676 | }; |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 2677 | { |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2678 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 2679 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen); |
| 2680 | } |
Alexey Bataev | cbdcbb7 | 2015-06-17 07:45:51 +0000 | [diff] [blame] | 2681 | |
| 2682 | // Emit an implicit barrier at the end. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2683 | if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2684 | CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for); |
Alexander Musman | f82886e | 2014-09-18 05:12:34 +0000 | [diff] [blame] | 2685 | } |
| 2686 | |
Alexey Bataev | 2df54a0 | 2015-03-12 08:53:29 +0000 | [diff] [blame] | 2687 | static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, |
| 2688 | const Twine &Name, |
| 2689 | llvm::Value *Init = nullptr) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2690 | LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); |
Alexey Bataev | 2df54a0 | 2015-03-12 08:53:29 +0000 | [diff] [blame] | 2691 | if (Init) |
Akira Hatanaka | 642f799 | 2016-10-18 19:05:41 +0000 | [diff] [blame] | 2692 | CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); |
Alexey Bataev | 2df54a0 | 2015-03-12 08:53:29 +0000 | [diff] [blame] | 2693 | return LVal; |
Alexey Bataev | d3f8dd2 | 2014-06-25 11:44:49 +0000 | [diff] [blame] | 2694 | } |
| 2695 | |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 2696 | void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2697 | const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt(); |
| 2698 | const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2699 | bool HasLastprivates = false; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2700 | auto &&CodeGen = [&S, CapturedStmt, CS, |
| 2701 | &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { |
| 2702 | ASTContext &C = CGF.getContext(); |
| 2703 | QualType KmpInt32Ty = |
| 2704 | C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2705 | // Emit helper vars inits. |
| 2706 | LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", |
| 2707 | CGF.Builder.getInt32(0)); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2708 | llvm::ConstantInt *GlobalUBVal = CS != nullptr |
| 2709 | ? CGF.Builder.getInt32(CS->size() - 1) |
| 2710 | : CGF.Builder.getInt32(0); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2711 | LValue UB = |
| 2712 | createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); |
| 2713 | LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", |
| 2714 | CGF.Builder.getInt32(1)); |
| 2715 | LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", |
| 2716 | CGF.Builder.getInt32(0)); |
| 2717 | // Loop counter. |
| 2718 | LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2719 | OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2720 | CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2721 | OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2722 | CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); |
| 2723 | // Generate condition for loop. |
| 2724 | BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue, |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2725 | OK_Ordinary, S.getBeginLoc(), FPOptions()); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2726 | // Increment for loop counter. |
| 2727 | UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary, |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2728 | S.getBeginLoc(), true); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2729 | auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2730 | // Iterate through all sections and emit a switch construct: |
| 2731 | // switch (IV) { |
| 2732 | // case 0: |
| 2733 | // <SectionStmt[0]>; |
| 2734 | // break; |
| 2735 | // ... |
| 2736 | // case <NumSection> - 1: |
| 2737 | // <SectionStmt[<NumSection> - 1]>; |
| 2738 | // break; |
| 2739 | // } |
| 2740 | // .omp.sections.exit: |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2741 | llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); |
| 2742 | llvm::SwitchInst *SwitchStmt = |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2743 | CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()), |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 2744 | ExitBB, CS == nullptr ? 1 : CS->size()); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2745 | if (CS) { |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 2746 | unsigned CaseNumber = 0; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2747 | for (const Stmt *SubStmt : CS->children()) { |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 2748 | auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); |
| 2749 | CGF.EmitBlock(CaseBB); |
| 2750 | SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); |
Benjamin Kramer | 642f173 | 2015-07-02 21:03:14 +0000 | [diff] [blame] | 2751 | CGF.EmitStmt(SubStmt); |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 2752 | CGF.EmitBranch(ExitBB); |
Benjamin Kramer | 642f173 | 2015-07-02 21:03:14 +0000 | [diff] [blame] | 2753 | ++CaseNumber; |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 2754 | } |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2755 | } else { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2756 | llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2757 | CGF.EmitBlock(CaseBB); |
| 2758 | SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2759 | CGF.EmitStmt(CapturedStmt); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2760 | CGF.EmitBranch(ExitBB); |
Alexey Bataev | 2cb9b95 | 2015-04-24 03:37:03 +0000 | [diff] [blame] | 2761 | } |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2762 | CGF.EmitBlock(ExitBB, /*IsFinished=*/true); |
Alexey Bataev | 2df54a0 | 2015-03-12 08:53:29 +0000 | [diff] [blame] | 2763 | }; |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 2764 | |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2765 | CodeGenFunction::OMPPrivateScope LoopScope(CGF); |
| 2766 | if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { |
Alexey Bataev | 9efc03b | 2015-04-27 04:34:03 +0000 | [diff] [blame] | 2767 | // Emit implicit barrier to synchronize threads and avoid data races on |
Alexey Bataev | cd8b6a2 | 2016-02-15 08:07:17 +0000 | [diff] [blame] | 2768 | // initialization of firstprivate variables and post-update of lastprivate |
| 2769 | // variables. |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2770 | CGF.CGM.getOpenMPRuntime().emitBarrierCall( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2771 | CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2772 | /*ForceSimpleCall=*/true); |
Alexey Bataev | 9efc03b | 2015-04-27 04:34:03 +0000 | [diff] [blame] | 2773 | } |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2774 | CGF.EmitOMPPrivateClause(S, LoopScope); |
| 2775 | HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); |
| 2776 | CGF.EmitOMPReductionClauseInit(S, LoopScope); |
| 2777 | (void)LoopScope.Privatize(); |
Alexey Bataev | 6070542 | 2018-10-30 15:50:12 +0000 | [diff] [blame] | 2778 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) |
| 2779 | CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); |
Alexey Bataev | 2cb9b95 | 2015-04-24 03:37:03 +0000 | [diff] [blame] | 2780 | |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2781 | // Emit static non-chunked loop. |
Alexey Bataev | 9ebd742 | 2016-05-10 09:57:36 +0000 | [diff] [blame] | 2782 | OpenMPScheduleTy ScheduleKind; |
| 2783 | ScheduleKind.Schedule = OMPC_SCHEDULE_static; |
Alexey Bataev | 0f87dbe | 2017-08-14 17:56:13 +0000 | [diff] [blame] | 2784 | CGOpenMPRuntime::StaticRTInput StaticInit( |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 2785 | /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF), |
| 2786 | LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF)); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2787 | CGF.CGM.getOpenMPRuntime().emitForStaticInit( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2788 | CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2789 | // UB = min(UB, GlobalUB); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2790 | llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc()); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2791 | llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2792 | CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); |
| 2793 | CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); |
| 2794 | // IV = LB; |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2795 | CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2796 | // while (idx <= UB) { BODY; ++idx; } |
| 2797 | CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen, |
| 2798 | [](CodeGenFunction &) {}); |
| 2799 | // Tell the runtime we are done. |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 2800 | auto &&CodeGen = [&S](CodeGenFunction &CGF) { |
Stephen Kelly | 1c301dc | 2018-08-09 21:09:38 +0000 | [diff] [blame] | 2801 | CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), |
Alexey Bataev | f43f714 | 2017-09-06 16:17:35 +0000 | [diff] [blame] | 2802 | S.getDirectiveKind()); |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 2803 | }; |
| 2804 | CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); |
Arpith Chacko Jacob | 101e8fb | 2017-02-16 16:20:16 +0000 | [diff] [blame] | 2805 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); |
Alexey Bataev | 6120507 | 2016-03-02 04:57:40 +0000 | [diff] [blame] | 2806 | // Emit post-update of the reduction variables if IsLastIter != 0. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2807 | emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { |
| 2808 | return CGF.Builder.CreateIsNotNull( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2809 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2810 | }); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2811 | |
| 2812 | // Emit final copy of the lastprivate variables if IsLastIter != 0. |
| 2813 | if (HasLastprivates) |
| 2814 | CGF.EmitOMPLastprivateClauseFinal( |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 2815 | S, /*NoFinals=*/false, |
| 2816 | CGF.Builder.CreateIsNotNull( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2817 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc()))); |
Alexey Bataev | 68adb7d | 2015-04-14 03:29:22 +0000 | [diff] [blame] | 2818 | }; |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2819 | |
| 2820 | bool HasCancel = false; |
| 2821 | if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S)) |
| 2822 | HasCancel = OSD->hasCancel(); |
| 2823 | else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S)) |
| 2824 | HasCancel = OPSD->hasCancel(); |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 2825 | OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel); |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2826 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen, |
| 2827 | HasCancel); |
| 2828 | // Emit barrier for lastprivates only if 'sections' directive has 'nowait' |
| 2829 | // clause. Otherwise the barrier will be generated by the codegen for the |
| 2830 | // directive. |
| 2831 | if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) { |
Alexey Bataev | 2cb9b95 | 2015-04-24 03:37:03 +0000 | [diff] [blame] | 2832 | // Emit implicit barrier to synchronize threads and avoid data races on |
| 2833 | // initialization of firstprivate variables. |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2834 | CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), |
Alexey Bataev | 3015bcc | 2016-01-22 08:56:50 +0000 | [diff] [blame] | 2835 | OMPD_unknown); |
Alexey Bataev | 2cb9b95 | 2015-04-24 03:37:03 +0000 | [diff] [blame] | 2836 | } |
Alexey Bataev | 68adb7d | 2015-04-14 03:29:22 +0000 | [diff] [blame] | 2837 | } |
Alexey Bataev | 2df54a0 | 2015-03-12 08:53:29 +0000 | [diff] [blame] | 2838 | |
Alexey Bataev | 68adb7d | 2015-04-14 03:29:22 +0000 | [diff] [blame] | 2839 | void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) { |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 2840 | { |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2841 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 2842 | EmitSections(S); |
| 2843 | } |
Alexey Bataev | 2df54a0 | 2015-03-12 08:53:29 +0000 | [diff] [blame] | 2844 | // Emit an implicit barrier at the end. |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 2845 | if (!S.getSingleClause<OMPNowaitClause>()) { |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2846 | CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 2847 | OMPD_sections); |
Alexey Bataev | f268568 | 2015-03-30 04:30:22 +0000 | [diff] [blame] | 2848 | } |
Alexey Bataev | 2df54a0 | 2015-03-12 08:53:29 +0000 | [diff] [blame] | 2849 | } |
| 2850 | |
| 2851 | void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) { |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 2852 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2853 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 2854 | }; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2855 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Alexey Bataev | 25e5b44 | 2015-09-15 12:52:43 +0000 | [diff] [blame] | 2856 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen, |
| 2857 | S.hasCancel()); |
Alexey Bataev | 1e0498a | 2014-06-26 08:21:58 +0000 | [diff] [blame] | 2858 | } |
| 2859 | |
Alexey Bataev | 6956e2e | 2015-02-05 06:35:41 +0000 | [diff] [blame] | 2860 | void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) { |
Alexey Bataev | a63048e | 2015-03-23 06:18:07 +0000 | [diff] [blame] | 2861 | llvm::SmallVector<const Expr *, 8> CopyprivateVars; |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 2862 | llvm::SmallVector<const Expr *, 8> DestExprs; |
Alexey Bataev | a63048e | 2015-03-23 06:18:07 +0000 | [diff] [blame] | 2863 | llvm::SmallVector<const Expr *, 8> SrcExprs; |
Alexey Bataev | a63048e | 2015-03-23 06:18:07 +0000 | [diff] [blame] | 2864 | llvm::SmallVector<const Expr *, 8> AssignmentOps; |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 2865 | // Check if there are any 'copyprivate' clauses associated with this |
Alexey Bataev | cd8b6a2 | 2016-02-15 08:07:17 +0000 | [diff] [blame] | 2866 | // 'single' construct. |
Alexey Bataev | a63048e | 2015-03-23 06:18:07 +0000 | [diff] [blame] | 2867 | // Build a list of copyprivate variables along with helper expressions |
| 2868 | // (<source>, <destination>, <destination>=<source> expressions) |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 2869 | for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) { |
Alexey Bataev | a63048e | 2015-03-23 06:18:07 +0000 | [diff] [blame] | 2870 | CopyprivateVars.append(C->varlists().begin(), C->varlists().end()); |
Alexey Bataev | 420d45b | 2015-04-14 05:11:24 +0000 | [diff] [blame] | 2871 | DestExprs.append(C->destination_exprs().begin(), |
| 2872 | C->destination_exprs().end()); |
Alexey Bataev | a63048e | 2015-03-23 06:18:07 +0000 | [diff] [blame] | 2873 | SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end()); |
Alexey Bataev | a63048e | 2015-03-23 06:18:07 +0000 | [diff] [blame] | 2874 | AssignmentOps.append(C->assignment_ops().begin(), |
| 2875 | C->assignment_ops().end()); |
| 2876 | } |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 2877 | // Emit code for 'single' region along with 'copyprivate' clauses |
| 2878 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 2879 | Action.Enter(CGF); |
| 2880 | OMPPrivateScope SingleScope(CGF); |
| 2881 | (void)CGF.EmitOMPFirstprivateClause(S, SingleScope); |
| 2882 | CGF.EmitOMPPrivateClause(S, SingleScope); |
| 2883 | (void)SingleScope.Privatize(); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2884 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 2885 | }; |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 2886 | { |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2887 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2888 | CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(), |
Alexey Bataev | 3392d76 | 2016-02-16 11:18:12 +0000 | [diff] [blame] | 2889 | CopyprivateVars, DestExprs, |
| 2890 | SrcExprs, AssignmentOps); |
| 2891 | } |
| 2892 | // Emit an implicit barrier at the end (to avoid data race on firstprivate |
| 2893 | // init or if no 'nowait' clause was specified and no 'copyprivate' clause). |
Alexey Bataev | 417089f | 2016-02-17 13:19:37 +0000 | [diff] [blame] | 2894 | if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) { |
Alexey Bataev | 5521d78 | 2015-04-24 04:21:15 +0000 | [diff] [blame] | 2895 | CGM.getOpenMPRuntime().emitBarrierCall( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2896 | *this, S.getBeginLoc(), |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 2897 | S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single); |
Alexey Bataev | f268568 | 2015-03-30 04:30:22 +0000 | [diff] [blame] | 2898 | } |
Alexey Bataev | d1e40fb | 2014-06-26 12:05:45 +0000 | [diff] [blame] | 2899 | } |
| 2900 | |
cchen | 47d6094 | 2019-12-05 13:43:48 -0500 | [diff] [blame] | 2901 | static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) { |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 2902 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 2903 | Action.Enter(CGF); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2904 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 2905 | }; |
cchen | 47d6094 | 2019-12-05 13:43:48 -0500 | [diff] [blame] | 2906 | CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); |
| 2907 | } |
| 2908 | |
| 2909 | void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) { |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2910 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
cchen | 47d6094 | 2019-12-05 13:43:48 -0500 | [diff] [blame] | 2911 | emitMaster(*this, S); |
Alexander Musman | 80c2289 | 2014-07-17 08:54:58 +0000 | [diff] [blame] | 2912 | } |
| 2913 | |
Alexey Bataev | 3a3bf0b | 2014-09-22 10:01:53 +0000 | [diff] [blame] | 2914 | void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) { |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 2915 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 2916 | Action.Enter(CGF); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2917 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 2918 | }; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 2919 | const Expr *Hint = nullptr; |
| 2920 | if (const auto *HintClause = S.getSingleClause<OMPHintClause>()) |
Alexey Bataev | fc57d16 | 2015-12-15 10:55:09 +0000 | [diff] [blame] | 2921 | Hint = HintClause->getHint(); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2922 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Alexey Bataev | fc57d16 | 2015-12-15 10:55:09 +0000 | [diff] [blame] | 2923 | CGM.getOpenMPRuntime().emitCriticalRegion(*this, |
| 2924 | S.getDirectiveName().getAsString(), |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 2925 | CodeGen, S.getBeginLoc(), Hint); |
Alexander Musman | d9ed09f | 2014-07-21 09:42:05 +0000 | [diff] [blame] | 2926 | } |
| 2927 | |
Alexey Bataev | 671605e | 2015-04-13 05:28:11 +0000 | [diff] [blame] | 2928 | void CodeGenFunction::EmitOMPParallelForDirective( |
| 2929 | const OMPParallelForDirective &S) { |
| 2930 | // Emit directive as a combined directive that consists of two implicit |
| 2931 | // directives: 'parallel' with 'for' directive. |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 2932 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 2933 | Action.Enter(CGF); |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 2934 | OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel()); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2935 | CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, |
| 2936 | emitDispatchForLoopBounds); |
Alexey Bataev | 671605e | 2015-04-13 05:28:11 +0000 | [diff] [blame] | 2937 | }; |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2938 | emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen, |
| 2939 | emitEmptyBoundParameters); |
Alexey Bataev | 4acb859 | 2014-07-07 13:01:15 +0000 | [diff] [blame] | 2940 | } |
| 2941 | |
Alexander Musman | e4e893b | 2014-09-23 09:33:00 +0000 | [diff] [blame] | 2942 | void CodeGenFunction::EmitOMPParallelForSimdDirective( |
Alexey Bataev | 3b5b5c4 | 2015-06-18 10:10:12 +0000 | [diff] [blame] | 2943 | const OMPParallelForSimdDirective &S) { |
| 2944 | // Emit directive as a combined directive that consists of two implicit |
| 2945 | // directives: 'parallel' with 'for' directive. |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 2946 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 2947 | Action.Enter(CGF); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2948 | CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, |
| 2949 | emitDispatchForLoopBounds); |
Alexey Bataev | 3b5b5c4 | 2015-06-18 10:10:12 +0000 | [diff] [blame] | 2950 | }; |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2951 | emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen, |
| 2952 | emitEmptyBoundParameters); |
Alexander Musman | e4e893b | 2014-09-23 09:33:00 +0000 | [diff] [blame] | 2953 | } |
| 2954 | |
cchen | 47d6094 | 2019-12-05 13:43:48 -0500 | [diff] [blame] | 2955 | void CodeGenFunction::EmitOMPParallelMasterDirective( |
| 2956 | const OMPParallelMasterDirective &S) { |
| 2957 | // Emit directive as a combined directive that consists of two implicit |
| 2958 | // directives: 'parallel' with 'master' directive. |
| 2959 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 2960 | Action.Enter(CGF); |
| 2961 | OMPPrivateScope PrivateScope(CGF); |
| 2962 | bool Copyins = CGF.EmitOMPCopyinClause(S); |
| 2963 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); |
| 2964 | if (Copyins) { |
| 2965 | // Emit implicit barrier to synchronize threads and avoid data races on |
| 2966 | // propagation master's thread values of threadprivate variables to local |
| 2967 | // instances of that variables of all other implicit threads. |
| 2968 | CGF.CGM.getOpenMPRuntime().emitBarrierCall( |
| 2969 | CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, |
| 2970 | /*ForceSimpleCall=*/true); |
| 2971 | } |
| 2972 | CGF.EmitOMPPrivateClause(S, PrivateScope); |
| 2973 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 2974 | (void)PrivateScope.Privatize(); |
| 2975 | emitMaster(CGF, S); |
| 2976 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); |
| 2977 | }; |
| 2978 | emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen, |
| 2979 | emitEmptyBoundParameters); |
| 2980 | emitPostUpdateForReductionClause(*this, S, |
| 2981 | [](CodeGenFunction &) { return nullptr; }); |
| 2982 | } |
| 2983 | |
Alexey Bataev | 84d0b3e | 2014-07-08 08:12:03 +0000 | [diff] [blame] | 2984 | void CodeGenFunction::EmitOMPParallelSectionsDirective( |
Alexey Bataev | 68adb7d | 2015-04-14 03:29:22 +0000 | [diff] [blame] | 2985 | const OMPParallelSectionsDirective &S) { |
| 2986 | // Emit directive as a combined directive that consists of two implicit |
| 2987 | // directives: 'parallel' with 'sections' directive. |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 2988 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 2989 | Action.Enter(CGF); |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 2990 | CGF.EmitSections(S); |
| 2991 | }; |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 2992 | emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen, |
| 2993 | emitEmptyBoundParameters); |
Alexey Bataev | 84d0b3e | 2014-07-08 08:12:03 +0000 | [diff] [blame] | 2994 | } |
| 2995 | |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 2996 | void CodeGenFunction::EmitOMPTaskBasedDirective( |
| 2997 | const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, |
| 2998 | const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, |
| 2999 | OMPTaskDataTy &Data) { |
Alexey Bataev | 62b63b1 | 2015-03-10 07:28:44 +0000 | [diff] [blame] | 3000 | // Emit outlined function for task construct. |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3001 | const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3002 | auto I = CS->getCapturedDecl()->param_begin(); |
| 3003 | auto PartId = std::next(I); |
| 3004 | auto TaskT = std::next(I, 4); |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 3005 | // Check if the task is final |
| 3006 | if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) { |
| 3007 | // If the condition constant folds and can be elided, try to avoid emitting |
| 3008 | // the condition and the dead arm of the if/else. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3009 | const Expr *Cond = Clause->getCondition(); |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 3010 | bool CondConstant; |
| 3011 | if (ConstantFoldsToSimpleInteger(Cond, CondConstant)) |
| 3012 | Data.Final.setInt(CondConstant); |
| 3013 | else |
| 3014 | Data.Final.setPointer(EvaluateExprAsBool(Cond)); |
| 3015 | } else { |
| 3016 | // By default the task is not final. |
| 3017 | Data.Final.setInt(/*IntVal=*/false); |
| 3018 | } |
Alexey Bataev | 1e1e286 | 2016-05-10 12:21:02 +0000 | [diff] [blame] | 3019 | // Check if the task has 'priority' clause. |
| 3020 | if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3021 | const Expr *Prio = Clause->getPriority(); |
Alexey Bataev | 5140e74 | 2016-07-19 04:21:09 +0000 | [diff] [blame] | 3022 | Data.Priority.setInt(/*IntVal=*/true); |
Alexey Bataev | ad537bb | 2016-05-30 09:06:50 +0000 | [diff] [blame] | 3023 | Data.Priority.setPointer(EmitScalarConversion( |
| 3024 | EmitScalarExpr(Prio), Prio->getType(), |
| 3025 | getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1), |
| 3026 | Prio->getExprLoc())); |
Alexey Bataev | 1e1e286 | 2016-05-10 12:21:02 +0000 | [diff] [blame] | 3027 | } |
Alexey Bataev | 62b63b1 | 2015-03-10 07:28:44 +0000 | [diff] [blame] | 3028 | // The first function argument for tasks is a thread id, the second one is a |
| 3029 | // part id (0 for tied tasks, >=0 for untied task). |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3030 | llvm::DenseSet<const VarDecl *> EmittedAsPrivate; |
| 3031 | // Get list of private variables. |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 3032 | for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) { |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3033 | auto IRef = C->varlist_begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3034 | for (const Expr *IInit : C->private_copies()) { |
| 3035 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3036 | if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 3037 | Data.PrivateVars.push_back(*IRef); |
| 3038 | Data.PrivateCopies.push_back(IInit); |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3039 | } |
| 3040 | ++IRef; |
| 3041 | } |
| 3042 | } |
| 3043 | EmittedAsPrivate.clear(); |
| 3044 | // Get list of firstprivate variables. |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 3045 | for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) { |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3046 | auto IRef = C->varlist_begin(); |
| 3047 | auto IElemInitRef = C->inits().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3048 | for (const Expr *IInit : C->private_copies()) { |
| 3049 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3050 | if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 3051 | Data.FirstprivateVars.push_back(*IRef); |
| 3052 | Data.FirstprivateCopies.push_back(IInit); |
| 3053 | Data.FirstprivateInits.push_back(*IElemInitRef); |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3054 | } |
Richard Trieu | cc3949d | 2016-02-18 22:34:54 +0000 | [diff] [blame] | 3055 | ++IRef; |
| 3056 | ++IElemInitRef; |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3057 | } |
| 3058 | } |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 3059 | // Get list of lastprivate variables (for taskloops). |
| 3060 | llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs; |
| 3061 | for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) { |
| 3062 | auto IRef = C->varlist_begin(); |
| 3063 | auto ID = C->destination_exprs().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3064 | for (const Expr *IInit : C->private_copies()) { |
| 3065 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 3066 | if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { |
| 3067 | Data.LastprivateVars.push_back(*IRef); |
| 3068 | Data.LastprivateCopies.push_back(IInit); |
| 3069 | } |
| 3070 | LastprivateDstsOrigs.insert( |
| 3071 | {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()), |
| 3072 | cast<DeclRefExpr>(*IRef)}); |
| 3073 | ++IRef; |
| 3074 | ++ID; |
| 3075 | } |
| 3076 | } |
Alexey Bataev | be5a8b4 | 2017-07-17 13:30:36 +0000 | [diff] [blame] | 3077 | SmallVector<const Expr *, 4> LHSs; |
| 3078 | SmallVector<const Expr *, 4> RHSs; |
| 3079 | for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) { |
| 3080 | auto IPriv = C->privates().begin(); |
| 3081 | auto IRed = C->reduction_ops().begin(); |
| 3082 | auto ILHS = C->lhs_exprs().begin(); |
| 3083 | auto IRHS = C->rhs_exprs().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3084 | for (const Expr *Ref : C->varlists()) { |
Alexey Bataev | be5a8b4 | 2017-07-17 13:30:36 +0000 | [diff] [blame] | 3085 | Data.ReductionVars.emplace_back(Ref); |
| 3086 | Data.ReductionCopies.emplace_back(*IPriv); |
| 3087 | Data.ReductionOps.emplace_back(*IRed); |
| 3088 | LHSs.emplace_back(*ILHS); |
| 3089 | RHSs.emplace_back(*IRHS); |
| 3090 | std::advance(IPriv, 1); |
| 3091 | std::advance(IRed, 1); |
| 3092 | std::advance(ILHS, 1); |
| 3093 | std::advance(IRHS, 1); |
| 3094 | } |
| 3095 | } |
| 3096 | Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3097 | *this, S.getBeginLoc(), LHSs, RHSs, Data); |
Alexey Bataev | 1d2353d | 2015-06-24 11:01:36 +0000 | [diff] [blame] | 3098 | // Build list of dependences. |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 3099 | for (const auto *C : S.getClausesOfKind<OMPDependClause>()) |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3100 | for (const Expr *IRef : C->varlists()) |
Alexey Bataev | 43a919f | 2018-04-13 17:48:43 +0000 | [diff] [blame] | 3101 | Data.Dependences.emplace_back(C->getDependencyKind(), IRef); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3102 | auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs, |
| 3103 | CapturedRegion](CodeGenFunction &CGF, |
| 3104 | PrePostActionTy &Action) { |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3105 | // Set proper addresses for generated private copies. |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 3106 | OMPPrivateScope Scope(CGF); |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 3107 | if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() || |
| 3108 | !Data.LastprivateVars.empty()) { |
James Y Knight | 9871db0 | 2019-02-05 16:42:33 +0000 | [diff] [blame] | 3109 | llvm::FunctionType *CopyFnTy = llvm::FunctionType::get( |
| 3110 | CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true); |
Alexey Bataev | 3c595a6 | 2017-08-14 15:01:03 +0000 | [diff] [blame] | 3111 | enum { PrivatesParam = 2, CopyFnParam = 3 }; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3112 | llvm::Value *CopyFn = CGF.Builder.CreateLoad( |
| 3113 | CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); |
| 3114 | llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( |
| 3115 | CS->getCapturedDecl()->getParam(PrivatesParam))); |
Alexey Bataev | 48591dd | 2016-04-20 04:01:36 +0000 | [diff] [blame] | 3116 | // Map privates. |
| 3117 | llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs; |
| 3118 | llvm::SmallVector<llvm::Value *, 16> CallArgs; |
| 3119 | CallArgs.push_back(PrivatesPtr); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3120 | for (const Expr *E : Data.PrivateVars) { |
| 3121 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); |
Alexey Bataev | 48591dd | 2016-04-20 04:01:36 +0000 | [diff] [blame] | 3122 | Address PrivatePtr = CGF.CreateMemTemp( |
| 3123 | CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3124 | PrivatePtrs.emplace_back(VD, PrivatePtr); |
Alexey Bataev | 48591dd | 2016-04-20 04:01:36 +0000 | [diff] [blame] | 3125 | CallArgs.push_back(PrivatePtr.getPointer()); |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3126 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3127 | for (const Expr *E : Data.FirstprivateVars) { |
| 3128 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); |
Alexey Bataev | 48591dd | 2016-04-20 04:01:36 +0000 | [diff] [blame] | 3129 | Address PrivatePtr = |
| 3130 | CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), |
| 3131 | ".firstpriv.ptr.addr"); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3132 | PrivatePtrs.emplace_back(VD, PrivatePtr); |
Alexey Bataev | 48591dd | 2016-04-20 04:01:36 +0000 | [diff] [blame] | 3133 | CallArgs.push_back(PrivatePtr.getPointer()); |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3134 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3135 | for (const Expr *E : Data.LastprivateVars) { |
| 3136 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 3137 | Address PrivatePtr = |
| 3138 | CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), |
| 3139 | ".lastpriv.ptr.addr"); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3140 | PrivatePtrs.emplace_back(VD, PrivatePtr); |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 3141 | CallArgs.push_back(PrivatePtr.getPointer()); |
| 3142 | } |
James Y Knight | 9871db0 | 2019-02-05 16:42:33 +0000 | [diff] [blame] | 3143 | CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( |
| 3144 | CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3145 | for (const auto &Pair : LastprivateDstsOrigs) { |
| 3146 | const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl()); |
Bruno Ricci | 5fc4db7 | 2018-12-21 14:10:18 +0000 | [diff] [blame] | 3147 | DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD), |
| 3148 | /*RefersToEnclosingVariableOrCapture=*/ |
| 3149 | CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, |
| 3150 | Pair.second->getType(), VK_LValue, |
| 3151 | Pair.second->getExprLoc()); |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 3152 | Scope.addPrivate(Pair.first, [&CGF, &DRE]() { |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 3153 | return CGF.EmitLValue(&DRE).getAddress(CGF); |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 3154 | }); |
| 3155 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3156 | for (const auto &Pair : PrivatePtrs) { |
Alexey Bataev | 48591dd | 2016-04-20 04:01:36 +0000 | [diff] [blame] | 3157 | Address Replacement(CGF.Builder.CreateLoad(Pair.second), |
| 3158 | CGF.getContext().getDeclAlign(Pair.first)); |
| 3159 | Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; }); |
| 3160 | } |
Alexey Bataev | 3ae88e2 | 2015-05-22 08:56:35 +0000 | [diff] [blame] | 3161 | } |
Alexey Bataev | be5a8b4 | 2017-07-17 13:30:36 +0000 | [diff] [blame] | 3162 | if (Data.Reductions) { |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3163 | OMPLexicalScope LexScope(CGF, S, CapturedRegion); |
Alexey Bataev | be5a8b4 | 2017-07-17 13:30:36 +0000 | [diff] [blame] | 3164 | ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies, |
| 3165 | Data.ReductionOps); |
| 3166 | llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( |
| 3167 | CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9))); |
| 3168 | for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) { |
| 3169 | RedCG.emitSharedLValue(CGF, Cnt); |
| 3170 | RedCG.emitAggregateType(CGF, Cnt); |
Alexey Bataev | 2e0cbe50 | 2018-03-08 15:24:08 +0000 | [diff] [blame] | 3171 | // FIXME: This must removed once the runtime library is fixed. |
| 3172 | // Emit required threadprivate variables for |
Raphael Isemann | b23ccec | 2018-12-10 12:37:46 +0000 | [diff] [blame] | 3173 | // initializer/combiner/finalizer. |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3174 | CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), |
Alexey Bataev | 2e0cbe50 | 2018-03-08 15:24:08 +0000 | [diff] [blame] | 3175 | RedCG, Cnt); |
Alexey Bataev | be5a8b4 | 2017-07-17 13:30:36 +0000 | [diff] [blame] | 3176 | Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3177 | CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); |
Alexey Bataev | be5a8b4 | 2017-07-17 13:30:36 +0000 | [diff] [blame] | 3178 | Replacement = |
| 3179 | Address(CGF.EmitScalarConversion( |
| 3180 | Replacement.getPointer(), CGF.getContext().VoidPtrTy, |
| 3181 | CGF.getContext().getPointerType( |
| 3182 | Data.ReductionCopies[Cnt]->getType()), |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 3183 | Data.ReductionCopies[Cnt]->getExprLoc()), |
Alexey Bataev | be5a8b4 | 2017-07-17 13:30:36 +0000 | [diff] [blame] | 3184 | Replacement.getAlignment()); |
| 3185 | Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); |
| 3186 | Scope.addPrivate(RedCG.getBaseDecl(Cnt), |
| 3187 | [Replacement]() { return Replacement; }); |
Alexey Bataev | be5a8b4 | 2017-07-17 13:30:36 +0000 | [diff] [blame] | 3188 | } |
| 3189 | } |
Alexey Bataev | 88202be | 2017-07-27 13:20:36 +0000 | [diff] [blame] | 3190 | // Privatize all private variables except for in_reduction items. |
Alexey Bataev | 48591dd | 2016-04-20 04:01:36 +0000 | [diff] [blame] | 3191 | (void)Scope.Privatize(); |
Alexey Bataev | 88202be | 2017-07-27 13:20:36 +0000 | [diff] [blame] | 3192 | SmallVector<const Expr *, 4> InRedVars; |
| 3193 | SmallVector<const Expr *, 4> InRedPrivs; |
| 3194 | SmallVector<const Expr *, 4> InRedOps; |
| 3195 | SmallVector<const Expr *, 4> TaskgroupDescriptors; |
| 3196 | for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) { |
| 3197 | auto IPriv = C->privates().begin(); |
| 3198 | auto IRed = C->reduction_ops().begin(); |
| 3199 | auto ITD = C->taskgroup_descriptors().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3200 | for (const Expr *Ref : C->varlists()) { |
Alexey Bataev | 88202be | 2017-07-27 13:20:36 +0000 | [diff] [blame] | 3201 | InRedVars.emplace_back(Ref); |
| 3202 | InRedPrivs.emplace_back(*IPriv); |
| 3203 | InRedOps.emplace_back(*IRed); |
| 3204 | TaskgroupDescriptors.emplace_back(*ITD); |
| 3205 | std::advance(IPriv, 1); |
| 3206 | std::advance(IRed, 1); |
| 3207 | std::advance(ITD, 1); |
| 3208 | } |
| 3209 | } |
| 3210 | // Privatize in_reduction items here, because taskgroup descriptors must be |
| 3211 | // privatized earlier. |
| 3212 | OMPPrivateScope InRedScope(CGF); |
| 3213 | if (!InRedVars.empty()) { |
| 3214 | ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps); |
| 3215 | for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) { |
| 3216 | RedCG.emitSharedLValue(CGF, Cnt); |
| 3217 | RedCG.emitAggregateType(CGF, Cnt); |
| 3218 | // The taskgroup descriptor variable is always implicit firstprivate and |
Raphael Isemann | b23ccec | 2018-12-10 12:37:46 +0000 | [diff] [blame] | 3219 | // privatized already during processing of the firstprivates. |
Alexey Bataev | 2e0cbe50 | 2018-03-08 15:24:08 +0000 | [diff] [blame] | 3220 | // FIXME: This must removed once the runtime library is fixed. |
| 3221 | // Emit required threadprivate variables for |
Raphael Isemann | b23ccec | 2018-12-10 12:37:46 +0000 | [diff] [blame] | 3222 | // initializer/combiner/finalizer. |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3223 | CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), |
Alexey Bataev | 2e0cbe50 | 2018-03-08 15:24:08 +0000 | [diff] [blame] | 3224 | RedCG, Cnt); |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 3225 | llvm::Value *ReductionsPtr = |
| 3226 | CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]), |
| 3227 | TaskgroupDescriptors[Cnt]->getExprLoc()); |
Alexey Bataev | 88202be | 2017-07-27 13:20:36 +0000 | [diff] [blame] | 3228 | Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3229 | CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); |
Alexey Bataev | 88202be | 2017-07-27 13:20:36 +0000 | [diff] [blame] | 3230 | Replacement = Address( |
| 3231 | CGF.EmitScalarConversion( |
| 3232 | Replacement.getPointer(), CGF.getContext().VoidPtrTy, |
| 3233 | CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 3234 | InRedPrivs[Cnt]->getExprLoc()), |
Alexey Bataev | 88202be | 2017-07-27 13:20:36 +0000 | [diff] [blame] | 3235 | Replacement.getAlignment()); |
| 3236 | Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); |
| 3237 | InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), |
| 3238 | [Replacement]() { return Replacement; }); |
Alexey Bataev | 88202be | 2017-07-27 13:20:36 +0000 | [diff] [blame] | 3239 | } |
| 3240 | } |
| 3241 | (void)InRedScope.Privatize(); |
Alexey Bataev | 48591dd | 2016-04-20 04:01:36 +0000 | [diff] [blame] | 3242 | |
| 3243 | Action.Enter(CGF); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 3244 | BodyGen(CGF); |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 3245 | }; |
James Y Knight | 9871db0 | 2019-02-05 16:42:33 +0000 | [diff] [blame] | 3246 | llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 3247 | S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied, |
| 3248 | Data.NumberOfParts); |
Alexey Bataev | 5bbcead | 2019-10-14 17:17:41 +0000 | [diff] [blame] | 3249 | OMPLexicalScope Scope(*this, S, llvm::None, |
Alexey Bataev | 6120582 | 2019-12-04 09:50:21 -0500 | [diff] [blame] | 3250 | !isOpenMPParallelDirective(S.getDirectiveKind()) && |
| 3251 | !isOpenMPSimdDirective(S.getDirectiveKind())); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 3252 | TaskGen(*this, OutlinedFn, Data); |
| 3253 | } |
| 3254 | |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3255 | static ImplicitParamDecl * |
| 3256 | createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 3257 | QualType Ty, CapturedDecl *CD, |
| 3258 | SourceLocation Loc) { |
| 3259 | auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty, |
| 3260 | ImplicitParamDecl::Other); |
| 3261 | auto *OrigRef = DeclRefExpr::Create( |
| 3262 | C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD, |
| 3263 | /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue); |
| 3264 | auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty, |
| 3265 | ImplicitParamDecl::Other); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3266 | auto *PrivateRef = DeclRefExpr::Create( |
| 3267 | C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD, |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 3268 | /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3269 | QualType ElemType = C.getBaseElementType(Ty); |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 3270 | auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType, |
| 3271 | ImplicitParamDecl::Other); |
| 3272 | auto *InitRef = DeclRefExpr::Create( |
| 3273 | C, NestedNameSpecifierLoc(), SourceLocation(), InitVD, |
| 3274 | /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3275 | PrivateVD->setInitStyle(VarDecl::CInit); |
| 3276 | PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue, |
| 3277 | InitRef, /*BasePath=*/nullptr, |
| 3278 | VK_RValue)); |
| 3279 | Data.FirstprivateVars.emplace_back(OrigRef); |
| 3280 | Data.FirstprivateCopies.emplace_back(PrivateRef); |
| 3281 | Data.FirstprivateInits.emplace_back(InitRef); |
| 3282 | return OrigVD; |
| 3283 | } |
| 3284 | |
| 3285 | void CodeGenFunction::EmitOMPTargetTaskBasedDirective( |
| 3286 | const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, |
| 3287 | OMPTargetDataInfo &InputInfo) { |
| 3288 | // Emit outlined function for task construct. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3289 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_task); |
| 3290 | Address CapturedStruct = GenerateCapturedStmtArgument(*CS); |
| 3291 | QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); |
| 3292 | auto I = CS->getCapturedDecl()->param_begin(); |
| 3293 | auto PartId = std::next(I); |
| 3294 | auto TaskT = std::next(I, 4); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3295 | OMPTaskDataTy Data; |
| 3296 | // The task is not final. |
| 3297 | Data.Final.setInt(/*IntVal=*/false); |
| 3298 | // Get list of firstprivate variables. |
| 3299 | for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) { |
| 3300 | auto IRef = C->varlist_begin(); |
| 3301 | auto IElemInitRef = C->inits().begin(); |
| 3302 | for (auto *IInit : C->private_copies()) { |
| 3303 | Data.FirstprivateVars.push_back(*IRef); |
| 3304 | Data.FirstprivateCopies.push_back(IInit); |
| 3305 | Data.FirstprivateInits.push_back(*IElemInitRef); |
| 3306 | ++IRef; |
| 3307 | ++IElemInitRef; |
| 3308 | } |
| 3309 | } |
| 3310 | OMPPrivateScope TargetScope(*this); |
| 3311 | VarDecl *BPVD = nullptr; |
| 3312 | VarDecl *PVD = nullptr; |
| 3313 | VarDecl *SVD = nullptr; |
| 3314 | if (InputInfo.NumberOfTargetItems > 0) { |
| 3315 | auto *CD = CapturedDecl::Create( |
| 3316 | getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0); |
| 3317 | llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems); |
| 3318 | QualType BaseAndPointersType = getContext().getConstantArrayType( |
Richard Smith | 772e266 | 2019-10-04 01:25:59 +0000 | [diff] [blame] | 3319 | getContext().VoidPtrTy, ArrSize, nullptr, ArrayType::Normal, |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3320 | /*IndexTypeQuals=*/0); |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 3321 | BPVD = createImplicitFirstprivateForType( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3322 | getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc()); |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 3323 | PVD = createImplicitFirstprivateForType( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3324 | getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc()); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3325 | QualType SizesType = getContext().getConstantArrayType( |
Alexey Bataev | a90fc66 | 2019-06-25 16:00:43 +0000 | [diff] [blame] | 3326 | getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1), |
Richard Smith | 772e266 | 2019-10-04 01:25:59 +0000 | [diff] [blame] | 3327 | ArrSize, nullptr, ArrayType::Normal, |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3328 | /*IndexTypeQuals=*/0); |
Alexey Bataev | a9b9cc0 | 2018-01-23 18:12:38 +0000 | [diff] [blame] | 3329 | SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD, |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3330 | S.getBeginLoc()); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3331 | TargetScope.addPrivate( |
| 3332 | BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; }); |
| 3333 | TargetScope.addPrivate(PVD, |
| 3334 | [&InputInfo]() { return InputInfo.PointersArray; }); |
| 3335 | TargetScope.addPrivate(SVD, |
| 3336 | [&InputInfo]() { return InputInfo.SizesArray; }); |
| 3337 | } |
| 3338 | (void)TargetScope.Privatize(); |
| 3339 | // Build list of dependences. |
| 3340 | for (const auto *C : S.getClausesOfKind<OMPDependClause>()) |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3341 | for (const Expr *IRef : C->varlists()) |
Alexey Bataev | 43a919f | 2018-04-13 17:48:43 +0000 | [diff] [blame] | 3342 | Data.Dependences.emplace_back(C->getDependencyKind(), IRef); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3343 | auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, |
| 3344 | &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 3345 | // Set proper addresses for generated private copies. |
| 3346 | OMPPrivateScope Scope(CGF); |
| 3347 | if (!Data.FirstprivateVars.empty()) { |
James Y Knight | 9871db0 | 2019-02-05 16:42:33 +0000 | [diff] [blame] | 3348 | llvm::FunctionType *CopyFnTy = llvm::FunctionType::get( |
| 3349 | CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3350 | enum { PrivatesParam = 2, CopyFnParam = 3 }; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3351 | llvm::Value *CopyFn = CGF.Builder.CreateLoad( |
| 3352 | CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); |
| 3353 | llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( |
| 3354 | CS->getCapturedDecl()->getParam(PrivatesParam))); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3355 | // Map privates. |
| 3356 | llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs; |
| 3357 | llvm::SmallVector<llvm::Value *, 16> CallArgs; |
| 3358 | CallArgs.push_back(PrivatesPtr); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3359 | for (const Expr *E : Data.FirstprivateVars) { |
| 3360 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3361 | Address PrivatePtr = |
| 3362 | CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), |
| 3363 | ".firstpriv.ptr.addr"); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3364 | PrivatePtrs.emplace_back(VD, PrivatePtr); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3365 | CallArgs.push_back(PrivatePtr.getPointer()); |
| 3366 | } |
James Y Knight | 9871db0 | 2019-02-05 16:42:33 +0000 | [diff] [blame] | 3367 | CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( |
| 3368 | CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3369 | for (const auto &Pair : PrivatePtrs) { |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3370 | Address Replacement(CGF.Builder.CreateLoad(Pair.second), |
| 3371 | CGF.getContext().getDeclAlign(Pair.first)); |
| 3372 | Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; }); |
| 3373 | } |
| 3374 | } |
| 3375 | // Privatize all private variables except for in_reduction items. |
| 3376 | (void)Scope.Privatize(); |
Alexey Bataev | 8451efa | 2018-01-15 19:06:12 +0000 | [diff] [blame] | 3377 | if (InputInfo.NumberOfTargetItems > 0) { |
| 3378 | InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( |
James Y Knight | 751fe28 | 2019-02-09 22:22:28 +0000 | [diff] [blame] | 3379 | CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0); |
Alexey Bataev | 8451efa | 2018-01-15 19:06:12 +0000 | [diff] [blame] | 3380 | InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( |
James Y Knight | 751fe28 | 2019-02-09 22:22:28 +0000 | [diff] [blame] | 3381 | CGF.GetAddrOfLocalVar(PVD), /*Index=*/0); |
Alexey Bataev | 8451efa | 2018-01-15 19:06:12 +0000 | [diff] [blame] | 3382 | InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( |
James Y Knight | 751fe28 | 2019-02-09 22:22:28 +0000 | [diff] [blame] | 3383 | CGF.GetAddrOfLocalVar(SVD), /*Index=*/0); |
Alexey Bataev | 8451efa | 2018-01-15 19:06:12 +0000 | [diff] [blame] | 3384 | } |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3385 | |
| 3386 | Action.Enter(CGF); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3387 | OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3388 | BodyGen(CGF); |
| 3389 | }; |
James Y Knight | 9871db0 | 2019-02-05 16:42:33 +0000 | [diff] [blame] | 3390 | llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction( |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3391 | S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true, |
| 3392 | Data.NumberOfParts); |
| 3393 | llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0); |
| 3394 | IntegerLiteral IfCond(getContext(), TrueOrFalse, |
| 3395 | getContext().getIntTypeForBitwidth(32, /*Signed=*/0), |
| 3396 | SourceLocation()); |
| 3397 | |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3398 | CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn, |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 3399 | SharedsTy, CapturedStruct, &IfCond, Data); |
| 3400 | } |
| 3401 | |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 3402 | void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { |
| 3403 | // Emit outlined function for task construct. |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3404 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_task); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3405 | Address CapturedStruct = GenerateCapturedStmtArgument(*CS); |
| 3406 | QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); |
Alexey Bataev | 1d67713 | 2015-04-22 13:57:31 +0000 | [diff] [blame] | 3407 | const Expr *IfCond = nullptr; |
Alexey Bataev | 7371aa3 | 2015-09-03 08:45:56 +0000 | [diff] [blame] | 3408 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { |
| 3409 | if (C->getNameModifier() == OMPD_unknown || |
| 3410 | C->getNameModifier() == OMPD_task) { |
| 3411 | IfCond = C->getCondition(); |
| 3412 | break; |
| 3413 | } |
Alexey Bataev | 1d67713 | 2015-04-22 13:57:31 +0000 | [diff] [blame] | 3414 | } |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 3415 | |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 3416 | OMPTaskDataTy Data; |
| 3417 | // Check if we should emit tied or untied task. |
| 3418 | Data.Tied = !S.getSingleClause<OMPUntiedClause>(); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 3419 | auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { |
| 3420 | CGF.EmitStmt(CS->getCapturedStmt()); |
| 3421 | }; |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 3422 | auto &&TaskGen = [&S, SharedsTy, CapturedStruct, |
James Y Knight | 9871db0 | 2019-02-05 16:42:33 +0000 | [diff] [blame] | 3423 | IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 3424 | const OMPTaskDataTy &Data) { |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3425 | CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn, |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 3426 | SharedsTy, CapturedStruct, IfCond, |
| 3427 | Data); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 3428 | }; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3429 | EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data); |
Alexey Bataev | 9c2e8ee | 2014-07-11 11:25:16 +0000 | [diff] [blame] | 3430 | } |
| 3431 | |
Alexey Bataev | 9f797f3 | 2015-02-05 05:57:51 +0000 | [diff] [blame] | 3432 | void CodeGenFunction::EmitOMPTaskyieldDirective( |
| 3433 | const OMPTaskyieldDirective &S) { |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3434 | CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc()); |
Alexey Bataev | 68446b7 | 2014-07-18 07:47:19 +0000 | [diff] [blame] | 3435 | } |
| 3436 | |
Alexey Bataev | 8f7c1b0 | 2014-12-05 04:09:23 +0000 | [diff] [blame] | 3437 | void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) { |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3438 | CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier); |
Alexey Bataev | 4d1dfea | 2014-07-18 09:11:51 +0000 | [diff] [blame] | 3439 | } |
| 3440 | |
Alexey Bataev | 8b8e202 | 2015-04-27 05:22:09 +0000 | [diff] [blame] | 3441 | void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) { |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3442 | CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc()); |
Alexey Bataev | 2df347a | 2014-07-18 10:17:07 +0000 | [diff] [blame] | 3443 | } |
| 3444 | |
Alexey Bataev | c30dd2d | 2015-06-18 12:14:09 +0000 | [diff] [blame] | 3445 | void CodeGenFunction::EmitOMPTaskgroupDirective( |
| 3446 | const OMPTaskgroupDirective &S) { |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 3447 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 3448 | Action.Enter(CGF); |
Alexey Bataev | 3b1b895 | 2017-07-25 15:53:26 +0000 | [diff] [blame] | 3449 | if (const Expr *E = S.getReductionRef()) { |
| 3450 | SmallVector<const Expr *, 4> LHSs; |
| 3451 | SmallVector<const Expr *, 4> RHSs; |
| 3452 | OMPTaskDataTy Data; |
| 3453 | for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) { |
| 3454 | auto IPriv = C->privates().begin(); |
| 3455 | auto IRed = C->reduction_ops().begin(); |
| 3456 | auto ILHS = C->lhs_exprs().begin(); |
| 3457 | auto IRHS = C->rhs_exprs().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3458 | for (const Expr *Ref : C->varlists()) { |
Alexey Bataev | 3b1b895 | 2017-07-25 15:53:26 +0000 | [diff] [blame] | 3459 | Data.ReductionVars.emplace_back(Ref); |
| 3460 | Data.ReductionCopies.emplace_back(*IPriv); |
| 3461 | Data.ReductionOps.emplace_back(*IRed); |
| 3462 | LHSs.emplace_back(*ILHS); |
| 3463 | RHSs.emplace_back(*IRHS); |
| 3464 | std::advance(IPriv, 1); |
| 3465 | std::advance(IRed, 1); |
| 3466 | std::advance(ILHS, 1); |
| 3467 | std::advance(IRHS, 1); |
| 3468 | } |
| 3469 | } |
| 3470 | llvm::Value *ReductionDesc = |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3471 | CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(), |
Alexey Bataev | 3b1b895 | 2017-07-25 15:53:26 +0000 | [diff] [blame] | 3472 | LHSs, RHSs, Data); |
| 3473 | const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); |
| 3474 | CGF.EmitVarDecl(*VD); |
| 3475 | CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), |
| 3476 | /*Volatile=*/false, E->getType()); |
| 3477 | } |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3478 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); |
Alexey Bataev | c30dd2d | 2015-06-18 12:14:09 +0000 | [diff] [blame] | 3479 | }; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3480 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3481 | CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc()); |
Alexey Bataev | c30dd2d | 2015-06-18 12:14:09 +0000 | [diff] [blame] | 3482 | } |
| 3483 | |
Alexey Bataev | cc37cc1 | 2014-11-20 04:34:54 +0000 | [diff] [blame] | 3484 | void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3485 | CGM.getOpenMPRuntime().emitFlush( |
| 3486 | *this, |
| 3487 | [&S]() -> ArrayRef<const Expr *> { |
| 3488 | if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>()) |
| 3489 | return llvm::makeArrayRef(FlushClause->varlist_begin(), |
| 3490 | FlushClause->varlist_end()); |
| 3491 | return llvm::None; |
| 3492 | }(), |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3493 | S.getBeginLoc()); |
Alexey Bataev | 6125da9 | 2014-07-21 11:26:11 +0000 | [diff] [blame] | 3494 | } |
| 3495 | |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 3496 | void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S, |
| 3497 | const CodeGenLoopTy &CodeGenLoop, |
| 3498 | Expr *IncExpr) { |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3499 | // Emit the loop iteration variable. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3500 | const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable()); |
| 3501 | const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl()); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3502 | EmitVarDecl(*IVDecl); |
| 3503 | |
| 3504 | // Emit the iterations count variable. |
| 3505 | // If it is not a variable, Sema decided to calculate iterations count on each |
| 3506 | // iteration (e.g., it is foldable into a constant). |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3507 | if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3508 | EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); |
| 3509 | // Emit calculation of the iterations count. |
| 3510 | EmitIgnoredExpr(S.getCalcLastIteration()); |
| 3511 | } |
| 3512 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3513 | CGOpenMPRuntime &RT = CGM.getOpenMPRuntime(); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3514 | |
Carlo Bertolli | 962bb80 | 2017-01-03 18:24:42 +0000 | [diff] [blame] | 3515 | bool HasLastprivateClause = false; |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3516 | // Check pre-condition. |
| 3517 | { |
Alexey Bataev | 5a3af13 | 2016-03-29 08:58:54 +0000 | [diff] [blame] | 3518 | OMPLoopScope PreInitScope(*this, S); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3519 | // Skip the entire loop if we don't meet the precondition. |
| 3520 | // If the condition constant folds and can be elided, avoid emitting the |
| 3521 | // whole loop. |
| 3522 | bool CondConstant; |
| 3523 | llvm::BasicBlock *ContBlock = nullptr; |
| 3524 | if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { |
| 3525 | if (!CondConstant) |
| 3526 | return; |
| 3527 | } else { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3528 | llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then"); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3529 | ContBlock = createBasicBlock("omp.precond.end"); |
| 3530 | emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock, |
| 3531 | getProfileCount(&S)); |
| 3532 | EmitBlock(ThenBlock); |
| 3533 | incrementProfileCounter(&S); |
| 3534 | } |
| 3535 | |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3536 | emitAlignedClause(*this, S); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3537 | // Emit 'then' code. |
| 3538 | { |
| 3539 | // Emit helper vars inits. |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 3540 | |
| 3541 | LValue LB = EmitOMPHelperVar( |
| 3542 | *this, cast<DeclRefExpr>( |
| 3543 | (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) |
| 3544 | ? S.getCombinedLowerBoundVariable() |
| 3545 | : S.getLowerBoundVariable()))); |
| 3546 | LValue UB = EmitOMPHelperVar( |
| 3547 | *this, cast<DeclRefExpr>( |
| 3548 | (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) |
| 3549 | ? S.getCombinedUpperBoundVariable() |
| 3550 | : S.getUpperBoundVariable()))); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3551 | LValue ST = |
| 3552 | EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable())); |
| 3553 | LValue IL = |
| 3554 | EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable())); |
| 3555 | |
| 3556 | OMPPrivateScope LoopScope(*this); |
Carlo Bertolli | 962bb80 | 2017-01-03 18:24:42 +0000 | [diff] [blame] | 3557 | if (EmitOMPFirstprivateClause(S, LoopScope)) { |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3558 | // Emit implicit barrier to synchronize threads and avoid data races |
| 3559 | // on initialization of firstprivate variables and post-update of |
Carlo Bertolli | 962bb80 | 2017-01-03 18:24:42 +0000 | [diff] [blame] | 3560 | // lastprivate variables. |
| 3561 | CGM.getOpenMPRuntime().emitBarrierCall( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3562 | *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3563 | /*ForceSimpleCall=*/true); |
Carlo Bertolli | 962bb80 | 2017-01-03 18:24:42 +0000 | [diff] [blame] | 3564 | } |
| 3565 | EmitOMPPrivateClause(S, LoopScope); |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3566 | if (isOpenMPSimdDirective(S.getDirectiveKind()) && |
Alexey Bataev | 999277a | 2017-12-06 14:31:09 +0000 | [diff] [blame] | 3567 | !isOpenMPParallelDirective(S.getDirectiveKind()) && |
| 3568 | !isOpenMPTeamsDirective(S.getDirectiveKind())) |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3569 | EmitOMPReductionClauseInit(S, LoopScope); |
Carlo Bertolli | 962bb80 | 2017-01-03 18:24:42 +0000 | [diff] [blame] | 3570 | HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); |
Alexey Bataev | 5dff95c | 2016-04-22 03:56:56 +0000 | [diff] [blame] | 3571 | EmitOMPPrivateLoopCounters(S, LoopScope); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3572 | (void)LoopScope.Privatize(); |
Alexey Bataev | 6070542 | 2018-10-30 15:50:12 +0000 | [diff] [blame] | 3573 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) |
| 3574 | CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3575 | |
| 3576 | // Detect the distribute schedule kind and chunk. |
| 3577 | llvm::Value *Chunk = nullptr; |
| 3578 | OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3579 | if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) { |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3580 | ScheduleKind = C->getDistScheduleKind(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3581 | if (const Expr *Ch = C->getChunkSize()) { |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3582 | Chunk = EmitScalarExpr(Ch); |
| 3583 | Chunk = EmitScalarConversion(Chunk, Ch->getType(), |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3584 | S.getIterationVariable()->getType(), |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3585 | S.getBeginLoc()); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3586 | } |
Gheorghe-Teodor Bercea | 02650d4 | 2018-09-27 19:22:56 +0000 | [diff] [blame] | 3587 | } else { |
| 3588 | // Default behaviour for dist_schedule clause. |
| 3589 | CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk( |
| 3590 | *this, S, ScheduleKind, Chunk); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3591 | } |
| 3592 | const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); |
| 3593 | const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); |
| 3594 | |
| 3595 | // OpenMP [2.10.8, distribute Construct, Description] |
| 3596 | // If dist_schedule is specified, kind must be static. If specified, |
| 3597 | // iterations are divided into chunks of size chunk_size, chunks are |
| 3598 | // assigned to the teams of the league in a round-robin fashion in the |
| 3599 | // order of the team number. When no chunk_size is specified, the |
| 3600 | // iteration space is divided into chunks that are approximately equal |
| 3601 | // in size, and at most one chunk is distributed to each team of the |
| 3602 | // league. The size of the chunks is unspecified in this case. |
Gheorghe-Teodor Bercea | e925676 | 2018-10-29 15:45:47 +0000 | [diff] [blame] | 3603 | bool StaticChunked = RT.isStaticChunked( |
| 3604 | ScheduleKind, /* Chunked */ Chunk != nullptr) && |
| 3605 | isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3606 | if (RT.isStaticNonchunked(ScheduleKind, |
Gheorghe-Teodor Bercea | e925676 | 2018-10-29 15:45:47 +0000 | [diff] [blame] | 3607 | /* Chunked */ Chunk != nullptr) || |
| 3608 | StaticChunked) { |
Alexey Bataev | 0f87dbe | 2017-08-14 17:56:13 +0000 | [diff] [blame] | 3609 | CGOpenMPRuntime::StaticRTInput StaticInit( |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 3610 | IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(*this), |
| 3611 | LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this), |
Gheorghe-Teodor Bercea | e925676 | 2018-10-29 15:45:47 +0000 | [diff] [blame] | 3612 | StaticChunked ? Chunk : nullptr); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3613 | RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, |
Alexey Bataev | 0f87dbe | 2017-08-14 17:56:13 +0000 | [diff] [blame] | 3614 | StaticInit); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3615 | JumpDest LoopExit = |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3616 | getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit")); |
| 3617 | // UB = min(UB, GlobalUB); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 3618 | EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) |
| 3619 | ? S.getCombinedEnsureUpperBound() |
| 3620 | : S.getEnsureUpperBound()); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3621 | // IV = LB; |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 3622 | EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) |
| 3623 | ? S.getCombinedInit() |
| 3624 | : S.getInit()); |
| 3625 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3626 | const Expr *Cond = |
| 3627 | isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()) |
| 3628 | ? S.getCombinedCond() |
| 3629 | : S.getCond(); |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 3630 | |
Gheorghe-Teodor Bercea | e925676 | 2018-10-29 15:45:47 +0000 | [diff] [blame] | 3631 | if (StaticChunked) |
| 3632 | Cond = S.getCombinedDistCond(); |
| 3633 | |
| 3634 | // For static unchunked schedules generate: |
| 3635 | // |
| 3636 | // 1. For distribute alone, codegen |
| 3637 | // while (idx <= UB) { |
| 3638 | // BODY; |
| 3639 | // ++idx; |
| 3640 | // } |
| 3641 | // |
| 3642 | // 2. When combined with 'for' (e.g. as in 'distribute parallel for') |
| 3643 | // while (idx <= UB) { |
| 3644 | // <CodeGen rest of pragma>(LB, UB); |
| 3645 | // idx += ST; |
| 3646 | // } |
| 3647 | // |
| 3648 | // For static chunk one schedule generate: |
| 3649 | // |
| 3650 | // while (IV <= GlobalUB) { |
| 3651 | // <CodeGen rest of pragma>(LB, UB); |
| 3652 | // LB += ST; |
| 3653 | // UB += ST; |
| 3654 | // UB = min(UB, GlobalUB); |
| 3655 | // IV = LB; |
| 3656 | // } |
| 3657 | // |
Alexey Bataev | 779a180 | 2019-12-06 12:21:31 -0500 | [diff] [blame] | 3658 | emitCommonSimdLoop( |
| 3659 | *this, S, |
| 3660 | [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 3661 | if (isOpenMPSimdDirective(S.getDirectiveKind())) |
| 3662 | CGF.EmitOMPSimdInit(S, /*IsMonotonic=*/true); |
| 3663 | }, |
| 3664 | [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop, |
| 3665 | StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) { |
| 3666 | CGF.EmitOMPInnerLoop( |
| 3667 | S, LoopScope.requiresCleanups(), Cond, IncExpr, |
| 3668 | [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { |
| 3669 | CodeGenLoop(CGF, S, LoopExit); |
| 3670 | }, |
| 3671 | [&S, StaticChunked](CodeGenFunction &CGF) { |
| 3672 | if (StaticChunked) { |
| 3673 | CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound()); |
| 3674 | CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound()); |
| 3675 | CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound()); |
| 3676 | CGF.EmitIgnoredExpr(S.getCombinedInit()); |
| 3677 | } |
| 3678 | }); |
| 3679 | }); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3680 | EmitBlock(LoopExit.getBlock()); |
| 3681 | // Tell the runtime we are done. |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3682 | RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind()); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3683 | } else { |
| 3684 | // Emit the outer loop, which requests its work chunk [LB..UB] from |
| 3685 | // runtime and runs the inner loop to process it. |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 3686 | const OMPLoopArguments LoopArguments = { |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 3687 | LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this), |
| 3688 | IL.getAddress(*this), Chunk}; |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 3689 | EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments, |
| 3690 | CodeGenLoop); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3691 | } |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3692 | if (isOpenMPSimdDirective(S.getDirectiveKind())) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3693 | EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3694 | return CGF.Builder.CreateIsNotNull( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3695 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3696 | }); |
| 3697 | } |
Carlo Bertolli | beda214 | 2018-02-22 19:38:14 +0000 | [diff] [blame] | 3698 | if (isOpenMPSimdDirective(S.getDirectiveKind()) && |
| 3699 | !isOpenMPParallelDirective(S.getDirectiveKind()) && |
| 3700 | !isOpenMPTeamsDirective(S.getDirectiveKind())) { |
Jonas Hahnfeld | 5aaaece | 2018-10-02 19:12:47 +0000 | [diff] [blame] | 3701 | EmitOMPReductionClauseFinal(S, OMPD_simd); |
Carlo Bertolli | beda214 | 2018-02-22 19:38:14 +0000 | [diff] [blame] | 3702 | // Emit post-update of the reduction variables if IsLastIter != 0. |
| 3703 | emitPostUpdateForReductionClause( |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3704 | *this, S, [IL, &S](CodeGenFunction &CGF) { |
Carlo Bertolli | beda214 | 2018-02-22 19:38:14 +0000 | [diff] [blame] | 3705 | return CGF.Builder.CreateIsNotNull( |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3706 | CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); |
Carlo Bertolli | beda214 | 2018-02-22 19:38:14 +0000 | [diff] [blame] | 3707 | }); |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3708 | } |
Carlo Bertolli | 962bb80 | 2017-01-03 18:24:42 +0000 | [diff] [blame] | 3709 | // Emit final copy of the lastprivate variables if IsLastIter != 0. |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3710 | if (HasLastprivateClause) { |
Carlo Bertolli | 962bb80 | 2017-01-03 18:24:42 +0000 | [diff] [blame] | 3711 | EmitOMPLastprivateClauseFinal( |
| 3712 | S, /*NoFinals=*/false, |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3713 | Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc()))); |
Alexey Bataev | 617db5f | 2017-12-04 15:38:33 +0000 | [diff] [blame] | 3714 | } |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3715 | } |
| 3716 | |
| 3717 | // We're now done with the loop, so jump to the continuation block. |
| 3718 | if (ContBlock) { |
| 3719 | EmitBranch(ContBlock); |
| 3720 | EmitBlock(ContBlock, true); |
| 3721 | } |
| 3722 | } |
| 3723 | } |
| 3724 | |
Carlo Bertolli | 6200a3d | 2015-12-14 14:51:25 +0000 | [diff] [blame] | 3725 | void CodeGenFunction::EmitOMPDistributeDirective( |
| 3726 | const OMPDistributeDirective &S) { |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 3727 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 3728 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); |
Carlo Bertolli | fc35ad2 | 2016-03-07 16:04:49 +0000 | [diff] [blame] | 3729 | }; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3730 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Alexey Bataev | 10a5431 | 2017-11-27 16:54:08 +0000 | [diff] [blame] | 3731 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen); |
Carlo Bertolli | 6200a3d | 2015-12-14 14:51:25 +0000 | [diff] [blame] | 3732 | } |
| 3733 | |
Alexey Bataev | 5f600d6 | 2015-09-29 03:48:57 +0000 | [diff] [blame] | 3734 | static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM, |
| 3735 | const CapturedStmt *S) { |
| 3736 | CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); |
| 3737 | CodeGenFunction::CGCapturedStmtInfo CapStmtInfo; |
| 3738 | CGF.CapturedStmtInfo = &CapStmtInfo; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3739 | llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S); |
Alexey Bataev | c0f879b | 2018-04-10 20:10:53 +0000 | [diff] [blame] | 3740 | Fn->setDoesNotRecurse(); |
Alexey Bataev | 5f600d6 | 2015-09-29 03:48:57 +0000 | [diff] [blame] | 3741 | return Fn; |
| 3742 | } |
| 3743 | |
Alexey Bataev | 98eb6e3 | 2015-04-22 11:15:40 +0000 | [diff] [blame] | 3744 | void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) { |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3745 | if (S.hasClausesOfKind<OMPDependClause>()) { |
| 3746 | assert(!S.getAssociatedStmt() && |
| 3747 | "No associated statement must be in ordered depend construct."); |
Alexey Bataev | 8b42706 | 2016-05-25 12:36:08 +0000 | [diff] [blame] | 3748 | for (const auto *DC : S.getClausesOfKind<OMPDependClause>()) |
| 3749 | CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC); |
Alexey Bataev | 8ef3141 | 2015-12-18 07:58:25 +0000 | [diff] [blame] | 3750 | return; |
Alexey Bataev | 8b42706 | 2016-05-25 12:36:08 +0000 | [diff] [blame] | 3751 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3752 | const auto *C = S.getSingleClause<OMPSIMDClause>(); |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 3753 | auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, |
| 3754 | PrePostActionTy &Action) { |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3755 | const CapturedStmt *CS = S.getInnermostCapturedStmt(); |
Alexey Bataev | 5f600d6 | 2015-09-29 03:48:57 +0000 | [diff] [blame] | 3756 | if (C) { |
Alexey Bataev | 5f600d6 | 2015-09-29 03:48:57 +0000 | [diff] [blame] | 3757 | llvm::SmallVector<llvm::Value *, 16> CapturedVars; |
| 3758 | CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3759 | llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3760 | CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(), |
Alexey Bataev | 3c595a6 | 2017-08-14 15:01:03 +0000 | [diff] [blame] | 3761 | OutlinedFn, CapturedVars); |
Alexey Bataev | 5f600d6 | 2015-09-29 03:48:57 +0000 | [diff] [blame] | 3762 | } else { |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 3763 | Action.Enter(CGF); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3764 | CGF.EmitStmt(CS->getCapturedStmt()); |
Alexey Bataev | 5f600d6 | 2015-09-29 03:48:57 +0000 | [diff] [blame] | 3765 | } |
Alexey Bataev | 98eb6e3 | 2015-04-22 11:15:40 +0000 | [diff] [blame] | 3766 | }; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 3767 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 3768 | CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C); |
Alexey Bataev | 9fb6e64 | 2014-07-22 06:45:04 +0000 | [diff] [blame] | 3769 | } |
| 3770 | |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 3771 | static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, |
Filipe Cabecinhas | 7af183d | 2015-08-11 04:19:28 +0000 | [diff] [blame] | 3772 | QualType SrcType, QualType DestType, |
| 3773 | SourceLocation Loc) { |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 3774 | assert(CGF.hasScalarEvaluationKind(DestType) && |
| 3775 | "DestType must have scalar evaluation kind."); |
| 3776 | assert(!Val.isAggregate() && "Must be a scalar or complex."); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3777 | return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, |
| 3778 | DestType, Loc) |
| 3779 | : CGF.EmitComplexToScalarConversion( |
| 3780 | Val.getComplexVal(), SrcType, DestType, Loc); |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 3781 | } |
| 3782 | |
| 3783 | static CodeGenFunction::ComplexPairTy |
| 3784 | convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, |
Filipe Cabecinhas | 7af183d | 2015-08-11 04:19:28 +0000 | [diff] [blame] | 3785 | QualType DestType, SourceLocation Loc) { |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 3786 | assert(CGF.getEvaluationKind(DestType) == TEK_Complex && |
| 3787 | "DestType must have complex evaluation kind."); |
| 3788 | CodeGenFunction::ComplexPairTy ComplexVal; |
| 3789 | if (Val.isScalar()) { |
| 3790 | // Convert the input element to the element type of the complex. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3791 | QualType DestElementType = |
| 3792 | DestType->castAs<ComplexType>()->getElementType(); |
| 3793 | llvm::Value *ScalarVal = CGF.EmitScalarConversion( |
| 3794 | Val.getScalarVal(), SrcType, DestElementType, Loc); |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 3795 | ComplexVal = CodeGenFunction::ComplexPairTy( |
| 3796 | ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType())); |
| 3797 | } else { |
| 3798 | assert(Val.isComplex() && "Must be a scalar or complex."); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3799 | QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType(); |
| 3800 | QualType DestElementType = |
| 3801 | DestType->castAs<ComplexType>()->getElementType(); |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 3802 | ComplexVal.first = CGF.EmitScalarConversion( |
Filipe Cabecinhas | 7af183d | 2015-08-11 04:19:28 +0000 | [diff] [blame] | 3803 | Val.getComplexVal().first, SrcElementType, DestElementType, Loc); |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 3804 | ComplexVal.second = CGF.EmitScalarConversion( |
Filipe Cabecinhas | 7af183d | 2015-08-11 04:19:28 +0000 | [diff] [blame] | 3805 | Val.getComplexVal().second, SrcElementType, DestElementType, Loc); |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 3806 | } |
| 3807 | return ComplexVal; |
| 3808 | } |
| 3809 | |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3810 | static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, |
| 3811 | LValue LVal, RValue RVal) { |
| 3812 | if (LVal.isGlobalReg()) { |
| 3813 | CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); |
| 3814 | } else { |
JF Bastien | 92f4ef1 | 2016-04-06 17:26:42 +0000 | [diff] [blame] | 3815 | CGF.EmitAtomicStore(RVal, LVal, |
| 3816 | IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent |
| 3817 | : llvm::AtomicOrdering::Monotonic, |
Rui Ueyama | 49a3ad2 | 2019-07-16 04:46:31 +0000 | [diff] [blame] | 3818 | LVal.isVolatile(), /*isInit=*/false); |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3819 | } |
| 3820 | } |
| 3821 | |
Alexey Bataev | 8524d15 | 2016-01-21 12:35:58 +0000 | [diff] [blame] | 3822 | void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal, |
| 3823 | QualType RValTy, SourceLocation Loc) { |
| 3824 | switch (getEvaluationKind(LVal.getType())) { |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3825 | case TEK_Scalar: |
Alexey Bataev | 8524d15 | 2016-01-21 12:35:58 +0000 | [diff] [blame] | 3826 | EmitStoreThroughLValue(RValue::get(convertToScalarValue( |
| 3827 | *this, RVal, RValTy, LVal.getType(), Loc)), |
| 3828 | LVal); |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3829 | break; |
| 3830 | case TEK_Complex: |
Alexey Bataev | 8524d15 | 2016-01-21 12:35:58 +0000 | [diff] [blame] | 3831 | EmitStoreOfComplex( |
| 3832 | convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal, |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3833 | /*isInit=*/false); |
| 3834 | break; |
| 3835 | case TEK_Aggregate: |
| 3836 | llvm_unreachable("Must be a scalar or complex."); |
| 3837 | } |
| 3838 | } |
| 3839 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3840 | static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 3841 | const Expr *X, const Expr *V, |
| 3842 | SourceLocation Loc) { |
| 3843 | // v = x; |
| 3844 | assert(V->isLValue() && "V of 'omp atomic read' is not lvalue"); |
| 3845 | assert(X->isLValue() && "X of 'omp atomic read' is not lvalue"); |
| 3846 | LValue XLValue = CGF.EmitLValue(X); |
| 3847 | LValue VLValue = CGF.EmitLValue(V); |
David Majnemer | a5b195a | 2015-02-14 01:35:12 +0000 | [diff] [blame] | 3848 | RValue Res = XLValue.isGlobalReg() |
| 3849 | ? CGF.EmitLoadOfLValue(XLValue, Loc) |
JF Bastien | 92f4ef1 | 2016-04-06 17:26:42 +0000 | [diff] [blame] | 3850 | : CGF.EmitAtomicLoad( |
| 3851 | XLValue, Loc, |
| 3852 | IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent |
| 3853 | : llvm::AtomicOrdering::Monotonic, |
| 3854 | XLValue.isVolatile()); |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 3855 | // OpenMP, 2.12.6, atomic Construct |
| 3856 | // Any atomic construct with a seq_cst clause forces the atomically |
| 3857 | // performed operation to include an implicit flush operation without a |
| 3858 | // list. |
| 3859 | if (IsSeqCst) |
Alexey Bataev | 3eff5f4 | 2015-02-25 08:32:46 +0000 | [diff] [blame] | 3860 | CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); |
Alexey Bataev | 8524d15 | 2016-01-21 12:35:58 +0000 | [diff] [blame] | 3861 | CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc); |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 3862 | } |
| 3863 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3864 | static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, |
Alexey Bataev | b832926 | 2015-02-27 06:33:30 +0000 | [diff] [blame] | 3865 | const Expr *X, const Expr *E, |
| 3866 | SourceLocation Loc) { |
| 3867 | // x = expr; |
| 3868 | assert(X->isLValue() && "X of 'omp atomic write' is not lvalue"); |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3869 | emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); |
Alexey Bataev | b832926 | 2015-02-27 06:33:30 +0000 | [diff] [blame] | 3870 | // OpenMP, 2.12.6, atomic Construct |
| 3871 | // Any atomic construct with a seq_cst clause forces the atomically |
| 3872 | // performed operation to include an implicit flush operation without a |
| 3873 | // list. |
| 3874 | if (IsSeqCst) |
| 3875 | CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); |
| 3876 | } |
| 3877 | |
Benjamin Kramer | 439ee9d | 2015-05-01 13:59:53 +0000 | [diff] [blame] | 3878 | static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, |
| 3879 | RValue Update, |
| 3880 | BinaryOperatorKind BO, |
| 3881 | llvm::AtomicOrdering AO, |
| 3882 | bool IsXLHSInRHSPart) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3883 | ASTContext &Context = CGF.getContext(); |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3884 | // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x' |
Alexey Bataev | b4505a7 | 2015-03-30 05:20:59 +0000 | [diff] [blame] | 3885 | // expression is simple and atomic is allowed for the given type for the |
| 3886 | // target platform. |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3887 | if (BO == BO_Comma || !Update.isScalar() || |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 3888 | !Update.getScalarVal()->getType()->isIntegerTy() || !X.isSimple() || |
| 3889 | (!isa<llvm::ConstantInt>(Update.getScalarVal()) && |
| 3890 | (Update.getScalarVal()->getType() != |
| 3891 | X.getAddress(CGF).getElementType())) || |
| 3892 | !X.getAddress(CGF).getElementType()->isIntegerTy() || |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3893 | !Context.getTargetInfo().hasBuiltinAtomic( |
| 3894 | Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment()))) |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3895 | return std::make_pair(false, RValue::get(nullptr)); |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3896 | |
| 3897 | llvm::AtomicRMWInst::BinOp RMWOp; |
| 3898 | switch (BO) { |
| 3899 | case BO_Add: |
| 3900 | RMWOp = llvm::AtomicRMWInst::Add; |
| 3901 | break; |
| 3902 | case BO_Sub: |
| 3903 | if (!IsXLHSInRHSPart) |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3904 | return std::make_pair(false, RValue::get(nullptr)); |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3905 | RMWOp = llvm::AtomicRMWInst::Sub; |
| 3906 | break; |
| 3907 | case BO_And: |
| 3908 | RMWOp = llvm::AtomicRMWInst::And; |
| 3909 | break; |
| 3910 | case BO_Or: |
| 3911 | RMWOp = llvm::AtomicRMWInst::Or; |
| 3912 | break; |
| 3913 | case BO_Xor: |
| 3914 | RMWOp = llvm::AtomicRMWInst::Xor; |
| 3915 | break; |
| 3916 | case BO_LT: |
| 3917 | RMWOp = X.getType()->hasSignedIntegerRepresentation() |
| 3918 | ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min |
| 3919 | : llvm::AtomicRMWInst::Max) |
| 3920 | : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin |
| 3921 | : llvm::AtomicRMWInst::UMax); |
| 3922 | break; |
| 3923 | case BO_GT: |
| 3924 | RMWOp = X.getType()->hasSignedIntegerRepresentation() |
| 3925 | ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max |
| 3926 | : llvm::AtomicRMWInst::Min) |
| 3927 | : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax |
| 3928 | : llvm::AtomicRMWInst::UMin); |
| 3929 | break; |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3930 | case BO_Assign: |
| 3931 | RMWOp = llvm::AtomicRMWInst::Xchg; |
| 3932 | break; |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3933 | case BO_Mul: |
| 3934 | case BO_Div: |
| 3935 | case BO_Rem: |
| 3936 | case BO_Shl: |
| 3937 | case BO_Shr: |
| 3938 | case BO_LAnd: |
| 3939 | case BO_LOr: |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3940 | return std::make_pair(false, RValue::get(nullptr)); |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3941 | case BO_PtrMemD: |
| 3942 | case BO_PtrMemI: |
| 3943 | case BO_LE: |
| 3944 | case BO_GE: |
| 3945 | case BO_EQ: |
| 3946 | case BO_NE: |
Richard Smith | c70f1d6 | 2017-12-14 15:16:18 +0000 | [diff] [blame] | 3947 | case BO_Cmp: |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3948 | case BO_AddAssign: |
| 3949 | case BO_SubAssign: |
| 3950 | case BO_AndAssign: |
| 3951 | case BO_OrAssign: |
| 3952 | case BO_XorAssign: |
| 3953 | case BO_MulAssign: |
| 3954 | case BO_DivAssign: |
| 3955 | case BO_RemAssign: |
| 3956 | case BO_ShlAssign: |
| 3957 | case BO_ShrAssign: |
| 3958 | case BO_Comma: |
| 3959 | llvm_unreachable("Unsupported atomic update operation"); |
| 3960 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3961 | llvm::Value *UpdateVal = Update.getScalarVal(); |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3962 | if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) { |
| 3963 | UpdateVal = CGF.Builder.CreateIntCast( |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 3964 | IC, X.getAddress(CGF).getElementType(), |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3965 | X.getType()->hasSignedIntegerRepresentation()); |
| 3966 | } |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3967 | llvm::Value *Res = |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 3968 | CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(CGF), UpdateVal, AO); |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3969 | return std::make_pair(true, RValue::get(Res)); |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3970 | } |
| 3971 | |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3972 | std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr( |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3973 | LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, |
| 3974 | llvm::AtomicOrdering AO, SourceLocation Loc, |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3975 | const llvm::function_ref<RValue(RValue)> CommonGen) { |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3976 | // Update expressions are allowed to have the following forms: |
| 3977 | // x binop= expr; -> xrval + expr; |
| 3978 | // x++, ++x -> xrval + 1; |
| 3979 | // x--, --x -> xrval - 1; |
| 3980 | // x = x binop expr; -> xrval binop expr |
| 3981 | // x = expr Op x; - > expr binop xrval; |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3982 | auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart); |
| 3983 | if (!Res.first) { |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 3984 | if (X.isGlobalReg()) { |
| 3985 | // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop |
| 3986 | // 'xrval'. |
| 3987 | EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X); |
| 3988 | } else { |
| 3989 | // Perform compare-and-swap procedure. |
| 3990 | EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified()); |
Alexey Bataev | b4505a7 | 2015-03-30 05:20:59 +0000 | [diff] [blame] | 3991 | } |
| 3992 | } |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 3993 | return Res; |
Alexey Bataev | b4505a7 | 2015-03-30 05:20:59 +0000 | [diff] [blame] | 3994 | } |
| 3995 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 3996 | static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, |
Alexey Bataev | b4505a7 | 2015-03-30 05:20:59 +0000 | [diff] [blame] | 3997 | const Expr *X, const Expr *E, |
| 3998 | const Expr *UE, bool IsXLHSInRHSPart, |
| 3999 | SourceLocation Loc) { |
| 4000 | assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) && |
| 4001 | "Update expr in 'atomic update' must be a binary operator."); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4002 | const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts()); |
Alexey Bataev | b4505a7 | 2015-03-30 05:20:59 +0000 | [diff] [blame] | 4003 | // Update expressions are allowed to have the following forms: |
| 4004 | // x binop= expr; -> xrval + expr; |
| 4005 | // x++, ++x -> xrval + 1; |
| 4006 | // x--, --x -> xrval - 1; |
| 4007 | // x = x binop expr; -> xrval binop expr |
| 4008 | // x = expr Op x; - > expr binop xrval; |
Alexey Bataev | 794ba0d | 2015-04-10 10:43:45 +0000 | [diff] [blame] | 4009 | assert(X->isLValue() && "X of 'omp atomic update' is not lvalue"); |
Alexey Bataev | b4505a7 | 2015-03-30 05:20:59 +0000 | [diff] [blame] | 4010 | LValue XLValue = CGF.EmitLValue(X); |
| 4011 | RValue ExprRValue = CGF.EmitAnyExpr(E); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4012 | llvm::AtomicOrdering AO = IsSeqCst |
| 4013 | ? llvm::AtomicOrdering::SequentiallyConsistent |
| 4014 | : llvm::AtomicOrdering::Monotonic; |
| 4015 | const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts()); |
| 4016 | const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts()); |
| 4017 | const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; |
| 4018 | const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; |
| 4019 | auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { |
| 4020 | CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); |
| 4021 | CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); |
| 4022 | return CGF.EmitAnyExpr(UE); |
| 4023 | }; |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4024 | (void)CGF.EmitOMPAtomicSimpleUpdateExpr( |
| 4025 | XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen); |
| 4026 | // OpenMP, 2.12.6, atomic Construct |
| 4027 | // Any atomic construct with a seq_cst clause forces the atomically |
| 4028 | // performed operation to include an implicit flush operation without a |
| 4029 | // list. |
| 4030 | if (IsSeqCst) |
| 4031 | CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); |
| 4032 | } |
| 4033 | |
| 4034 | static RValue convertToType(CodeGenFunction &CGF, RValue Value, |
Filipe Cabecinhas | 7af183d | 2015-08-11 04:19:28 +0000 | [diff] [blame] | 4035 | QualType SourceType, QualType ResType, |
| 4036 | SourceLocation Loc) { |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4037 | switch (CGF.getEvaluationKind(ResType)) { |
| 4038 | case TEK_Scalar: |
Filipe Cabecinhas | 7af183d | 2015-08-11 04:19:28 +0000 | [diff] [blame] | 4039 | return RValue::get( |
| 4040 | convertToScalarValue(CGF, Value, SourceType, ResType, Loc)); |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4041 | case TEK_Complex: { |
Filipe Cabecinhas | 7af183d | 2015-08-11 04:19:28 +0000 | [diff] [blame] | 4042 | auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc); |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4043 | return RValue::getComplex(Res.first, Res.second); |
| 4044 | } |
| 4045 | case TEK_Aggregate: |
| 4046 | break; |
| 4047 | } |
| 4048 | llvm_unreachable("Must be a scalar or complex."); |
| 4049 | } |
| 4050 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4051 | static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4052 | bool IsPostfixUpdate, const Expr *V, |
| 4053 | const Expr *X, const Expr *E, |
| 4054 | const Expr *UE, bool IsXLHSInRHSPart, |
| 4055 | SourceLocation Loc) { |
| 4056 | assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue"); |
| 4057 | assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue"); |
| 4058 | RValue NewVVal; |
| 4059 | LValue VLValue = CGF.EmitLValue(V); |
| 4060 | LValue XLValue = CGF.EmitLValue(X); |
| 4061 | RValue ExprRValue = CGF.EmitAnyExpr(E); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4062 | llvm::AtomicOrdering AO = IsSeqCst |
| 4063 | ? llvm::AtomicOrdering::SequentiallyConsistent |
| 4064 | : llvm::AtomicOrdering::Monotonic; |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4065 | QualType NewVValType; |
| 4066 | if (UE) { |
| 4067 | // 'x' is updated with some additional value. |
| 4068 | assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) && |
| 4069 | "Update expr in 'atomic capture' must be a binary operator."); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4070 | const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts()); |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4071 | // Update expressions are allowed to have the following forms: |
| 4072 | // x binop= expr; -> xrval + expr; |
| 4073 | // x++, ++x -> xrval + 1; |
| 4074 | // x--, --x -> xrval - 1; |
| 4075 | // x = x binop expr; -> xrval binop expr |
| 4076 | // x = expr Op x; - > expr binop xrval; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4077 | const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts()); |
| 4078 | const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts()); |
| 4079 | const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4080 | NewVValType = XRValExpr->getType(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4081 | const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4082 | auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4083 | IsPostfixUpdate](RValue XRValue) { |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4084 | CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); |
| 4085 | CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); |
| 4086 | RValue Res = CGF.EmitAnyExpr(UE); |
| 4087 | NewVVal = IsPostfixUpdate ? XRValue : Res; |
| 4088 | return Res; |
| 4089 | }; |
| 4090 | auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( |
| 4091 | XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen); |
| 4092 | if (Res.first) { |
| 4093 | // 'atomicrmw' instruction was generated. |
| 4094 | if (IsPostfixUpdate) { |
| 4095 | // Use old value from 'atomicrmw'. |
| 4096 | NewVVal = Res.second; |
| 4097 | } else { |
| 4098 | // 'atomicrmw' does not provide new value, so evaluate it using old |
| 4099 | // value of 'x'. |
| 4100 | CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); |
| 4101 | CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); |
| 4102 | NewVVal = CGF.EmitAnyExpr(UE); |
| 4103 | } |
| 4104 | } |
| 4105 | } else { |
| 4106 | // 'x' is simply rewritten with some 'expr'. |
| 4107 | NewVValType = X->getType().getNonReferenceType(); |
| 4108 | ExprRValue = convertToType(CGF, ExprRValue, E->getType(), |
Filipe Cabecinhas | 7af183d | 2015-08-11 04:19:28 +0000 | [diff] [blame] | 4109 | X->getType().getNonReferenceType(), Loc); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4110 | auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) { |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4111 | NewVVal = XRValue; |
| 4112 | return ExprRValue; |
| 4113 | }; |
| 4114 | // Try to perform atomicrmw xchg, otherwise simple exchange. |
| 4115 | auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( |
| 4116 | XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO, |
| 4117 | Loc, Gen); |
| 4118 | if (Res.first) { |
| 4119 | // 'atomicrmw' instruction was generated. |
| 4120 | NewVVal = IsPostfixUpdate ? Res.second : ExprRValue; |
| 4121 | } |
| 4122 | } |
| 4123 | // Emit post-update store to 'v' of old/new 'x' value. |
Alexey Bataev | 8524d15 | 2016-01-21 12:35:58 +0000 | [diff] [blame] | 4124 | CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc); |
Alexey Bataev | b4505a7 | 2015-03-30 05:20:59 +0000 | [diff] [blame] | 4125 | // OpenMP, 2.12.6, atomic Construct |
| 4126 | // Any atomic construct with a seq_cst clause forces the atomically |
| 4127 | // performed operation to include an implicit flush operation without a |
| 4128 | // list. |
| 4129 | if (IsSeqCst) |
| 4130 | CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); |
| 4131 | } |
| 4132 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4133 | static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4134 | bool IsSeqCst, bool IsPostfixUpdate, |
| 4135 | const Expr *X, const Expr *V, const Expr *E, |
| 4136 | const Expr *UE, bool IsXLHSInRHSPart, |
| 4137 | SourceLocation Loc) { |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 4138 | switch (Kind) { |
| 4139 | case OMPC_read: |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4140 | emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc); |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 4141 | break; |
| 4142 | case OMPC_write: |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4143 | emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc); |
Alexey Bataev | b832926 | 2015-02-27 06:33:30 +0000 | [diff] [blame] | 4144 | break; |
Alexey Bataev | b4505a7 | 2015-03-30 05:20:59 +0000 | [diff] [blame] | 4145 | case OMPC_unknown: |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 4146 | case OMPC_update: |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4147 | emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc); |
Alexey Bataev | b4505a7 | 2015-03-30 05:20:59 +0000 | [diff] [blame] | 4148 | break; |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 4149 | case OMPC_capture: |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4150 | emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE, |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4151 | IsXLHSInRHSPart, Loc); |
| 4152 | break; |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 4153 | case OMPC_if: |
| 4154 | case OMPC_final: |
| 4155 | case OMPC_num_threads: |
| 4156 | case OMPC_private: |
| 4157 | case OMPC_firstprivate: |
| 4158 | case OMPC_lastprivate: |
| 4159 | case OMPC_reduction: |
Alexey Bataev | 169d96a | 2017-07-18 20:17:46 +0000 | [diff] [blame] | 4160 | case OMPC_task_reduction: |
Alexey Bataev | fa312f3 | 2017-07-21 18:48:21 +0000 | [diff] [blame] | 4161 | case OMPC_in_reduction: |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 4162 | case OMPC_safelen: |
Alexey Bataev | 66b15b5 | 2015-08-21 11:14:16 +0000 | [diff] [blame] | 4163 | case OMPC_simdlen: |
Alexey Bataev | 9cc10fc | 2019-03-12 18:52:33 +0000 | [diff] [blame] | 4164 | case OMPC_allocator: |
Alexey Bataev | e04483e | 2019-03-27 14:14:31 +0000 | [diff] [blame] | 4165 | case OMPC_allocate: |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 4166 | case OMPC_collapse: |
| 4167 | case OMPC_default: |
| 4168 | case OMPC_seq_cst: |
| 4169 | case OMPC_shared: |
| 4170 | case OMPC_linear: |
| 4171 | case OMPC_aligned: |
| 4172 | case OMPC_copyin: |
| 4173 | case OMPC_copyprivate: |
| 4174 | case OMPC_flush: |
| 4175 | case OMPC_proc_bind: |
| 4176 | case OMPC_schedule: |
| 4177 | case OMPC_ordered: |
| 4178 | case OMPC_nowait: |
| 4179 | case OMPC_untied: |
| 4180 | case OMPC_threadprivate: |
Alexey Bataev | 1c2cfbc | 2015-06-23 14:25:19 +0000 | [diff] [blame] | 4181 | case OMPC_depend: |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 4182 | case OMPC_mergeable: |
Michael Wong | e710d54 | 2015-08-07 16:16:36 +0000 | [diff] [blame] | 4183 | case OMPC_device: |
Alexey Bataev | 346265e | 2015-09-25 10:37:12 +0000 | [diff] [blame] | 4184 | case OMPC_threads: |
Alexey Bataev | d14d1e6 | 2015-09-28 06:39:35 +0000 | [diff] [blame] | 4185 | case OMPC_simd: |
Kelvin Li | 0bff7af | 2015-11-23 05:32:03 +0000 | [diff] [blame] | 4186 | case OMPC_map: |
Kelvin Li | 099bb8c | 2015-11-24 20:50:12 +0000 | [diff] [blame] | 4187 | case OMPC_num_teams: |
Kelvin Li | a15fb1a | 2015-11-27 18:47:36 +0000 | [diff] [blame] | 4188 | case OMPC_thread_limit: |
Alexey Bataev | a056935 | 2015-12-01 10:17:31 +0000 | [diff] [blame] | 4189 | case OMPC_priority: |
Alexey Bataev | 1fd4aed | 2015-12-07 12:52:51 +0000 | [diff] [blame] | 4190 | case OMPC_grainsize: |
Alexey Bataev | b825de1 | 2015-12-07 10:51:44 +0000 | [diff] [blame] | 4191 | case OMPC_nogroup: |
Alexey Bataev | 382967a | 2015-12-08 12:06:20 +0000 | [diff] [blame] | 4192 | case OMPC_num_tasks: |
Alexey Bataev | 28c7541 | 2015-12-15 08:19:24 +0000 | [diff] [blame] | 4193 | case OMPC_hint: |
Carlo Bertolli | b4adf55 | 2016-01-15 18:50:31 +0000 | [diff] [blame] | 4194 | case OMPC_dist_schedule: |
Arpith Chacko Jacob | 3cf8904 | 2016-01-26 16:37:23 +0000 | [diff] [blame] | 4195 | case OMPC_defaultmap: |
Alexey Bataev | e48a5fc | 2016-04-12 05:28:34 +0000 | [diff] [blame] | 4196 | case OMPC_uniform: |
Samuel Antao | 661c090 | 2016-05-26 17:39:58 +0000 | [diff] [blame] | 4197 | case OMPC_to: |
Samuel Antao | ec172c6 | 2016-05-26 17:49:04 +0000 | [diff] [blame] | 4198 | case OMPC_from: |
Carlo Bertolli | 2404b17 | 2016-07-13 15:37:16 +0000 | [diff] [blame] | 4199 | case OMPC_use_device_ptr: |
Carlo Bertolli | 70594e9 | 2016-07-13 17:16:49 +0000 | [diff] [blame] | 4200 | case OMPC_is_device_ptr: |
Kelvin Li | 1408f91 | 2018-09-26 04:28:39 +0000 | [diff] [blame] | 4201 | case OMPC_unified_address: |
Alexey Bataev | 94c5064 | 2018-10-01 14:26:31 +0000 | [diff] [blame] | 4202 | case OMPC_unified_shared_memory: |
Patrick Lyster | 6bdf63b | 2018-10-03 20:07:58 +0000 | [diff] [blame] | 4203 | case OMPC_reverse_offload: |
Patrick Lyster | 3fe9e39 | 2018-10-11 14:41:10 +0000 | [diff] [blame] | 4204 | case OMPC_dynamic_allocators: |
Patrick Lyster | 7a2a27c | 2018-11-02 12:18:11 +0000 | [diff] [blame] | 4205 | case OMPC_atomic_default_mem_order: |
Alexey Bataev | 729e242 | 2019-08-23 16:11:14 +0000 | [diff] [blame] | 4206 | case OMPC_device_type: |
Alexey Bataev | dba792c | 2019-09-23 18:13:31 +0000 | [diff] [blame] | 4207 | case OMPC_match: |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 4208 | llvm_unreachable("Clause is not allowed in 'omp atomic'."); |
| 4209 | } |
| 4210 | } |
| 4211 | |
| 4212 | void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) { |
Benjamin Kramer | fc600dc | 2015-08-30 15:12:28 +0000 | [diff] [blame] | 4213 | bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>(); |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 4214 | OpenMPClauseKind Kind = OMPC_unknown; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4215 | for (const OMPClause *C : S.clauses()) { |
Alexey Bataev | b57056f | 2015-01-22 06:17:56 +0000 | [diff] [blame] | 4216 | // Find first clause (skip seq_cst clause, if it is first). |
| 4217 | if (C->getClauseKind() != OMPC_seq_cst) { |
| 4218 | Kind = C->getClauseKind(); |
| 4219 | break; |
| 4220 | } |
| 4221 | } |
Alexey Bataev | 10fec57 | 2015-03-11 04:48:56 +0000 | [diff] [blame] | 4222 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4223 | const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers(); |
Bill Wendling | 7c44da2 | 2018-10-31 03:48:47 +0000 | [diff] [blame] | 4224 | if (const auto *FE = dyn_cast<FullExpr>(CS)) |
| 4225 | enterFullExpression(FE); |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4226 | // Processing for statements under 'atomic capture'. |
| 4227 | if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4228 | for (const Stmt *C : Compound->body()) { |
Bill Wendling | 7c44da2 | 2018-10-31 03:48:47 +0000 | [diff] [blame] | 4229 | if (const auto *FE = dyn_cast<FullExpr>(C)) |
| 4230 | enterFullExpression(FE); |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4231 | } |
| 4232 | } |
Alexey Bataev | 10fec57 | 2015-03-11 04:48:56 +0000 | [diff] [blame] | 4233 | |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 4234 | auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF, |
| 4235 | PrePostActionTy &) { |
Alexey Bataev | 33c5640 | 2015-12-14 09:26:19 +0000 | [diff] [blame] | 4236 | CGF.EmitStopPoint(CS); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4237 | emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(), |
Alexey Bataev | 5e018f9 | 2015-04-23 06:35:10 +0000 | [diff] [blame] | 4238 | S.getV(), S.getExpr(), S.getUpdateExpr(), |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 4239 | S.isXLHSInRHSPart(), S.getBeginLoc()); |
Alexey Bataev | 6f1ffc0 | 2015-04-10 04:50:10 +0000 | [diff] [blame] | 4240 | }; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 4241 | OMPLexicalScope Scope(*this, S, OMPD_unknown); |
Alexey Bataev | 81c7ea0 | 2015-07-03 09:56:58 +0000 | [diff] [blame] | 4242 | CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen); |
Alexey Bataev | 0162e45 | 2014-07-22 10:10:35 +0000 | [diff] [blame] | 4243 | } |
| 4244 | |
Arpith Chacko Jacob | 43a8b7b | 2017-01-16 15:26:02 +0000 | [diff] [blame] | 4245 | static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, |
| 4246 | const OMPExecutableDirective &S, |
| 4247 | const RegionCodeGenTy &CodeGen) { |
| 4248 | assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind())); |
| 4249 | CodeGenModule &CGM = CGF.CGM; |
Samuel Antao | bed3c46 | 2015-10-02 16:14:20 +0000 | [diff] [blame] | 4250 | |
Alexey Bataev | 4f4bf7c | 2018-03-15 15:47:20 +0000 | [diff] [blame] | 4251 | // On device emit this construct as inlined code. |
| 4252 | if (CGM.getLangOpts().OpenMPIsDevice) { |
| 4253 | OMPLexicalScope Scope(CGF, S, OMPD_target); |
| 4254 | CGM.getOpenMPRuntime().emitInlinedDirective( |
| 4255 | CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
Alexey Bataev | 4ac68a2 | 2018-05-16 15:08:32 +0000 | [diff] [blame] | 4256 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); |
Alexey Bataev | 4f4bf7c | 2018-03-15 15:47:20 +0000 | [diff] [blame] | 4257 | }); |
| 4258 | return; |
| 4259 | } |
| 4260 | |
Samuel Antao | ee8fb30 | 2016-01-06 13:42:12 +0000 | [diff] [blame] | 4261 | llvm::Function *Fn = nullptr; |
| 4262 | llvm::Constant *FnID = nullptr; |
Samuel Antao | bed3c46 | 2015-10-02 16:14:20 +0000 | [diff] [blame] | 4263 | |
Samuel Antao | bed3c46 | 2015-10-02 16:14:20 +0000 | [diff] [blame] | 4264 | const Expr *IfCond = nullptr; |
Arpith Chacko Jacob | fe4890a | 2017-01-18 20:40:48 +0000 | [diff] [blame] | 4265 | // Check for the at most one if clause associated with the target region. |
| 4266 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { |
| 4267 | if (C->getNameModifier() == OMPD_unknown || |
| 4268 | C->getNameModifier() == OMPD_target) { |
| 4269 | IfCond = C->getCondition(); |
| 4270 | break; |
| 4271 | } |
Samuel Antao | bed3c46 | 2015-10-02 16:14:20 +0000 | [diff] [blame] | 4272 | } |
| 4273 | |
| 4274 | // Check if we have any device clause associated with the directive. |
| 4275 | const Expr *Device = nullptr; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4276 | if (auto *C = S.getSingleClause<OMPDeviceClause>()) |
Samuel Antao | bed3c46 | 2015-10-02 16:14:20 +0000 | [diff] [blame] | 4277 | Device = C->getDevice(); |
Samuel Antao | bed3c46 | 2015-10-02 16:14:20 +0000 | [diff] [blame] | 4278 | |
Samuel Antao | ee8fb30 | 2016-01-06 13:42:12 +0000 | [diff] [blame] | 4279 | // Check if we have an if clause whose conditional always evaluates to false |
| 4280 | // or if we do not have any targets specified. If so the target region is not |
| 4281 | // an offload entry point. |
| 4282 | bool IsOffloadEntry = true; |
| 4283 | if (IfCond) { |
| 4284 | bool Val; |
Arpith Chacko Jacob | 43a8b7b | 2017-01-16 15:26:02 +0000 | [diff] [blame] | 4285 | if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val) |
Samuel Antao | ee8fb30 | 2016-01-06 13:42:12 +0000 | [diff] [blame] | 4286 | IsOffloadEntry = false; |
| 4287 | } |
| 4288 | if (CGM.getLangOpts().OMPTargetTriples.empty()) |
| 4289 | IsOffloadEntry = false; |
| 4290 | |
Arpith Chacko Jacob | 43a8b7b | 2017-01-16 15:26:02 +0000 | [diff] [blame] | 4291 | assert(CGF.CurFuncDecl && "No parent declaration for target region!"); |
Samuel Antao | ee8fb30 | 2016-01-06 13:42:12 +0000 | [diff] [blame] | 4292 | StringRef ParentName; |
| 4293 | // In case we have Ctors/Dtors we use the complete type variant to produce |
| 4294 | // the mangling of the device outlined kernel. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4295 | if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl)) |
Samuel Antao | ee8fb30 | 2016-01-06 13:42:12 +0000 | [diff] [blame] | 4296 | ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete)); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4297 | else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl)) |
Samuel Antao | ee8fb30 | 2016-01-06 13:42:12 +0000 | [diff] [blame] | 4298 | ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete)); |
| 4299 | else |
| 4300 | ParentName = |
Arpith Chacko Jacob | 43a8b7b | 2017-01-16 15:26:02 +0000 | [diff] [blame] | 4301 | CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl))); |
Samuel Antao | ee8fb30 | 2016-01-06 13:42:12 +0000 | [diff] [blame] | 4302 | |
Arpith Chacko Jacob | 43a8b7b | 2017-01-16 15:26:02 +0000 | [diff] [blame] | 4303 | // Emit target region as a standalone region. |
| 4304 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID, |
| 4305 | IsOffloadEntry, CodeGen); |
Alexey Bataev | 8451efa | 2018-01-15 19:06:12 +0000 | [diff] [blame] | 4306 | OMPLexicalScope Scope(CGF, S, OMPD_task); |
Alexey Bataev | ec7946e | 2019-09-23 14:06:51 +0000 | [diff] [blame] | 4307 | auto &&SizeEmitter = |
| 4308 | [IsOffloadEntry](CodeGenFunction &CGF, |
| 4309 | const OMPLoopDirective &D) -> llvm::Value * { |
| 4310 | if (IsOffloadEntry) { |
| 4311 | OMPLoopScope(CGF, D); |
| 4312 | // Emit calculation of the iterations count. |
| 4313 | llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations()); |
| 4314 | NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty, |
| 4315 | /*isSigned=*/false); |
| 4316 | return NumIterations; |
| 4317 | } |
| 4318 | return nullptr; |
Alexey Bataev | 7bb3353 | 2019-01-07 21:30:43 +0000 | [diff] [blame] | 4319 | }; |
Alexey Bataev | ec7946e | 2019-09-23 14:06:51 +0000 | [diff] [blame] | 4320 | CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device, |
| 4321 | SizeEmitter); |
Alexey Bataev | 0bd520b | 2014-09-19 08:19:49 +0000 | [diff] [blame] | 4322 | } |
| 4323 | |
Arpith Chacko Jacob | 43a8b7b | 2017-01-16 15:26:02 +0000 | [diff] [blame] | 4324 | static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, |
| 4325 | PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4326 | Action.Enter(CGF); |
Arpith Chacko Jacob | 43a8b7b | 2017-01-16 15:26:02 +0000 | [diff] [blame] | 4327 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); |
| 4328 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); |
| 4329 | CGF.EmitOMPPrivateClause(S, PrivateScope); |
| 4330 | (void)PrivateScope.Privatize(); |
Alexey Bataev | 6070542 | 2018-10-30 15:50:12 +0000 | [diff] [blame] | 4331 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) |
| 4332 | CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); |
Arpith Chacko Jacob | 43a8b7b | 2017-01-16 15:26:02 +0000 | [diff] [blame] | 4333 | |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 4334 | CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); |
Arpith Chacko Jacob | 43a8b7b | 2017-01-16 15:26:02 +0000 | [diff] [blame] | 4335 | } |
| 4336 | |
| 4337 | void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM, |
| 4338 | StringRef ParentName, |
| 4339 | const OMPTargetDirective &S) { |
| 4340 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4341 | emitTargetRegion(CGF, S, Action); |
| 4342 | }; |
| 4343 | llvm::Function *Fn; |
| 4344 | llvm::Constant *Addr; |
| 4345 | // Emit target region as a standalone region. |
| 4346 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( |
| 4347 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); |
| 4348 | assert(Fn && Addr && "Target device function emission failed."); |
| 4349 | } |
| 4350 | |
| 4351 | void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) { |
| 4352 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4353 | emitTargetRegion(CGF, S, Action); |
| 4354 | }; |
| 4355 | emitCommonOMPTargetDirective(*this, S, CodeGen); |
| 4356 | } |
| 4357 | |
Carlo Bertolli | 430d8ec | 2016-03-03 20:34:23 +0000 | [diff] [blame] | 4358 | static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, |
| 4359 | const OMPExecutableDirective &S, |
| 4360 | OpenMPDirectiveKind InnermostKind, |
| 4361 | const RegionCodeGenTy &CodeGen) { |
Arpith Chacko Jacob | 19b911c | 2017-01-18 18:18:53 +0000 | [diff] [blame] | 4362 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams); |
James Y Knight | 9871db0 | 2019-02-05 16:42:33 +0000 | [diff] [blame] | 4363 | llvm::Function *OutlinedFn = |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4364 | CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( |
| 4365 | S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen); |
Samuel Antao | b68e2db | 2016-03-03 16:20:23 +0000 | [diff] [blame] | 4366 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4367 | const auto *NT = S.getSingleClause<OMPNumTeamsClause>(); |
| 4368 | const auto *TL = S.getSingleClause<OMPThreadLimitClause>(); |
Carlo Bertolli | 430d8ec | 2016-03-03 20:34:23 +0000 | [diff] [blame] | 4369 | if (NT || TL) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4370 | const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr; |
| 4371 | const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr; |
Carlo Bertolli | 430d8ec | 2016-03-03 20:34:23 +0000 | [diff] [blame] | 4372 | |
Carlo Bertolli | c687225 | 2016-04-04 15:55:02 +0000 | [diff] [blame] | 4373 | CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit, |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 4374 | S.getBeginLoc()); |
Carlo Bertolli | 430d8ec | 2016-03-03 20:34:23 +0000 | [diff] [blame] | 4375 | } |
| 4376 | |
Arpith Chacko Jacob | 99a1e0e | 2017-01-25 02:18:43 +0000 | [diff] [blame] | 4377 | OMPTeamsScope Scope(CGF, S); |
Alexey Bataev | 14fa1c6 | 2016-03-29 05:34:15 +0000 | [diff] [blame] | 4378 | llvm::SmallVector<llvm::Value *, 16> CapturedVars; |
| 4379 | CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 4380 | CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn, |
Carlo Bertolli | 430d8ec | 2016-03-03 20:34:23 +0000 | [diff] [blame] | 4381 | CapturedVars); |
| 4382 | } |
| 4383 | |
| 4384 | void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) { |
Kelvin Li | 51336dd | 2016-12-15 17:55:32 +0000 | [diff] [blame] | 4385 | // Emit teams region as a standalone region. |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 4386 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4387 | Action.Enter(CGF); |
Carlo Bertolli | 430d8ec | 2016-03-03 20:34:23 +0000 | [diff] [blame] | 4388 | OMPPrivateScope PrivateScope(CGF); |
Carlo Bertolli | 6ad7b5a | 2016-03-03 22:09:40 +0000 | [diff] [blame] | 4389 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); |
| 4390 | CGF.EmitOMPPrivateClause(S, PrivateScope); |
Arpith Chacko Jacob | fc711b1 | 2017-02-16 16:48:49 +0000 | [diff] [blame] | 4391 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
Carlo Bertolli | 430d8ec | 2016-03-03 20:34:23 +0000 | [diff] [blame] | 4392 | (void)PrivateScope.Privatize(); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 4393 | CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); |
Arpith Chacko Jacob | fc711b1 | 2017-02-16 16:48:49 +0000 | [diff] [blame] | 4394 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); |
Carlo Bertolli | 430d8ec | 2016-03-03 20:34:23 +0000 | [diff] [blame] | 4395 | }; |
Alexey Bataev | 2139ed6 | 2017-11-16 18:20:21 +0000 | [diff] [blame] | 4396 | emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4397 | emitPostUpdateForReductionClause(*this, S, |
| 4398 | [](CodeGenFunction &) { return nullptr; }); |
Alexey Bataev | 13314bf | 2014-10-09 04:18:56 +0000 | [diff] [blame] | 4399 | } |
Alexey Bataev | 6d4ed05 | 2015-07-01 06:57:41 +0000 | [diff] [blame] | 4400 | |
Arpith Chacko Jacob | 99a1e0e | 2017-01-25 02:18:43 +0000 | [diff] [blame] | 4401 | static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, |
| 4402 | const OMPTargetTeamsDirective &S) { |
| 4403 | auto *CS = S.getCapturedStmt(OMPD_teams); |
| 4404 | Action.Enter(CGF); |
Alexey Bataev | f9fc42e | 2017-11-22 14:25:55 +0000 | [diff] [blame] | 4405 | // Emit teams region as a standalone region. |
| 4406 | auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4407 | Action.Enter(CGF); |
Alexey Bataev | f9fc42e | 2017-11-22 14:25:55 +0000 | [diff] [blame] | 4408 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); |
| 4409 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); |
| 4410 | CGF.EmitOMPPrivateClause(S, PrivateScope); |
| 4411 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 4412 | (void)PrivateScope.Privatize(); |
Alexey Bataev | 6070542 | 2018-10-30 15:50:12 +0000 | [diff] [blame] | 4413 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) |
| 4414 | CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); |
Arpith Chacko Jacob | 99a1e0e | 2017-01-25 02:18:43 +0000 | [diff] [blame] | 4415 | CGF.EmitStmt(CS->getCapturedStmt()); |
Alexey Bataev | f9fc42e | 2017-11-22 14:25:55 +0000 | [diff] [blame] | 4416 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); |
Arpith Chacko Jacob | 99a1e0e | 2017-01-25 02:18:43 +0000 | [diff] [blame] | 4417 | }; |
| 4418 | emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4419 | emitPostUpdateForReductionClause(CGF, S, |
| 4420 | [](CodeGenFunction &) { return nullptr; }); |
Arpith Chacko Jacob | 99a1e0e | 2017-01-25 02:18:43 +0000 | [diff] [blame] | 4421 | } |
| 4422 | |
| 4423 | void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction( |
| 4424 | CodeGenModule &CGM, StringRef ParentName, |
| 4425 | const OMPTargetTeamsDirective &S) { |
| 4426 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4427 | emitTargetTeamsRegion(CGF, Action, S); |
| 4428 | }; |
| 4429 | llvm::Function *Fn; |
| 4430 | llvm::Constant *Addr; |
| 4431 | // Emit target region as a standalone region. |
| 4432 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( |
| 4433 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); |
| 4434 | assert(Fn && Addr && "Target device function emission failed."); |
| 4435 | } |
| 4436 | |
| 4437 | void CodeGenFunction::EmitOMPTargetTeamsDirective( |
| 4438 | const OMPTargetTeamsDirective &S) { |
| 4439 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4440 | emitTargetTeamsRegion(CGF, Action, S); |
| 4441 | }; |
| 4442 | emitCommonOMPTargetDirective(*this, S, CodeGen); |
| 4443 | } |
| 4444 | |
Alexey Bataev | dfa430f | 2017-12-08 15:03:50 +0000 | [diff] [blame] | 4445 | static void |
| 4446 | emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, |
| 4447 | const OMPTargetTeamsDistributeDirective &S) { |
| 4448 | Action.Enter(CGF); |
| 4449 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 4450 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); |
| 4451 | }; |
| 4452 | |
| 4453 | // Emit teams region as a standalone region. |
| 4454 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 4455 | PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4456 | Action.Enter(CGF); |
Alexey Bataev | dfa430f | 2017-12-08 15:03:50 +0000 | [diff] [blame] | 4457 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); |
| 4458 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 4459 | (void)PrivateScope.Privatize(); |
| 4460 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, |
| 4461 | CodeGenDistribute); |
| 4462 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); |
| 4463 | }; |
| 4464 | emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); |
| 4465 | emitPostUpdateForReductionClause(CGF, S, |
| 4466 | [](CodeGenFunction &) { return nullptr; }); |
| 4467 | } |
| 4468 | |
| 4469 | void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction( |
| 4470 | CodeGenModule &CGM, StringRef ParentName, |
| 4471 | const OMPTargetTeamsDistributeDirective &S) { |
| 4472 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4473 | emitTargetTeamsDistributeRegion(CGF, Action, S); |
| 4474 | }; |
| 4475 | llvm::Function *Fn; |
| 4476 | llvm::Constant *Addr; |
| 4477 | // Emit target region as a standalone region. |
| 4478 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( |
| 4479 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); |
| 4480 | assert(Fn && Addr && "Target device function emission failed."); |
| 4481 | } |
| 4482 | |
| 4483 | void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective( |
| 4484 | const OMPTargetTeamsDistributeDirective &S) { |
| 4485 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4486 | emitTargetTeamsDistributeRegion(CGF, Action, S); |
| 4487 | }; |
| 4488 | emitCommonOMPTargetDirective(*this, S, CodeGen); |
| 4489 | } |
| 4490 | |
Alexey Bataev | fbe17fb | 2017-12-13 19:45:06 +0000 | [diff] [blame] | 4491 | static void emitTargetTeamsDistributeSimdRegion( |
| 4492 | CodeGenFunction &CGF, PrePostActionTy &Action, |
| 4493 | const OMPTargetTeamsDistributeSimdDirective &S) { |
| 4494 | Action.Enter(CGF); |
| 4495 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 4496 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); |
| 4497 | }; |
| 4498 | |
| 4499 | // Emit teams region as a standalone region. |
| 4500 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 4501 | PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4502 | Action.Enter(CGF); |
Alexey Bataev | fbe17fb | 2017-12-13 19:45:06 +0000 | [diff] [blame] | 4503 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); |
| 4504 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 4505 | (void)PrivateScope.Privatize(); |
| 4506 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, |
| 4507 | CodeGenDistribute); |
| 4508 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); |
| 4509 | }; |
| 4510 | emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); |
| 4511 | emitPostUpdateForReductionClause(CGF, S, |
| 4512 | [](CodeGenFunction &) { return nullptr; }); |
| 4513 | } |
| 4514 | |
| 4515 | void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction( |
| 4516 | CodeGenModule &CGM, StringRef ParentName, |
| 4517 | const OMPTargetTeamsDistributeSimdDirective &S) { |
| 4518 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4519 | emitTargetTeamsDistributeSimdRegion(CGF, Action, S); |
| 4520 | }; |
| 4521 | llvm::Function *Fn; |
| 4522 | llvm::Constant *Addr; |
| 4523 | // Emit target region as a standalone region. |
| 4524 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( |
| 4525 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); |
| 4526 | assert(Fn && Addr && "Target device function emission failed."); |
| 4527 | } |
| 4528 | |
| 4529 | void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective( |
| 4530 | const OMPTargetTeamsDistributeSimdDirective &S) { |
| 4531 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4532 | emitTargetTeamsDistributeSimdRegion(CGF, Action, S); |
| 4533 | }; |
| 4534 | emitCommonOMPTargetDirective(*this, S, CodeGen); |
| 4535 | } |
| 4536 | |
Carlo Bertolli | ba1487b | 2017-10-04 14:12:09 +0000 | [diff] [blame] | 4537 | void CodeGenFunction::EmitOMPTeamsDistributeDirective( |
| 4538 | const OMPTeamsDistributeDirective &S) { |
| 4539 | |
| 4540 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 4541 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); |
| 4542 | }; |
| 4543 | |
| 4544 | // Emit teams region as a standalone region. |
| 4545 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 4546 | PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4547 | Action.Enter(CGF); |
Carlo Bertolli | ba1487b | 2017-10-04 14:12:09 +0000 | [diff] [blame] | 4548 | OMPPrivateScope PrivateScope(CGF); |
| 4549 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 4550 | (void)PrivateScope.Privatize(); |
| 4551 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, |
| 4552 | CodeGenDistribute); |
| 4553 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); |
| 4554 | }; |
Alexey Bataev | 95c6dd4 | 2017-11-29 15:14:16 +0000 | [diff] [blame] | 4555 | emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen); |
Carlo Bertolli | ba1487b | 2017-10-04 14:12:09 +0000 | [diff] [blame] | 4556 | emitPostUpdateForReductionClause(*this, S, |
| 4557 | [](CodeGenFunction &) { return nullptr; }); |
| 4558 | } |
| 4559 | |
Alexey Bataev | 999277a | 2017-12-06 14:31:09 +0000 | [diff] [blame] | 4560 | void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective( |
| 4561 | const OMPTeamsDistributeSimdDirective &S) { |
| 4562 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 4563 | CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); |
| 4564 | }; |
| 4565 | |
| 4566 | // Emit teams region as a standalone region. |
| 4567 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 4568 | PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4569 | Action.Enter(CGF); |
Alexey Bataev | 999277a | 2017-12-06 14:31:09 +0000 | [diff] [blame] | 4570 | OMPPrivateScope PrivateScope(CGF); |
| 4571 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 4572 | (void)PrivateScope.Privatize(); |
| 4573 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, |
| 4574 | CodeGenDistribute); |
| 4575 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); |
| 4576 | }; |
| 4577 | emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen); |
| 4578 | emitPostUpdateForReductionClause(*this, S, |
| 4579 | [](CodeGenFunction &) { return nullptr; }); |
| 4580 | } |
| 4581 | |
Carlo Bertolli | 62fae15 | 2017-11-20 20:46:39 +0000 | [diff] [blame] | 4582 | void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective( |
| 4583 | const OMPTeamsDistributeParallelForDirective &S) { |
| 4584 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 4585 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, |
| 4586 | S.getDistInc()); |
| 4587 | }; |
| 4588 | |
| 4589 | // Emit teams region as a standalone region. |
| 4590 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 4591 | PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4592 | Action.Enter(CGF); |
Carlo Bertolli | 62fae15 | 2017-11-20 20:46:39 +0000 | [diff] [blame] | 4593 | OMPPrivateScope PrivateScope(CGF); |
| 4594 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 4595 | (void)PrivateScope.Privatize(); |
Alexey Bataev | 10a5431 | 2017-11-27 16:54:08 +0000 | [diff] [blame] | 4596 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, |
| 4597 | CodeGenDistribute); |
Carlo Bertolli | 62fae15 | 2017-11-20 20:46:39 +0000 | [diff] [blame] | 4598 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); |
| 4599 | }; |
| 4600 | emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen); |
| 4601 | emitPostUpdateForReductionClause(*this, S, |
| 4602 | [](CodeGenFunction &) { return nullptr; }); |
| 4603 | } |
| 4604 | |
Carlo Bertolli | 56a2aa4 | 2017-12-04 20:57:19 +0000 | [diff] [blame] | 4605 | void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective( |
| 4606 | const OMPTeamsDistributeParallelForSimdDirective &S) { |
| 4607 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 4608 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, |
| 4609 | S.getDistInc()); |
| 4610 | }; |
| 4611 | |
| 4612 | // Emit teams region as a standalone region. |
| 4613 | auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 4614 | PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4615 | Action.Enter(CGF); |
Carlo Bertolli | 56a2aa4 | 2017-12-04 20:57:19 +0000 | [diff] [blame] | 4616 | OMPPrivateScope PrivateScope(CGF); |
| 4617 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 4618 | (void)PrivateScope.Privatize(); |
| 4619 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective( |
| 4620 | CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); |
| 4621 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); |
| 4622 | }; |
| 4623 | emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen); |
| 4624 | emitPostUpdateForReductionClause(*this, S, |
| 4625 | [](CodeGenFunction &) { return nullptr; }); |
| 4626 | } |
| 4627 | |
Carlo Bertolli | 52978c3 | 2018-01-03 21:12:44 +0000 | [diff] [blame] | 4628 | static void emitTargetTeamsDistributeParallelForRegion( |
| 4629 | CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, |
| 4630 | PrePostActionTy &Action) { |
Carlo Bertolli | 7971209 | 2018-02-28 20:48:35 +0000 | [diff] [blame] | 4631 | Action.Enter(CGF); |
Carlo Bertolli | 52978c3 | 2018-01-03 21:12:44 +0000 | [diff] [blame] | 4632 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 4633 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, |
| 4634 | S.getDistInc()); |
| 4635 | }; |
| 4636 | |
| 4637 | // Emit teams region as a standalone region. |
| 4638 | auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 4639 | PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4640 | Action.Enter(CGF); |
Carlo Bertolli | 52978c3 | 2018-01-03 21:12:44 +0000 | [diff] [blame] | 4641 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); |
| 4642 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 4643 | (void)PrivateScope.Privatize(); |
| 4644 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective( |
| 4645 | CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); |
| 4646 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); |
| 4647 | }; |
| 4648 | |
| 4649 | emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, |
| 4650 | CodeGenTeams); |
| 4651 | emitPostUpdateForReductionClause(CGF, S, |
| 4652 | [](CodeGenFunction &) { return nullptr; }); |
| 4653 | } |
| 4654 | |
| 4655 | void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction( |
| 4656 | CodeGenModule &CGM, StringRef ParentName, |
| 4657 | const OMPTargetTeamsDistributeParallelForDirective &S) { |
| 4658 | // Emit SPMD target teams distribute parallel for region as a standalone |
| 4659 | // region. |
| 4660 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4661 | emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); |
| 4662 | }; |
| 4663 | llvm::Function *Fn; |
| 4664 | llvm::Constant *Addr; |
| 4665 | // Emit target region as a standalone region. |
| 4666 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( |
| 4667 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); |
| 4668 | assert(Fn && Addr && "Target device function emission failed."); |
| 4669 | } |
| 4670 | |
| 4671 | void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective( |
| 4672 | const OMPTargetTeamsDistributeParallelForDirective &S) { |
| 4673 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4674 | emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); |
| 4675 | }; |
| 4676 | emitCommonOMPTargetDirective(*this, S, CodeGen); |
| 4677 | } |
| 4678 | |
Alexey Bataev | 647dd84 | 2018-01-15 20:59:40 +0000 | [diff] [blame] | 4679 | static void emitTargetTeamsDistributeParallelForSimdRegion( |
| 4680 | CodeGenFunction &CGF, |
| 4681 | const OMPTargetTeamsDistributeParallelForSimdDirective &S, |
| 4682 | PrePostActionTy &Action) { |
Carlo Bertolli | 7971209 | 2018-02-28 20:48:35 +0000 | [diff] [blame] | 4683 | Action.Enter(CGF); |
Alexey Bataev | 647dd84 | 2018-01-15 20:59:40 +0000 | [diff] [blame] | 4684 | auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 4685 | CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, |
| 4686 | S.getDistInc()); |
| 4687 | }; |
| 4688 | |
| 4689 | // Emit teams region as a standalone region. |
| 4690 | auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 4691 | PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4692 | Action.Enter(CGF); |
Alexey Bataev | 647dd84 | 2018-01-15 20:59:40 +0000 | [diff] [blame] | 4693 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); |
| 4694 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 4695 | (void)PrivateScope.Privatize(); |
| 4696 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective( |
| 4697 | CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); |
| 4698 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); |
| 4699 | }; |
| 4700 | |
| 4701 | emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, |
| 4702 | CodeGenTeams); |
| 4703 | emitPostUpdateForReductionClause(CGF, S, |
| 4704 | [](CodeGenFunction &) { return nullptr; }); |
| 4705 | } |
| 4706 | |
| 4707 | void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction( |
| 4708 | CodeGenModule &CGM, StringRef ParentName, |
| 4709 | const OMPTargetTeamsDistributeParallelForSimdDirective &S) { |
| 4710 | // Emit SPMD target teams distribute parallel for simd region as a standalone |
| 4711 | // region. |
| 4712 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4713 | emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); |
| 4714 | }; |
| 4715 | llvm::Function *Fn; |
| 4716 | llvm::Constant *Addr; |
| 4717 | // Emit target region as a standalone region. |
| 4718 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( |
| 4719 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); |
| 4720 | assert(Fn && Addr && "Target device function emission failed."); |
| 4721 | } |
| 4722 | |
| 4723 | void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective( |
| 4724 | const OMPTargetTeamsDistributeParallelForSimdDirective &S) { |
| 4725 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4726 | emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); |
| 4727 | }; |
| 4728 | emitCommonOMPTargetDirective(*this, S, CodeGen); |
| 4729 | } |
| 4730 | |
Alexey Bataev | 6d4ed05 | 2015-07-01 06:57:41 +0000 | [diff] [blame] | 4731 | void CodeGenFunction::EmitOMPCancellationPointDirective( |
| 4732 | const OMPCancellationPointDirective &S) { |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 4733 | CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(), |
Alexey Bataev | 0f34da1 | 2015-07-02 04:17:07 +0000 | [diff] [blame] | 4734 | S.getCancelRegion()); |
Alexey Bataev | 6d4ed05 | 2015-07-01 06:57:41 +0000 | [diff] [blame] | 4735 | } |
| 4736 | |
Alexey Bataev | 8090987 | 2015-07-02 11:25:17 +0000 | [diff] [blame] | 4737 | void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) { |
Alexey Bataev | 87933c7 | 2015-09-18 08:07:34 +0000 | [diff] [blame] | 4738 | const Expr *IfCond = nullptr; |
| 4739 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { |
| 4740 | if (C->getNameModifier() == OMPD_unknown || |
| 4741 | C->getNameModifier() == OMPD_cancel) { |
| 4742 | IfCond = C->getCondition(); |
| 4743 | break; |
| 4744 | } |
| 4745 | } |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 4746 | CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond, |
Alexey Bataev | 7d5d33e | 2015-07-06 05:50:32 +0000 | [diff] [blame] | 4747 | S.getCancelRegion()); |
Alexey Bataev | 8090987 | 2015-07-02 11:25:17 +0000 | [diff] [blame] | 4748 | } |
| 4749 | |
Alexey Bataev | 81c7ea0 | 2015-07-03 09:56:58 +0000 | [diff] [blame] | 4750 | CodeGenFunction::JumpDest |
| 4751 | CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) { |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 4752 | if (Kind == OMPD_parallel || Kind == OMPD_task || |
| 4753 | Kind == OMPD_target_parallel) |
Alexey Bataev | 81c7ea0 | 2015-07-03 09:56:58 +0000 | [diff] [blame] | 4754 | return ReturnBlock; |
Alexey Bataev | 25e5b44 | 2015-09-15 12:52:43 +0000 | [diff] [blame] | 4755 | assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections || |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 4756 | Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for || |
| 4757 | Kind == OMPD_distribute_parallel_for || |
Alexey Bataev | dcb4b8fb | 2017-11-22 20:19:50 +0000 | [diff] [blame] | 4758 | Kind == OMPD_target_parallel_for || |
Alexey Bataev | 16e7988 | 2017-11-22 21:12:03 +0000 | [diff] [blame] | 4759 | Kind == OMPD_teams_distribute_parallel_for || |
| 4760 | Kind == OMPD_target_teams_distribute_parallel_for); |
Alexey Bataev | 957d856 | 2016-11-17 15:12:05 +0000 | [diff] [blame] | 4761 | return OMPCancelStack.getExitBlock(); |
Alexey Bataev | 81c7ea0 | 2015-07-03 09:56:58 +0000 | [diff] [blame] | 4762 | } |
Michael Wong | 65f367f | 2015-07-21 13:44:28 +0000 | [diff] [blame] | 4763 | |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4764 | void CodeGenFunction::EmitOMPUseDevicePtrClause( |
| 4765 | const OMPClause &NC, OMPPrivateScope &PrivateScope, |
| 4766 | const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) { |
| 4767 | const auto &C = cast<OMPUseDevicePtrClause>(NC); |
| 4768 | auto OrigVarIt = C.varlist_begin(); |
| 4769 | auto InitIt = C.inits().begin(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4770 | for (const Expr *PvtVarIt : C.private_copies()) { |
| 4771 | const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl()); |
| 4772 | const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl()); |
| 4773 | const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl()); |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4774 | |
| 4775 | // In order to identify the right initializer we need to match the |
| 4776 | // declaration used by the mapping logic. In some cases we may get |
| 4777 | // OMPCapturedExprDecl that refers to the original declaration. |
| 4778 | const ValueDecl *MatchingVD = OrigVD; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4779 | if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) { |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4780 | // OMPCapturedExprDecl are used to privative fields of the current |
| 4781 | // structure. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4782 | const auto *ME = cast<MemberExpr>(OED->getInit()); |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4783 | assert(isa<CXXThisExpr>(ME->getBase()) && |
| 4784 | "Base should be the current struct!"); |
| 4785 | MatchingVD = ME->getMemberDecl(); |
| 4786 | } |
| 4787 | |
| 4788 | // If we don't have information about the current list item, move on to |
| 4789 | // the next one. |
| 4790 | auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD); |
| 4791 | if (InitAddrIt == CaptureDeviceAddrMap.end()) |
| 4792 | continue; |
| 4793 | |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4794 | bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD, |
| 4795 | InitAddrIt, InitVD, |
| 4796 | PvtVD]() { |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4797 | // Initialize the temporary initialization variable with the address we |
| 4798 | // get from the runtime library. We have to cast the source address |
| 4799 | // because it is always a void *. References are materialized in the |
| 4800 | // privatization scope, so the initialization here disregards the fact |
| 4801 | // the original variable is a reference. |
| 4802 | QualType AddrQTy = |
| 4803 | getContext().getPointerType(OrigVD->getType().getNonReferenceType()); |
| 4804 | llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy); |
| 4805 | Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy); |
| 4806 | setAddrOfLocalVar(InitVD, InitAddr); |
| 4807 | |
| 4808 | // Emit private declaration, it will be initialized by the value we |
| 4809 | // declaration we just added to the local declarations map. |
| 4810 | EmitDecl(*PvtVD); |
| 4811 | |
| 4812 | // The initialization variables reached its purpose in the emission |
Alexander Kornienko | 2a8c18d | 2018-04-06 15:14:32 +0000 | [diff] [blame] | 4813 | // of the previous declaration, so we don't need it anymore. |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4814 | LocalDeclMap.erase(InitVD); |
| 4815 | |
| 4816 | // Return the address of the private variable. |
| 4817 | return GetAddrOfLocalVar(PvtVD); |
| 4818 | }); |
| 4819 | assert(IsRegistered && "firstprivate var already registered as private"); |
| 4820 | // Silence the warning about unused variable. |
| 4821 | (void)IsRegistered; |
| 4822 | |
| 4823 | ++OrigVarIt; |
| 4824 | ++InitIt; |
| 4825 | } |
| 4826 | } |
| 4827 | |
Michael Wong | 65f367f | 2015-07-21 13:44:28 +0000 | [diff] [blame] | 4828 | // Generate the instructions for '#pragma omp target data' directive. |
| 4829 | void CodeGenFunction::EmitOMPTargetDataDirective( |
| 4830 | const OMPTargetDataDirective &S) { |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4831 | CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true); |
| 4832 | |
| 4833 | // Create a pre/post action to signal the privatization of the device pointer. |
| 4834 | // This action can be replaced by the OpenMP runtime code generation to |
| 4835 | // deactivate privatization. |
| 4836 | bool PrivatizeDevicePointers = false; |
| 4837 | class DevicePointerPrivActionTy : public PrePostActionTy { |
| 4838 | bool &PrivatizeDevicePointers; |
| 4839 | |
| 4840 | public: |
| 4841 | explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers) |
| 4842 | : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {} |
| 4843 | void Enter(CodeGenFunction &CGF) override { |
| 4844 | PrivatizeDevicePointers = true; |
| 4845 | } |
Samuel Antao | df158d5 | 2016-04-27 22:58:19 +0000 | [diff] [blame] | 4846 | }; |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4847 | DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers); |
| 4848 | |
| 4849 | auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers]( |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 4850 | CodeGenFunction &CGF, PrePostActionTy &Action) { |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4851 | auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 4852 | CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4853 | }; |
| 4854 | |
Alexander Kornienko | 2a8c18d | 2018-04-06 15:14:32 +0000 | [diff] [blame] | 4855 | // Codegen that selects whether to generate the privatization code or not. |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4856 | auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers, |
| 4857 | &InnermostCodeGen](CodeGenFunction &CGF, |
| 4858 | PrePostActionTy &Action) { |
| 4859 | RegionCodeGenTy RCG(InnermostCodeGen); |
| 4860 | PrivatizeDevicePointers = false; |
| 4861 | |
| 4862 | // Call the pre-action to change the status of PrivatizeDevicePointers if |
| 4863 | // needed. |
| 4864 | Action.Enter(CGF); |
| 4865 | |
| 4866 | if (PrivatizeDevicePointers) { |
| 4867 | OMPPrivateScope PrivateScope(CGF); |
| 4868 | // Emit all instances of the use_device_ptr clause. |
| 4869 | for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>()) |
| 4870 | CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, |
| 4871 | Info.CaptureDeviceAddrMap); |
| 4872 | (void)PrivateScope.Privatize(); |
| 4873 | RCG(CGF); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4874 | } else { |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4875 | RCG(CGF); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4876 | } |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4877 | }; |
| 4878 | |
| 4879 | // Forward the provided action to the privatization codegen. |
| 4880 | RegionCodeGenTy PrivRCG(PrivCodeGen); |
| 4881 | PrivRCG.setAction(Action); |
| 4882 | |
| 4883 | // Notwithstanding the body of the region is emitted as inlined directive, |
| 4884 | // we don't use an inline scope as changes in the references inside the |
| 4885 | // region are expected to be visible outside, so we do not privative them. |
| 4886 | OMPLexicalScope Scope(CGF, S); |
| 4887 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, |
| 4888 | PrivRCG); |
| 4889 | }; |
| 4890 | |
| 4891 | RegionCodeGenTy RCG(CodeGen); |
Samuel Antao | df158d5 | 2016-04-27 22:58:19 +0000 | [diff] [blame] | 4892 | |
| 4893 | // If we don't have target devices, don't bother emitting the data mapping |
| 4894 | // code. |
| 4895 | if (CGM.getLangOpts().OMPTargetTriples.empty()) { |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4896 | RCG(*this); |
Samuel Antao | df158d5 | 2016-04-27 22:58:19 +0000 | [diff] [blame] | 4897 | return; |
| 4898 | } |
| 4899 | |
| 4900 | // Check if we have any if clause associated with the directive. |
| 4901 | const Expr *IfCond = nullptr; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4902 | if (const auto *C = S.getSingleClause<OMPIfClause>()) |
Samuel Antao | df158d5 | 2016-04-27 22:58:19 +0000 | [diff] [blame] | 4903 | IfCond = C->getCondition(); |
| 4904 | |
| 4905 | // Check if we have any device clause associated with the directive. |
| 4906 | const Expr *Device = nullptr; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4907 | if (const auto *C = S.getSingleClause<OMPDeviceClause>()) |
Samuel Antao | df158d5 | 2016-04-27 22:58:19 +0000 | [diff] [blame] | 4908 | Device = C->getDevice(); |
| 4909 | |
Samuel Antao | cc10b85 | 2016-07-28 14:23:26 +0000 | [diff] [blame] | 4910 | // Set the action to signal privatization of device pointers. |
| 4911 | RCG.setAction(PrivAction); |
| 4912 | |
| 4913 | // Emit region code. |
| 4914 | CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG, |
| 4915 | Info); |
Michael Wong | 65f367f | 2015-07-21 13:44:28 +0000 | [diff] [blame] | 4916 | } |
Alexey Bataev | 49f6e78 | 2015-12-01 04:18:41 +0000 | [diff] [blame] | 4917 | |
Samuel Antao | df67fc4 | 2016-01-19 19:15:56 +0000 | [diff] [blame] | 4918 | void CodeGenFunction::EmitOMPTargetEnterDataDirective( |
| 4919 | const OMPTargetEnterDataDirective &S) { |
Samuel Antao | bd0ae2e | 2016-04-27 23:07:29 +0000 | [diff] [blame] | 4920 | // If we don't have target devices, don't bother emitting the data mapping |
| 4921 | // code. |
| 4922 | if (CGM.getLangOpts().OMPTargetTriples.empty()) |
| 4923 | return; |
| 4924 | |
| 4925 | // Check if we have any if clause associated with the directive. |
| 4926 | const Expr *IfCond = nullptr; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4927 | if (const auto *C = S.getSingleClause<OMPIfClause>()) |
Samuel Antao | bd0ae2e | 2016-04-27 23:07:29 +0000 | [diff] [blame] | 4928 | IfCond = C->getCondition(); |
| 4929 | |
| 4930 | // Check if we have any device clause associated with the directive. |
| 4931 | const Expr *Device = nullptr; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4932 | if (const auto *C = S.getSingleClause<OMPDeviceClause>()) |
Samuel Antao | bd0ae2e | 2016-04-27 23:07:29 +0000 | [diff] [blame] | 4933 | Device = C->getDevice(); |
| 4934 | |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 4935 | OMPLexicalScope Scope(*this, S, OMPD_task); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 4936 | CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); |
Samuel Antao | df67fc4 | 2016-01-19 19:15:56 +0000 | [diff] [blame] | 4937 | } |
| 4938 | |
Samuel Antao | 7259076 | 2016-01-19 20:04:50 +0000 | [diff] [blame] | 4939 | void CodeGenFunction::EmitOMPTargetExitDataDirective( |
| 4940 | const OMPTargetExitDataDirective &S) { |
Samuel Antao | 8dd6628 | 2016-04-27 23:14:30 +0000 | [diff] [blame] | 4941 | // If we don't have target devices, don't bother emitting the data mapping |
| 4942 | // code. |
| 4943 | if (CGM.getLangOpts().OMPTargetTriples.empty()) |
| 4944 | return; |
| 4945 | |
| 4946 | // Check if we have any if clause associated with the directive. |
| 4947 | const Expr *IfCond = nullptr; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4948 | if (const auto *C = S.getSingleClause<OMPIfClause>()) |
Samuel Antao | 8dd6628 | 2016-04-27 23:14:30 +0000 | [diff] [blame] | 4949 | IfCond = C->getCondition(); |
| 4950 | |
| 4951 | // Check if we have any device clause associated with the directive. |
| 4952 | const Expr *Device = nullptr; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4953 | if (const auto *C = S.getSingleClause<OMPDeviceClause>()) |
Samuel Antao | 8dd6628 | 2016-04-27 23:14:30 +0000 | [diff] [blame] | 4954 | Device = C->getDevice(); |
| 4955 | |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 4956 | OMPLexicalScope Scope(*this, S, OMPD_task); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 4957 | CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); |
Samuel Antao | 7259076 | 2016-01-19 20:04:50 +0000 | [diff] [blame] | 4958 | } |
| 4959 | |
Arpith Chacko Jacob | 19b911c | 2017-01-18 18:18:53 +0000 | [diff] [blame] | 4960 | static void emitTargetParallelRegion(CodeGenFunction &CGF, |
| 4961 | const OMPTargetParallelDirective &S, |
| 4962 | PrePostActionTy &Action) { |
| 4963 | // Get the captured statement associated with the 'parallel' region. |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4964 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel); |
Arpith Chacko Jacob | 19b911c | 2017-01-18 18:18:53 +0000 | [diff] [blame] | 4965 | Action.Enter(CGF); |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 4966 | auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { |
Alexey Bataev | 63cc8e9 | 2018-03-20 14:45:59 +0000 | [diff] [blame] | 4967 | Action.Enter(CGF); |
Arpith Chacko Jacob | 101e8fb | 2017-02-16 16:20:16 +0000 | [diff] [blame] | 4968 | CodeGenFunction::OMPPrivateScope PrivateScope(CGF); |
| 4969 | (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); |
| 4970 | CGF.EmitOMPPrivateClause(S, PrivateScope); |
| 4971 | CGF.EmitOMPReductionClauseInit(S, PrivateScope); |
| 4972 | (void)PrivateScope.Privatize(); |
Alexey Bataev | 6070542 | 2018-10-30 15:50:12 +0000 | [diff] [blame] | 4973 | if (isOpenMPTargetExecutionDirective(S.getDirectiveKind())) |
| 4974 | CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); |
Arpith Chacko Jacob | 19b911c | 2017-01-18 18:18:53 +0000 | [diff] [blame] | 4975 | // TODO: Add support for clauses. |
| 4976 | CGF.EmitStmt(CS->getCapturedStmt()); |
Arpith Chacko Jacob | 101e8fb | 2017-02-16 16:20:16 +0000 | [diff] [blame] | 4977 | CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); |
Arpith Chacko Jacob | 19b911c | 2017-01-18 18:18:53 +0000 | [diff] [blame] | 4978 | }; |
Carlo Bertolli | b0ff0a6 | 2017-04-25 17:52:12 +0000 | [diff] [blame] | 4979 | emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen, |
| 4980 | emitEmptyBoundParameters); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 4981 | emitPostUpdateForReductionClause(CGF, S, |
| 4982 | [](CodeGenFunction &) { return nullptr; }); |
Arpith Chacko Jacob | 19b911c | 2017-01-18 18:18:53 +0000 | [diff] [blame] | 4983 | } |
| 4984 | |
| 4985 | void CodeGenFunction::EmitOMPTargetParallelDeviceFunction( |
| 4986 | CodeGenModule &CGM, StringRef ParentName, |
| 4987 | const OMPTargetParallelDirective &S) { |
| 4988 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 4989 | emitTargetParallelRegion(CGF, S, Action); |
| 4990 | }; |
| 4991 | llvm::Function *Fn; |
| 4992 | llvm::Constant *Addr; |
| 4993 | // Emit target region as a standalone region. |
| 4994 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( |
| 4995 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); |
| 4996 | assert(Fn && Addr && "Target device function emission failed."); |
| 4997 | } |
| 4998 | |
Arpith Chacko Jacob | e955b3d | 2016-01-26 18:48:41 +0000 | [diff] [blame] | 4999 | void CodeGenFunction::EmitOMPTargetParallelDirective( |
| 5000 | const OMPTargetParallelDirective &S) { |
Arpith Chacko Jacob | 19b911c | 2017-01-18 18:18:53 +0000 | [diff] [blame] | 5001 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 5002 | emitTargetParallelRegion(CGF, S, Action); |
| 5003 | }; |
| 5004 | emitCommonOMPTargetDirective(*this, S, CodeGen); |
Arpith Chacko Jacob | e955b3d | 2016-01-26 18:48:41 +0000 | [diff] [blame] | 5005 | } |
| 5006 | |
Alexey Bataev | fb0ebec | 2017-11-08 20:16:14 +0000 | [diff] [blame] | 5007 | static void emitTargetParallelForRegion(CodeGenFunction &CGF, |
| 5008 | const OMPTargetParallelForDirective &S, |
| 5009 | PrePostActionTy &Action) { |
| 5010 | Action.Enter(CGF); |
| 5011 | // Emit directive as a combined directive that consists of two implicit |
| 5012 | // directives: 'parallel' with 'for' directive. |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 5013 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 5014 | Action.Enter(CGF); |
Alexey Bataev | 2139ed6 | 2017-11-16 18:20:21 +0000 | [diff] [blame] | 5015 | CodeGenFunction::OMPCancelStackRAII CancelRegion( |
| 5016 | CGF, OMPD_target_parallel_for, S.hasCancel()); |
Alexey Bataev | fb0ebec | 2017-11-08 20:16:14 +0000 | [diff] [blame] | 5017 | CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, |
| 5018 | emitDispatchForLoopBounds); |
| 5019 | }; |
| 5020 | emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, |
| 5021 | emitEmptyBoundParameters); |
| 5022 | } |
| 5023 | |
| 5024 | void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction( |
| 5025 | CodeGenModule &CGM, StringRef ParentName, |
| 5026 | const OMPTargetParallelForDirective &S) { |
| 5027 | // Emit SPMD target parallel for region as a standalone region. |
| 5028 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 5029 | emitTargetParallelForRegion(CGF, S, Action); |
| 5030 | }; |
| 5031 | llvm::Function *Fn; |
| 5032 | llvm::Constant *Addr; |
| 5033 | // Emit target region as a standalone region. |
| 5034 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( |
| 5035 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); |
| 5036 | assert(Fn && Addr && "Target device function emission failed."); |
| 5037 | } |
| 5038 | |
Arpith Chacko Jacob | 05bebb5 | 2016-02-03 15:46:42 +0000 | [diff] [blame] | 5039 | void CodeGenFunction::EmitOMPTargetParallelForDirective( |
| 5040 | const OMPTargetParallelForDirective &S) { |
Alexey Bataev | fb0ebec | 2017-11-08 20:16:14 +0000 | [diff] [blame] | 5041 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 5042 | emitTargetParallelForRegion(CGF, S, Action); |
| 5043 | }; |
| 5044 | emitCommonOMPTargetDirective(*this, S, CodeGen); |
Arpith Chacko Jacob | 05bebb5 | 2016-02-03 15:46:42 +0000 | [diff] [blame] | 5045 | } |
| 5046 | |
Alexey Bataev | 5d7edca | 2017-11-09 17:32:15 +0000 | [diff] [blame] | 5047 | static void |
| 5048 | emitTargetParallelForSimdRegion(CodeGenFunction &CGF, |
| 5049 | const OMPTargetParallelForSimdDirective &S, |
| 5050 | PrePostActionTy &Action) { |
| 5051 | Action.Enter(CGF); |
| 5052 | // Emit directive as a combined directive that consists of two implicit |
| 5053 | // directives: 'parallel' with 'for' directive. |
Alexey Bataev | c99042b | 2018-03-15 18:10:54 +0000 | [diff] [blame] | 5054 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 5055 | Action.Enter(CGF); |
Alexey Bataev | 5d7edca | 2017-11-09 17:32:15 +0000 | [diff] [blame] | 5056 | CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, |
| 5057 | emitDispatchForLoopBounds); |
| 5058 | }; |
| 5059 | emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, |
| 5060 | emitEmptyBoundParameters); |
| 5061 | } |
| 5062 | |
| 5063 | void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction( |
| 5064 | CodeGenModule &CGM, StringRef ParentName, |
| 5065 | const OMPTargetParallelForSimdDirective &S) { |
| 5066 | // Emit SPMD target parallel for region as a standalone region. |
| 5067 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 5068 | emitTargetParallelForSimdRegion(CGF, S, Action); |
| 5069 | }; |
| 5070 | llvm::Function *Fn; |
| 5071 | llvm::Constant *Addr; |
| 5072 | // Emit target region as a standalone region. |
| 5073 | CGM.getOpenMPRuntime().emitTargetOutlinedFunction( |
| 5074 | S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen); |
| 5075 | assert(Fn && Addr && "Target device function emission failed."); |
| 5076 | } |
| 5077 | |
| 5078 | void CodeGenFunction::EmitOMPTargetParallelForSimdDirective( |
| 5079 | const OMPTargetParallelForSimdDirective &S) { |
| 5080 | auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 5081 | emitTargetParallelForSimdRegion(CGF, S, Action); |
| 5082 | }; |
| 5083 | emitCommonOMPTargetDirective(*this, S, CodeGen); |
| 5084 | } |
| 5085 | |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5086 | /// Emit a helper variable and return corresponding lvalue. |
| 5087 | static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, |
| 5088 | const ImplicitParamDecl *PVD, |
| 5089 | CodeGenFunction::OMPPrivateScope &Privates) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 5090 | const auto *VDecl = cast<VarDecl>(Helper->getDecl()); |
| 5091 | Privates.addPrivate(VDecl, |
| 5092 | [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); }); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5093 | } |
| 5094 | |
| 5095 | void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) { |
| 5096 | assert(isOpenMPTaskLoopDirective(S.getDirectiveKind())); |
| 5097 | // Emit outlined function for task construct. |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 5098 | const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 5099 | Address CapturedStruct = GenerateCapturedStmtArgument(*CS); |
| 5100 | QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5101 | const Expr *IfCond = nullptr; |
| 5102 | for (const auto *C : S.getClausesOfKind<OMPIfClause>()) { |
| 5103 | if (C->getNameModifier() == OMPD_unknown || |
| 5104 | C->getNameModifier() == OMPD_taskloop) { |
| 5105 | IfCond = C->getCondition(); |
| 5106 | break; |
| 5107 | } |
| 5108 | } |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 5109 | |
| 5110 | OMPTaskDataTy Data; |
| 5111 | // Check if taskloop must be emitted without taskgroup. |
| 5112 | Data.Nogroup = S.getSingleClause<OMPNogroupClause>(); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5113 | // TODO: Check if we should emit tied or untied task. |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 5114 | Data.Tied = true; |
| 5115 | // Set scheduling for taskloop |
Alexey Bataev | 2b19a6f | 2016-04-28 09:15:06 +0000 | [diff] [blame] | 5116 | if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) { |
| 5117 | // grainsize clause |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 5118 | Data.Schedule.setInt(/*IntVal=*/false); |
| 5119 | Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize())); |
Alexey Bataev | 2b19a6f | 2016-04-28 09:15:06 +0000 | [diff] [blame] | 5120 | } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) { |
| 5121 | // num_tasks clause |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 5122 | Data.Schedule.setInt(/*IntVal=*/true); |
| 5123 | Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks())); |
Alexey Bataev | 2b19a6f | 2016-04-28 09:15:06 +0000 | [diff] [blame] | 5124 | } |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5125 | |
| 5126 | auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 5127 | // if (PreCond) { |
| 5128 | // for (IV in 0..LastIteration) BODY; |
| 5129 | // <Final counter/linear vars updates>; |
| 5130 | // } |
| 5131 | // |
| 5132 | |
| 5133 | // Emit: if (PreCond) - begin. |
| 5134 | // If the condition constant folds and can be elided, avoid emitting the |
| 5135 | // whole loop. |
| 5136 | bool CondConstant; |
| 5137 | llvm::BasicBlock *ContBlock = nullptr; |
| 5138 | OMPLoopScope PreInitScope(CGF, S); |
| 5139 | if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { |
| 5140 | if (!CondConstant) |
| 5141 | return; |
| 5142 | } else { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 5143 | llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5144 | ContBlock = CGF.createBasicBlock("taskloop.if.end"); |
| 5145 | emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, |
| 5146 | CGF.getProfileCount(&S)); |
| 5147 | CGF.EmitBlock(ThenBlock); |
| 5148 | CGF.incrementProfileCounter(&S); |
| 5149 | } |
| 5150 | |
Alexey Bataev | 6120582 | 2019-12-04 09:50:21 -0500 | [diff] [blame] | 5151 | (void)CGF.EmitOMPLinearClauseInit(S); |
Alexey Bataev | 1e73ef3 | 2016-04-28 12:14:51 +0000 | [diff] [blame] | 5152 | |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5153 | OMPPrivateScope LoopScope(CGF); |
| 5154 | // Emit helper vars inits. |
| 5155 | enum { LowerBound = 5, UpperBound, Stride, LastIter }; |
| 5156 | auto *I = CS->getCapturedDecl()->param_begin(); |
| 5157 | auto *LBP = std::next(I, LowerBound); |
| 5158 | auto *UBP = std::next(I, UpperBound); |
| 5159 | auto *STP = std::next(I, Stride); |
| 5160 | auto *LIP = std::next(I, LastIter); |
| 5161 | mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP, |
| 5162 | LoopScope); |
| 5163 | mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP, |
| 5164 | LoopScope); |
| 5165 | mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope); |
| 5166 | mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP, |
| 5167 | LoopScope); |
| 5168 | CGF.EmitOMPPrivateLoopCounters(S, LoopScope); |
Alexey Bataev | 14a388f | 2019-10-25 10:27:13 -0400 | [diff] [blame] | 5169 | CGF.EmitOMPLinearClause(S, LoopScope); |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 5170 | bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5171 | (void)LoopScope.Privatize(); |
| 5172 | // Emit the loop iteration variable. |
| 5173 | const Expr *IVExpr = S.getIterationVariable(); |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 5174 | const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl()); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5175 | CGF.EmitVarDecl(*IVDecl); |
| 5176 | CGF.EmitIgnoredExpr(S.getInit()); |
| 5177 | |
| 5178 | // Emit the iterations count variable. |
| 5179 | // If it is not a variable, Sema decided to calculate iterations count on |
| 5180 | // each iteration (e.g., it is foldable into a constant). |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 5181 | if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5182 | CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); |
| 5183 | // Emit calculation of the iterations count. |
| 5184 | CGF.EmitIgnoredExpr(S.getCalcLastIteration()); |
| 5185 | } |
| 5186 | |
Alexey Bataev | 6120582 | 2019-12-04 09:50:21 -0500 | [diff] [blame] | 5187 | { |
| 5188 | OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false); |
| 5189 | emitCommonSimdLoop( |
| 5190 | CGF, S, |
| 5191 | [&S](CodeGenFunction &CGF, PrePostActionTy &) { |
| 5192 | if (isOpenMPSimdDirective(S.getDirectiveKind())) |
| 5193 | CGF.EmitOMPSimdInit(S); |
| 5194 | }, |
| 5195 | [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { |
| 5196 | CGF.EmitOMPInnerLoop( |
| 5197 | S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(), |
| 5198 | [&S](CodeGenFunction &CGF) { |
| 5199 | CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest()); |
| 5200 | CGF.EmitStopPoint(&S); |
| 5201 | }, |
| 5202 | [](CodeGenFunction &) {}); |
| 5203 | }); |
| 5204 | } |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5205 | // Emit: if (PreCond) - end. |
| 5206 | if (ContBlock) { |
| 5207 | CGF.EmitBranch(ContBlock); |
| 5208 | CGF.EmitBlock(ContBlock, true); |
| 5209 | } |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 5210 | // Emit final copy of the lastprivate variables if IsLastIter != 0. |
| 5211 | if (HasLastprivateClause) { |
| 5212 | CGF.EmitOMPLastprivateClauseFinal( |
| 5213 | S, isOpenMPSimdDirective(S.getDirectiveKind()), |
| 5214 | CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar( |
| 5215 | CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 5216 | (*LIP)->getType(), S.getBeginLoc()))); |
Alexey Bataev | f93095a | 2016-05-05 08:46:22 +0000 | [diff] [blame] | 5217 | } |
Alexey Bataev | 14a388f | 2019-10-25 10:27:13 -0400 | [diff] [blame] | 5218 | CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) { |
| 5219 | return CGF.Builder.CreateIsNotNull( |
| 5220 | CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, |
| 5221 | (*LIP)->getType(), S.getBeginLoc())); |
| 5222 | }); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5223 | }; |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 5224 | auto &&TaskGen = [&S, SharedsTy, CapturedStruct, |
James Y Knight | 9871db0 | 2019-02-05 16:42:33 +0000 | [diff] [blame] | 5225 | IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 5226 | const OMPTaskDataTy &Data) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 5227 | auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond, |
| 5228 | &Data](CodeGenFunction &CGF, PrePostActionTy &) { |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5229 | OMPLoopScope PreInitScope(CGF, S); |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 5230 | CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S, |
Alexey Bataev | 24b5bae | 2016-04-28 09:23:51 +0000 | [diff] [blame] | 5231 | OutlinedFn, SharedsTy, |
| 5232 | CapturedStruct, IfCond, Data); |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5233 | }; |
| 5234 | CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, |
| 5235 | CodeGen); |
| 5236 | }; |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 5237 | if (Data.Nogroup) { |
| 5238 | EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data); |
| 5239 | } else { |
Alexey Bataev | 3344603 | 2017-07-12 18:09:32 +0000 | [diff] [blame] | 5240 | CGM.getOpenMPRuntime().emitTaskgroupRegion( |
| 5241 | *this, |
| 5242 | [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF, |
| 5243 | PrePostActionTy &Action) { |
| 5244 | Action.Enter(CGF); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 5245 | CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, |
| 5246 | Data); |
Alexey Bataev | 3344603 | 2017-07-12 18:09:32 +0000 | [diff] [blame] | 5247 | }, |
Stephen Kelly | f2ceec4 | 2018-08-09 21:08:08 +0000 | [diff] [blame] | 5248 | S.getBeginLoc()); |
Alexey Bataev | 3344603 | 2017-07-12 18:09:32 +0000 | [diff] [blame] | 5249 | } |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5250 | } |
| 5251 | |
Alexey Bataev | 49f6e78 | 2015-12-01 04:18:41 +0000 | [diff] [blame] | 5252 | void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) { |
Alexey Bataev | 7292c29 | 2016-04-25 12:22:29 +0000 | [diff] [blame] | 5253 | EmitOMPTaskLoopBasedDirective(S); |
Alexey Bataev | 49f6e78 | 2015-12-01 04:18:41 +0000 | [diff] [blame] | 5254 | } |
| 5255 | |
Alexey Bataev | 0a6ed84 | 2015-12-03 09:40:15 +0000 | [diff] [blame] | 5256 | void CodeGenFunction::EmitOMPTaskLoopSimdDirective( |
| 5257 | const OMPTaskLoopSimdDirective &S) { |
Alexey Bataev | 6120582 | 2019-12-04 09:50:21 -0500 | [diff] [blame] | 5258 | OMPLexicalScope Scope(*this, S); |
Alexey Bataev | 1e73ef3 | 2016-04-28 12:14:51 +0000 | [diff] [blame] | 5259 | EmitOMPTaskLoopBasedDirective(S); |
Alexey Bataev | 0a6ed84 | 2015-12-03 09:40:15 +0000 | [diff] [blame] | 5260 | } |
Samuel Antao | 686c70c | 2016-05-26 17:30:50 +0000 | [diff] [blame] | 5261 | |
Alexey Bataev | 60e51c4 | 2019-10-10 20:13:02 +0000 | [diff] [blame] | 5262 | void CodeGenFunction::EmitOMPMasterTaskLoopDirective( |
| 5263 | const OMPMasterTaskLoopDirective &S) { |
| 5264 | auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 5265 | Action.Enter(CGF); |
| 5266 | EmitOMPTaskLoopBasedDirective(S); |
| 5267 | }; |
| 5268 | OMPLexicalScope Scope(*this, S, llvm::None, /*EmitPreInitStmt=*/false); |
| 5269 | CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc()); |
| 5270 | } |
| 5271 | |
Alexey Bataev | b8552ab | 2019-10-18 16:47:35 +0000 | [diff] [blame] | 5272 | void CodeGenFunction::EmitOMPMasterTaskLoopSimdDirective( |
| 5273 | const OMPMasterTaskLoopSimdDirective &S) { |
| 5274 | auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 5275 | Action.Enter(CGF); |
| 5276 | EmitOMPTaskLoopBasedDirective(S); |
| 5277 | }; |
Alexey Bataev | 6120582 | 2019-12-04 09:50:21 -0500 | [diff] [blame] | 5278 | OMPLexicalScope Scope(*this, S); |
Alexey Bataev | b8552ab | 2019-10-18 16:47:35 +0000 | [diff] [blame] | 5279 | CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc()); |
| 5280 | } |
| 5281 | |
Alexey Bataev | 5bbcead | 2019-10-14 17:17:41 +0000 | [diff] [blame] | 5282 | void CodeGenFunction::EmitOMPParallelMasterTaskLoopDirective( |
| 5283 | const OMPParallelMasterTaskLoopDirective &S) { |
| 5284 | auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 5285 | auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, |
| 5286 | PrePostActionTy &Action) { |
| 5287 | Action.Enter(CGF); |
| 5288 | CGF.EmitOMPTaskLoopBasedDirective(S); |
| 5289 | }; |
| 5290 | OMPLexicalScope Scope(CGF, S, llvm::None, /*EmitPreInitStmt=*/false); |
| 5291 | CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, |
| 5292 | S.getBeginLoc()); |
| 5293 | }; |
| 5294 | emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop, CodeGen, |
| 5295 | emitEmptyBoundParameters); |
| 5296 | } |
| 5297 | |
Alexey Bataev | 14a388f | 2019-10-25 10:27:13 -0400 | [diff] [blame] | 5298 | void CodeGenFunction::EmitOMPParallelMasterTaskLoopSimdDirective( |
| 5299 | const OMPParallelMasterTaskLoopSimdDirective &S) { |
| 5300 | auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { |
| 5301 | auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, |
| 5302 | PrePostActionTy &Action) { |
| 5303 | Action.Enter(CGF); |
| 5304 | CGF.EmitOMPTaskLoopBasedDirective(S); |
| 5305 | }; |
| 5306 | OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); |
| 5307 | CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, |
| 5308 | S.getBeginLoc()); |
| 5309 | }; |
| 5310 | emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop_simd, CodeGen, |
| 5311 | emitEmptyBoundParameters); |
| 5312 | } |
| 5313 | |
Samuel Antao | 686c70c | 2016-05-26 17:30:50 +0000 | [diff] [blame] | 5314 | // Generate the instructions for '#pragma omp target update' directive. |
| 5315 | void CodeGenFunction::EmitOMPTargetUpdateDirective( |
| 5316 | const OMPTargetUpdateDirective &S) { |
Samuel Antao | 8d2d730 | 2016-05-26 18:30:22 +0000 | [diff] [blame] | 5317 | // If we don't have target devices, don't bother emitting the data mapping |
| 5318 | // code. |
| 5319 | if (CGM.getLangOpts().OMPTargetTriples.empty()) |
| 5320 | return; |
| 5321 | |
| 5322 | // Check if we have any if clause associated with the directive. |
| 5323 | const Expr *IfCond = nullptr; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 5324 | if (const auto *C = S.getSingleClause<OMPIfClause>()) |
Samuel Antao | 8d2d730 | 2016-05-26 18:30:22 +0000 | [diff] [blame] | 5325 | IfCond = C->getCondition(); |
| 5326 | |
| 5327 | // Check if we have any device clause associated with the directive. |
| 5328 | const Expr *Device = nullptr; |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 5329 | if (const auto *C = S.getSingleClause<OMPDeviceClause>()) |
Samuel Antao | 8d2d730 | 2016-05-26 18:30:22 +0000 | [diff] [blame] | 5330 | Device = C->getDevice(); |
| 5331 | |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 5332 | OMPLexicalScope Scope(*this, S, OMPD_task); |
Alexey Bataev | d2202ca | 2017-12-27 17:58:32 +0000 | [diff] [blame] | 5333 | CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device); |
Samuel Antao | 686c70c | 2016-05-26 17:30:50 +0000 | [diff] [blame] | 5334 | } |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 5335 | |
| 5336 | void CodeGenFunction::EmitSimpleOMPExecutableDirective( |
| 5337 | const OMPExecutableDirective &D) { |
| 5338 | if (!D.hasAssociatedStmt() || !D.getAssociatedStmt()) |
| 5339 | return; |
Akira Hatanaka | 9f37c0e | 2019-12-03 13:07:22 -0800 | [diff] [blame] | 5340 | auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 5341 | if (isOpenMPSimdDirective(D.getDirectiveKind())) { |
| 5342 | emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action); |
| 5343 | } else { |
Alexey Bataev | 6ab5bb1 | 2018-10-29 15:01:58 +0000 | [diff] [blame] | 5344 | OMPPrivateScope LoopGlobals(CGF); |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 5345 | if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) { |
Alexey Bataev | ddf3db9 | 2018-04-13 17:31:06 +0000 | [diff] [blame] | 5346 | for (const Expr *E : LD->counters()) { |
Alexey Bataev | 6ab5bb1 | 2018-10-29 15:01:58 +0000 | [diff] [blame] | 5347 | const auto *VD = dyn_cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); |
| 5348 | if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) { |
| 5349 | LValue GlobLVal = CGF.EmitLValue(E); |
| 5350 | LoopGlobals.addPrivate( |
Akira Hatanaka | f139ae3 | 2019-12-03 15:17:01 -0800 | [diff] [blame] | 5351 | VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); }); |
Alexey Bataev | 6ab5bb1 | 2018-10-29 15:01:58 +0000 | [diff] [blame] | 5352 | } |
Bjorn Pettersson | 6c2d83b | 2018-10-30 08:49:26 +0000 | [diff] [blame] | 5353 | if (isa<OMPCapturedExprDecl>(VD)) { |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 5354 | // Emit only those that were not explicitly referenced in clauses. |
| 5355 | if (!CGF.LocalDeclMap.count(VD)) |
| 5356 | CGF.EmitVarDecl(*VD); |
| 5357 | } |
| 5358 | } |
Alexey Bataev | f138fda | 2018-08-13 19:04:24 +0000 | [diff] [blame] | 5359 | for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) { |
| 5360 | if (!C->getNumForLoops()) |
| 5361 | continue; |
| 5362 | for (unsigned I = LD->getCollapsedNumber(), |
| 5363 | E = C->getLoopNumIterations().size(); |
| 5364 | I < E; ++I) { |
| 5365 | if (const auto *VD = dyn_cast<OMPCapturedExprDecl>( |
Mike Rice | 0ed4666 | 2018-09-20 17:19:41 +0000 | [diff] [blame] | 5366 | cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) { |
Alexey Bataev | f138fda | 2018-08-13 19:04:24 +0000 | [diff] [blame] | 5367 | // Emit only those that were not explicitly referenced in clauses. |
| 5368 | if (!CGF.LocalDeclMap.count(VD)) |
| 5369 | CGF.EmitVarDecl(*VD); |
| 5370 | } |
| 5371 | } |
| 5372 | } |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 5373 | } |
Alexey Bataev | 6ab5bb1 | 2018-10-29 15:01:58 +0000 | [diff] [blame] | 5374 | LoopGlobals.Privatize(); |
Alexey Bataev | 475a744 | 2018-01-12 19:39:11 +0000 | [diff] [blame] | 5375 | CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); |
Alexey Bataev | a8a9153a | 2017-12-29 18:07:07 +0000 | [diff] [blame] | 5376 | } |
| 5377 | }; |
| 5378 | OMPSimdLexicalScope Scope(*this, D); |
| 5379 | CGM.getOpenMPRuntime().emitInlinedDirective( |
| 5380 | *this, |
| 5381 | isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd |
| 5382 | : D.getDirectiveKind(), |
| 5383 | CodeGen); |
| 5384 | } |