blob: 72b9a3738b34a42b2ffb068ef916916c609d725e [file] [log] [blame]
Alexey Bataev9959db52014-05-06 10:08:46 +00001//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This provides a class for OpenMP runtime code generation.
11//
12//===----------------------------------------------------------------------===//
13
Samuel Antaoee8fb302016-01-06 13:42:12 +000014#include "CGCXXABI.h"
15#include "CGCleanup.h"
Alexey Bataev9959db52014-05-06 10:08:46 +000016#include "CGOpenMPRuntime.h"
17#include "CodeGenFunction.h"
John McCall5ad74072017-03-02 20:04:19 +000018#include "clang/CodeGen/ConstantInitBuilder.h"
Alexey Bataev9959db52014-05-06 10:08:46 +000019#include "clang/AST/Decl.h"
Chandler Carruth0d9593d2015-01-14 11:29:14 +000020#include "clang/AST/StmtOpenMP.h"
Alexey Bataev9959db52014-05-06 10:08:46 +000021#include "llvm/ADT/ArrayRef.h"
Alexey Bataev0f87dbe2017-08-14 17:56:13 +000022#include "llvm/ADT/BitmaskEnum.h"
Teresa Johnsonffc4e242016-11-11 05:35:12 +000023#include "llvm/Bitcode/BitcodeReader.h"
Alexey Bataevd74d0602014-10-13 06:02:40 +000024#include "llvm/IR/CallSite.h"
Alexey Bataev9959db52014-05-06 10:08:46 +000025#include "llvm/IR/DerivedTypes.h"
26#include "llvm/IR/GlobalValue.h"
27#include "llvm/IR/Value.h"
Samuel Antaoee8fb302016-01-06 13:42:12 +000028#include "llvm/Support/Format.h"
Alexey Bataev9959db52014-05-06 10:08:46 +000029#include "llvm/Support/raw_ostream.h"
Alexey Bataev23b69422014-06-18 07:08:49 +000030#include <cassert>
Alexey Bataev9959db52014-05-06 10:08:46 +000031
32using namespace clang;
33using namespace CodeGen;
34
Benjamin Kramerc52193f2014-10-10 13:57:57 +000035namespace {
Alexey Bataev8cbe0a62015-02-26 10:27:34 +000036/// \brief Base class for handling code generation inside OpenMP regions.
Alexey Bataev18095712014-10-10 12:19:54 +000037class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
38public:
Alexey Bataev6f1ffc02015-04-10 04:50:10 +000039 /// \brief Kinds of OpenMP regions used in codegen.
40 enum CGOpenMPRegionKind {
41 /// \brief Region with outlined function for standalone 'parallel'
42 /// directive.
43 ParallelOutlinedRegion,
44 /// \brief Region with outlined function for standalone 'task' directive.
45 TaskOutlinedRegion,
46 /// \brief Region for constructs that do not require function outlining,
47 /// like 'for', 'sections', 'atomic' etc. directives.
48 InlinedRegion,
Samuel Antaobed3c462015-10-02 16:14:20 +000049 /// \brief Region with outlined function for standalone 'target' directive.
50 TargetRegion,
Alexey Bataev6f1ffc02015-04-10 04:50:10 +000051 };
Alexey Bataev18095712014-10-10 12:19:54 +000052
Alexey Bataev6f1ffc02015-04-10 04:50:10 +000053 CGOpenMPRegionInfo(const CapturedStmt &CS,
54 const CGOpenMPRegionKind RegionKind,
Alexey Bataev25e5b442015-09-15 12:52:43 +000055 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
56 bool HasCancel)
Alexey Bataev6f1ffc02015-04-10 04:50:10 +000057 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
Alexey Bataev25e5b442015-09-15 12:52:43 +000058 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
Alexey Bataev6f1ffc02015-04-10 04:50:10 +000059
60 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
Alexey Bataev25e5b442015-09-15 12:52:43 +000061 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
62 bool HasCancel)
Alexey Bataev81c7ea02015-07-03 09:56:58 +000063 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
Alexey Bataev25e5b442015-09-15 12:52:43 +000064 Kind(Kind), HasCancel(HasCancel) {}
Alexey Bataev8cbe0a62015-02-26 10:27:34 +000065
66 /// \brief Get a variable or parameter for storing global thread id
Alexey Bataev18095712014-10-10 12:19:54 +000067 /// inside OpenMP construct.
Alexey Bataev8cbe0a62015-02-26 10:27:34 +000068 virtual const VarDecl *getThreadIDVariable() const = 0;
Alexey Bataev18095712014-10-10 12:19:54 +000069
Alexey Bataev6f1ffc02015-04-10 04:50:10 +000070 /// \brief Emit the captured statement body.
Hans Wennborg7eb54642015-09-10 17:07:54 +000071 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
Alexey Bataev6f1ffc02015-04-10 04:50:10 +000072
Alexey Bataev8cbe0a62015-02-26 10:27:34 +000073 /// \brief Get an LValue for the current ThreadID variable.
Alexey Bataev62b63b12015-03-10 07:28:44 +000074 /// \return LValue for thread id variable. This LValue always has type int32*.
75 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
Alexey Bataev18095712014-10-10 12:19:54 +000076
Alexey Bataev48591dd2016-04-20 04:01:36 +000077 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
78
Alexey Bataev6f1ffc02015-04-10 04:50:10 +000079 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
Alexey Bataev8cbe0a62015-02-26 10:27:34 +000080
Alexey Bataev81c7ea02015-07-03 09:56:58 +000081 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
82
Alexey Bataev25e5b442015-09-15 12:52:43 +000083 bool hasCancel() const { return HasCancel; }
84
Alexey Bataev18095712014-10-10 12:19:54 +000085 static bool classof(const CGCapturedStmtInfo *Info) {
86 return Info->getKind() == CR_OpenMP;
87 }
Alexey Bataev6f1ffc02015-04-10 04:50:10 +000088
Alexey Bataev48591dd2016-04-20 04:01:36 +000089 ~CGOpenMPRegionInfo() override = default;
90
Alexey Bataev8cbe0a62015-02-26 10:27:34 +000091protected:
Alexey Bataev6f1ffc02015-04-10 04:50:10 +000092 CGOpenMPRegionKind RegionKind;
Hans Wennborg45c74392016-01-12 20:54:36 +000093 RegionCodeGenTy CodeGen;
Alexey Bataev81c7ea02015-07-03 09:56:58 +000094 OpenMPDirectiveKind Kind;
Alexey Bataev25e5b442015-09-15 12:52:43 +000095 bool HasCancel;
Alexey Bataev8cbe0a62015-02-26 10:27:34 +000096};
Alexey Bataev18095712014-10-10 12:19:54 +000097
Alexey Bataev8cbe0a62015-02-26 10:27:34 +000098/// \brief API for captured statement code generation in OpenMP constructs.
Alexey Bataev48591dd2016-04-20 04:01:36 +000099class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000100public:
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000101 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
Alexey Bataev81c7ea02015-07-03 09:56:58 +0000102 const RegionCodeGenTy &CodeGen,
Arpith Chacko Jacobbb36fe82017-01-10 15:42:51 +0000103 OpenMPDirectiveKind Kind, bool HasCancel,
104 StringRef HelperName)
Alexey Bataev25e5b442015-09-15 12:52:43 +0000105 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
106 HasCancel),
Arpith Chacko Jacobbb36fe82017-01-10 15:42:51 +0000107 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000108 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
109 }
Eugene Zelenko0a4f3f42016-02-10 19:11:58 +0000110
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000111 /// \brief Get a variable or parameter for storing global thread id
112 /// inside OpenMP construct.
Alexander Kornienko34eb2072015-04-11 02:00:23 +0000113 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000114
Alexey Bataev18095712014-10-10 12:19:54 +0000115 /// \brief Get the name of the capture helper.
Arpith Chacko Jacobbb36fe82017-01-10 15:42:51 +0000116 StringRef getHelperName() const override { return HelperName; }
Alexey Bataev18095712014-10-10 12:19:54 +0000117
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000118 static bool classof(const CGCapturedStmtInfo *Info) {
119 return CGOpenMPRegionInfo::classof(Info) &&
120 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
121 ParallelOutlinedRegion;
122 }
123
Alexey Bataev18095712014-10-10 12:19:54 +0000124private:
125 /// \brief A variable or parameter storing global thread id for OpenMP
126 /// constructs.
127 const VarDecl *ThreadIDVar;
Arpith Chacko Jacobbb36fe82017-01-10 15:42:51 +0000128 StringRef HelperName;
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000129};
130
Alexey Bataev62b63b12015-03-10 07:28:44 +0000131/// \brief API for captured statement code generation in OpenMP constructs.
Alexey Bataev48591dd2016-04-20 04:01:36 +0000132class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
Alexey Bataev62b63b12015-03-10 07:28:44 +0000133public:
Alexey Bataev48591dd2016-04-20 04:01:36 +0000134 class UntiedTaskActionTy final : public PrePostActionTy {
135 bool Untied;
136 const VarDecl *PartIDVar;
137 const RegionCodeGenTy UntiedCodeGen;
138 llvm::SwitchInst *UntiedSwitch = nullptr;
139
140 public:
141 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
142 const RegionCodeGenTy &UntiedCodeGen)
143 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
144 void Enter(CodeGenFunction &CGF) override {
145 if (Untied) {
146 // Emit task switching point.
147 auto PartIdLVal = CGF.EmitLoadOfPointerLValue(
148 CGF.GetAddrOfLocalVar(PartIDVar),
149 PartIDVar->getType()->castAs<PointerType>());
150 auto *Res = CGF.EmitLoadOfScalar(PartIdLVal, SourceLocation());
151 auto *DoneBB = CGF.createBasicBlock(".untied.done.");
152 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
153 CGF.EmitBlock(DoneBB);
154 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
155 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
156 UntiedSwitch->addCase(CGF.Builder.getInt32(0),
157 CGF.Builder.GetInsertBlock());
158 emitUntiedSwitch(CGF);
159 }
160 }
161 void emitUntiedSwitch(CodeGenFunction &CGF) const {
162 if (Untied) {
163 auto PartIdLVal = CGF.EmitLoadOfPointerLValue(
164 CGF.GetAddrOfLocalVar(PartIDVar),
165 PartIDVar->getType()->castAs<PointerType>());
166 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
167 PartIdLVal);
168 UntiedCodeGen(CGF);
169 CodeGenFunction::JumpDest CurPoint =
170 CGF.getJumpDestInCurrentScope(".untied.next.");
171 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
172 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
173 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
174 CGF.Builder.GetInsertBlock());
175 CGF.EmitBranchThroughCleanup(CurPoint);
176 CGF.EmitBlock(CurPoint.getBlock());
177 }
178 }
179 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
180 };
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000181 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
Alexey Bataev62b63b12015-03-10 07:28:44 +0000182 const VarDecl *ThreadIDVar,
Alexey Bataev81c7ea02015-07-03 09:56:58 +0000183 const RegionCodeGenTy &CodeGen,
Alexey Bataev48591dd2016-04-20 04:01:36 +0000184 OpenMPDirectiveKind Kind, bool HasCancel,
185 const UntiedTaskActionTy &Action)
Alexey Bataev25e5b442015-09-15 12:52:43 +0000186 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
Alexey Bataev48591dd2016-04-20 04:01:36 +0000187 ThreadIDVar(ThreadIDVar), Action(Action) {
Alexey Bataev62b63b12015-03-10 07:28:44 +0000188 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
189 }
Eugene Zelenko0a4f3f42016-02-10 19:11:58 +0000190
Alexey Bataev62b63b12015-03-10 07:28:44 +0000191 /// \brief Get a variable or parameter for storing global thread id
192 /// inside OpenMP construct.
Alexander Kornienko34eb2072015-04-11 02:00:23 +0000193 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
Alexey Bataev62b63b12015-03-10 07:28:44 +0000194
195 /// \brief Get an LValue for the current ThreadID variable.
Alexander Kornienko34eb2072015-04-11 02:00:23 +0000196 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
Alexey Bataev62b63b12015-03-10 07:28:44 +0000197
Alexey Bataev62b63b12015-03-10 07:28:44 +0000198 /// \brief Get the name of the capture helper.
199 StringRef getHelperName() const override { return ".omp_outlined."; }
200
Alexey Bataev48591dd2016-04-20 04:01:36 +0000201 void emitUntiedSwitch(CodeGenFunction &CGF) override {
202 Action.emitUntiedSwitch(CGF);
203 }
204
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000205 static bool classof(const CGCapturedStmtInfo *Info) {
206 return CGOpenMPRegionInfo::classof(Info) &&
207 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
208 TaskOutlinedRegion;
209 }
210
Alexey Bataev62b63b12015-03-10 07:28:44 +0000211private:
212 /// \brief A variable or parameter storing global thread id for OpenMP
213 /// constructs.
214 const VarDecl *ThreadIDVar;
Alexey Bataev48591dd2016-04-20 04:01:36 +0000215 /// Action for emitting code for untied tasks.
216 const UntiedTaskActionTy &Action;
Alexey Bataev62b63b12015-03-10 07:28:44 +0000217};
218
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000219/// \brief API for inlined captured statement code generation in OpenMP
220/// constructs.
221class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
222public:
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000223 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
Alexey Bataev81c7ea02015-07-03 09:56:58 +0000224 const RegionCodeGenTy &CodeGen,
Alexey Bataev25e5b442015-09-15 12:52:43 +0000225 OpenMPDirectiveKind Kind, bool HasCancel)
226 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
227 OldCSI(OldCSI),
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000228 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
Eugene Zelenko0a4f3f42016-02-10 19:11:58 +0000229
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000230 // \brief Retrieve the value of the context parameter.
Alexander Kornienko34eb2072015-04-11 02:00:23 +0000231 llvm::Value *getContextValue() const override {
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000232 if (OuterRegionInfo)
233 return OuterRegionInfo->getContextValue();
234 llvm_unreachable("No context value for inlined OpenMP region");
235 }
Eugene Zelenko0a4f3f42016-02-10 19:11:58 +0000236
Hans Wennborg7eb54642015-09-10 17:07:54 +0000237 void setContextValue(llvm::Value *V) override {
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000238 if (OuterRegionInfo) {
239 OuterRegionInfo->setContextValue(V);
240 return;
241 }
242 llvm_unreachable("No context value for inlined OpenMP region");
243 }
Eugene Zelenko0a4f3f42016-02-10 19:11:58 +0000244
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000245 /// \brief Lookup the captured field decl for a variable.
Alexander Kornienko34eb2072015-04-11 02:00:23 +0000246 const FieldDecl *lookup(const VarDecl *VD) const override {
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000247 if (OuterRegionInfo)
248 return OuterRegionInfo->lookup(VD);
Alexey Bataev69c62a92015-04-15 04:52:20 +0000249 // If there is no outer outlined region,no need to lookup in a list of
250 // captured variables, we can use the original one.
251 return nullptr;
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000252 }
Eugene Zelenko0a4f3f42016-02-10 19:11:58 +0000253
Alexander Kornienko34eb2072015-04-11 02:00:23 +0000254 FieldDecl *getThisFieldDecl() const override {
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000255 if (OuterRegionInfo)
256 return OuterRegionInfo->getThisFieldDecl();
257 return nullptr;
258 }
Eugene Zelenko0a4f3f42016-02-10 19:11:58 +0000259
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000260 /// \brief Get a variable or parameter for storing global thread id
261 /// inside OpenMP construct.
Alexander Kornienko34eb2072015-04-11 02:00:23 +0000262 const VarDecl *getThreadIDVariable() const override {
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000263 if (OuterRegionInfo)
264 return OuterRegionInfo->getThreadIDVariable();
265 return nullptr;
266 }
Alexey Bataev62b63b12015-03-10 07:28:44 +0000267
Alexey Bataev311a9282017-10-12 13:51:32 +0000268 /// \brief Get an LValue for the current ThreadID variable.
269 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
270 if (OuterRegionInfo)
271 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
272 llvm_unreachable("No LValue for inlined OpenMP construct");
273 }
274
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000275 /// \brief Get the name of the capture helper.
Alexander Kornienko34eb2072015-04-11 02:00:23 +0000276 StringRef getHelperName() const override {
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000277 if (auto *OuterRegionInfo = getOldCSI())
278 return OuterRegionInfo->getHelperName();
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000279 llvm_unreachable("No helper name for inlined OpenMP construct");
280 }
281
Alexey Bataev48591dd2016-04-20 04:01:36 +0000282 void emitUntiedSwitch(CodeGenFunction &CGF) override {
283 if (OuterRegionInfo)
284 OuterRegionInfo->emitUntiedSwitch(CGF);
285 }
286
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000287 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
288
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000289 static bool classof(const CGCapturedStmtInfo *Info) {
290 return CGOpenMPRegionInfo::classof(Info) &&
291 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
292 }
293
Alexey Bataev48591dd2016-04-20 04:01:36 +0000294 ~CGOpenMPInlinedRegionInfo() override = default;
295
Alexey Bataev8cbe0a62015-02-26 10:27:34 +0000296private:
297 /// \brief CodeGen info about outer OpenMP region.
298 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
299 CGOpenMPRegionInfo *OuterRegionInfo;
Alexey Bataev18095712014-10-10 12:19:54 +0000300};
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000301
Samuel Antaobed3c462015-10-02 16:14:20 +0000302/// \brief API for captured statement code generation in OpenMP target
303/// constructs. For this captures, implicit parameters are used instead of the
Samuel Antaoee8fb302016-01-06 13:42:12 +0000304/// captured fields. The name of the target region has to be unique in a given
305/// application so it is provided by the client, because only the client has
306/// the information to generate that.
Alexey Bataev48591dd2016-04-20 04:01:36 +0000307class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
Samuel Antaobed3c462015-10-02 16:14:20 +0000308public:
309 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
Samuel Antaoee8fb302016-01-06 13:42:12 +0000310 const RegionCodeGenTy &CodeGen, StringRef HelperName)
Samuel Antaobed3c462015-10-02 16:14:20 +0000311 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
Samuel Antaoee8fb302016-01-06 13:42:12 +0000312 /*HasCancel=*/false),
313 HelperName(HelperName) {}
Samuel Antaobed3c462015-10-02 16:14:20 +0000314
315 /// \brief This is unused for target regions because each starts executing
316 /// with a single thread.
317 const VarDecl *getThreadIDVariable() const override { return nullptr; }
318
319 /// \brief Get the name of the capture helper.
Samuel Antaoee8fb302016-01-06 13:42:12 +0000320 StringRef getHelperName() const override { return HelperName; }
Samuel Antaobed3c462015-10-02 16:14:20 +0000321
322 static bool classof(const CGCapturedStmtInfo *Info) {
323 return CGOpenMPRegionInfo::classof(Info) &&
324 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
325 }
Samuel Antaoee8fb302016-01-06 13:42:12 +0000326
327private:
328 StringRef HelperName;
Samuel Antaobed3c462015-10-02 16:14:20 +0000329};
330
Alexey Bataev14fa1c62016-03-29 05:34:15 +0000331static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
Samuel Antaob68e2db2016-03-03 16:20:23 +0000332 llvm_unreachable("No codegen for expressions");
333}
334/// \brief API for generation of expressions captured in a innermost OpenMP
335/// region.
Alexey Bataev48591dd2016-04-20 04:01:36 +0000336class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
Samuel Antaob68e2db2016-03-03 16:20:23 +0000337public:
338 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
339 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
340 OMPD_unknown,
341 /*HasCancel=*/false),
342 PrivScope(CGF) {
343 // Make sure the globals captured in the provided statement are local by
344 // using the privatization logic. We assume the same variable is not
345 // captured more than once.
346 for (auto &C : CS.captures()) {
347 if (!C.capturesVariable() && !C.capturesVariableByCopy())
348 continue;
349
350 const VarDecl *VD = C.getCapturedVar();
351 if (VD->isLocalVarDeclOrParm())
352 continue;
353
354 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
355 /*RefersToEnclosingVariableOrCapture=*/false,
356 VD->getType().getNonReferenceType(), VK_LValue,
357 SourceLocation());
358 PrivScope.addPrivate(VD, [&CGF, &DRE]() -> Address {
359 return CGF.EmitLValue(&DRE).getAddress();
360 });
361 }
362 (void)PrivScope.Privatize();
363 }
364
365 /// \brief Lookup the captured field decl for a variable.
366 const FieldDecl *lookup(const VarDecl *VD) const override {
367 if (auto *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
368 return FD;
369 return nullptr;
370 }
371
372 /// \brief Emit the captured statement body.
373 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
374 llvm_unreachable("No body for expressions");
375 }
376
377 /// \brief Get a variable or parameter for storing global thread id
378 /// inside OpenMP construct.
379 const VarDecl *getThreadIDVariable() const override {
380 llvm_unreachable("No thread id for expressions");
381 }
382
383 /// \brief Get the name of the capture helper.
384 StringRef getHelperName() const override {
385 llvm_unreachable("No helper name for expressions");
386 }
387
388 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
389
390private:
391 /// Private scope to capture global variables.
392 CodeGenFunction::OMPPrivateScope PrivScope;
393};
394
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000395/// \brief RAII for emitting code of OpenMP constructs.
396class InlinedOpenMPRegionRAII {
397 CodeGenFunction &CGF;
Alexey Bataev4ba78a42016-04-27 07:56:03 +0000398 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
399 FieldDecl *LambdaThisCaptureField = nullptr;
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000400
401public:
402 /// \brief Constructs region for combined constructs.
403 /// \param CodeGen Code generation sequence for combined directives. Includes
404 /// a list of functions used for code generation of implicitly inlined
405 /// regions.
Alexey Bataev81c7ea02015-07-03 09:56:58 +0000406 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
Alexey Bataev25e5b442015-09-15 12:52:43 +0000407 OpenMPDirectiveKind Kind, bool HasCancel)
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000408 : CGF(CGF) {
409 // Start emission for the construct.
Alexey Bataev25e5b442015-09-15 12:52:43 +0000410 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
411 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
Alexey Bataev4ba78a42016-04-27 07:56:03 +0000412 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
413 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
414 CGF.LambdaThisCaptureField = nullptr;
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000415 }
Eugene Zelenko0a4f3f42016-02-10 19:11:58 +0000416
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000417 ~InlinedOpenMPRegionRAII() {
418 // Restore original CapturedStmtInfo only if we're done with code emission.
419 auto *OldCSI =
420 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
421 delete CGF.CapturedStmtInfo;
422 CGF.CapturedStmtInfo = OldCSI;
Alexey Bataev4ba78a42016-04-27 07:56:03 +0000423 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
424 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
Alexey Bataev6f1ffc02015-04-10 04:50:10 +0000425 }
426};
427
Alexey Bataev50b3c952016-02-19 10:38:26 +0000428/// \brief Values for bit flags used in the ident_t to describe the fields.
429/// All enumeric elements are named and described in accordance with the code
430/// from http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h
Alexey Bataev0f87dbe2017-08-14 17:56:13 +0000431enum OpenMPLocationFlags : unsigned {
Alexey Bataev50b3c952016-02-19 10:38:26 +0000432 /// \brief Use trampoline for internal microtask.
433 OMP_IDENT_IMD = 0x01,
434 /// \brief Use c-style ident structure.
435 OMP_IDENT_KMPC = 0x02,
436 /// \brief Atomic reduction option for kmpc_reduce.
437 OMP_ATOMIC_REDUCE = 0x10,
438 /// \brief Explicit 'barrier' directive.
439 OMP_IDENT_BARRIER_EXPL = 0x20,
440 /// \brief Implicit barrier in code.
441 OMP_IDENT_BARRIER_IMPL = 0x40,
442 /// \brief Implicit barrier in 'for' directive.
443 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
444 /// \brief Implicit barrier in 'sections' directive.
445 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
446 /// \brief Implicit barrier in 'single' directive.
Alexey Bataev0f87dbe2017-08-14 17:56:13 +0000447 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
448 /// Call of __kmp_for_static_init for static loop.
449 OMP_IDENT_WORK_LOOP = 0x200,
450 /// Call of __kmp_for_static_init for sections.
451 OMP_IDENT_WORK_SECTIONS = 0x400,
452 /// Call of __kmp_for_static_init for distribute.
453 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
454 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)
Alexey Bataev50b3c952016-02-19 10:38:26 +0000455};
456
457/// \brief Describes ident structure that describes a source location.
458/// All descriptions are taken from
459/// http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h
460/// Original structure:
461/// typedef struct ident {
462/// kmp_int32 reserved_1; /**< might be used in Fortran;
463/// see above */
464/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
465/// KMP_IDENT_KMPC identifies this union
466/// member */
467/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
468/// see above */
469///#if USE_ITT_BUILD
470/// /* but currently used for storing
471/// region-specific ITT */
472/// /* contextual information. */
473///#endif /* USE_ITT_BUILD */
474/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
475/// C++ */
476/// char const *psource; /**< String describing the source location.
477/// The string is composed of semi-colon separated
478// fields which describe the source file,
479/// the function and a pair of line numbers that
480/// delimit the construct.
481/// */
482/// } ident_t;
483enum IdentFieldIndex {
484 /// \brief might be used in Fortran
485 IdentField_Reserved_1,
486 /// \brief OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
487 IdentField_Flags,
488 /// \brief Not really used in Fortran any more
489 IdentField_Reserved_2,
490 /// \brief Source[4] in Fortran, do not use for C++
491 IdentField_Reserved_3,
492 /// \brief String describing the source location. The string is composed of
493 /// semi-colon separated fields which describe the source file, the function
494 /// and a pair of line numbers that delimit the construct.
495 IdentField_PSource
496};
497
498/// \brief Schedule types for 'omp for' loops (these enumerators are taken from
499/// the enum sched_type in kmp.h).
500enum OpenMPSchedType {
501 /// \brief Lower bound for default (unordered) versions.
502 OMP_sch_lower = 32,
503 OMP_sch_static_chunked = 33,
504 OMP_sch_static = 34,
505 OMP_sch_dynamic_chunked = 35,
506 OMP_sch_guided_chunked = 36,
507 OMP_sch_runtime = 37,
508 OMP_sch_auto = 38,
Alexey Bataev6cff6242016-05-30 13:05:14 +0000509 /// static with chunk adjustment (e.g., simd)
Samuel Antao4c8035b2016-12-12 18:00:20 +0000510 OMP_sch_static_balanced_chunked = 45,
Alexey Bataev50b3c952016-02-19 10:38:26 +0000511 /// \brief Lower bound for 'ordered' versions.
512 OMP_ord_lower = 64,
513 OMP_ord_static_chunked = 65,
514 OMP_ord_static = 66,
515 OMP_ord_dynamic_chunked = 67,
516 OMP_ord_guided_chunked = 68,
517 OMP_ord_runtime = 69,
518 OMP_ord_auto = 70,
519 OMP_sch_default = OMP_sch_static,
Carlo Bertollifc35ad22016-03-07 16:04:49 +0000520 /// \brief dist_schedule types
521 OMP_dist_sch_static_chunked = 91,
522 OMP_dist_sch_static = 92,
Alexey Bataev9ebd7422016-05-10 09:57:36 +0000523 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
524 /// Set if the monotonic schedule modifier was present.
525 OMP_sch_modifier_monotonic = (1 << 29),
526 /// Set if the nonmonotonic schedule modifier was present.
527 OMP_sch_modifier_nonmonotonic = (1 << 30),
Alexey Bataev50b3c952016-02-19 10:38:26 +0000528};
529
530enum OpenMPRTLFunction {
531 /// \brief Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
532 /// kmpc_micro microtask, ...);
533 OMPRTL__kmpc_fork_call,
534 /// \brief Call to void *__kmpc_threadprivate_cached(ident_t *loc,
535 /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
536 OMPRTL__kmpc_threadprivate_cached,
537 /// \brief Call to void __kmpc_threadprivate_register( ident_t *,
538 /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
539 OMPRTL__kmpc_threadprivate_register,
540 // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
541 OMPRTL__kmpc_global_thread_num,
542 // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
543 // kmp_critical_name *crit);
544 OMPRTL__kmpc_critical,
545 // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
546 // global_tid, kmp_critical_name *crit, uintptr_t hint);
547 OMPRTL__kmpc_critical_with_hint,
548 // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
549 // kmp_critical_name *crit);
550 OMPRTL__kmpc_end_critical,
551 // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
552 // global_tid);
553 OMPRTL__kmpc_cancel_barrier,
554 // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
555 OMPRTL__kmpc_barrier,
556 // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
557 OMPRTL__kmpc_for_static_fini,
558 // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
559 // global_tid);
560 OMPRTL__kmpc_serialized_parallel,
561 // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
562 // global_tid);
563 OMPRTL__kmpc_end_serialized_parallel,
564 // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
565 // kmp_int32 num_threads);
566 OMPRTL__kmpc_push_num_threads,
567 // Call to void __kmpc_flush(ident_t *loc);
568 OMPRTL__kmpc_flush,
569 // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
570 OMPRTL__kmpc_master,
571 // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
572 OMPRTL__kmpc_end_master,
573 // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
574 // int end_part);
575 OMPRTL__kmpc_omp_taskyield,
576 // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
577 OMPRTL__kmpc_single,
578 // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
579 OMPRTL__kmpc_end_single,
580 // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
581 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
582 // kmp_routine_entry_t *task_entry);
583 OMPRTL__kmpc_omp_task_alloc,
584 // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
585 // new_task);
586 OMPRTL__kmpc_omp_task,
587 // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
588 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
589 // kmp_int32 didit);
590 OMPRTL__kmpc_copyprivate,
591 // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
592 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
593 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
594 OMPRTL__kmpc_reduce,
595 // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
596 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
597 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
598 // *lck);
599 OMPRTL__kmpc_reduce_nowait,
600 // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
601 // kmp_critical_name *lck);
602 OMPRTL__kmpc_end_reduce,
603 // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
604 // kmp_critical_name *lck);
605 OMPRTL__kmpc_end_reduce_nowait,
606 // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
607 // kmp_task_t * new_task);
608 OMPRTL__kmpc_omp_task_begin_if0,
609 // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
610 // kmp_task_t * new_task);
611 OMPRTL__kmpc_omp_task_complete_if0,
612 // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
613 OMPRTL__kmpc_ordered,
614 // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
615 OMPRTL__kmpc_end_ordered,
616 // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
617 // global_tid);
618 OMPRTL__kmpc_omp_taskwait,
619 // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
620 OMPRTL__kmpc_taskgroup,
621 // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
622 OMPRTL__kmpc_end_taskgroup,
623 // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
624 // int proc_bind);
625 OMPRTL__kmpc_push_proc_bind,
626 // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
627 // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
628 // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
629 OMPRTL__kmpc_omp_task_with_deps,
630 // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
631 // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
632 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
633 OMPRTL__kmpc_omp_wait_deps,
634 // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
635 // global_tid, kmp_int32 cncl_kind);
636 OMPRTL__kmpc_cancellationpoint,
637 // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
638 // kmp_int32 cncl_kind);
639 OMPRTL__kmpc_cancel,
Carlo Bertolli430d8ec2016-03-03 20:34:23 +0000640 // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
641 // kmp_int32 num_teams, kmp_int32 thread_limit);
642 OMPRTL__kmpc_push_num_teams,
Alexey Bataev7292c292016-04-25 12:22:29 +0000643 // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
644 // microtask, ...);
Carlo Bertolli430d8ec2016-03-03 20:34:23 +0000645 OMPRTL__kmpc_fork_teams,
Alexey Bataev7292c292016-04-25 12:22:29 +0000646 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
647 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
648 // sched, kmp_uint64 grainsize, void *task_dup);
649 OMPRTL__kmpc_taskloop,
Alexey Bataev8b427062016-05-25 12:36:08 +0000650 // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
651 // num_dims, struct kmp_dim *dims);
652 OMPRTL__kmpc_doacross_init,
653 // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
654 OMPRTL__kmpc_doacross_fini,
655 // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
656 // *vec);
657 OMPRTL__kmpc_doacross_post,
658 // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
659 // *vec);
660 OMPRTL__kmpc_doacross_wait,
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000661 // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
662 // *data);
663 OMPRTL__kmpc_task_reduction_init,
664 // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
665 // *d);
666 OMPRTL__kmpc_task_reduction_get_th_data,
Alexey Bataev50b3c952016-02-19 10:38:26 +0000667
668 //
669 // Offloading related calls
670 //
George Rokos63bc9d62017-11-21 18:25:12 +0000671 // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
672 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
Alexey Bataev50b3c952016-02-19 10:38:26 +0000673 // *arg_types);
674 OMPRTL__tgt_target,
George Rokos63bc9d62017-11-21 18:25:12 +0000675 // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
676 // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
677 // *arg_types, int32_t num_teams, int32_t thread_limit);
Samuel Antaob68e2db2016-03-03 16:20:23 +0000678 OMPRTL__tgt_target_teams,
Alexey Bataev50b3c952016-02-19 10:38:26 +0000679 // Call to void __tgt_register_lib(__tgt_bin_desc *desc);
680 OMPRTL__tgt_register_lib,
681 // Call to void __tgt_unregister_lib(__tgt_bin_desc *desc);
682 OMPRTL__tgt_unregister_lib,
George Rokos63bc9d62017-11-21 18:25:12 +0000683 // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
684 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
Samuel Antaodf158d52016-04-27 22:58:19 +0000685 OMPRTL__tgt_target_data_begin,
George Rokos63bc9d62017-11-21 18:25:12 +0000686 // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
687 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
Samuel Antaodf158d52016-04-27 22:58:19 +0000688 OMPRTL__tgt_target_data_end,
George Rokos63bc9d62017-11-21 18:25:12 +0000689 // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
690 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
Samuel Antao8d2d7302016-05-26 18:30:22 +0000691 OMPRTL__tgt_target_data_update,
Alexey Bataev50b3c952016-02-19 10:38:26 +0000692};
693
Alexey Bataev14fa1c62016-03-29 05:34:15 +0000694/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
695/// region.
696class CleanupTy final : public EHScopeStack::Cleanup {
697 PrePostActionTy *Action;
698
699public:
700 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
701 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
702 if (!CGF.HaveInsertPoint())
703 return;
704 Action->Exit(CGF);
705 }
706};
707
Hans Wennborg7eb54642015-09-10 17:07:54 +0000708} // anonymous namespace
Alexey Bataev18095712014-10-10 12:19:54 +0000709
Alexey Bataev14fa1c62016-03-29 05:34:15 +0000710void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
711 CodeGenFunction::RunCleanupsScope Scope(CGF);
712 if (PrePostAction) {
713 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
714 Callback(CodeGen, CGF, *PrePostAction);
715 } else {
716 PrePostActionTy Action;
717 Callback(CodeGen, CGF, Action);
718 }
719}
720
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000721/// Check if the combiner is a call to UDR combiner and if it is so return the
722/// UDR decl used for reduction.
723static const OMPDeclareReductionDecl *
724getReductionInit(const Expr *ReductionOp) {
725 if (auto *CE = dyn_cast<CallExpr>(ReductionOp))
726 if (auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
727 if (auto *DRE =
728 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
729 if (auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
730 return DRD;
731 return nullptr;
732}
733
734static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
735 const OMPDeclareReductionDecl *DRD,
736 const Expr *InitOp,
737 Address Private, Address Original,
738 QualType Ty) {
739 if (DRD->getInitializer()) {
740 std::pair<llvm::Function *, llvm::Function *> Reduction =
741 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
742 auto *CE = cast<CallExpr>(InitOp);
743 auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
744 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
745 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
746 auto *LHSDRE = cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
747 auto *RHSDRE = cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
748 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
749 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
750 [=]() -> Address { return Private; });
751 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
752 [=]() -> Address { return Original; });
753 (void)PrivateScope.Privatize();
754 RValue Func = RValue::get(Reduction.second);
755 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
756 CGF.EmitIgnoredExpr(InitOp);
757 } else {
758 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
759 auto *GV = new llvm::GlobalVariable(
760 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
761 llvm::GlobalValue::PrivateLinkage, Init, ".init");
762 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
763 RValue InitRVal;
764 switch (CGF.getEvaluationKind(Ty)) {
765 case TEK_Scalar:
766 InitRVal = CGF.EmitLoadOfLValue(LV, SourceLocation());
767 break;
768 case TEK_Complex:
769 InitRVal =
770 RValue::getComplex(CGF.EmitLoadOfComplex(LV, SourceLocation()));
771 break;
772 case TEK_Aggregate:
773 InitRVal = RValue::getAggregate(LV.getAddress());
774 break;
775 }
776 OpaqueValueExpr OVE(SourceLocation(), Ty, VK_RValue);
777 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
778 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
779 /*IsInitializer=*/false);
780 }
781}
782
783/// \brief Emit initialization of arrays of complex types.
784/// \param DestAddr Address of the array.
785/// \param Type Type of array.
786/// \param Init Initial expression of array.
787/// \param SrcAddr Address of the original array.
788static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
Alexey Bataeva7b19152017-10-12 20:03:39 +0000789 QualType Type, bool EmitDeclareReductionInit,
790 const Expr *Init,
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000791 const OMPDeclareReductionDecl *DRD,
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000792 Address SrcAddr = Address::invalid()) {
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000793 // Perform element-by-element initialization.
794 QualType ElementTy;
795
796 // Drill down to the base element type on both arrays.
797 auto ArrayTy = Type->getAsArrayTypeUnsafe();
798 auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
799 DestAddr =
800 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
801 if (DRD)
802 SrcAddr =
803 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
804
805 llvm::Value *SrcBegin = nullptr;
806 if (DRD)
807 SrcBegin = SrcAddr.getPointer();
808 auto DestBegin = DestAddr.getPointer();
809 // Cast from pointer to array type to pointer to single element.
810 auto DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
811 // The basic structure here is a while-do loop.
812 auto BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
813 auto DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
814 auto IsEmpty =
815 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
816 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
817
818 // Enter the loop body, making that address the current address.
819 auto EntryBB = CGF.Builder.GetInsertBlock();
820 CGF.EmitBlock(BodyBB);
821
822 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
823
824 llvm::PHINode *SrcElementPHI = nullptr;
825 Address SrcElementCurrent = Address::invalid();
826 if (DRD) {
827 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
828 "omp.arraycpy.srcElementPast");
829 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
830 SrcElementCurrent =
831 Address(SrcElementPHI,
832 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
833 }
834 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
835 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
836 DestElementPHI->addIncoming(DestBegin, EntryBB);
837 Address DestElementCurrent =
838 Address(DestElementPHI,
839 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
840
841 // Emit copy.
842 {
843 CodeGenFunction::RunCleanupsScope InitScope(CGF);
Alexey Bataeva7b19152017-10-12 20:03:39 +0000844 if (EmitDeclareReductionInit) {
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000845 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
846 SrcElementCurrent, ElementTy);
847 } else
848 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
849 /*IsInitializer=*/false);
850 }
851
852 if (DRD) {
853 // Shift the address forward by one element.
854 auto SrcElementNext = CGF.Builder.CreateConstGEP1_32(
855 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
856 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
857 }
858
859 // Shift the address forward by one element.
860 auto DestElementNext = CGF.Builder.CreateConstGEP1_32(
861 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
862 // Check whether we've reached the end.
863 auto Done =
864 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
865 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
866 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
867
868 // Done.
869 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
870}
871
872LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
Alexey Bataevf47c4b42017-09-26 13:47:31 +0000873 return CGF.EmitOMPSharedLValue(E);
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000874}
875
876LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
877 const Expr *E) {
878 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
879 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
880 return LValue();
881}
882
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000883void ReductionCodeGen::emitAggregateInitialization(
884 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
885 const OMPDeclareReductionDecl *DRD) {
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000886 // Emit VarDecl with copy init for arrays.
887 // Get the address of the original variable captured in current
888 // captured region.
889 auto *PrivateVD =
890 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
Alexey Bataeva7b19152017-10-12 20:03:39 +0000891 bool EmitDeclareReductionInit =
892 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000893 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
Alexey Bataeva7b19152017-10-12 20:03:39 +0000894 EmitDeclareReductionInit,
895 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
896 : PrivateVD->getInit(),
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000897 DRD, SharedLVal.getAddress());
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000898}
899
900ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
901 ArrayRef<const Expr *> Privates,
902 ArrayRef<const Expr *> ReductionOps) {
903 ClausesData.reserve(Shareds.size());
904 SharedAddresses.reserve(Shareds.size());
905 Sizes.reserve(Shareds.size());
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000906 BaseDecls.reserve(Shareds.size());
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000907 auto IPriv = Privates.begin();
908 auto IRed = ReductionOps.begin();
909 for (const auto *Ref : Shareds) {
910 ClausesData.emplace_back(Ref, *IPriv, *IRed);
911 std::advance(IPriv, 1);
912 std::advance(IRed, 1);
913 }
914}
915
916void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
917 assert(SharedAddresses.size() == N &&
918 "Number of generated lvalues must be exactly N.");
Jonas Hahnfeld4525c822017-10-23 19:01:35 +0000919 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
920 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
921 SharedAddresses.emplace_back(First, Second);
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000922}
923
924void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
925 auto *PrivateVD =
926 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
927 QualType PrivateType = PrivateVD->getType();
928 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
Jonas Hahnfeld4525c822017-10-23 19:01:35 +0000929 if (!PrivateType->isVariablyModifiedType()) {
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000930 Sizes.emplace_back(
931 CGF.getTypeSize(
932 SharedAddresses[N].first.getType().getNonReferenceType()),
933 nullptr);
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000934 return;
935 }
936 llvm::Value *Size;
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000937 llvm::Value *SizeInChars;
938 llvm::Type *ElemType =
939 cast<llvm::PointerType>(SharedAddresses[N].first.getPointer()->getType())
940 ->getElementType();
941 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000942 if (AsArraySection) {
943 Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(),
944 SharedAddresses[N].first.getPointer());
945 Size = CGF.Builder.CreateNUWAdd(
946 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000947 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000948 } else {
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000949 SizeInChars = CGF.getTypeSize(
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000950 SharedAddresses[N].first.getType().getNonReferenceType());
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000951 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000952 }
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000953 Sizes.emplace_back(SizeInChars, Size);
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000954 CodeGenFunction::OpaqueValueMapping OpaqueMap(
955 CGF,
956 cast<OpaqueValueExpr>(
957 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
958 RValue::get(Size));
959 CGF.EmitVariablyModifiedType(PrivateType);
960}
961
962void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
963 llvm::Value *Size) {
964 auto *PrivateVD =
965 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
966 QualType PrivateType = PrivateVD->getType();
Jonas Hahnfeld4525c822017-10-23 19:01:35 +0000967 if (!PrivateType->isVariablyModifiedType()) {
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000968 assert(!Size && !Sizes[N].second &&
Jonas Hahnfeld4525c822017-10-23 19:01:35 +0000969 "Size should be nullptr for non-variably modified reduction "
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000970 "items.");
971 return;
972 }
973 CodeGenFunction::OpaqueValueMapping OpaqueMap(
974 CGF,
975 cast<OpaqueValueExpr>(
976 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
977 RValue::get(Size));
978 CGF.EmitVariablyModifiedType(PrivateType);
979}
980
981void ReductionCodeGen::emitInitialization(
982 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
983 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
984 assert(SharedAddresses.size() > N && "No variable was generated");
985 auto *PrivateVD =
986 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
987 auto *DRD = getReductionInit(ClausesData[N].ReductionOp);
988 QualType PrivateType = PrivateVD->getType();
989 PrivateAddr = CGF.Builder.CreateElementBitCast(
990 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
991 QualType SharedType = SharedAddresses[N].first.getType();
992 SharedLVal = CGF.MakeAddrLValue(
993 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(),
994 CGF.ConvertTypeForMem(SharedType)),
Ivan A. Kosarevf5f20462017-10-12 11:29:46 +0000995 SharedType, SharedAddresses[N].first.getBaseInfo(),
Ivan A. Kosarevb9c59f32017-10-31 11:05:34 +0000996 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
Jonas Hahnfeld4525c822017-10-23 19:01:35 +0000997 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
Alexey Bataevbe5a8b42017-07-17 13:30:36 +0000998 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000999 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1000 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1001 PrivateAddr, SharedLVal.getAddress(),
1002 SharedLVal.getType());
1003 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1004 !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1005 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1006 PrivateVD->getType().getQualifiers(),
1007 /*IsInitializer=*/false);
1008 }
1009}
1010
1011bool ReductionCodeGen::needCleanups(unsigned N) {
1012 auto *PrivateVD =
1013 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1014 QualType PrivateType = PrivateVD->getType();
1015 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1016 return DTorKind != QualType::DK_none;
1017}
1018
1019void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1020 Address PrivateAddr) {
1021 auto *PrivateVD =
1022 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1023 QualType PrivateType = PrivateVD->getType();
1024 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1025 if (needCleanups(N)) {
1026 PrivateAddr = CGF.Builder.CreateElementBitCast(
1027 PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1028 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1029 }
1030}
1031
1032static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1033 LValue BaseLV) {
1034 BaseTy = BaseTy.getNonReferenceType();
1035 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1036 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1037 if (auto *PtrTy = BaseTy->getAs<PointerType>())
1038 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
1039 else {
Ivan A. Kosarev9f9d1572017-10-30 11:49:31 +00001040 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);
1041 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001042 }
1043 BaseTy = BaseTy->getPointeeType();
1044 }
1045 return CGF.MakeAddrLValue(
1046 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(),
1047 CGF.ConvertTypeForMem(ElTy)),
Ivan A. Kosarevf5f20462017-10-12 11:29:46 +00001048 BaseLV.getType(), BaseLV.getBaseInfo(),
Ivan A. Kosarevb9c59f32017-10-31 11:05:34 +00001049 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001050}
1051
1052static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1053 llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1054 llvm::Value *Addr) {
1055 Address Tmp = Address::invalid();
1056 Address TopTmp = Address::invalid();
1057 Address MostTopTmp = Address::invalid();
1058 BaseTy = BaseTy.getNonReferenceType();
1059 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1060 !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1061 Tmp = CGF.CreateMemTemp(BaseTy);
1062 if (TopTmp.isValid())
1063 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1064 else
1065 MostTopTmp = Tmp;
1066 TopTmp = Tmp;
1067 BaseTy = BaseTy->getPointeeType();
1068 }
1069 llvm::Type *Ty = BaseLVType;
1070 if (Tmp.isValid())
1071 Ty = Tmp.getElementType();
1072 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1073 if (Tmp.isValid()) {
1074 CGF.Builder.CreateStore(Addr, Tmp);
1075 return MostTopTmp;
1076 }
1077 return Address(Addr, BaseLVAlignment);
1078}
1079
1080Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1081 Address PrivateAddr) {
1082 const DeclRefExpr *DE;
1083 const VarDecl *OrigVD = nullptr;
1084 if (auto *OASE = dyn_cast<OMPArraySectionExpr>(ClausesData[N].Ref)) {
1085 auto *Base = OASE->getBase()->IgnoreParenImpCasts();
1086 while (auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1087 Base = TempOASE->getBase()->IgnoreParenImpCasts();
1088 while (auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1089 Base = TempASE->getBase()->IgnoreParenImpCasts();
1090 DE = cast<DeclRefExpr>(Base);
1091 OrigVD = cast<VarDecl>(DE->getDecl());
1092 } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(ClausesData[N].Ref)) {
1093 auto *Base = ASE->getBase()->IgnoreParenImpCasts();
1094 while (auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1095 Base = TempASE->getBase()->IgnoreParenImpCasts();
1096 DE = cast<DeclRefExpr>(Base);
1097 OrigVD = cast<VarDecl>(DE->getDecl());
1098 }
1099 if (OrigVD) {
1100 BaseDecls.emplace_back(OrigVD);
1101 auto OriginalBaseLValue = CGF.EmitLValue(DE);
1102 LValue BaseLValue =
1103 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1104 OriginalBaseLValue);
1105 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1106 BaseLValue.getPointer(), SharedAddresses[N].first.getPointer());
Jonas Hahnfeld273d2612017-12-06 19:15:28 +00001107 llvm::Value *PrivatePointer =
1108 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1109 PrivateAddr.getPointer(),
1110 SharedAddresses[N].first.getAddress().getType());
1111 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001112 return castToBase(CGF, OrigVD->getType(),
1113 SharedAddresses[N].first.getType(),
Jonas Hahnfeld273d2612017-12-06 19:15:28 +00001114 OriginalBaseLValue.getAddress().getType(),
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001115 OriginalBaseLValue.getAlignment(), Ptr);
1116 }
1117 BaseDecls.emplace_back(
1118 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1119 return PrivateAddr;
1120}
1121
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00001122bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1123 auto *DRD = getReductionInit(ClausesData[N].ReductionOp);
1124 return DRD && DRD->getInitializer();
1125}
1126
Alexey Bataev18095712014-10-10 12:19:54 +00001127LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
Alexey Bataev31300ed2016-02-04 11:27:03 +00001128 return CGF.EmitLoadOfPointerLValue(
1129 CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1130 getThreadIDVariable()->getType()->castAs<PointerType>());
Alexey Bataev18095712014-10-10 12:19:54 +00001131}
1132
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001133void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001134 if (!CGF.HaveInsertPoint())
1135 return;
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001136 // 1.2.2 OpenMP Language Terminology
1137 // Structured block - An executable statement with a single entry at the
1138 // top and a single exit at the bottom.
1139 // The point of exit cannot be a branch out of the structured block.
1140 // longjmp() and throw() must not violate the entry/exit criteria.
1141 CGF.EHStack.pushTerminate();
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001142 CodeGen(CGF);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001143 CGF.EHStack.popTerminate();
Alexey Bataev4a5bb772014-10-08 14:01:46 +00001144}
1145
Alexey Bataev62b63b12015-03-10 07:28:44 +00001146LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1147 CodeGenFunction &CGF) {
Alexey Bataev2377fe92015-09-10 08:12:02 +00001148 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1149 getThreadIDVariable()->getType(),
Ivan A. Kosarev5f8c0ca2017-10-10 09:39:32 +00001150 AlignmentSource::Decl);
Alexey Bataev62b63b12015-03-10 07:28:44 +00001151}
1152
Alexey Bataev9959db52014-05-06 10:08:46 +00001153CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
Alexey Bataevc5b1d322016-03-04 09:22:22 +00001154 : CGM(CGM), OffloadEntriesInfoManager(CGM) {
Alexey Bataev9959db52014-05-06 10:08:46 +00001155 IdentTy = llvm::StructType::create(
1156 "ident_t", CGM.Int32Ty /* reserved_1 */, CGM.Int32Ty /* flags */,
1157 CGM.Int32Ty /* reserved_2 */, CGM.Int32Ty /* reserved_3 */,
Serge Guelton1d993272017-05-09 19:31:30 +00001158 CGM.Int8PtrTy /* psource */);
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00001159 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
Samuel Antaoee8fb302016-01-06 13:42:12 +00001160
1161 loadOffloadInfoMetadata();
Alexey Bataev9959db52014-05-06 10:08:46 +00001162}
1163
Alexey Bataev91797552015-03-18 04:13:55 +00001164void CGOpenMPRuntime::clear() {
1165 InternalVars.clear();
1166}
1167
Alexey Bataevc5b1d322016-03-04 09:22:22 +00001168static llvm::Function *
1169emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1170 const Expr *CombinerInitializer, const VarDecl *In,
1171 const VarDecl *Out, bool IsCombiner) {
1172 // void .omp_combiner.(Ty *in, Ty *out);
1173 auto &C = CGM.getContext();
1174 QualType PtrTy = C.getPointerType(Ty).withRestrict();
1175 FunctionArgList Args;
Alexey Bataevc5b1d322016-03-04 09:22:22 +00001176 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
Alexey Bataev56223232017-06-09 13:40:18 +00001177 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
Alexey Bataeva839ddd2016-03-17 10:19:46 +00001178 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
Alexey Bataev56223232017-06-09 13:40:18 +00001179 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
Alexey Bataevc5b1d322016-03-04 09:22:22 +00001180 Args.push_back(&OmpOutParm);
Alexey Bataeva839ddd2016-03-17 10:19:46 +00001181 Args.push_back(&OmpInParm);
Alexey Bataevc5b1d322016-03-04 09:22:22 +00001182 auto &FnInfo =
John McCallc56a8b32016-03-11 04:30:31 +00001183 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
Alexey Bataevc5b1d322016-03-04 09:22:22 +00001184 auto *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1185 auto *Fn = llvm::Function::Create(
1186 FnTy, llvm::GlobalValue::InternalLinkage,
1187 IsCombiner ? ".omp_combiner." : ".omp_initializer.", &CGM.getModule());
1188 CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, FnInfo);
Chandler Carruthfcd33142016-12-23 01:24:49 +00001189 Fn->removeFnAttr(llvm::Attribute::NoInline);
Mehdi Amini6aa9e9b2017-05-29 05:38:20 +00001190 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
Alexey Bataeva839ddd2016-03-17 10:19:46 +00001191 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
Alexey Bataevc5b1d322016-03-04 09:22:22 +00001192 CodeGenFunction CGF(CGM);
1193 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1194 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1195 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args);
1196 CodeGenFunction::OMPPrivateScope Scope(CGF);
1197 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1198 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() -> Address {
1199 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1200 .getAddress();
1201 });
1202 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1203 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() -> Address {
1204 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1205 .getAddress();
1206 });
1207 (void)Scope.Privatize();
Alexey Bataev070f43a2017-09-06 14:49:58 +00001208 if (!IsCombiner && Out->hasInit() &&
1209 !CGF.isTrivialInitializer(Out->getInit())) {
1210 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1211 Out->getType().getQualifiers(),
1212 /*IsInitializer=*/true);
1213 }
1214 if (CombinerInitializer)
1215 CGF.EmitIgnoredExpr(CombinerInitializer);
Alexey Bataevc5b1d322016-03-04 09:22:22 +00001216 Scope.ForceCleanup();
1217 CGF.FinishFunction();
1218 return Fn;
1219}
1220
1221void CGOpenMPRuntime::emitUserDefinedReduction(
1222 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1223 if (UDRMap.count(D) > 0)
1224 return;
1225 auto &C = CGM.getContext();
1226 if (!In || !Out) {
1227 In = &C.Idents.get("omp_in");
1228 Out = &C.Idents.get("omp_out");
1229 }
1230 llvm::Function *Combiner = emitCombinerOrInitializer(
1231 CGM, D->getType(), D->getCombiner(), cast<VarDecl>(D->lookup(In).front()),
1232 cast<VarDecl>(D->lookup(Out).front()),
1233 /*IsCombiner=*/true);
1234 llvm::Function *Initializer = nullptr;
1235 if (auto *Init = D->getInitializer()) {
1236 if (!Priv || !Orig) {
1237 Priv = &C.Idents.get("omp_priv");
1238 Orig = &C.Idents.get("omp_orig");
1239 }
1240 Initializer = emitCombinerOrInitializer(
Alexey Bataev070f43a2017-09-06 14:49:58 +00001241 CGM, D->getType(),
1242 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1243 : nullptr,
1244 cast<VarDecl>(D->lookup(Orig).front()),
Alexey Bataevc5b1d322016-03-04 09:22:22 +00001245 cast<VarDecl>(D->lookup(Priv).front()),
1246 /*IsCombiner=*/false);
1247 }
1248 UDRMap.insert(std::make_pair(D, std::make_pair(Combiner, Initializer)));
1249 if (CGF) {
1250 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1251 Decls.second.push_back(D);
1252 }
1253}
1254
Alexey Bataeva839ddd2016-03-17 10:19:46 +00001255std::pair<llvm::Function *, llvm::Function *>
1256CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1257 auto I = UDRMap.find(D);
1258 if (I != UDRMap.end())
1259 return I->second;
1260 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1261 return UDRMap.lookup(D);
1262}
1263
John McCall7f416cc2015-09-08 08:05:57 +00001264// Layout information for ident_t.
1265static CharUnits getIdentAlign(CodeGenModule &CGM) {
1266 return CGM.getPointerAlign();
1267}
1268static CharUnits getIdentSize(CodeGenModule &CGM) {
1269 assert((4 * CGM.getPointerSize()).isMultipleOf(CGM.getPointerAlign()));
1270 return CharUnits::fromQuantity(16) + CGM.getPointerSize();
1271}
Alexey Bataev50b3c952016-02-19 10:38:26 +00001272static CharUnits getOffsetOfIdentField(IdentFieldIndex Field) {
John McCall7f416cc2015-09-08 08:05:57 +00001273 // All the fields except the last are i32, so this works beautifully.
1274 return unsigned(Field) * CharUnits::fromQuantity(4);
1275}
1276static Address createIdentFieldGEP(CodeGenFunction &CGF, Address Addr,
Alexey Bataev50b3c952016-02-19 10:38:26 +00001277 IdentFieldIndex Field,
John McCall7f416cc2015-09-08 08:05:57 +00001278 const llvm::Twine &Name = "") {
1279 auto Offset = getOffsetOfIdentField(Field);
1280 return CGF.Builder.CreateStructGEP(Addr, Field, Offset, Name);
1281}
1282
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00001283static llvm::Value *emitParallelOrTeamsOutlinedFunction(
1284 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1285 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1286 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
Alexey Bataev62b63b12015-03-10 07:28:44 +00001287 assert(ThreadIDVar->getType()->isPointerType() &&
1288 "thread id variable must be of type kmp_int32 *");
Alexey Bataev18095712014-10-10 12:19:54 +00001289 CodeGenFunction CGF(CGM, true);
Alexey Bataev25e5b442015-09-15 12:52:43 +00001290 bool HasCancel = false;
1291 if (auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1292 HasCancel = OPD->hasCancel();
1293 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1294 HasCancel = OPSD->hasCancel();
1295 else if (auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1296 HasCancel = OPFD->hasCancel();
Alexey Bataev2139ed62017-11-16 18:20:21 +00001297 else if (auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1298 HasCancel = OPFD->hasCancel();
Alexey Bataev10a54312017-11-27 16:54:08 +00001299 else if (auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1300 HasCancel = OPFD->hasCancel();
1301 else if (auto *OPFD = dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1302 HasCancel = OPFD->hasCancel();
1303 else if (auto *OPFD =
1304 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1305 HasCancel = OPFD->hasCancel();
Alexey Bataev25e5b442015-09-15 12:52:43 +00001306 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00001307 HasCancel, OutlinedHelperName);
Alexey Bataevd157d472015-06-24 03:35:38 +00001308 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
Alexey Bataev2377fe92015-09-10 08:12:02 +00001309 return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
Alexey Bataev18095712014-10-10 12:19:54 +00001310}
1311
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00001312llvm::Value *CGOpenMPRuntime::emitParallelOutlinedFunction(
1313 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1314 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1315 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1316 return emitParallelOrTeamsOutlinedFunction(
1317 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1318}
1319
1320llvm::Value *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1321 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1322 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1323 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1324 return emitParallelOrTeamsOutlinedFunction(
1325 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1326}
1327
Alexey Bataev81c7ea02015-07-03 09:56:58 +00001328llvm::Value *CGOpenMPRuntime::emitTaskOutlinedFunction(
1329 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
Alexey Bataev48591dd2016-04-20 04:01:36 +00001330 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1331 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1332 bool Tied, unsigned &NumberOfParts) {
1333 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1334 PrePostActionTy &) {
1335 auto *ThreadID = getThreadID(CGF, D.getLocStart());
1336 auto *UpLoc = emitUpdateLocation(CGF, D.getLocStart());
1337 llvm::Value *TaskArgs[] = {
1338 UpLoc, ThreadID,
1339 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1340 TaskTVar->getType()->castAs<PointerType>())
1341 .getPointer()};
1342 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1343 };
1344 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1345 UntiedCodeGen);
1346 CodeGen.setAction(Action);
Alexey Bataev62b63b12015-03-10 07:28:44 +00001347 assert(!ThreadIDVar->getType()->isPointerType() &&
1348 "thread id variable must be of type kmp_int32 for tasks");
1349 auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
Alexey Bataev7292c292016-04-25 12:22:29 +00001350 auto *TD = dyn_cast<OMPTaskDirective>(&D);
Alexey Bataev62b63b12015-03-10 07:28:44 +00001351 CodeGenFunction CGF(CGM, true);
Alexey Bataev7292c292016-04-25 12:22:29 +00001352 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1353 InnermostKind,
1354 TD ? TD->hasCancel() : false, Action);
Alexey Bataevd157d472015-06-24 03:35:38 +00001355 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
Alexey Bataev48591dd2016-04-20 04:01:36 +00001356 auto *Res = CGF.GenerateCapturedStmtFunction(*CS);
1357 if (!Tied)
1358 NumberOfParts = Action.getNumberOfParts();
1359 return Res;
Alexey Bataev62b63b12015-03-10 07:28:44 +00001360}
1361
Alexey Bataev50b3c952016-02-19 10:38:26 +00001362Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
John McCall7f416cc2015-09-08 08:05:57 +00001363 CharUnits Align = getIdentAlign(CGM);
Alexey Bataev15007ba2014-05-07 06:18:01 +00001364 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags);
Alexey Bataev9959db52014-05-06 10:08:46 +00001365 if (!Entry) {
1366 if (!DefaultOpenMPPSource) {
1367 // Initialize default location for psource field of ident_t structure of
1368 // all ident_t objects. Format is ";file;function;line;column;;".
1369 // Taken from
1370 // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp_str.c
1371 DefaultOpenMPPSource =
John McCall7f416cc2015-09-08 08:05:57 +00001372 CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
Alexey Bataev9959db52014-05-06 10:08:46 +00001373 DefaultOpenMPPSource =
1374 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1375 }
Alexey Bataev9959db52014-05-06 10:08:46 +00001376
John McCall23c9dc62016-11-28 22:18:27 +00001377 ConstantInitBuilder builder(CGM);
John McCall6c9f1fdb2016-11-19 08:17:24 +00001378 auto fields = builder.beginStruct(IdentTy);
1379 fields.addInt(CGM.Int32Ty, 0);
1380 fields.addInt(CGM.Int32Ty, Flags);
1381 fields.addInt(CGM.Int32Ty, 0);
1382 fields.addInt(CGM.Int32Ty, 0);
1383 fields.add(DefaultOpenMPPSource);
1384 auto DefaultOpenMPLocation =
1385 fields.finishAndCreateGlobal("", Align, /*isConstant*/ true,
1386 llvm::GlobalValue::PrivateLinkage);
1387 DefaultOpenMPLocation->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1388
John McCall7f416cc2015-09-08 08:05:57 +00001389 OpenMPDefaultLocMap[Flags] = Entry = DefaultOpenMPLocation;
Alexey Bataev9959db52014-05-06 10:08:46 +00001390 }
John McCall7f416cc2015-09-08 08:05:57 +00001391 return Address(Entry, Align);
Alexey Bataev9959db52014-05-06 10:08:46 +00001392}
1393
Alexey Bataev3eff5f42015-02-25 08:32:46 +00001394llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1395 SourceLocation Loc,
Alexey Bataev50b3c952016-02-19 10:38:26 +00001396 unsigned Flags) {
1397 Flags |= OMP_IDENT_KMPC;
Alexey Bataev9959db52014-05-06 10:08:46 +00001398 // If no debug info is generated - return global default location.
Benjamin Kramer8c305922016-02-02 11:06:51 +00001399 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
Alexey Bataev9959db52014-05-06 10:08:46 +00001400 Loc.isInvalid())
John McCall7f416cc2015-09-08 08:05:57 +00001401 return getOrCreateDefaultLocation(Flags).getPointer();
Alexey Bataev9959db52014-05-06 10:08:46 +00001402
1403 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1404
John McCall7f416cc2015-09-08 08:05:57 +00001405 Address LocValue = Address::invalid();
Alexey Bataev1e4b7132014-12-03 12:11:24 +00001406 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1407 if (I != OpenMPLocThreadIDMap.end())
John McCall7f416cc2015-09-08 08:05:57 +00001408 LocValue = Address(I->second.DebugLoc, getIdentAlign(CGF.CGM));
1409
Alexander Musmanc6388682014-12-15 07:07:06 +00001410 // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1411 // GetOpenMPThreadID was called before this routine.
John McCall7f416cc2015-09-08 08:05:57 +00001412 if (!LocValue.isValid()) {
Alexey Bataev15007ba2014-05-07 06:18:01 +00001413 // Generate "ident_t .kmpc_loc.addr;"
John McCall7f416cc2015-09-08 08:05:57 +00001414 Address AI = CGF.CreateTempAlloca(IdentTy, getIdentAlign(CGF.CGM),
1415 ".kmpc_loc.addr");
Alexey Bataev18095712014-10-10 12:19:54 +00001416 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
John McCall7f416cc2015-09-08 08:05:57 +00001417 Elem.second.DebugLoc = AI.getPointer();
Alexey Bataev9959db52014-05-06 10:08:46 +00001418 LocValue = AI;
1419
1420 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1421 CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
Alexey Bataev3eff5f42015-02-25 08:32:46 +00001422 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
John McCall7f416cc2015-09-08 08:05:57 +00001423 CGM.getSize(getIdentSize(CGF.CGM)));
Alexey Bataev9959db52014-05-06 10:08:46 +00001424 }
1425
1426 // char **psource = &.kmpc_loc_<flags>.addr.psource;
John McCall7f416cc2015-09-08 08:05:57 +00001427 Address PSource = createIdentFieldGEP(CGF, LocValue, IdentField_PSource);
Alexey Bataev9959db52014-05-06 10:08:46 +00001428
Alexey Bataevf002aca2014-05-30 05:48:40 +00001429 auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1430 if (OMPDebugLoc == nullptr) {
1431 SmallString<128> Buffer2;
1432 llvm::raw_svector_ostream OS2(Buffer2);
1433 // Build debug location
1434 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1435 OS2 << ";" << PLoc.getFilename() << ";";
1436 if (const FunctionDecl *FD =
1437 dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) {
1438 OS2 << FD->getQualifiedNameAsString();
1439 }
1440 OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1441 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1442 OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
Alexey Bataev9959db52014-05-06 10:08:46 +00001443 }
Alexey Bataev9959db52014-05-06 10:08:46 +00001444 // *psource = ";<File>;<Function>;<Line>;<Column>;;";
Alexey Bataevf002aca2014-05-30 05:48:40 +00001445 CGF.Builder.CreateStore(OMPDebugLoc, PSource);
1446
John McCall7f416cc2015-09-08 08:05:57 +00001447 // Our callers always pass this to a runtime function, so for
1448 // convenience, go ahead and return a naked pointer.
1449 return LocValue.getPointer();
Alexey Bataev9959db52014-05-06 10:08:46 +00001450}
1451
Alexey Bataev3eff5f42015-02-25 08:32:46 +00001452llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1453 SourceLocation Loc) {
Alexey Bataev9959db52014-05-06 10:08:46 +00001454 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1455
Alexey Bataev4a5bb772014-10-08 14:01:46 +00001456 llvm::Value *ThreadID = nullptr;
Alexey Bataev18095712014-10-10 12:19:54 +00001457 // Check whether we've already cached a load of the thread id in this
1458 // function.
Alexey Bataev1e4b7132014-12-03 12:11:24 +00001459 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
Alexey Bataev18095712014-10-10 12:19:54 +00001460 if (I != OpenMPLocThreadIDMap.end()) {
1461 ThreadID = I->second.ThreadID;
Alexey Bataev03b340a2014-10-21 03:16:40 +00001462 if (ThreadID != nullptr)
1463 return ThreadID;
1464 }
Alexey Bataevaee18552017-08-16 14:01:00 +00001465 // If exceptions are enabled, do not use parameter to avoid possible crash.
Alexey Bataev5d2c9a42017-11-02 18:55:05 +00001466 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1467 !CGF.getLangOpts().CXXExceptions ||
Alexey Bataev0e1b4582017-11-02 14:25:34 +00001468 CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
Alexey Bataevaee18552017-08-16 14:01:00 +00001469 if (auto *OMPRegionInfo =
1470 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1471 if (OMPRegionInfo->getThreadIDVariable()) {
1472 // Check if this an outlined function with thread id passed as argument.
1473 auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1474 ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal();
1475 // If value loaded in entry block, cache it and use it everywhere in
1476 // function.
1477 if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1478 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1479 Elem.second.ThreadID = ThreadID;
1480 }
1481 return ThreadID;
Alexey Bataev8cbe0a62015-02-26 10:27:34 +00001482 }
Alexey Bataevd6c57552014-07-25 07:55:17 +00001483 }
Alexey Bataev9959db52014-05-06 10:08:46 +00001484 }
Alexey Bataev8cbe0a62015-02-26 10:27:34 +00001485
1486 // This is not an outlined function region - need to call __kmpc_int32
1487 // kmpc_global_thread_num(ident_t *loc).
1488 // Generate thread id value and cache this value for use across the
1489 // function.
1490 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1491 CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
Alexey Bataev0e1b4582017-11-02 14:25:34 +00001492 auto *Call = CGF.Builder.CreateCall(
1493 createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1494 emitUpdateLocation(CGF, Loc));
1495 Call->setCallingConv(CGF.getRuntimeCC());
Alexey Bataev8cbe0a62015-02-26 10:27:34 +00001496 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
Alexey Bataev0e1b4582017-11-02 14:25:34 +00001497 Elem.second.ThreadID = Call;
1498 return Call;
Alexey Bataev9959db52014-05-06 10:08:46 +00001499}
1500
Alexey Bataev3eff5f42015-02-25 08:32:46 +00001501void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
Alexey Bataev9959db52014-05-06 10:08:46 +00001502 assert(CGF.CurFn && "No function in current CodeGenFunction.");
Alexey Bataev03b340a2014-10-21 03:16:40 +00001503 if (OpenMPLocThreadIDMap.count(CGF.CurFn))
1504 OpenMPLocThreadIDMap.erase(CGF.CurFn);
Alexey Bataevc5b1d322016-03-04 09:22:22 +00001505 if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1506 for(auto *D : FunctionUDRMap[CGF.CurFn]) {
1507 UDRMap.erase(D);
1508 }
1509 FunctionUDRMap.erase(CGF.CurFn);
1510 }
Alexey Bataev9959db52014-05-06 10:08:46 +00001511}
1512
1513llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001514 if (!IdentTy) {
1515 }
Alexey Bataev9959db52014-05-06 10:08:46 +00001516 return llvm::PointerType::getUnqual(IdentTy);
1517}
1518
1519llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001520 if (!Kmpc_MicroTy) {
1521 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1522 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1523 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1524 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1525 }
Alexey Bataev9959db52014-05-06 10:08:46 +00001526 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1527}
1528
1529llvm::Constant *
Alexey Bataev50b3c952016-02-19 10:38:26 +00001530CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
Alexey Bataev9959db52014-05-06 10:08:46 +00001531 llvm::Constant *RTLFn = nullptr;
Alexey Bataev50b3c952016-02-19 10:38:26 +00001532 switch (static_cast<OpenMPRTLFunction>(Function)) {
Alexey Bataev9959db52014-05-06 10:08:46 +00001533 case OMPRTL__kmpc_fork_call: {
1534 // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1535 // microtask, ...);
Alexey Bataev23b69422014-06-18 07:08:49 +00001536 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1537 getKmpc_MicroPointerTy()};
Alexey Bataev9959db52014-05-06 10:08:46 +00001538 llvm::FunctionType *FnTy =
Alexey Bataevd74d0602014-10-13 06:02:40 +00001539 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
Alexey Bataev9959db52014-05-06 10:08:46 +00001540 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1541 break;
1542 }
1543 case OMPRTL__kmpc_global_thread_num: {
1544 // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
Alexey Bataev23b69422014-06-18 07:08:49 +00001545 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
Alexey Bataev9959db52014-05-06 10:08:46 +00001546 llvm::FunctionType *FnTy =
Alexey Bataevd74d0602014-10-13 06:02:40 +00001547 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
Alexey Bataev9959db52014-05-06 10:08:46 +00001548 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1549 break;
1550 }
Alexey Bataev97720002014-11-11 04:05:39 +00001551 case OMPRTL__kmpc_threadprivate_cached: {
1552 // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1553 // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1554 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1555 CGM.VoidPtrTy, CGM.SizeTy,
1556 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1557 llvm::FunctionType *FnTy =
1558 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1559 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1560 break;
1561 }
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00001562 case OMPRTL__kmpc_critical: {
Alexey Bataevf9472182014-09-22 12:32:31 +00001563 // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1564 // kmp_critical_name *crit);
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00001565 llvm::Type *TypeParams[] = {
1566 getIdentTyPointerTy(), CGM.Int32Ty,
1567 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1568 llvm::FunctionType *FnTy =
1569 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1570 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1571 break;
1572 }
Alexey Bataevfc57d162015-12-15 10:55:09 +00001573 case OMPRTL__kmpc_critical_with_hint: {
1574 // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1575 // kmp_critical_name *crit, uintptr_t hint);
1576 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1577 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1578 CGM.IntPtrTy};
1579 llvm::FunctionType *FnTy =
1580 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1581 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1582 break;
1583 }
Alexey Bataev97720002014-11-11 04:05:39 +00001584 case OMPRTL__kmpc_threadprivate_register: {
1585 // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1586 // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1587 // typedef void *(*kmpc_ctor)(void *);
1588 auto KmpcCtorTy =
1589 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1590 /*isVarArg*/ false)->getPointerTo();
1591 // typedef void *(*kmpc_cctor)(void *, void *);
1592 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1593 auto KmpcCopyCtorTy =
1594 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1595 /*isVarArg*/ false)->getPointerTo();
1596 // typedef void (*kmpc_dtor)(void *);
1597 auto KmpcDtorTy =
1598 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1599 ->getPointerTo();
1600 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1601 KmpcCopyCtorTy, KmpcDtorTy};
1602 auto FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1603 /*isVarArg*/ false);
1604 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1605 break;
1606 }
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00001607 case OMPRTL__kmpc_end_critical: {
Alexey Bataevf9472182014-09-22 12:32:31 +00001608 // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1609 // kmp_critical_name *crit);
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00001610 llvm::Type *TypeParams[] = {
1611 getIdentTyPointerTy(), CGM.Int32Ty,
1612 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1613 llvm::FunctionType *FnTy =
1614 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1615 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1616 break;
1617 }
Alexey Bataev8f7c1b02014-12-05 04:09:23 +00001618 case OMPRTL__kmpc_cancel_barrier: {
1619 // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1620 // global_tid);
Alexey Bataev4a5bb772014-10-08 14:01:46 +00001621 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1622 llvm::FunctionType *FnTy =
Alexey Bataev8f7c1b02014-12-05 04:09:23 +00001623 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1624 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
Alexey Bataev4a5bb772014-10-08 14:01:46 +00001625 break;
1626 }
Alexey Bataev81c7ea02015-07-03 09:56:58 +00001627 case OMPRTL__kmpc_barrier: {
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00001628 // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
Alexey Bataev81c7ea02015-07-03 09:56:58 +00001629 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1630 llvm::FunctionType *FnTy =
1631 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1632 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1633 break;
1634 }
Alexander Musmanc6388682014-12-15 07:07:06 +00001635 case OMPRTL__kmpc_for_static_fini: {
1636 // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1637 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1638 llvm::FunctionType *FnTy =
1639 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1640 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1641 break;
1642 }
Alexey Bataevb2059782014-10-13 08:23:51 +00001643 case OMPRTL__kmpc_push_num_threads: {
1644 // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1645 // kmp_int32 num_threads)
1646 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1647 CGM.Int32Ty};
1648 llvm::FunctionType *FnTy =
1649 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1650 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1651 break;
1652 }
Alexey Bataevd74d0602014-10-13 06:02:40 +00001653 case OMPRTL__kmpc_serialized_parallel: {
1654 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1655 // global_tid);
1656 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1657 llvm::FunctionType *FnTy =
1658 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1659 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1660 break;
1661 }
1662 case OMPRTL__kmpc_end_serialized_parallel: {
1663 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1664 // global_tid);
1665 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1666 llvm::FunctionType *FnTy =
1667 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1668 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1669 break;
1670 }
Alexey Bataevcc37cc12014-11-20 04:34:54 +00001671 case OMPRTL__kmpc_flush: {
Alexey Bataevd76df6d2015-02-24 12:55:09 +00001672 // Build void __kmpc_flush(ident_t *loc);
Alexey Bataevcc37cc12014-11-20 04:34:54 +00001673 llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1674 llvm::FunctionType *FnTy =
Alexey Bataevd76df6d2015-02-24 12:55:09 +00001675 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
Alexey Bataevcc37cc12014-11-20 04:34:54 +00001676 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
1677 break;
1678 }
Alexey Bataev8d690652014-12-04 07:23:53 +00001679 case OMPRTL__kmpc_master: {
1680 // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
1681 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1682 llvm::FunctionType *FnTy =
1683 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1684 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
1685 break;
1686 }
1687 case OMPRTL__kmpc_end_master: {
1688 // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
1689 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1690 llvm::FunctionType *FnTy =
1691 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1692 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
1693 break;
1694 }
Alexey Bataev9f797f32015-02-05 05:57:51 +00001695 case OMPRTL__kmpc_omp_taskyield: {
1696 // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
1697 // int end_part);
1698 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1699 llvm::FunctionType *FnTy =
1700 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1701 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
1702 break;
1703 }
Alexey Bataev6956e2e2015-02-05 06:35:41 +00001704 case OMPRTL__kmpc_single: {
1705 // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
1706 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1707 llvm::FunctionType *FnTy =
1708 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1709 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
1710 break;
1711 }
1712 case OMPRTL__kmpc_end_single: {
1713 // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
1714 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1715 llvm::FunctionType *FnTy =
1716 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1717 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
1718 break;
1719 }
Alexey Bataev62b63b12015-03-10 07:28:44 +00001720 case OMPRTL__kmpc_omp_task_alloc: {
1721 // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
1722 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1723 // kmp_routine_entry_t *task_entry);
1724 assert(KmpRoutineEntryPtrTy != nullptr &&
1725 "Type kmp_routine_entry_t must be created.");
1726 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1727 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
1728 // Return void * and then cast to particular kmp_task_t type.
1729 llvm::FunctionType *FnTy =
1730 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
1731 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
1732 break;
1733 }
1734 case OMPRTL__kmpc_omp_task: {
1735 // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1736 // *new_task);
1737 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1738 CGM.VoidPtrTy};
1739 llvm::FunctionType *FnTy =
1740 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1741 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
1742 break;
1743 }
Alexey Bataeva63048e2015-03-23 06:18:07 +00001744 case OMPRTL__kmpc_copyprivate: {
1745 // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
Alexey Bataev66beaa92015-04-30 03:47:32 +00001746 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
Alexey Bataeva63048e2015-03-23 06:18:07 +00001747 // kmp_int32 didit);
1748 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1749 auto *CpyFnTy =
1750 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
Alexey Bataev66beaa92015-04-30 03:47:32 +00001751 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
Alexey Bataeva63048e2015-03-23 06:18:07 +00001752 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
1753 CGM.Int32Ty};
1754 llvm::FunctionType *FnTy =
1755 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1756 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
1757 break;
1758 }
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001759 case OMPRTL__kmpc_reduce: {
1760 // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
1761 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
1762 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
1763 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1764 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1765 /*isVarArg=*/false);
1766 llvm::Type *TypeParams[] = {
1767 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1768 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1769 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1770 llvm::FunctionType *FnTy =
1771 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1772 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
1773 break;
1774 }
1775 case OMPRTL__kmpc_reduce_nowait: {
1776 // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
1777 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
1778 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
1779 // *lck);
1780 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1781 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1782 /*isVarArg=*/false);
1783 llvm::Type *TypeParams[] = {
1784 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1785 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1786 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1787 llvm::FunctionType *FnTy =
1788 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1789 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
1790 break;
1791 }
1792 case OMPRTL__kmpc_end_reduce: {
1793 // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
1794 // kmp_critical_name *lck);
1795 llvm::Type *TypeParams[] = {
1796 getIdentTyPointerTy(), CGM.Int32Ty,
1797 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1798 llvm::FunctionType *FnTy =
1799 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1800 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
1801 break;
1802 }
1803 case OMPRTL__kmpc_end_reduce_nowait: {
1804 // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
1805 // kmp_critical_name *lck);
1806 llvm::Type *TypeParams[] = {
1807 getIdentTyPointerTy(), CGM.Int32Ty,
1808 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1809 llvm::FunctionType *FnTy =
1810 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1811 RTLFn =
1812 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
1813 break;
1814 }
Alexey Bataev1d677132015-04-22 13:57:31 +00001815 case OMPRTL__kmpc_omp_task_begin_if0: {
1816 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1817 // *new_task);
1818 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1819 CGM.VoidPtrTy};
1820 llvm::FunctionType *FnTy =
1821 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1822 RTLFn =
1823 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
1824 break;
1825 }
1826 case OMPRTL__kmpc_omp_task_complete_if0: {
1827 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1828 // *new_task);
1829 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1830 CGM.VoidPtrTy};
1831 llvm::FunctionType *FnTy =
1832 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1833 RTLFn = CGM.CreateRuntimeFunction(FnTy,
1834 /*Name=*/"__kmpc_omp_task_complete_if0");
1835 break;
1836 }
Alexey Bataev98eb6e32015-04-22 11:15:40 +00001837 case OMPRTL__kmpc_ordered: {
1838 // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
1839 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1840 llvm::FunctionType *FnTy =
1841 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1842 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
1843 break;
1844 }
1845 case OMPRTL__kmpc_end_ordered: {
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00001846 // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
Alexey Bataev98eb6e32015-04-22 11:15:40 +00001847 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1848 llvm::FunctionType *FnTy =
1849 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1850 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
1851 break;
1852 }
Alexey Bataev8b8e2022015-04-27 05:22:09 +00001853 case OMPRTL__kmpc_omp_taskwait: {
1854 // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
1855 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1856 llvm::FunctionType *FnTy =
1857 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1858 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
1859 break;
1860 }
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00001861 case OMPRTL__kmpc_taskgroup: {
1862 // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
1863 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1864 llvm::FunctionType *FnTy =
1865 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1866 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
1867 break;
1868 }
1869 case OMPRTL__kmpc_end_taskgroup: {
1870 // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
1871 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1872 llvm::FunctionType *FnTy =
1873 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1874 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
1875 break;
1876 }
Alexey Bataev7f210c62015-06-18 13:40:03 +00001877 case OMPRTL__kmpc_push_proc_bind: {
1878 // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
1879 // int proc_bind)
1880 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1881 llvm::FunctionType *FnTy =
1882 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1883 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
1884 break;
1885 }
Alexey Bataev1d2353d2015-06-24 11:01:36 +00001886 case OMPRTL__kmpc_omp_task_with_deps: {
1887 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
1888 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
1889 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
1890 llvm::Type *TypeParams[] = {
1891 getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
1892 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
1893 llvm::FunctionType *FnTy =
1894 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1895 RTLFn =
1896 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
1897 break;
1898 }
1899 case OMPRTL__kmpc_omp_wait_deps: {
1900 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
1901 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
1902 // kmp_depend_info_t *noalias_dep_list);
1903 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1904 CGM.Int32Ty, CGM.VoidPtrTy,
1905 CGM.Int32Ty, CGM.VoidPtrTy};
1906 llvm::FunctionType *FnTy =
1907 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1908 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
1909 break;
1910 }
Alexey Bataev0f34da12015-07-02 04:17:07 +00001911 case OMPRTL__kmpc_cancellationpoint: {
1912 // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
1913 // global_tid, kmp_int32 cncl_kind)
1914 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1915 llvm::FunctionType *FnTy =
1916 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1917 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
1918 break;
1919 }
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00001920 case OMPRTL__kmpc_cancel: {
1921 // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
1922 // kmp_int32 cncl_kind)
1923 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1924 llvm::FunctionType *FnTy =
1925 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1926 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
1927 break;
1928 }
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00001929 case OMPRTL__kmpc_push_num_teams: {
1930 // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
1931 // kmp_int32 num_teams, kmp_int32 num_threads)
1932 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1933 CGM.Int32Ty};
1934 llvm::FunctionType *FnTy =
1935 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1936 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
1937 break;
1938 }
1939 case OMPRTL__kmpc_fork_teams: {
1940 // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
1941 // microtask, ...);
1942 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1943 getKmpc_MicroPointerTy()};
1944 llvm::FunctionType *FnTy =
1945 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1946 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
1947 break;
1948 }
Alexey Bataev7292c292016-04-25 12:22:29 +00001949 case OMPRTL__kmpc_taskloop: {
1950 // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
1951 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
1952 // sched, kmp_uint64 grainsize, void *task_dup);
1953 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
1954 CGM.IntTy,
1955 CGM.VoidPtrTy,
1956 CGM.IntTy,
1957 CGM.Int64Ty->getPointerTo(),
1958 CGM.Int64Ty->getPointerTo(),
1959 CGM.Int64Ty,
1960 CGM.IntTy,
1961 CGM.IntTy,
1962 CGM.Int64Ty,
1963 CGM.VoidPtrTy};
1964 llvm::FunctionType *FnTy =
1965 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1966 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
1967 break;
1968 }
Alexey Bataev8b427062016-05-25 12:36:08 +00001969 case OMPRTL__kmpc_doacross_init: {
1970 // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
1971 // num_dims, struct kmp_dim *dims);
1972 llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
1973 CGM.Int32Ty,
1974 CGM.Int32Ty,
1975 CGM.VoidPtrTy};
1976 llvm::FunctionType *FnTy =
1977 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1978 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
1979 break;
1980 }
1981 case OMPRTL__kmpc_doacross_fini: {
1982 // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
1983 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1984 llvm::FunctionType *FnTy =
1985 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1986 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
1987 break;
1988 }
1989 case OMPRTL__kmpc_doacross_post: {
1990 // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
1991 // *vec);
1992 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1993 CGM.Int64Ty->getPointerTo()};
1994 llvm::FunctionType *FnTy =
1995 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1996 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
1997 break;
1998 }
1999 case OMPRTL__kmpc_doacross_wait: {
2000 // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2001 // *vec);
2002 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2003 CGM.Int64Ty->getPointerTo()};
2004 llvm::FunctionType *FnTy =
2005 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2006 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2007 break;
2008 }
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00002009 case OMPRTL__kmpc_task_reduction_init: {
2010 // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2011 // *data);
2012 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2013 llvm::FunctionType *FnTy =
2014 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2015 RTLFn =
2016 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2017 break;
2018 }
2019 case OMPRTL__kmpc_task_reduction_get_th_data: {
2020 // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2021 // *d);
2022 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2023 llvm::FunctionType *FnTy =
2024 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2025 RTLFn = CGM.CreateRuntimeFunction(
2026 FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2027 break;
2028 }
Samuel Antaobed3c462015-10-02 16:14:20 +00002029 case OMPRTL__tgt_target: {
George Rokos63bc9d62017-11-21 18:25:12 +00002030 // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2031 // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
Samuel Antaobed3c462015-10-02 16:14:20 +00002032 // *arg_types);
George Rokos63bc9d62017-11-21 18:25:12 +00002033 llvm::Type *TypeParams[] = {CGM.Int64Ty,
Samuel Antaobed3c462015-10-02 16:14:20 +00002034 CGM.VoidPtrTy,
2035 CGM.Int32Ty,
2036 CGM.VoidPtrPtrTy,
2037 CGM.VoidPtrPtrTy,
2038 CGM.SizeTy->getPointerTo(),
George Rokos63bc9d62017-11-21 18:25:12 +00002039 CGM.Int64Ty->getPointerTo()};
Samuel Antaobed3c462015-10-02 16:14:20 +00002040 llvm::FunctionType *FnTy =
2041 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2042 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2043 break;
2044 }
Samuel Antaob68e2db2016-03-03 16:20:23 +00002045 case OMPRTL__tgt_target_teams: {
George Rokos63bc9d62017-11-21 18:25:12 +00002046 // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
Samuel Antaob68e2db2016-03-03 16:20:23 +00002047 // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes,
George Rokos63bc9d62017-11-21 18:25:12 +00002048 // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2049 llvm::Type *TypeParams[] = {CGM.Int64Ty,
Samuel Antaob68e2db2016-03-03 16:20:23 +00002050 CGM.VoidPtrTy,
2051 CGM.Int32Ty,
2052 CGM.VoidPtrPtrTy,
2053 CGM.VoidPtrPtrTy,
2054 CGM.SizeTy->getPointerTo(),
George Rokos63bc9d62017-11-21 18:25:12 +00002055 CGM.Int64Ty->getPointerTo(),
Samuel Antaob68e2db2016-03-03 16:20:23 +00002056 CGM.Int32Ty,
2057 CGM.Int32Ty};
2058 llvm::FunctionType *FnTy =
2059 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2060 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2061 break;
2062 }
Samuel Antaoee8fb302016-01-06 13:42:12 +00002063 case OMPRTL__tgt_register_lib: {
2064 // Build void __tgt_register_lib(__tgt_bin_desc *desc);
2065 QualType ParamTy =
2066 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2067 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2068 llvm::FunctionType *FnTy =
2069 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2070 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_lib");
2071 break;
2072 }
2073 case OMPRTL__tgt_unregister_lib: {
2074 // Build void __tgt_unregister_lib(__tgt_bin_desc *desc);
2075 QualType ParamTy =
2076 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2077 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2078 llvm::FunctionType *FnTy =
2079 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2080 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_unregister_lib");
2081 break;
2082 }
Samuel Antaodf158d52016-04-27 22:58:19 +00002083 case OMPRTL__tgt_target_data_begin: {
George Rokos63bc9d62017-11-21 18:25:12 +00002084 // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2085 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2086 llvm::Type *TypeParams[] = {CGM.Int64Ty,
Samuel Antaodf158d52016-04-27 22:58:19 +00002087 CGM.Int32Ty,
2088 CGM.VoidPtrPtrTy,
2089 CGM.VoidPtrPtrTy,
2090 CGM.SizeTy->getPointerTo(),
George Rokos63bc9d62017-11-21 18:25:12 +00002091 CGM.Int64Ty->getPointerTo()};
Samuel Antaodf158d52016-04-27 22:58:19 +00002092 llvm::FunctionType *FnTy =
2093 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2094 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2095 break;
2096 }
2097 case OMPRTL__tgt_target_data_end: {
George Rokos63bc9d62017-11-21 18:25:12 +00002098 // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2099 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2100 llvm::Type *TypeParams[] = {CGM.Int64Ty,
Samuel Antaodf158d52016-04-27 22:58:19 +00002101 CGM.Int32Ty,
2102 CGM.VoidPtrPtrTy,
2103 CGM.VoidPtrPtrTy,
2104 CGM.SizeTy->getPointerTo(),
George Rokos63bc9d62017-11-21 18:25:12 +00002105 CGM.Int64Ty->getPointerTo()};
Samuel Antaodf158d52016-04-27 22:58:19 +00002106 llvm::FunctionType *FnTy =
2107 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2108 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2109 break;
2110 }
Samuel Antao8d2d7302016-05-26 18:30:22 +00002111 case OMPRTL__tgt_target_data_update: {
George Rokos63bc9d62017-11-21 18:25:12 +00002112 // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2113 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2114 llvm::Type *TypeParams[] = {CGM.Int64Ty,
Samuel Antao8d2d7302016-05-26 18:30:22 +00002115 CGM.Int32Ty,
2116 CGM.VoidPtrPtrTy,
2117 CGM.VoidPtrPtrTy,
2118 CGM.SizeTy->getPointerTo(),
George Rokos63bc9d62017-11-21 18:25:12 +00002119 CGM.Int64Ty->getPointerTo()};
Samuel Antao8d2d7302016-05-26 18:30:22 +00002120 llvm::FunctionType *FnTy =
2121 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2122 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2123 break;
2124 }
Alexey Bataev9959db52014-05-06 10:08:46 +00002125 }
Alexey Bataev50b3c952016-02-19 10:38:26 +00002126 assert(RTLFn && "Unable to find OpenMP runtime function");
Alexey Bataev9959db52014-05-06 10:08:46 +00002127 return RTLFn;
2128}
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00002129
Alexander Musman21212e42015-03-13 10:38:23 +00002130llvm::Constant *CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize,
2131 bool IVSigned) {
2132 assert((IVSize == 32 || IVSize == 64) &&
2133 "IV size is not compatible with the omp runtime");
2134 auto Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2135 : "__kmpc_for_static_init_4u")
2136 : (IVSigned ? "__kmpc_for_static_init_8"
2137 : "__kmpc_for_static_init_8u");
2138 auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2139 auto PtrTy = llvm::PointerType::getUnqual(ITy);
2140 llvm::Type *TypeParams[] = {
2141 getIdentTyPointerTy(), // loc
2142 CGM.Int32Ty, // tid
2143 CGM.Int32Ty, // schedtype
2144 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2145 PtrTy, // p_lower
2146 PtrTy, // p_upper
2147 PtrTy, // p_stride
2148 ITy, // incr
2149 ITy // chunk
2150 };
2151 llvm::FunctionType *FnTy =
2152 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2153 return CGM.CreateRuntimeFunction(FnTy, Name);
2154}
2155
Alexander Musman92bdaab2015-03-12 13:37:50 +00002156llvm::Constant *CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize,
2157 bool IVSigned) {
2158 assert((IVSize == 32 || IVSize == 64) &&
2159 "IV size is not compatible with the omp runtime");
2160 auto Name =
2161 IVSize == 32
2162 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2163 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2164 auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2165 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2166 CGM.Int32Ty, // tid
2167 CGM.Int32Ty, // schedtype
2168 ITy, // lower
2169 ITy, // upper
2170 ITy, // stride
2171 ITy // chunk
2172 };
2173 llvm::FunctionType *FnTy =
2174 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2175 return CGM.CreateRuntimeFunction(FnTy, Name);
2176}
2177
Alexey Bataev98eb6e32015-04-22 11:15:40 +00002178llvm::Constant *CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize,
2179 bool IVSigned) {
2180 assert((IVSize == 32 || IVSize == 64) &&
2181 "IV size is not compatible with the omp runtime");
2182 auto Name =
2183 IVSize == 32
2184 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2185 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2186 llvm::Type *TypeParams[] = {
2187 getIdentTyPointerTy(), // loc
2188 CGM.Int32Ty, // tid
2189 };
2190 llvm::FunctionType *FnTy =
2191 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2192 return CGM.CreateRuntimeFunction(FnTy, Name);
2193}
2194
Alexander Musman92bdaab2015-03-12 13:37:50 +00002195llvm::Constant *CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize,
2196 bool IVSigned) {
2197 assert((IVSize == 32 || IVSize == 64) &&
2198 "IV size is not compatible with the omp runtime");
2199 auto Name =
2200 IVSize == 32
2201 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2202 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2203 auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2204 auto PtrTy = llvm::PointerType::getUnqual(ITy);
2205 llvm::Type *TypeParams[] = {
2206 getIdentTyPointerTy(), // loc
2207 CGM.Int32Ty, // tid
2208 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2209 PtrTy, // p_lower
2210 PtrTy, // p_upper
2211 PtrTy // p_stride
2212 };
2213 llvm::FunctionType *FnTy =
2214 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2215 return CGM.CreateRuntimeFunction(FnTy, Name);
2216}
2217
Alexey Bataev97720002014-11-11 04:05:39 +00002218llvm::Constant *
2219CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
Samuel Antaof8b50122015-07-13 22:54:53 +00002220 assert(!CGM.getLangOpts().OpenMPUseTLS ||
2221 !CGM.getContext().getTargetInfo().isTLSSupported());
Alexey Bataev97720002014-11-11 04:05:39 +00002222 // Lookup the entry, lazily creating it if necessary.
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002223 return getOrCreateInternalVariable(CGM.Int8PtrPtrTy,
Alexey Bataev97720002014-11-11 04:05:39 +00002224 Twine(CGM.getMangledName(VD)) + ".cache.");
2225}
2226
John McCall7f416cc2015-09-08 08:05:57 +00002227Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2228 const VarDecl *VD,
2229 Address VDAddr,
2230 SourceLocation Loc) {
Samuel Antaof8b50122015-07-13 22:54:53 +00002231 if (CGM.getLangOpts().OpenMPUseTLS &&
2232 CGM.getContext().getTargetInfo().isTLSSupported())
2233 return VDAddr;
2234
John McCall7f416cc2015-09-08 08:05:57 +00002235 auto VarTy = VDAddr.getElementType();
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002236 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
John McCall7f416cc2015-09-08 08:05:57 +00002237 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2238 CGM.Int8PtrTy),
Alexey Bataev97720002014-11-11 04:05:39 +00002239 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2240 getOrCreateThreadPrivateCache(VD)};
John McCall7f416cc2015-09-08 08:05:57 +00002241 return Address(CGF.EmitRuntimeCall(
2242 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2243 VDAddr.getAlignment());
Alexey Bataev97720002014-11-11 04:05:39 +00002244}
2245
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002246void CGOpenMPRuntime::emitThreadPrivateVarInit(
John McCall7f416cc2015-09-08 08:05:57 +00002247 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
Alexey Bataev97720002014-11-11 04:05:39 +00002248 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2249 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2250 // library.
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002251 auto OMPLoc = emitUpdateLocation(CGF, Loc);
2252 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
Alexey Bataev97720002014-11-11 04:05:39 +00002253 OMPLoc);
2254 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2255 // to register constructor/destructor for variable.
2256 llvm::Value *Args[] = {OMPLoc,
John McCall7f416cc2015-09-08 08:05:57 +00002257 CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2258 CGM.VoidPtrTy),
Alexey Bataev97720002014-11-11 04:05:39 +00002259 Ctor, CopyCtor, Dtor};
Alexey Bataev1e4b7132014-12-03 12:11:24 +00002260 CGF.EmitRuntimeCall(
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002261 createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
Alexey Bataev97720002014-11-11 04:05:39 +00002262}
2263
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002264llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
John McCall7f416cc2015-09-08 08:05:57 +00002265 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
Alexey Bataev97720002014-11-11 04:05:39 +00002266 bool PerformInit, CodeGenFunction *CGF) {
Samuel Antaof8b50122015-07-13 22:54:53 +00002267 if (CGM.getLangOpts().OpenMPUseTLS &&
2268 CGM.getContext().getTargetInfo().isTLSSupported())
2269 return nullptr;
2270
Alexey Bataev97720002014-11-11 04:05:39 +00002271 VD = VD->getDefinition(CGM.getContext());
2272 if (VD && ThreadPrivateWithDefinition.count(VD) == 0) {
2273 ThreadPrivateWithDefinition.insert(VD);
2274 QualType ASTTy = VD->getType();
2275
2276 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2277 auto Init = VD->getAnyInitializer();
2278 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2279 // Generate function that re-emits the declaration's initializer into the
2280 // threadprivate copy of the variable VD
2281 CodeGenFunction CtorCGF(CGM);
2282 FunctionArgList Args;
Alexey Bataev56223232017-06-09 13:40:18 +00002283 ImplicitParamDecl Dst(CGM.getContext(), CGM.getContext().VoidPtrTy,
2284 ImplicitParamDecl::Other);
Alexey Bataev97720002014-11-11 04:05:39 +00002285 Args.push_back(&Dst);
2286
John McCallc56a8b32016-03-11 04:30:31 +00002287 auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2288 CGM.getContext().VoidPtrTy, Args);
Alexey Bataev97720002014-11-11 04:05:39 +00002289 auto FTy = CGM.getTypes().GetFunctionType(FI);
2290 auto Fn = CGM.CreateGlobalInitOrDestructFunction(
Akira Hatanaka7791f1a42015-10-31 01:28:07 +00002291 FTy, ".__kmpc_global_ctor_.", FI, Loc);
Alexey Bataev97720002014-11-11 04:05:39 +00002292 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2293 Args, SourceLocation());
2294 auto ArgVal = CtorCGF.EmitLoadOfScalar(
John McCall7f416cc2015-09-08 08:05:57 +00002295 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
Alexey Bataev97720002014-11-11 04:05:39 +00002296 CGM.getContext().VoidPtrTy, Dst.getLocation());
John McCall7f416cc2015-09-08 08:05:57 +00002297 Address Arg = Address(ArgVal, VDAddr.getAlignment());
2298 Arg = CtorCGF.Builder.CreateElementBitCast(Arg,
2299 CtorCGF.ConvertTypeForMem(ASTTy));
Alexey Bataev97720002014-11-11 04:05:39 +00002300 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2301 /*IsInitializer=*/true);
2302 ArgVal = CtorCGF.EmitLoadOfScalar(
John McCall7f416cc2015-09-08 08:05:57 +00002303 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
Alexey Bataev97720002014-11-11 04:05:39 +00002304 CGM.getContext().VoidPtrTy, Dst.getLocation());
2305 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2306 CtorCGF.FinishFunction();
2307 Ctor = Fn;
2308 }
2309 if (VD->getType().isDestructedType() != QualType::DK_none) {
2310 // Generate function that emits destructor call for the threadprivate copy
2311 // of the variable VD
2312 CodeGenFunction DtorCGF(CGM);
2313 FunctionArgList Args;
Alexey Bataev56223232017-06-09 13:40:18 +00002314 ImplicitParamDecl Dst(CGM.getContext(), CGM.getContext().VoidPtrTy,
2315 ImplicitParamDecl::Other);
Alexey Bataev97720002014-11-11 04:05:39 +00002316 Args.push_back(&Dst);
2317
John McCallc56a8b32016-03-11 04:30:31 +00002318 auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2319 CGM.getContext().VoidTy, Args);
Alexey Bataev97720002014-11-11 04:05:39 +00002320 auto FTy = CGM.getTypes().GetFunctionType(FI);
2321 auto Fn = CGM.CreateGlobalInitOrDestructFunction(
Akira Hatanaka7791f1a42015-10-31 01:28:07 +00002322 FTy, ".__kmpc_global_dtor_.", FI, Loc);
Adrian Prantl1858c662016-04-24 22:22:29 +00002323 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
Alexey Bataev97720002014-11-11 04:05:39 +00002324 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2325 SourceLocation());
Adrian Prantl1858c662016-04-24 22:22:29 +00002326 // Create a scope with an artificial location for the body of this function.
2327 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
Alexey Bataev97720002014-11-11 04:05:39 +00002328 auto ArgVal = DtorCGF.EmitLoadOfScalar(
2329 DtorCGF.GetAddrOfLocalVar(&Dst),
John McCall7f416cc2015-09-08 08:05:57 +00002330 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2331 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
Alexey Bataev97720002014-11-11 04:05:39 +00002332 DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2333 DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2334 DtorCGF.FinishFunction();
2335 Dtor = Fn;
2336 }
2337 // Do not emit init function if it is not required.
2338 if (!Ctor && !Dtor)
2339 return nullptr;
2340
2341 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2342 auto CopyCtorTy =
2343 llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2344 /*isVarArg=*/false)->getPointerTo();
2345 // Copying constructor for the threadprivate variable.
2346 // Must be NULL - reserved by runtime, but currently it requires that this
2347 // parameter is always NULL. Otherwise it fires assertion.
2348 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2349 if (Ctor == nullptr) {
2350 auto CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2351 /*isVarArg=*/false)->getPointerTo();
2352 Ctor = llvm::Constant::getNullValue(CtorTy);
2353 }
2354 if (Dtor == nullptr) {
2355 auto DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2356 /*isVarArg=*/false)->getPointerTo();
2357 Dtor = llvm::Constant::getNullValue(DtorTy);
2358 }
2359 if (!CGF) {
2360 auto InitFunctionTy =
2361 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2362 auto InitFunction = CGM.CreateGlobalInitOrDestructFunction(
Akira Hatanaka7791f1a42015-10-31 01:28:07 +00002363 InitFunctionTy, ".__omp_threadprivate_init_.",
2364 CGM.getTypes().arrangeNullaryFunction());
Alexey Bataev97720002014-11-11 04:05:39 +00002365 CodeGenFunction InitCGF(CGM);
2366 FunctionArgList ArgList;
2367 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2368 CGM.getTypes().arrangeNullaryFunction(), ArgList,
2369 Loc);
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002370 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
Alexey Bataev97720002014-11-11 04:05:39 +00002371 InitCGF.FinishFunction();
2372 return InitFunction;
2373 }
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002374 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
Alexey Bataev97720002014-11-11 04:05:39 +00002375 }
2376 return nullptr;
2377}
2378
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00002379Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2380 QualType VarType,
2381 StringRef Name) {
2382 llvm::Twine VarName(Name, ".artificial.");
2383 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
2384 llvm::Value *GAddr = getOrCreateInternalVariable(VarLVType, VarName);
2385 llvm::Value *Args[] = {
2386 emitUpdateLocation(CGF, SourceLocation()),
2387 getThreadID(CGF, SourceLocation()),
2388 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2389 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2390 /*IsSigned=*/false),
2391 getOrCreateInternalVariable(CGM.VoidPtrPtrTy, VarName + ".cache.")};
2392 return Address(
2393 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2394 CGF.EmitRuntimeCall(
2395 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2396 VarLVType->getPointerTo(/*AddrSpace=*/0)),
2397 CGM.getPointerAlign());
2398}
2399
Alexey Bataev1d677132015-04-22 13:57:31 +00002400/// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen
2401/// function. Here is the logic:
2402/// if (Cond) {
2403/// ThenGen();
2404/// } else {
2405/// ElseGen();
2406/// }
Arpith Chacko Jacobbb36fe82017-01-10 15:42:51 +00002407void CGOpenMPRuntime::emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
2408 const RegionCodeGenTy &ThenGen,
2409 const RegionCodeGenTy &ElseGen) {
Alexey Bataev1d677132015-04-22 13:57:31 +00002410 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
2411
2412 // If the condition constant folds and can be elided, try to avoid emitting
2413 // the condition and the dead arm of the if/else.
2414 bool CondConstant;
2415 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002416 if (CondConstant)
Alexey Bataev1d677132015-04-22 13:57:31 +00002417 ThenGen(CGF);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002418 else
Alexey Bataev1d677132015-04-22 13:57:31 +00002419 ElseGen(CGF);
Alexey Bataev1d677132015-04-22 13:57:31 +00002420 return;
2421 }
2422
2423 // Otherwise, the condition did not fold, or we couldn't elide it. Just
2424 // emit the conditional branch.
2425 auto ThenBlock = CGF.createBasicBlock("omp_if.then");
2426 auto ElseBlock = CGF.createBasicBlock("omp_if.else");
2427 auto ContBlock = CGF.createBasicBlock("omp_if.end");
2428 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
2429
2430 // Emit the 'then' code.
2431 CGF.EmitBlock(ThenBlock);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002432 ThenGen(CGF);
Alexey Bataev1d677132015-04-22 13:57:31 +00002433 CGF.EmitBranch(ContBlock);
2434 // Emit the 'else' code if present.
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002435 // There is no need to emit line number for unconditional branch.
2436 (void)ApplyDebugLocation::CreateEmpty(CGF);
2437 CGF.EmitBlock(ElseBlock);
2438 ElseGen(CGF);
2439 // There is no need to emit line number for unconditional branch.
2440 (void)ApplyDebugLocation::CreateEmpty(CGF);
2441 CGF.EmitBranch(ContBlock);
Alexey Bataev1d677132015-04-22 13:57:31 +00002442 // Emit the continuation block for code after the if.
2443 CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
Alexey Bataev4a5bb772014-10-08 14:01:46 +00002444}
2445
Alexey Bataev1d677132015-04-22 13:57:31 +00002446void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
2447 llvm::Value *OutlinedFn,
Alexey Bataev2377fe92015-09-10 08:12:02 +00002448 ArrayRef<llvm::Value *> CapturedVars,
Alexey Bataev1d677132015-04-22 13:57:31 +00002449 const Expr *IfCond) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00002450 if (!CGF.HaveInsertPoint())
2451 return;
Alexey Bataev1d677132015-04-22 13:57:31 +00002452 auto *RTLoc = emitUpdateLocation(CGF, Loc);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002453 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
2454 PrePostActionTy &) {
Alexey Bataev2377fe92015-09-10 08:12:02 +00002455 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002456 auto &RT = CGF.CGM.getOpenMPRuntime();
Alexey Bataev2377fe92015-09-10 08:12:02 +00002457 llvm::Value *Args[] = {
2458 RTLoc,
2459 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002460 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
Alexey Bataev2377fe92015-09-10 08:12:02 +00002461 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2462 RealArgs.append(std::begin(Args), std::end(Args));
2463 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2464
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002465 auto RTLFn = RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
Alexey Bataev2377fe92015-09-10 08:12:02 +00002466 CGF.EmitRuntimeCall(RTLFn, RealArgs);
2467 };
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002468 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
2469 PrePostActionTy &) {
2470 auto &RT = CGF.CGM.getOpenMPRuntime();
2471 auto ThreadID = RT.getThreadID(CGF, Loc);
Alexey Bataev1d677132015-04-22 13:57:31 +00002472 // Build calls:
2473 // __kmpc_serialized_parallel(&Loc, GTid);
2474 llvm::Value *Args[] = {RTLoc, ThreadID};
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002475 CGF.EmitRuntimeCall(
2476 RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
Alexey Bataevd74d0602014-10-13 06:02:40 +00002477
Alexey Bataev1d677132015-04-22 13:57:31 +00002478 // OutlinedFn(&GTid, &zero, CapturedStruct);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002479 auto ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
John McCall7f416cc2015-09-08 08:05:57 +00002480 Address ZeroAddr =
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002481 CGF.CreateTempAlloca(CGF.Int32Ty, CharUnits::fromQuantity(4),
2482 /*Name*/ ".zero.addr");
Alexey Bataev1d677132015-04-22 13:57:31 +00002483 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
Alexey Bataev2377fe92015-09-10 08:12:02 +00002484 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2485 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
2486 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
2487 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
Alexey Bataev3c595a62017-08-14 15:01:03 +00002488 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
Alexey Bataevd74d0602014-10-13 06:02:40 +00002489
Alexey Bataev1d677132015-04-22 13:57:31 +00002490 // __kmpc_end_serialized_parallel(&Loc, GTid);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002491 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
Alexey Bataev1d677132015-04-22 13:57:31 +00002492 CGF.EmitRuntimeCall(
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002493 RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
2494 EndArgs);
Alexey Bataev1d677132015-04-22 13:57:31 +00002495 };
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002496 if (IfCond)
Alexey Bataev1d677132015-04-22 13:57:31 +00002497 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002498 else {
2499 RegionCodeGenTy ThenRCG(ThenGen);
2500 ThenRCG(CGF);
Alexey Bataevf539faa2016-03-28 12:58:34 +00002501 }
Alexey Bataevd74d0602014-10-13 06:02:40 +00002502}
2503
NAKAMURA Takumi59c74b222014-10-27 08:08:18 +00002504// If we're inside an (outlined) parallel region, use the region info's
Alexey Bataevd74d0602014-10-13 06:02:40 +00002505// thread-ID variable (it is passed in a first argument of the outlined function
2506// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2507// regular serial code region, get thread ID by calling kmp_int32
2508// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2509// return the address of that temp.
John McCall7f416cc2015-09-08 08:05:57 +00002510Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2511 SourceLocation Loc) {
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002512 if (auto *OMPRegionInfo =
Alexey Bataevd74d0602014-10-13 06:02:40 +00002513 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
Alexey Bataev8cbe0a62015-02-26 10:27:34 +00002514 if (OMPRegionInfo->getThreadIDVariable())
Alexey Bataev62b63b12015-03-10 07:28:44 +00002515 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
Alexey Bataev8cbe0a62015-02-26 10:27:34 +00002516
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002517 auto ThreadID = getThreadID(CGF, Loc);
Alexey Bataevd74d0602014-10-13 06:02:40 +00002518 auto Int32Ty =
2519 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2520 auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
2521 CGF.EmitStoreOfScalar(ThreadID,
John McCall7f416cc2015-09-08 08:05:57 +00002522 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
Alexey Bataevd74d0602014-10-13 06:02:40 +00002523
2524 return ThreadIDTemp;
2525}
2526
Alexey Bataev97720002014-11-11 04:05:39 +00002527llvm::Constant *
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002528CGOpenMPRuntime::getOrCreateInternalVariable(llvm::Type *Ty,
Alexey Bataev97720002014-11-11 04:05:39 +00002529 const llvm::Twine &Name) {
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00002530 SmallString<256> Buffer;
2531 llvm::raw_svector_ostream Out(Buffer);
Alexey Bataev97720002014-11-11 04:05:39 +00002532 Out << Name;
2533 auto RuntimeName = Out.str();
David Blaikie13156b62014-11-19 03:06:06 +00002534 auto &Elem = *InternalVars.insert(std::make_pair(RuntimeName, nullptr)).first;
2535 if (Elem.second) {
2536 assert(Elem.second->getType()->getPointerElementType() == Ty &&
Alexey Bataev97720002014-11-11 04:05:39 +00002537 "OMP internal variable has different type than requested");
David Blaikie13156b62014-11-19 03:06:06 +00002538 return &*Elem.second;
Alexey Bataev97720002014-11-11 04:05:39 +00002539 }
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00002540
David Blaikie13156b62014-11-19 03:06:06 +00002541 return Elem.second = new llvm::GlobalVariable(
2542 CGM.getModule(), Ty, /*IsConstant*/ false,
2543 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
2544 Elem.first());
Alexey Bataev97720002014-11-11 04:05:39 +00002545}
2546
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002547llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
Alexey Bataev97720002014-11-11 04:05:39 +00002548 llvm::Twine Name(".gomp_critical_user_", CriticalName);
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002549 return getOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var"));
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00002550}
2551
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002552namespace {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002553/// Common pre(post)-action for different OpenMP constructs.
2554class CommonActionTy final : public PrePostActionTy {
2555 llvm::Value *EnterCallee;
2556 ArrayRef<llvm::Value *> EnterArgs;
2557 llvm::Value *ExitCallee;
2558 ArrayRef<llvm::Value *> ExitArgs;
2559 bool Conditional;
2560 llvm::BasicBlock *ContBlock = nullptr;
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002561
2562public:
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002563 CommonActionTy(llvm::Value *EnterCallee, ArrayRef<llvm::Value *> EnterArgs,
2564 llvm::Value *ExitCallee, ArrayRef<llvm::Value *> ExitArgs,
2565 bool Conditional = false)
2566 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2567 ExitArgs(ExitArgs), Conditional(Conditional) {}
2568 void Enter(CodeGenFunction &CGF) override {
2569 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
2570 if (Conditional) {
2571 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
2572 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
2573 ContBlock = CGF.createBasicBlock("omp_if.end");
2574 // Generate the branch (If-stmt)
2575 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2576 CGF.EmitBlock(ThenBlock);
2577 }
Alexey Bataeva744ff52015-05-05 09:24:37 +00002578 }
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002579 void Done(CodeGenFunction &CGF) {
2580 // Emit the rest of blocks/branches
2581 CGF.EmitBranch(ContBlock);
2582 CGF.EmitBlock(ContBlock, true);
2583 }
2584 void Exit(CodeGenFunction &CGF) override {
2585 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
Alexey Bataev3e6124b2015-04-10 07:48:12 +00002586 }
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002587};
Hans Wennborg7eb54642015-09-10 17:07:54 +00002588} // anonymous namespace
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002589
2590void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2591 StringRef CriticalName,
2592 const RegionCodeGenTy &CriticalOpGen,
Alexey Bataevfc57d162015-12-15 10:55:09 +00002593 SourceLocation Loc, const Expr *Hint) {
2594 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
Alexey Bataev75ddfab2014-12-01 11:32:38 +00002595 // CriticalOpGen();
2596 // __kmpc_end_critical(ident_t *, gtid, Lock);
2597 // Prepare arguments and build a call to __kmpc_critical
Alexey Bataev8ef31412015-12-18 07:58:25 +00002598 if (!CGF.HaveInsertPoint())
2599 return;
Alexey Bataevfc57d162015-12-15 10:55:09 +00002600 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2601 getCriticalRegionLock(CriticalName)};
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002602 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
2603 std::end(Args));
Alexey Bataevfc57d162015-12-15 10:55:09 +00002604 if (Hint) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002605 EnterArgs.push_back(CGF.Builder.CreateIntCast(
2606 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
2607 }
2608 CommonActionTy Action(
2609 createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
2610 : OMPRTL__kmpc_critical),
2611 EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
2612 CriticalOpGen.setAction(Action);
Alexey Bataevfc57d162015-12-15 10:55:09 +00002613 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00002614}
Alexey Bataev4a5bb772014-10-08 14:01:46 +00002615
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002616void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002617 const RegionCodeGenTy &MasterOpGen,
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002618 SourceLocation Loc) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00002619 if (!CGF.HaveInsertPoint())
2620 return;
Alexey Bataev8d690652014-12-04 07:23:53 +00002621 // if(__kmpc_master(ident_t *, gtid)) {
2622 // MasterOpGen();
2623 // __kmpc_end_master(ident_t *, gtid);
2624 // }
2625 // Prepare arguments and build a call to __kmpc_master
Alexey Bataevd7614fb2015-04-10 06:33:45 +00002626 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002627 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
2628 createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
2629 /*Conditional=*/true);
2630 MasterOpGen.setAction(Action);
2631 emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
2632 Action.Done(CGF);
Alexey Bataev8d690652014-12-04 07:23:53 +00002633}
2634
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002635void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2636 SourceLocation Loc) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00002637 if (!CGF.HaveInsertPoint())
2638 return;
Alexey Bataev9f797f32015-02-05 05:57:51 +00002639 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2640 llvm::Value *Args[] = {
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002641 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
Alexey Bataev9f797f32015-02-05 05:57:51 +00002642 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002643 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
Alexey Bataev48591dd2016-04-20 04:01:36 +00002644 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2645 Region->emitUntiedSwitch(CGF);
Alexey Bataev9f797f32015-02-05 05:57:51 +00002646}
2647
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00002648void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2649 const RegionCodeGenTy &TaskgroupOpGen,
2650 SourceLocation Loc) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00002651 if (!CGF.HaveInsertPoint())
2652 return;
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00002653 // __kmpc_taskgroup(ident_t *, gtid);
2654 // TaskgroupOpGen();
2655 // __kmpc_end_taskgroup(ident_t *, gtid);
2656 // Prepare arguments and build a call to __kmpc_taskgroup
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002657 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2658 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
2659 createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
2660 Args);
2661 TaskgroupOpGen.setAction(Action);
2662 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00002663}
2664
John McCall7f416cc2015-09-08 08:05:57 +00002665/// Given an array of pointers to variables, project the address of a
2666/// given variable.
Alexey Bataevf24e7b12015-10-08 09:10:53 +00002667static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2668 unsigned Index, const VarDecl *Var) {
John McCall7f416cc2015-09-08 08:05:57 +00002669 // Pull out the pointer to the variable.
2670 Address PtrAddr =
Alexey Bataevf24e7b12015-10-08 09:10:53 +00002671 CGF.Builder.CreateConstArrayGEP(Array, Index, CGF.getPointerSize());
John McCall7f416cc2015-09-08 08:05:57 +00002672 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
2673
2674 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
Alexey Bataevf24e7b12015-10-08 09:10:53 +00002675 Addr = CGF.Builder.CreateElementBitCast(
2676 Addr, CGF.ConvertTypeForMem(Var->getType()));
John McCall7f416cc2015-09-08 08:05:57 +00002677 return Addr;
2678}
2679
Alexey Bataeva63048e2015-03-23 06:18:07 +00002680static llvm::Value *emitCopyprivateCopyFunction(
Alexey Bataev420d45b2015-04-14 05:11:24 +00002681 CodeGenModule &CGM, llvm::Type *ArgsType,
2682 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2683 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps) {
Alexey Bataeva63048e2015-03-23 06:18:07 +00002684 auto &C = CGM.getContext();
2685 // void copy_func(void *LHSArg, void *RHSArg);
2686 FunctionArgList Args;
Alexey Bataev56223232017-06-09 13:40:18 +00002687 ImplicitParamDecl LHSArg(C, C.VoidPtrTy, ImplicitParamDecl::Other);
2688 ImplicitParamDecl RHSArg(C, C.VoidPtrTy, ImplicitParamDecl::Other);
Alexey Bataeva63048e2015-03-23 06:18:07 +00002689 Args.push_back(&LHSArg);
2690 Args.push_back(&RHSArg);
John McCallc56a8b32016-03-11 04:30:31 +00002691 auto &CGFI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
Alexey Bataeva63048e2015-03-23 06:18:07 +00002692 auto *Fn = llvm::Function::Create(
2693 CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
2694 ".omp.copyprivate.copy_func", &CGM.getModule());
Akira Hatanaka44a59f82015-10-28 02:30:47 +00002695 CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, CGFI);
Alexey Bataeva63048e2015-03-23 06:18:07 +00002696 CodeGenFunction CGF(CGM);
2697 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args);
Alexey Bataev420d45b2015-04-14 05:11:24 +00002698 // Dest = (void*[n])(LHSArg);
Alexey Bataeva63048e2015-03-23 06:18:07 +00002699 // Src = (void*[n])(RHSArg);
John McCall7f416cc2015-09-08 08:05:57 +00002700 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2701 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
2702 ArgsType), CGF.getPointerAlign());
2703 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2704 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
2705 ArgsType), CGF.getPointerAlign());
Alexey Bataeva63048e2015-03-23 06:18:07 +00002706 // *(Type0*)Dst[0] = *(Type0*)Src[0];
2707 // *(Type1*)Dst[1] = *(Type1*)Src[1];
2708 // ...
2709 // *(Typen*)Dst[n] = *(Typen*)Src[n];
Alexey Bataeva63048e2015-03-23 06:18:07 +00002710 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
John McCall7f416cc2015-09-08 08:05:57 +00002711 auto DestVar = cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
2712 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
2713
2714 auto SrcVar = cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
2715 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
2716
Alexey Bataev1d9c15c2015-05-19 12:31:28 +00002717 auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
2718 QualType Type = VD->getType();
John McCall7f416cc2015-09-08 08:05:57 +00002719 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
Alexey Bataeva63048e2015-03-23 06:18:07 +00002720 }
Alexey Bataeva63048e2015-03-23 06:18:07 +00002721 CGF.FinishFunction();
2722 return Fn;
2723}
2724
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002725void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002726 const RegionCodeGenTy &SingleOpGen,
Alexey Bataeva63048e2015-03-23 06:18:07 +00002727 SourceLocation Loc,
2728 ArrayRef<const Expr *> CopyprivateVars,
2729 ArrayRef<const Expr *> SrcExprs,
2730 ArrayRef<const Expr *> DstExprs,
2731 ArrayRef<const Expr *> AssignmentOps) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00002732 if (!CGF.HaveInsertPoint())
2733 return;
Alexey Bataeva63048e2015-03-23 06:18:07 +00002734 assert(CopyprivateVars.size() == SrcExprs.size() &&
2735 CopyprivateVars.size() == DstExprs.size() &&
2736 CopyprivateVars.size() == AssignmentOps.size());
2737 auto &C = CGM.getContext();
2738 // int32 did_it = 0;
Alexey Bataev6956e2e2015-02-05 06:35:41 +00002739 // if(__kmpc_single(ident_t *, gtid)) {
2740 // SingleOpGen();
2741 // __kmpc_end_single(ident_t *, gtid);
Alexey Bataeva63048e2015-03-23 06:18:07 +00002742 // did_it = 1;
Alexey Bataev6956e2e2015-02-05 06:35:41 +00002743 // }
Alexey Bataeva63048e2015-03-23 06:18:07 +00002744 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2745 // <copy_func>, did_it);
2746
John McCall7f416cc2015-09-08 08:05:57 +00002747 Address DidIt = Address::invalid();
Alexey Bataeva63048e2015-03-23 06:18:07 +00002748 if (!CopyprivateVars.empty()) {
2749 // int32 did_it = 0;
2750 auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2751 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
John McCall7f416cc2015-09-08 08:05:57 +00002752 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
Alexey Bataeva63048e2015-03-23 06:18:07 +00002753 }
Alexey Bataev6956e2e2015-02-05 06:35:41 +00002754 // Prepare arguments and build a call to __kmpc_single
Alexey Bataevd7614fb2015-04-10 06:33:45 +00002755 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002756 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
2757 createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
2758 /*Conditional=*/true);
2759 SingleOpGen.setAction(Action);
2760 emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
2761 if (DidIt.isValid()) {
2762 // did_it = 1;
2763 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
2764 }
2765 Action.Done(CGF);
Alexey Bataeva63048e2015-03-23 06:18:07 +00002766 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2767 // <copy_func>, did_it);
John McCall7f416cc2015-09-08 08:05:57 +00002768 if (DidIt.isValid()) {
Alexey Bataeva63048e2015-03-23 06:18:07 +00002769 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
2770 auto CopyprivateArrayTy =
2771 C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
2772 /*IndexTypeQuals=*/0);
2773 // Create a list of all private variables for copyprivate.
John McCall7f416cc2015-09-08 08:05:57 +00002774 Address CopyprivateList =
Alexey Bataeva63048e2015-03-23 06:18:07 +00002775 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
2776 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
John McCall7f416cc2015-09-08 08:05:57 +00002777 Address Elem = CGF.Builder.CreateConstArrayGEP(
2778 CopyprivateList, I, CGF.getPointerSize());
2779 CGF.Builder.CreateStore(
Alexey Bataeva63048e2015-03-23 06:18:07 +00002780 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
John McCall7f416cc2015-09-08 08:05:57 +00002781 CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy),
2782 Elem);
Alexey Bataeva63048e2015-03-23 06:18:07 +00002783 }
2784 // Build function that copies private values from single region to all other
2785 // threads in the corresponding parallel region.
2786 auto *CpyFn = emitCopyprivateCopyFunction(
2787 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
Alexey Bataev420d45b2015-04-14 05:11:24 +00002788 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps);
Alexey Bataev1189bd02016-01-26 12:20:39 +00002789 auto *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
John McCall7f416cc2015-09-08 08:05:57 +00002790 Address CL =
2791 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
2792 CGF.VoidPtrTy);
2793 auto *DidItVal = CGF.Builder.CreateLoad(DidIt);
Alexey Bataeva63048e2015-03-23 06:18:07 +00002794 llvm::Value *Args[] = {
2795 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2796 getThreadID(CGF, Loc), // i32 <gtid>
Alexey Bataev66beaa92015-04-30 03:47:32 +00002797 BufSize, // size_t <buf_size>
John McCall7f416cc2015-09-08 08:05:57 +00002798 CL.getPointer(), // void *<copyprivate list>
Alexey Bataeva63048e2015-03-23 06:18:07 +00002799 CpyFn, // void (*) (void *, void *) <copy_func>
2800 DidItVal // i32 did_it
2801 };
2802 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
2803 }
Alexey Bataev6956e2e2015-02-05 06:35:41 +00002804}
2805
Alexey Bataev98eb6e32015-04-22 11:15:40 +00002806void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2807 const RegionCodeGenTy &OrderedOpGen,
Alexey Bataev5f600d62015-09-29 03:48:57 +00002808 SourceLocation Loc, bool IsThreads) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00002809 if (!CGF.HaveInsertPoint())
2810 return;
Alexey Bataev98eb6e32015-04-22 11:15:40 +00002811 // __kmpc_ordered(ident_t *, gtid);
2812 // OrderedOpGen();
2813 // __kmpc_end_ordered(ident_t *, gtid);
2814 // Prepare arguments and build a call to __kmpc_ordered
Alexey Bataev5f600d62015-09-29 03:48:57 +00002815 if (IsThreads) {
Alexey Bataev98eb6e32015-04-22 11:15:40 +00002816 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002817 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
2818 createRuntimeFunction(OMPRTL__kmpc_end_ordered),
2819 Args);
2820 OrderedOpGen.setAction(Action);
2821 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
2822 return;
Alexey Bataev98eb6e32015-04-22 11:15:40 +00002823 }
Alexey Bataev5f600d62015-09-29 03:48:57 +00002824 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
Alexey Bataev98eb6e32015-04-22 11:15:40 +00002825}
2826
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002827void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
Alexey Bataev25e5b442015-09-15 12:52:43 +00002828 OpenMPDirectiveKind Kind, bool EmitChecks,
2829 bool ForceSimpleCall) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00002830 if (!CGF.HaveInsertPoint())
2831 return;
Alexey Bataev8f7c1b02014-12-05 04:09:23 +00002832 // Build call __kmpc_cancel_barrier(loc, thread_id);
Alexey Bataev81c7ea02015-07-03 09:56:58 +00002833 // Build call __kmpc_barrier(loc, thread_id);
Alexey Bataev50b3c952016-02-19 10:38:26 +00002834 unsigned Flags;
2835 if (Kind == OMPD_for)
2836 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2837 else if (Kind == OMPD_sections)
2838 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2839 else if (Kind == OMPD_single)
2840 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2841 else if (Kind == OMPD_barrier)
2842 Flags = OMP_IDENT_BARRIER_EXPL;
2843 else
2844 Flags = OMP_IDENT_BARRIER_IMPL;
Alexey Bataev81c7ea02015-07-03 09:56:58 +00002845 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
2846 // thread_id);
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002847 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2848 getThreadID(CGF, Loc)};
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002849 if (auto *OMPRegionInfo =
2850 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
Alexey Bataev25e5b442015-09-15 12:52:43 +00002851 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
Alexey Bataev81c7ea02015-07-03 09:56:58 +00002852 auto *Result = CGF.EmitRuntimeCall(
2853 createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
Alexey Bataev25e5b442015-09-15 12:52:43 +00002854 if (EmitChecks) {
Alexey Bataev81c7ea02015-07-03 09:56:58 +00002855 // if (__kmpc_cancel_barrier()) {
2856 // exit from construct;
2857 // }
2858 auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
2859 auto *ContBB = CGF.createBasicBlock(".cancel.continue");
2860 auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
2861 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2862 CGF.EmitBlock(ExitBB);
2863 // exit from construct;
Alexey Bataev25e5b442015-09-15 12:52:43 +00002864 auto CancelDestination =
2865 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
Alexey Bataev81c7ea02015-07-03 09:56:58 +00002866 CGF.EmitBranchThroughCleanup(CancelDestination);
2867 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
2868 }
2869 return;
2870 }
2871 }
2872 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
Alexey Bataev4a5bb772014-10-08 14:01:46 +00002873}
2874
Alexander Musmanc6388682014-12-15 07:07:06 +00002875/// \brief Map the OpenMP loop schedule to the runtime enumeration.
2876static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00002877 bool Chunked, bool Ordered) {
Alexander Musmanc6388682014-12-15 07:07:06 +00002878 switch (ScheduleKind) {
2879 case OMPC_SCHEDULE_static:
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00002880 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2881 : (Ordered ? OMP_ord_static : OMP_sch_static);
Alexander Musmanc6388682014-12-15 07:07:06 +00002882 case OMPC_SCHEDULE_dynamic:
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00002883 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
Alexander Musmanc6388682014-12-15 07:07:06 +00002884 case OMPC_SCHEDULE_guided:
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00002885 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
Alexander Musmanc6388682014-12-15 07:07:06 +00002886 case OMPC_SCHEDULE_runtime:
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00002887 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2888 case OMPC_SCHEDULE_auto:
2889 return Ordered ? OMP_ord_auto : OMP_sch_auto;
Alexander Musmanc6388682014-12-15 07:07:06 +00002890 case OMPC_SCHEDULE_unknown:
2891 assert(!Chunked && "chunk was specified but schedule kind not known");
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00002892 return Ordered ? OMP_ord_static : OMP_sch_static;
Alexander Musmanc6388682014-12-15 07:07:06 +00002893 }
2894 llvm_unreachable("Unexpected runtime schedule");
2895}
2896
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002897/// \brief Map the OpenMP distribute schedule to the runtime enumeration.
2898static OpenMPSchedType
2899getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
2900 // only static is allowed for dist_schedule
2901 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2902}
2903
Alexander Musmanc6388682014-12-15 07:07:06 +00002904bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
2905 bool Chunked) const {
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00002906 auto Schedule = getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
Alexander Musmanc6388682014-12-15 07:07:06 +00002907 return Schedule == OMP_sch_static;
2908}
2909
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002910bool CGOpenMPRuntime::isStaticNonchunked(
2911 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2912 auto Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2913 return Schedule == OMP_dist_sch_static;
2914}
2915
2916
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002917bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00002918 auto Schedule =
2919 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002920 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
2921 return Schedule != OMP_sch_static;
2922}
2923
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002924static int addMonoNonMonoModifier(OpenMPSchedType Schedule,
2925 OpenMPScheduleClauseModifier M1,
2926 OpenMPScheduleClauseModifier M2) {
Alexey Bataev6cff6242016-05-30 13:05:14 +00002927 int Modifier = 0;
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002928 switch (M1) {
2929 case OMPC_SCHEDULE_MODIFIER_monotonic:
Alexey Bataev6cff6242016-05-30 13:05:14 +00002930 Modifier = OMP_sch_modifier_monotonic;
2931 break;
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002932 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
Alexey Bataev6cff6242016-05-30 13:05:14 +00002933 Modifier = OMP_sch_modifier_nonmonotonic;
2934 break;
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002935 case OMPC_SCHEDULE_MODIFIER_simd:
Alexey Bataev6cff6242016-05-30 13:05:14 +00002936 if (Schedule == OMP_sch_static_chunked)
2937 Schedule = OMP_sch_static_balanced_chunked;
2938 break;
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002939 case OMPC_SCHEDULE_MODIFIER_last:
2940 case OMPC_SCHEDULE_MODIFIER_unknown:
2941 break;
2942 }
2943 switch (M2) {
2944 case OMPC_SCHEDULE_MODIFIER_monotonic:
Alexey Bataev6cff6242016-05-30 13:05:14 +00002945 Modifier = OMP_sch_modifier_monotonic;
2946 break;
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002947 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
Alexey Bataev6cff6242016-05-30 13:05:14 +00002948 Modifier = OMP_sch_modifier_nonmonotonic;
2949 break;
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002950 case OMPC_SCHEDULE_MODIFIER_simd:
Alexey Bataev6cff6242016-05-30 13:05:14 +00002951 if (Schedule == OMP_sch_static_chunked)
2952 Schedule = OMP_sch_static_balanced_chunked;
2953 break;
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002954 case OMPC_SCHEDULE_MODIFIER_last:
2955 case OMPC_SCHEDULE_MODIFIER_unknown:
2956 break;
2957 }
Alexey Bataev6cff6242016-05-30 13:05:14 +00002958 return Schedule | Modifier;
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002959}
2960
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002961void CGOpenMPRuntime::emitForDispatchInit(
2962 CodeGenFunction &CGF, SourceLocation Loc,
2963 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
2964 bool Ordered, const DispatchRTInput &DispatchValues) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00002965 if (!CGF.HaveInsertPoint())
2966 return;
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002967 OpenMPSchedType Schedule = getRuntimeSchedule(
2968 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
John McCall7f416cc2015-09-08 08:05:57 +00002969 assert(Ordered ||
2970 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
Alexey Bataev6cff6242016-05-30 13:05:14 +00002971 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2972 Schedule != OMP_sch_static_balanced_chunked));
John McCall7f416cc2015-09-08 08:05:57 +00002973 // Call __kmpc_dispatch_init(
2974 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
2975 // kmp_int[32|64] lower, kmp_int[32|64] upper,
2976 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
Alexander Musmanc6388682014-12-15 07:07:06 +00002977
John McCall7f416cc2015-09-08 08:05:57 +00002978 // If the Chunk was not specified in the clause - use default value 1.
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002979 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
2980 : CGF.Builder.getIntN(IVSize, 1);
John McCall7f416cc2015-09-08 08:05:57 +00002981 llvm::Value *Args[] = {
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002982 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2983 CGF.Builder.getInt32(addMonoNonMonoModifier(
2984 Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002985 DispatchValues.LB, // Lower
2986 DispatchValues.UB, // Upper
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002987 CGF.Builder.getIntN(IVSize, 1), // Stride
2988 Chunk // Chunk
John McCall7f416cc2015-09-08 08:05:57 +00002989 };
2990 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
2991}
2992
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002993static void emitForStaticInitCall(
2994 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2995 llvm::Constant *ForStaticInitFunction, OpenMPSchedType Schedule,
2996 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00002997 const CGOpenMPRuntime::StaticRTInput &Values) {
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002998 if (!CGF.HaveInsertPoint())
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00002999 return;
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003000
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003001 assert(!Values.Ordered);
3002 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
3003 Schedule == OMP_sch_static_balanced_chunked ||
3004 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
3005 Schedule == OMP_dist_sch_static ||
3006 Schedule == OMP_dist_sch_static_chunked);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003007
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003008 // Call __kmpc_for_static_init(
3009 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3010 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3011 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3012 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
3013 llvm::Value *Chunk = Values.Chunk;
3014 if (Chunk == nullptr) {
3015 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
3016 Schedule == OMP_dist_sch_static) &&
3017 "expected static non-chunked schedule");
3018 // If the Chunk was not specified in the clause - use default value 1.
3019 Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3020 } else {
3021 assert((Schedule == OMP_sch_static_chunked ||
3022 Schedule == OMP_sch_static_balanced_chunked ||
3023 Schedule == OMP_ord_static_chunked ||
3024 Schedule == OMP_dist_sch_static_chunked) &&
3025 "expected static chunked schedule");
3026 }
3027 llvm::Value *Args[] = {
3028 UpdateLocation,
3029 ThreadId,
3030 CGF.Builder.getInt32(addMonoNonMonoModifier(Schedule, M1,
3031 M2)), // Schedule type
3032 Values.IL.getPointer(), // &isLastIter
3033 Values.LB.getPointer(), // &LB
3034 Values.UB.getPointer(), // &UB
3035 Values.ST.getPointer(), // &Stride
3036 CGF.Builder.getIntN(Values.IVSize, 1), // Incr
3037 Chunk // Chunk
3038 };
3039 CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003040}
3041
John McCall7f416cc2015-09-08 08:05:57 +00003042void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3043 SourceLocation Loc,
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003044 OpenMPDirectiveKind DKind,
Alexey Bataev9ebd7422016-05-10 09:57:36 +00003045 const OpenMPScheduleTy &ScheduleKind,
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003046 const StaticRTInput &Values) {
3047 OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3048 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3049 assert(isOpenMPWorksharingDirective(DKind) &&
3050 "Expected loop-based or sections-based directive.");
3051 auto *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3052 isOpenMPLoopDirective(DKind)
3053 ? OMP_IDENT_WORK_LOOP
3054 : OMP_IDENT_WORK_SECTIONS);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003055 auto *ThreadId = getThreadID(CGF, Loc);
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003056 auto *StaticInitFunction =
3057 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
Alexey Bataev9ebd7422016-05-10 09:57:36 +00003058 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003059 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003060}
John McCall7f416cc2015-09-08 08:05:57 +00003061
Alexey Bataev9ebd7422016-05-10 09:57:36 +00003062void CGOpenMPRuntime::emitDistributeStaticInit(
3063 CodeGenFunction &CGF, SourceLocation Loc,
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003064 OpenMPDistScheduleClauseKind SchedKind,
3065 const CGOpenMPRuntime::StaticRTInput &Values) {
3066 OpenMPSchedType ScheduleNum =
3067 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3068 auto *UpdatedLocation =
3069 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003070 auto *ThreadId = getThreadID(CGF, Loc);
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003071 auto *StaticInitFunction =
3072 createForStaticInitFunction(Values.IVSize, Values.IVSigned);
Alexey Bataev9ebd7422016-05-10 09:57:36 +00003073 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3074 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003075 OMPC_SCHEDULE_MODIFIER_unknown, Values);
Alexander Musmanc6388682014-12-15 07:07:06 +00003076}
3077
Alexey Bataev98eb6e32015-04-22 11:15:40 +00003078void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
Alexey Bataevf43f7142017-09-06 16:17:35 +00003079 SourceLocation Loc,
3080 OpenMPDirectiveKind DKind) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00003081 if (!CGF.HaveInsertPoint())
3082 return;
Alexander Musmanc6388682014-12-15 07:07:06 +00003083 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
Alexey Bataevf43f7142017-09-06 16:17:35 +00003084 llvm::Value *Args[] = {
3085 emitUpdateLocation(CGF, Loc,
3086 isOpenMPDistributeDirective(DKind)
3087 ? OMP_IDENT_WORK_DISTRIBUTE
3088 : isOpenMPLoopDirective(DKind)
3089 ? OMP_IDENT_WORK_LOOP
3090 : OMP_IDENT_WORK_SECTIONS),
3091 getThreadID(CGF, Loc)};
Alexey Bataev3eff5f42015-02-25 08:32:46 +00003092 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3093 Args);
Alexander Musmanc6388682014-12-15 07:07:06 +00003094}
3095
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00003096void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3097 SourceLocation Loc,
3098 unsigned IVSize,
3099 bool IVSigned) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00003100 if (!CGF.HaveInsertPoint())
3101 return;
Alexey Bataev98eb6e32015-04-22 11:15:40 +00003102 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
Alexey Bataev50b3c952016-02-19 10:38:26 +00003103 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
Alexey Bataev98eb6e32015-04-22 11:15:40 +00003104 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3105}
3106
Alexander Musman92bdaab2015-03-12 13:37:50 +00003107llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3108 SourceLocation Loc, unsigned IVSize,
John McCall7f416cc2015-09-08 08:05:57 +00003109 bool IVSigned, Address IL,
3110 Address LB, Address UB,
3111 Address ST) {
Alexander Musman92bdaab2015-03-12 13:37:50 +00003112 // Call __kmpc_dispatch_next(
3113 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3114 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3115 // kmp_int[32|64] *p_stride);
3116 llvm::Value *Args[] = {
Alexey Bataev50b3c952016-02-19 10:38:26 +00003117 emitUpdateLocation(CGF, Loc),
3118 getThreadID(CGF, Loc),
John McCall7f416cc2015-09-08 08:05:57 +00003119 IL.getPointer(), // &isLastIter
3120 LB.getPointer(), // &Lower
3121 UB.getPointer(), // &Upper
3122 ST.getPointer() // &Stride
Alexander Musman92bdaab2015-03-12 13:37:50 +00003123 };
3124 llvm::Value *Call =
3125 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3126 return CGF.EmitScalarConversion(
3127 Call, CGF.getContext().getIntTypeForBitwidth(32, /* Signed */ true),
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003128 CGF.getContext().BoolTy, Loc);
Alexander Musman92bdaab2015-03-12 13:37:50 +00003129}
3130
Alexey Bataev3eff5f42015-02-25 08:32:46 +00003131void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3132 llvm::Value *NumThreads,
3133 SourceLocation Loc) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00003134 if (!CGF.HaveInsertPoint())
3135 return;
Alexey Bataevb2059782014-10-13 08:23:51 +00003136 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3137 llvm::Value *Args[] = {
Alexey Bataev3eff5f42015-02-25 08:32:46 +00003138 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
Alexey Bataevb2059782014-10-13 08:23:51 +00003139 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
Alexey Bataev3eff5f42015-02-25 08:32:46 +00003140 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3141 Args);
Alexey Bataevb2059782014-10-13 08:23:51 +00003142}
3143
Alexey Bataev7f210c62015-06-18 13:40:03 +00003144void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3145 OpenMPProcBindClauseKind ProcBind,
3146 SourceLocation Loc) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00003147 if (!CGF.HaveInsertPoint())
3148 return;
Alexey Bataev7f210c62015-06-18 13:40:03 +00003149 // Constants for proc bind value accepted by the runtime.
3150 enum ProcBindTy {
3151 ProcBindFalse = 0,
3152 ProcBindTrue,
3153 ProcBindMaster,
3154 ProcBindClose,
3155 ProcBindSpread,
3156 ProcBindIntel,
3157 ProcBindDefault
3158 } RuntimeProcBind;
3159 switch (ProcBind) {
3160 case OMPC_PROC_BIND_master:
3161 RuntimeProcBind = ProcBindMaster;
3162 break;
3163 case OMPC_PROC_BIND_close:
3164 RuntimeProcBind = ProcBindClose;
3165 break;
3166 case OMPC_PROC_BIND_spread:
3167 RuntimeProcBind = ProcBindSpread;
3168 break;
3169 case OMPC_PROC_BIND_unknown:
3170 llvm_unreachable("Unsupported proc_bind value.");
3171 }
3172 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3173 llvm::Value *Args[] = {
3174 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3175 llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)};
3176 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3177}
3178
Alexey Bataev3eff5f42015-02-25 08:32:46 +00003179void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3180 SourceLocation Loc) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00003181 if (!CGF.HaveInsertPoint())
3182 return;
Alexey Bataevd76df6d2015-02-24 12:55:09 +00003183 // Build call void __kmpc_flush(ident_t *loc)
Alexey Bataev3eff5f42015-02-25 08:32:46 +00003184 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3185 emitUpdateLocation(CGF, Loc));
Alexey Bataevcc37cc12014-11-20 04:34:54 +00003186}
Alexey Bataev3eff5f42015-02-25 08:32:46 +00003187
Alexey Bataev62b63b12015-03-10 07:28:44 +00003188namespace {
3189/// \brief Indexes of fields for type kmp_task_t.
3190enum KmpTaskTFields {
3191 /// \brief List of shared variables.
3192 KmpTaskTShareds,
3193 /// \brief Task routine.
3194 KmpTaskTRoutine,
3195 /// \brief Partition id for the untied tasks.
3196 KmpTaskTPartId,
Alexey Bataevad537bb2016-05-30 09:06:50 +00003197 /// Function with call of destructors for private variables.
3198 Data1,
3199 /// Task priority.
3200 Data2,
Alexey Bataev7292c292016-04-25 12:22:29 +00003201 /// (Taskloops only) Lower bound.
3202 KmpTaskTLowerBound,
3203 /// (Taskloops only) Upper bound.
3204 KmpTaskTUpperBound,
3205 /// (Taskloops only) Stride.
3206 KmpTaskTStride,
3207 /// (Taskloops only) Is last iteration flag.
3208 KmpTaskTLastIter,
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003209 /// (Taskloops only) Reduction data.
3210 KmpTaskTReductions,
Alexey Bataev62b63b12015-03-10 07:28:44 +00003211};
Hans Wennborg7eb54642015-09-10 17:07:54 +00003212} // anonymous namespace
Alexey Bataev62b63b12015-03-10 07:28:44 +00003213
Samuel Antaoee8fb302016-01-06 13:42:12 +00003214bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3215 // FIXME: Add other entries type when they become supported.
3216 return OffloadEntriesTargetRegion.empty();
3217}
3218
3219/// \brief Initialize target region entry.
3220void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3221 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3222 StringRef ParentName, unsigned LineNum,
Samuel Antao2de62b02016-02-13 23:35:10 +00003223 unsigned Order) {
Samuel Antaoee8fb302016-01-06 13:42:12 +00003224 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
3225 "only required for the device "
3226 "code generation.");
Samuel Antao2de62b02016-02-13 23:35:10 +00003227 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
Samuel Antaof83efdb2017-01-05 16:02:49 +00003228 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3229 /*Flags=*/0);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003230 ++OffloadingEntriesNum;
3231}
3232
3233void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3234 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3235 StringRef ParentName, unsigned LineNum,
Samuel Antaof83efdb2017-01-05 16:02:49 +00003236 llvm::Constant *Addr, llvm::Constant *ID,
3237 int32_t Flags) {
Samuel Antaoee8fb302016-01-06 13:42:12 +00003238 // If we are emitting code for a target, the entry is already initialized,
3239 // only has to be registered.
3240 if (CGM.getLangOpts().OpenMPIsDevice) {
Samuel Antao2de62b02016-02-13 23:35:10 +00003241 assert(hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) &&
Samuel Antaoee8fb302016-01-06 13:42:12 +00003242 "Entry must exist.");
Samuel Antao2de62b02016-02-13 23:35:10 +00003243 auto &Entry =
3244 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
Samuel Antaoee8fb302016-01-06 13:42:12 +00003245 assert(Entry.isValid() && "Entry not initialized!");
3246 Entry.setAddress(Addr);
3247 Entry.setID(ID);
Samuel Antaof83efdb2017-01-05 16:02:49 +00003248 Entry.setFlags(Flags);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003249 return;
3250 } else {
Samuel Antaof83efdb2017-01-05 16:02:49 +00003251 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum++, Addr, ID, Flags);
Samuel Antao2de62b02016-02-13 23:35:10 +00003252 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
Samuel Antaoee8fb302016-01-06 13:42:12 +00003253 }
3254}
3255
3256bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
Samuel Antao2de62b02016-02-13 23:35:10 +00003257 unsigned DeviceID, unsigned FileID, StringRef ParentName,
3258 unsigned LineNum) const {
Samuel Antaoee8fb302016-01-06 13:42:12 +00003259 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3260 if (PerDevice == OffloadEntriesTargetRegion.end())
3261 return false;
3262 auto PerFile = PerDevice->second.find(FileID);
3263 if (PerFile == PerDevice->second.end())
3264 return false;
3265 auto PerParentName = PerFile->second.find(ParentName);
3266 if (PerParentName == PerFile->second.end())
3267 return false;
3268 auto PerLine = PerParentName->second.find(LineNum);
3269 if (PerLine == PerParentName->second.end())
3270 return false;
Samuel Antaoee8fb302016-01-06 13:42:12 +00003271 // Fail if this entry is already registered.
Samuel Antao2de62b02016-02-13 23:35:10 +00003272 if (PerLine->second.getAddress() || PerLine->second.getID())
Samuel Antaoee8fb302016-01-06 13:42:12 +00003273 return false;
3274 return true;
3275}
3276
3277void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3278 const OffloadTargetRegionEntryInfoActTy &Action) {
3279 // Scan all target region entries and perform the provided action.
3280 for (auto &D : OffloadEntriesTargetRegion)
3281 for (auto &F : D.second)
3282 for (auto &P : F.second)
3283 for (auto &L : P.second)
Samuel Antao2de62b02016-02-13 23:35:10 +00003284 Action(D.first, F.first, P.first(), L.first, L.second);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003285}
3286
3287/// \brief Create a Ctor/Dtor-like function whose body is emitted through
3288/// \a Codegen. This is used to emit the two functions that register and
3289/// unregister the descriptor of the current compilation unit.
3290static llvm::Function *
3291createOffloadingBinaryDescriptorFunction(CodeGenModule &CGM, StringRef Name,
3292 const RegionCodeGenTy &Codegen) {
3293 auto &C = CGM.getContext();
3294 FunctionArgList Args;
Alexey Bataev56223232017-06-09 13:40:18 +00003295 ImplicitParamDecl DummyPtr(C, C.VoidPtrTy, ImplicitParamDecl::Other);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003296 Args.push_back(&DummyPtr);
3297
3298 CodeGenFunction CGF(CGM);
John McCallc56a8b32016-03-11 04:30:31 +00003299 auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003300 auto FTy = CGM.getTypes().GetFunctionType(FI);
3301 auto *Fn =
3302 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, SourceLocation());
3303 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FI, Args, SourceLocation());
3304 Codegen(CGF);
3305 CGF.FinishFunction();
3306 return Fn;
3307}
3308
3309llvm::Function *
3310CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() {
3311
3312 // If we don't have entries or if we are emitting code for the device, we
3313 // don't need to do anything.
3314 if (CGM.getLangOpts().OpenMPIsDevice || OffloadEntriesInfoManager.empty())
3315 return nullptr;
3316
3317 auto &M = CGM.getModule();
3318 auto &C = CGM.getContext();
3319
3320 // Get list of devices we care about
3321 auto &Devices = CGM.getLangOpts().OMPTargetTriples;
3322
3323 // We should be creating an offloading descriptor only if there are devices
3324 // specified.
3325 assert(!Devices.empty() && "No OpenMP offloading devices??");
3326
3327 // Create the external variables that will point to the begin and end of the
3328 // host entries section. These will be defined by the linker.
3329 auto *OffloadEntryTy =
3330 CGM.getTypes().ConvertTypeForMem(getTgtOffloadEntryQTy());
3331 llvm::GlobalVariable *HostEntriesBegin = new llvm::GlobalVariable(
3332 M, OffloadEntryTy, /*isConstant=*/true,
Eugene Zelenko1660a5d2016-01-26 19:01:06 +00003333 llvm::GlobalValue::ExternalLinkage, /*Initializer=*/nullptr,
Samuel Antaoee8fb302016-01-06 13:42:12 +00003334 ".omp_offloading.entries_begin");
3335 llvm::GlobalVariable *HostEntriesEnd = new llvm::GlobalVariable(
3336 M, OffloadEntryTy, /*isConstant=*/true,
Eugene Zelenko1660a5d2016-01-26 19:01:06 +00003337 llvm::GlobalValue::ExternalLinkage, /*Initializer=*/nullptr,
Samuel Antaoee8fb302016-01-06 13:42:12 +00003338 ".omp_offloading.entries_end");
3339
3340 // Create all device images
Samuel Antaoee8fb302016-01-06 13:42:12 +00003341 auto *DeviceImageTy = cast<llvm::StructType>(
3342 CGM.getTypes().ConvertTypeForMem(getTgtDeviceImageQTy()));
John McCall23c9dc62016-11-28 22:18:27 +00003343 ConstantInitBuilder DeviceImagesBuilder(CGM);
John McCall6c9f1fdb2016-11-19 08:17:24 +00003344 auto DeviceImagesEntries = DeviceImagesBuilder.beginArray(DeviceImageTy);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003345
3346 for (unsigned i = 0; i < Devices.size(); ++i) {
3347 StringRef T = Devices[i].getTriple();
3348 auto *ImgBegin = new llvm::GlobalVariable(
3349 M, CGM.Int8Ty, /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage,
Eugene Zelenko1660a5d2016-01-26 19:01:06 +00003350 /*Initializer=*/nullptr,
3351 Twine(".omp_offloading.img_start.") + Twine(T));
Samuel Antaoee8fb302016-01-06 13:42:12 +00003352 auto *ImgEnd = new llvm::GlobalVariable(
3353 M, CGM.Int8Ty, /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage,
Eugene Zelenko1660a5d2016-01-26 19:01:06 +00003354 /*Initializer=*/nullptr, Twine(".omp_offloading.img_end.") + Twine(T));
Samuel Antaoee8fb302016-01-06 13:42:12 +00003355
John McCall6c9f1fdb2016-11-19 08:17:24 +00003356 auto Dev = DeviceImagesEntries.beginStruct(DeviceImageTy);
3357 Dev.add(ImgBegin);
3358 Dev.add(ImgEnd);
3359 Dev.add(HostEntriesBegin);
3360 Dev.add(HostEntriesEnd);
John McCallf1788632016-11-28 22:18:30 +00003361 Dev.finishAndAddTo(DeviceImagesEntries);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003362 }
3363
3364 // Create device images global array.
John McCall6c9f1fdb2016-11-19 08:17:24 +00003365 llvm::GlobalVariable *DeviceImages =
3366 DeviceImagesEntries.finishAndCreateGlobal(".omp_offloading.device_images",
3367 CGM.getPointerAlign(),
3368 /*isConstant=*/true);
Peter Collingbournebcf909d2016-06-14 21:02:05 +00003369 DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003370
3371 // This is a Zero array to be used in the creation of the constant expressions
3372 llvm::Constant *Index[] = {llvm::Constant::getNullValue(CGM.Int32Ty),
3373 llvm::Constant::getNullValue(CGM.Int32Ty)};
3374
3375 // Create the target region descriptor.
3376 auto *BinaryDescriptorTy = cast<llvm::StructType>(
3377 CGM.getTypes().ConvertTypeForMem(getTgtBinaryDescriptorQTy()));
John McCall23c9dc62016-11-28 22:18:27 +00003378 ConstantInitBuilder DescBuilder(CGM);
John McCall6c9f1fdb2016-11-19 08:17:24 +00003379 auto DescInit = DescBuilder.beginStruct(BinaryDescriptorTy);
3380 DescInit.addInt(CGM.Int32Ty, Devices.size());
3381 DescInit.add(llvm::ConstantExpr::getGetElementPtr(DeviceImages->getValueType(),
3382 DeviceImages,
3383 Index));
3384 DescInit.add(HostEntriesBegin);
3385 DescInit.add(HostEntriesEnd);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003386
John McCall6c9f1fdb2016-11-19 08:17:24 +00003387 auto *Desc = DescInit.finishAndCreateGlobal(".omp_offloading.descriptor",
3388 CGM.getPointerAlign(),
3389 /*isConstant=*/true);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003390
3391 // Emit code to register or unregister the descriptor at execution
3392 // startup or closing, respectively.
3393
3394 // Create a variable to drive the registration and unregistration of the
3395 // descriptor, so we can reuse the logic that emits Ctors and Dtors.
3396 auto *IdentInfo = &C.Idents.get(".omp_offloading.reg_unreg_var");
3397 ImplicitParamDecl RegUnregVar(C, C.getTranslationUnitDecl(), SourceLocation(),
Alexey Bataev56223232017-06-09 13:40:18 +00003398 IdentInfo, C.CharTy, ImplicitParamDecl::Other);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003399
3400 auto *UnRegFn = createOffloadingBinaryDescriptorFunction(
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003401 CGM, ".omp_offloading.descriptor_unreg",
3402 [&](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev3c595a62017-08-14 15:01:03 +00003403 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_unregister_lib),
3404 Desc);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003405 });
3406 auto *RegFn = createOffloadingBinaryDescriptorFunction(
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003407 CGM, ".omp_offloading.descriptor_reg",
3408 [&](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev3c595a62017-08-14 15:01:03 +00003409 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_lib),
3410 Desc);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003411 CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
3412 });
George Rokos29d0f002017-05-27 03:03:13 +00003413 if (CGM.supportsCOMDAT()) {
3414 // It is sufficient to call registration function only once, so create a
3415 // COMDAT group for registration/unregistration functions and associated
3416 // data. That would reduce startup time and code size. Registration
3417 // function serves as a COMDAT group key.
3418 auto ComdatKey = M.getOrInsertComdat(RegFn->getName());
3419 RegFn->setLinkage(llvm::GlobalValue::LinkOnceAnyLinkage);
3420 RegFn->setVisibility(llvm::GlobalValue::HiddenVisibility);
3421 RegFn->setComdat(ComdatKey);
3422 UnRegFn->setComdat(ComdatKey);
3423 DeviceImages->setComdat(ComdatKey);
3424 Desc->setComdat(ComdatKey);
3425 }
Samuel Antaoee8fb302016-01-06 13:42:12 +00003426 return RegFn;
3427}
3428
Samuel Antao2de62b02016-02-13 23:35:10 +00003429void CGOpenMPRuntime::createOffloadEntry(llvm::Constant *ID,
Samuel Antaof83efdb2017-01-05 16:02:49 +00003430 llvm::Constant *Addr, uint64_t Size,
3431 int32_t Flags) {
Samuel Antao2de62b02016-02-13 23:35:10 +00003432 StringRef Name = Addr->getName();
Samuel Antaoee8fb302016-01-06 13:42:12 +00003433 auto *TgtOffloadEntryType = cast<llvm::StructType>(
3434 CGM.getTypes().ConvertTypeForMem(getTgtOffloadEntryQTy()));
3435 llvm::LLVMContext &C = CGM.getModule().getContext();
3436 llvm::Module &M = CGM.getModule();
3437
3438 // Make sure the address has the right type.
Samuel Antao2de62b02016-02-13 23:35:10 +00003439 llvm::Constant *AddrPtr = llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003440
3441 // Create constant string with the name.
3442 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
3443
3444 llvm::GlobalVariable *Str =
3445 new llvm::GlobalVariable(M, StrPtrInit->getType(), /*isConstant=*/true,
3446 llvm::GlobalValue::InternalLinkage, StrPtrInit,
3447 ".omp_offloading.entry_name");
Peter Collingbournebcf909d2016-06-14 21:02:05 +00003448 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003449 llvm::Constant *StrPtr = llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy);
3450
John McCall6c9f1fdb2016-11-19 08:17:24 +00003451 // We can't have any padding between symbols, so we need to have 1-byte
3452 // alignment.
3453 auto Align = CharUnits::fromQuantity(1);
3454
Samuel Antaoee8fb302016-01-06 13:42:12 +00003455 // Create the entry struct.
John McCall23c9dc62016-11-28 22:18:27 +00003456 ConstantInitBuilder EntryBuilder(CGM);
John McCall6c9f1fdb2016-11-19 08:17:24 +00003457 auto EntryInit = EntryBuilder.beginStruct(TgtOffloadEntryType);
3458 EntryInit.add(AddrPtr);
3459 EntryInit.add(StrPtr);
3460 EntryInit.addInt(CGM.SizeTy, Size);
Samuel Antaof83efdb2017-01-05 16:02:49 +00003461 EntryInit.addInt(CGM.Int32Ty, Flags);
3462 EntryInit.addInt(CGM.Int32Ty, 0);
John McCall6c9f1fdb2016-11-19 08:17:24 +00003463 llvm::GlobalVariable *Entry =
3464 EntryInit.finishAndCreateGlobal(".omp_offloading.entry",
3465 Align,
3466 /*constant*/ true,
3467 llvm::GlobalValue::ExternalLinkage);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003468
3469 // The entry has to be created in the section the linker expects it to be.
3470 Entry->setSection(".omp_offloading.entries");
Samuel Antaoee8fb302016-01-06 13:42:12 +00003471}
3472
3473void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
3474 // Emit the offloading entries and metadata so that the device codegen side
Samuel Antao4c8035b2016-12-12 18:00:20 +00003475 // can easily figure out what to emit. The produced metadata looks like
3476 // this:
Samuel Antaoee8fb302016-01-06 13:42:12 +00003477 //
3478 // !omp_offload.info = !{!1, ...}
3479 //
3480 // Right now we only generate metadata for function that contain target
3481 // regions.
3482
3483 // If we do not have entries, we dont need to do anything.
3484 if (OffloadEntriesInfoManager.empty())
3485 return;
3486
3487 llvm::Module &M = CGM.getModule();
3488 llvm::LLVMContext &C = M.getContext();
3489 SmallVector<OffloadEntriesInfoManagerTy::OffloadEntryInfo *, 16>
3490 OrderedEntries(OffloadEntriesInfoManager.size());
3491
3492 // Create the offloading info metadata node.
3493 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
3494
Simon Pilgrim2c518802017-03-30 14:13:19 +00003495 // Auxiliary methods to create metadata values and strings.
Samuel Antaoee8fb302016-01-06 13:42:12 +00003496 auto getMDInt = [&](unsigned v) {
3497 return llvm::ConstantAsMetadata::get(
3498 llvm::ConstantInt::get(llvm::Type::getInt32Ty(C), v));
3499 };
3500
3501 auto getMDString = [&](StringRef v) { return llvm::MDString::get(C, v); };
3502
3503 // Create function that emits metadata for each target region entry;
3504 auto &&TargetRegionMetadataEmitter = [&](
3505 unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned Line,
Samuel Antaoee8fb302016-01-06 13:42:12 +00003506 OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
3507 llvm::SmallVector<llvm::Metadata *, 32> Ops;
3508 // Generate metadata for target regions. Each entry of this metadata
3509 // contains:
3510 // - Entry 0 -> Kind of this type of metadata (0).
3511 // - Entry 1 -> Device ID of the file where the entry was identified.
3512 // - Entry 2 -> File ID of the file where the entry was identified.
3513 // - Entry 3 -> Mangled name of the function where the entry was identified.
3514 // - Entry 4 -> Line in the file where the entry was identified.
Samuel Antao2de62b02016-02-13 23:35:10 +00003515 // - Entry 5 -> Order the entry was created.
Samuel Antaoee8fb302016-01-06 13:42:12 +00003516 // The first element of the metadata node is the kind.
3517 Ops.push_back(getMDInt(E.getKind()));
3518 Ops.push_back(getMDInt(DeviceID));
3519 Ops.push_back(getMDInt(FileID));
3520 Ops.push_back(getMDString(ParentName));
3521 Ops.push_back(getMDInt(Line));
Samuel Antaoee8fb302016-01-06 13:42:12 +00003522 Ops.push_back(getMDInt(E.getOrder()));
3523
3524 // Save this entry in the right position of the ordered entries array.
3525 OrderedEntries[E.getOrder()] = &E;
3526
3527 // Add metadata to the named metadata node.
3528 MD->addOperand(llvm::MDNode::get(C, Ops));
3529 };
3530
3531 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
3532 TargetRegionMetadataEmitter);
3533
3534 for (auto *E : OrderedEntries) {
3535 assert(E && "All ordered entries must exist!");
3536 if (auto *CE =
3537 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
3538 E)) {
3539 assert(CE->getID() && CE->getAddress() &&
3540 "Entry ID and Addr are invalid!");
Samuel Antao2de62b02016-02-13 23:35:10 +00003541 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0);
Samuel Antaoee8fb302016-01-06 13:42:12 +00003542 } else
3543 llvm_unreachable("Unsupported entry kind.");
3544 }
3545}
3546
3547/// \brief Loads all the offload entries information from the host IR
3548/// metadata.
3549void CGOpenMPRuntime::loadOffloadInfoMetadata() {
3550 // If we are in target mode, load the metadata from the host IR. This code has
3551 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
3552
3553 if (!CGM.getLangOpts().OpenMPIsDevice)
3554 return;
3555
3556 if (CGM.getLangOpts().OMPHostIRFile.empty())
3557 return;
3558
3559 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
3560 if (Buf.getError())
3561 return;
3562
3563 llvm::LLVMContext C;
Peter Collingbourned9445c42016-11-13 07:00:17 +00003564 auto ME = expectedToErrorOrAndEmitErrors(
3565 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
Samuel Antaoee8fb302016-01-06 13:42:12 +00003566
3567 if (ME.getError())
3568 return;
3569
3570 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
3571 if (!MD)
3572 return;
3573
3574 for (auto I : MD->operands()) {
3575 llvm::MDNode *MN = cast<llvm::MDNode>(I);
3576
3577 auto getMDInt = [&](unsigned Idx) {
3578 llvm::ConstantAsMetadata *V =
3579 cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
3580 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
3581 };
3582
3583 auto getMDString = [&](unsigned Idx) {
3584 llvm::MDString *V = cast<llvm::MDString>(MN->getOperand(Idx));
3585 return V->getString();
3586 };
3587
3588 switch (getMDInt(0)) {
3589 default:
3590 llvm_unreachable("Unexpected metadata!");
3591 break;
3592 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
3593 OFFLOAD_ENTRY_INFO_TARGET_REGION:
3594 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
3595 /*DeviceID=*/getMDInt(1), /*FileID=*/getMDInt(2),
3596 /*ParentName=*/getMDString(3), /*Line=*/getMDInt(4),
Samuel Antao2de62b02016-02-13 23:35:10 +00003597 /*Order=*/getMDInt(5));
Samuel Antaoee8fb302016-01-06 13:42:12 +00003598 break;
3599 }
3600 }
3601}
3602
Alexey Bataev62b63b12015-03-10 07:28:44 +00003603void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
3604 if (!KmpRoutineEntryPtrTy) {
3605 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
3606 auto &C = CGM.getContext();
3607 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3608 FunctionProtoType::ExtProtoInfo EPI;
3609 KmpRoutineEntryPtrQTy = C.getPointerType(
3610 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
3611 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
3612 }
3613}
3614
Alexey Bataevc71a4092015-09-11 10:29:41 +00003615static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
3616 QualType FieldTy) {
Alexey Bataev62b63b12015-03-10 07:28:44 +00003617 auto *Field = FieldDecl::Create(
3618 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
3619 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
3620 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
3621 Field->setAccess(AS_public);
3622 DC->addDecl(Field);
Alexey Bataevc71a4092015-09-11 10:29:41 +00003623 return Field;
Alexey Bataev62b63b12015-03-10 07:28:44 +00003624}
3625
Samuel Antaoee8fb302016-01-06 13:42:12 +00003626QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
3627
3628 // Make sure the type of the entry is already created. This is the type we
3629 // have to create:
3630 // struct __tgt_offload_entry{
3631 // void *addr; // Pointer to the offload entry info.
3632 // // (function or global)
3633 // char *name; // Name of the function or global.
3634 // size_t size; // Size of the entry info (0 if it a function).
Samuel Antaof83efdb2017-01-05 16:02:49 +00003635 // int32_t flags; // Flags associated with the entry, e.g. 'link'.
3636 // int32_t reserved; // Reserved, to use by the runtime library.
Samuel Antaoee8fb302016-01-06 13:42:12 +00003637 // };
3638 if (TgtOffloadEntryQTy.isNull()) {
3639 ASTContext &C = CGM.getContext();
3640 auto *RD = C.buildImplicitRecord("__tgt_offload_entry");
3641 RD->startDefinition();
3642 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3643 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
3644 addFieldToRecordDecl(C, RD, C.getSizeType());
Samuel Antaof83efdb2017-01-05 16:02:49 +00003645 addFieldToRecordDecl(
3646 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
3647 addFieldToRecordDecl(
3648 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
Samuel Antaoee8fb302016-01-06 13:42:12 +00003649 RD->completeDefinition();
3650 TgtOffloadEntryQTy = C.getRecordType(RD);
3651 }
3652 return TgtOffloadEntryQTy;
3653}
3654
3655QualType CGOpenMPRuntime::getTgtDeviceImageQTy() {
3656 // These are the types we need to build:
3657 // struct __tgt_device_image{
3658 // void *ImageStart; // Pointer to the target code start.
3659 // void *ImageEnd; // Pointer to the target code end.
3660 // // We also add the host entries to the device image, as it may be useful
3661 // // for the target runtime to have access to that information.
3662 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all
3663 // // the entries.
3664 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
3665 // // entries (non inclusive).
3666 // };
3667 if (TgtDeviceImageQTy.isNull()) {
3668 ASTContext &C = CGM.getContext();
3669 auto *RD = C.buildImplicitRecord("__tgt_device_image");
3670 RD->startDefinition();
3671 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3672 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3673 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
3674 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
3675 RD->completeDefinition();
3676 TgtDeviceImageQTy = C.getRecordType(RD);
3677 }
3678 return TgtDeviceImageQTy;
3679}
3680
3681QualType CGOpenMPRuntime::getTgtBinaryDescriptorQTy() {
3682 // struct __tgt_bin_desc{
3683 // int32_t NumDevices; // Number of devices supported.
3684 // __tgt_device_image *DeviceImages; // Arrays of device images
3685 // // (one per device).
3686 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all the
3687 // // entries.
3688 // __tgt_offload_entry *EntriesEnd; // End of the table with all the
3689 // // entries (non inclusive).
3690 // };
3691 if (TgtBinaryDescriptorQTy.isNull()) {
3692 ASTContext &C = CGM.getContext();
3693 auto *RD = C.buildImplicitRecord("__tgt_bin_desc");
3694 RD->startDefinition();
3695 addFieldToRecordDecl(
3696 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
3697 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtDeviceImageQTy()));
3698 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
3699 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
3700 RD->completeDefinition();
3701 TgtBinaryDescriptorQTy = C.getRecordType(RD);
3702 }
3703 return TgtBinaryDescriptorQTy;
3704}
3705
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003706namespace {
Alexey Bataev9e034042015-05-05 04:05:12 +00003707struct PrivateHelpersTy {
3708 PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
3709 const VarDecl *PrivateElemInit)
3710 : Original(Original), PrivateCopy(PrivateCopy),
3711 PrivateElemInit(PrivateElemInit) {}
3712 const VarDecl *Original;
3713 const VarDecl *PrivateCopy;
3714 const VarDecl *PrivateElemInit;
3715};
3716typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
Hans Wennborg7eb54642015-09-10 17:07:54 +00003717} // anonymous namespace
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003718
Alexey Bataev9e034042015-05-05 04:05:12 +00003719static RecordDecl *
Craig Topper8674c5c2015-09-29 04:30:07 +00003720createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003721 if (!Privates.empty()) {
3722 auto &C = CGM.getContext();
3723 // Build struct .kmp_privates_t. {
3724 // /* private vars */
3725 // };
3726 auto *RD = C.buildImplicitRecord(".kmp_privates.t");
3727 RD->startDefinition();
3728 for (auto &&Pair : Privates) {
Alexey Bataevc71a4092015-09-11 10:29:41 +00003729 auto *VD = Pair.second.Original;
3730 auto Type = VD->getType();
Alexey Bataev1d9c15c2015-05-19 12:31:28 +00003731 Type = Type.getNonReferenceType();
Alexey Bataevc71a4092015-09-11 10:29:41 +00003732 auto *FD = addFieldToRecordDecl(C, RD, Type);
3733 if (VD->hasAttrs()) {
3734 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
3735 E(VD->getAttrs().end());
3736 I != E; ++I)
3737 FD->addAttr(*I);
3738 }
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003739 }
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003740 RD->completeDefinition();
3741 return RD;
3742 }
3743 return nullptr;
3744}
3745
Alexey Bataev9e034042015-05-05 04:05:12 +00003746static RecordDecl *
Alexey Bataev7292c292016-04-25 12:22:29 +00003747createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
3748 QualType KmpInt32Ty,
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00003749 QualType KmpRoutineEntryPointerQTy) {
Alexey Bataev62b63b12015-03-10 07:28:44 +00003750 auto &C = CGM.getContext();
3751 // Build struct kmp_task_t {
3752 // void * shareds;
3753 // kmp_routine_entry_t routine;
3754 // kmp_int32 part_id;
Alexey Bataevad537bb2016-05-30 09:06:50 +00003755 // kmp_cmplrdata_t data1;
3756 // kmp_cmplrdata_t data2;
Alexey Bataev7292c292016-04-25 12:22:29 +00003757 // For taskloops additional fields:
3758 // kmp_uint64 lb;
3759 // kmp_uint64 ub;
3760 // kmp_int64 st;
3761 // kmp_int32 liter;
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003762 // void * reductions;
Alexey Bataev62b63b12015-03-10 07:28:44 +00003763 // };
Alexey Bataevad537bb2016-05-30 09:06:50 +00003764 auto *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
3765 UD->startDefinition();
3766 addFieldToRecordDecl(C, UD, KmpInt32Ty);
3767 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
3768 UD->completeDefinition();
3769 QualType KmpCmplrdataTy = C.getRecordType(UD);
Alexey Bataev62b63b12015-03-10 07:28:44 +00003770 auto *RD = C.buildImplicitRecord("kmp_task_t");
3771 RD->startDefinition();
3772 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
3773 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
3774 addFieldToRecordDecl(C, RD, KmpInt32Ty);
Alexey Bataevad537bb2016-05-30 09:06:50 +00003775 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
3776 addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
Alexey Bataev7292c292016-04-25 12:22:29 +00003777 if (isOpenMPTaskLoopDirective(Kind)) {
3778 QualType KmpUInt64Ty =
3779 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
3780 QualType KmpInt64Ty =
3781 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
3782 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3783 addFieldToRecordDecl(C, RD, KmpUInt64Ty);
3784 addFieldToRecordDecl(C, RD, KmpInt64Ty);
3785 addFieldToRecordDecl(C, RD, KmpInt32Ty);
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003786 addFieldToRecordDecl(C, RD, C.VoidPtrTy);
Alexey Bataev7292c292016-04-25 12:22:29 +00003787 }
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00003788 RD->completeDefinition();
3789 return RD;
3790}
3791
3792static RecordDecl *
3793createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
Craig Topper8674c5c2015-09-29 04:30:07 +00003794 ArrayRef<PrivateDataTy> Privates) {
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00003795 auto &C = CGM.getContext();
3796 // Build struct kmp_task_t_with_privates {
3797 // kmp_task_t task_data;
3798 // .kmp_privates_t. privates;
3799 // };
3800 auto *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
3801 RD->startDefinition();
3802 addFieldToRecordDecl(C, RD, KmpTaskTQTy);
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003803 if (auto *PrivateRD = createPrivatesRecordDecl(CGM, Privates)) {
3804 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
3805 }
Alexey Bataev62b63b12015-03-10 07:28:44 +00003806 RD->completeDefinition();
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003807 return RD;
Alexey Bataev62b63b12015-03-10 07:28:44 +00003808}
3809
3810/// \brief Emit a proxy function which accepts kmp_task_t as the second
3811/// argument.
3812/// \code
3813/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
Alexey Bataev48591dd2016-04-20 04:01:36 +00003814/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
Alexey Bataev7292c292016-04-25 12:22:29 +00003815/// For taskloops:
3816/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003817/// tt->reductions, tt->shareds);
Alexey Bataev62b63b12015-03-10 07:28:44 +00003818/// return 0;
3819/// }
3820/// \endcode
3821static llvm::Value *
3822emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
Alexey Bataev7292c292016-04-25 12:22:29 +00003823 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
3824 QualType KmpTaskTWithPrivatesPtrQTy,
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00003825 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003826 QualType SharedsPtrTy, llvm::Value *TaskFunction,
3827 llvm::Value *TaskPrivatesMap) {
Alexey Bataev62b63b12015-03-10 07:28:44 +00003828 auto &C = CGM.getContext();
3829 FunctionArgList Args;
Alexey Bataev56223232017-06-09 13:40:18 +00003830 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
3831 ImplicitParamDecl::Other);
3832 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3833 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
3834 ImplicitParamDecl::Other);
Alexey Bataev62b63b12015-03-10 07:28:44 +00003835 Args.push_back(&GtidArg);
3836 Args.push_back(&TaskTypeArg);
Alexey Bataev62b63b12015-03-10 07:28:44 +00003837 auto &TaskEntryFnInfo =
John McCallc56a8b32016-03-11 04:30:31 +00003838 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
Alexey Bataev62b63b12015-03-10 07:28:44 +00003839 auto *TaskEntryTy = CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
3840 auto *TaskEntry =
3841 llvm::Function::Create(TaskEntryTy, llvm::GlobalValue::InternalLinkage,
3842 ".omp_task_entry.", &CGM.getModule());
Akira Hatanaka44a59f82015-10-28 02:30:47 +00003843 CGM.SetInternalFunctionAttributes(/*D=*/nullptr, TaskEntry, TaskEntryFnInfo);
Alexey Bataev62b63b12015-03-10 07:28:44 +00003844 CodeGenFunction CGF(CGM);
Alexey Bataev62b63b12015-03-10 07:28:44 +00003845 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args);
3846
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003847 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
Alexey Bataev7292c292016-04-25 12:22:29 +00003848 // tt,
3849 // For taskloops:
3850 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3851 // tt->task_data.shareds);
Alexey Bataev62b63b12015-03-10 07:28:44 +00003852 auto *GtidParam = CGF.EmitLoadOfScalar(
John McCall7f416cc2015-09-08 08:05:57 +00003853 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
Alexey Bataev31300ed2016-02-04 11:27:03 +00003854 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3855 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3856 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00003857 auto *KmpTaskTWithPrivatesQTyRD =
3858 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003859 LValue Base =
3860 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00003861 auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
3862 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3863 auto PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
Alexey Bataev48591dd2016-04-20 04:01:36 +00003864 auto *PartidParam = PartIdLVal.getPointer();
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00003865
3866 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3867 auto SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003868 auto *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00003869 CGF.EmitLoadOfLValue(SharedsLVal, Loc).getScalarVal(),
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003870 CGF.ConvertTypeForMem(SharedsPtrTy));
3871
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003872 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3873 llvm::Value *PrivatesParam;
3874 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3875 auto PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
3876 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
John McCall7f416cc2015-09-08 08:05:57 +00003877 PrivatesLVal.getPointer(), CGF.VoidPtrTy);
Alexey Bataev7292c292016-04-25 12:22:29 +00003878 } else
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003879 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003880
Alexey Bataev7292c292016-04-25 12:22:29 +00003881 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
3882 TaskPrivatesMap,
3883 CGF.Builder
3884 .CreatePointerBitCastOrAddrSpaceCast(
3885 TDBase.getAddress(), CGF.VoidPtrTy)
3886 .getPointer()};
3887 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
3888 std::end(CommonArgs));
3889 if (isOpenMPTaskLoopDirective(Kind)) {
3890 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3891 auto LBLVal = CGF.EmitLValueForField(Base, *LBFI);
3892 auto *LBParam = CGF.EmitLoadOfLValue(LBLVal, Loc).getScalarVal();
3893 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3894 auto UBLVal = CGF.EmitLValueForField(Base, *UBFI);
3895 auto *UBParam = CGF.EmitLoadOfLValue(UBLVal, Loc).getScalarVal();
3896 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3897 auto StLVal = CGF.EmitLValueForField(Base, *StFI);
3898 auto *StParam = CGF.EmitLoadOfLValue(StLVal, Loc).getScalarVal();
3899 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3900 auto LILVal = CGF.EmitLValueForField(Base, *LIFI);
3901 auto *LIParam = CGF.EmitLoadOfLValue(LILVal, Loc).getScalarVal();
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003902 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
3903 auto RLVal = CGF.EmitLValueForField(Base, *RFI);
3904 auto *RParam = CGF.EmitLoadOfLValue(RLVal, Loc).getScalarVal();
Alexey Bataev7292c292016-04-25 12:22:29 +00003905 CallArgs.push_back(LBParam);
3906 CallArgs.push_back(UBParam);
3907 CallArgs.push_back(StParam);
3908 CallArgs.push_back(LIParam);
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003909 CallArgs.push_back(RParam);
Alexey Bataev7292c292016-04-25 12:22:29 +00003910 }
3911 CallArgs.push_back(SharedsParam);
3912
Alexey Bataev3c595a62017-08-14 15:01:03 +00003913 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
3914 CallArgs);
Alexey Bataev62b63b12015-03-10 07:28:44 +00003915 CGF.EmitStoreThroughLValue(
3916 RValue::get(CGF.Builder.getInt32(/*C=*/0)),
John McCall7f416cc2015-09-08 08:05:57 +00003917 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
Alexey Bataev62b63b12015-03-10 07:28:44 +00003918 CGF.FinishFunction();
3919 return TaskEntry;
3920}
3921
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00003922static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
3923 SourceLocation Loc,
3924 QualType KmpInt32Ty,
3925 QualType KmpTaskTWithPrivatesPtrQTy,
3926 QualType KmpTaskTWithPrivatesQTy) {
Alexey Bataev62b63b12015-03-10 07:28:44 +00003927 auto &C = CGM.getContext();
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003928 FunctionArgList Args;
Alexey Bataev56223232017-06-09 13:40:18 +00003929 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
3930 ImplicitParamDecl::Other);
3931 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3932 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
3933 ImplicitParamDecl::Other);
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003934 Args.push_back(&GtidArg);
3935 Args.push_back(&TaskTypeArg);
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003936 auto &DestructorFnInfo =
John McCallc56a8b32016-03-11 04:30:31 +00003937 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003938 auto *DestructorFnTy = CGM.getTypes().GetFunctionType(DestructorFnInfo);
3939 auto *DestructorFn =
3940 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
3941 ".omp_task_destructor.", &CGM.getModule());
Akira Hatanaka44a59f82015-10-28 02:30:47 +00003942 CGM.SetInternalFunctionAttributes(/*D=*/nullptr, DestructorFn,
3943 DestructorFnInfo);
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003944 CodeGenFunction CGF(CGM);
3945 CGF.disableDebugInfo();
3946 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
3947 Args);
3948
Alexey Bataev31300ed2016-02-04 11:27:03 +00003949 LValue Base = CGF.EmitLoadOfPointerLValue(
3950 CGF.GetAddrOfLocalVar(&TaskTypeArg),
3951 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00003952 auto *KmpTaskTWithPrivatesQTyRD =
3953 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
3954 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
Alexey Bataev36c1eb92015-04-30 06:51:57 +00003955 Base = CGF.EmitLValueForField(Base, *FI);
3956 for (auto *Field :
3957 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3958 if (auto DtorKind = Field->getType().isDestructedType()) {
3959 auto FieldLValue = CGF.EmitLValueForField(Base, Field);
3960 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
3961 }
3962 }
3963 CGF.FinishFunction();
3964 return DestructorFn;
3965}
3966
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003967/// \brief Emit a privates mapping function for correct handling of private and
3968/// firstprivate variables.
3969/// \code
3970/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
3971/// **noalias priv1,..., <tyn> **noalias privn) {
3972/// *priv1 = &.privates.priv1;
3973/// ...;
3974/// *privn = &.privates.privn;
3975/// }
3976/// \endcode
3977static llvm::Value *
3978emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
Craig Topper8674c5c2015-09-29 04:30:07 +00003979 ArrayRef<const Expr *> PrivateVars,
3980 ArrayRef<const Expr *> FirstprivateVars,
Alexey Bataevf93095a2016-05-05 08:46:22 +00003981 ArrayRef<const Expr *> LastprivateVars,
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003982 QualType PrivatesQTy,
Craig Topper8674c5c2015-09-29 04:30:07 +00003983 ArrayRef<PrivateDataTy> Privates) {
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003984 auto &C = CGM.getContext();
3985 FunctionArgList Args;
3986 ImplicitParamDecl TaskPrivatesArg(
3987 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
Alexey Bataev56223232017-06-09 13:40:18 +00003988 C.getPointerType(PrivatesQTy).withConst().withRestrict(),
3989 ImplicitParamDecl::Other);
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003990 Args.push_back(&TaskPrivatesArg);
3991 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
3992 unsigned Counter = 1;
3993 for (auto *E: PrivateVars) {
3994 Args.push_back(ImplicitParamDecl::Create(
Alexey Bataev56223232017-06-09 13:40:18 +00003995 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
3996 C.getPointerType(C.getPointerType(E->getType()))
3997 .withConst()
3998 .withRestrict(),
3999 ImplicitParamDecl::Other));
Alexey Bataev3ae88e22015-05-22 08:56:35 +00004000 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4001 PrivateVarsPos[VD] = Counter;
4002 ++Counter;
4003 }
4004 for (auto *E : FirstprivateVars) {
4005 Args.push_back(ImplicitParamDecl::Create(
Alexey Bataev56223232017-06-09 13:40:18 +00004006 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4007 C.getPointerType(C.getPointerType(E->getType()))
4008 .withConst()
4009 .withRestrict(),
4010 ImplicitParamDecl::Other));
Alexey Bataev3ae88e22015-05-22 08:56:35 +00004011 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4012 PrivateVarsPos[VD] = Counter;
4013 ++Counter;
4014 }
Alexey Bataevf93095a2016-05-05 08:46:22 +00004015 for (auto *E: LastprivateVars) {
4016 Args.push_back(ImplicitParamDecl::Create(
Alexey Bataev56223232017-06-09 13:40:18 +00004017 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4018 C.getPointerType(C.getPointerType(E->getType()))
4019 .withConst()
4020 .withRestrict(),
4021 ImplicitParamDecl::Other));
Alexey Bataevf93095a2016-05-05 08:46:22 +00004022 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4023 PrivateVarsPos[VD] = Counter;
4024 ++Counter;
4025 }
Alexey Bataev3ae88e22015-05-22 08:56:35 +00004026 auto &TaskPrivatesMapFnInfo =
John McCallc56a8b32016-03-11 04:30:31 +00004027 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
Alexey Bataev3ae88e22015-05-22 08:56:35 +00004028 auto *TaskPrivatesMapTy =
4029 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4030 auto *TaskPrivatesMap = llvm::Function::Create(
4031 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage,
4032 ".omp_task_privates_map.", &CGM.getModule());
Akira Hatanaka44a59f82015-10-28 02:30:47 +00004033 CGM.SetInternalFunctionAttributes(/*D=*/nullptr, TaskPrivatesMap,
4034 TaskPrivatesMapFnInfo);
Chandler Carruthfcd33142016-12-23 01:24:49 +00004035 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
Mehdi Amini6aa9e9b2017-05-29 05:38:20 +00004036 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
Evgeniy Stepanov6b2a61d2015-09-14 21:35:16 +00004037 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
Alexey Bataev3ae88e22015-05-22 08:56:35 +00004038 CodeGenFunction CGF(CGM);
4039 CGF.disableDebugInfo();
4040 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4041 TaskPrivatesMapFnInfo, Args);
4042
4043 // *privi = &.privates.privi;
Alexey Bataev31300ed2016-02-04 11:27:03 +00004044 LValue Base = CGF.EmitLoadOfPointerLValue(
4045 CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4046 TaskPrivatesArg.getType()->castAs<PointerType>());
Alexey Bataev3ae88e22015-05-22 08:56:35 +00004047 auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4048 Counter = 0;
4049 for (auto *Field : PrivatesQTyRD->fields()) {
4050 auto FieldLVal = CGF.EmitLValueForField(Base, Field);
4051 auto *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
John McCall7f416cc2015-09-08 08:05:57 +00004052 auto RefLVal = CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
Alexey Bataev31300ed2016-02-04 11:27:03 +00004053 auto RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4054 RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>());
Alexey Bataev2377fe92015-09-10 08:12:02 +00004055 CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal);
Alexey Bataev3ae88e22015-05-22 08:56:35 +00004056 ++Counter;
4057 }
4058 CGF.FinishFunction();
4059 return TaskPrivatesMap;
4060}
4061
Mandeep Singh Grangb14fb6a22017-11-28 20:41:13 +00004062static bool stable_sort_comparator(const PrivateDataTy P1,
4063 const PrivateDataTy P2) {
4064 return P1.first > P2.first;
Alexey Bataev36c1eb92015-04-30 06:51:57 +00004065}
4066
Alexey Bataevf93095a2016-05-05 08:46:22 +00004067/// Emit initialization for private variables in task-based directives.
Alexey Bataev8a831592016-05-10 10:36:51 +00004068static void emitPrivatesInit(CodeGenFunction &CGF,
Alexey Bataevf93095a2016-05-05 08:46:22 +00004069 const OMPExecutableDirective &D,
4070 Address KmpTaskSharedsPtr, LValue TDBase,
4071 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4072 QualType SharedsTy, QualType SharedsPtrTy,
4073 const OMPTaskDataTy &Data,
4074 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4075 auto &C = CGF.getContext();
Alexey Bataevf93095a2016-05-05 08:46:22 +00004076 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4077 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4078 LValue SrcBase;
4079 if (!Data.FirstprivateVars.empty()) {
4080 SrcBase = CGF.MakeAddrLValue(
4081 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4082 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4083 SharedsTy);
4084 }
4085 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(
4086 cast<CapturedStmt>(*D.getAssociatedStmt()));
4087 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4088 for (auto &&Pair : Privates) {
4089 auto *VD = Pair.second.PrivateCopy;
4090 auto *Init = VD->getAnyInitializer();
Alexey Bataevf93095a2016-05-05 08:46:22 +00004091 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4092 !CGF.isTrivialInitializer(Init)))) {
Alexey Bataev8a831592016-05-10 10:36:51 +00004093 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
Alexey Bataevf93095a2016-05-05 08:46:22 +00004094 if (auto *Elem = Pair.second.PrivateElemInit) {
4095 auto *OriginalVD = Pair.second.Original;
4096 auto *SharedField = CapturesInfo.lookup(OriginalVD);
4097 auto SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4098 SharedRefLValue = CGF.MakeAddrLValue(
4099 Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
Krzysztof Parzyszek8f248232017-05-18 17:07:11 +00004100 SharedRefLValue.getType(),
Ivan A. Kosarevb9c59f32017-10-31 11:05:34 +00004101 LValueBaseInfo(AlignmentSource::Decl),
4102 SharedRefLValue.getTBAAInfo());
Alexey Bataevf93095a2016-05-05 08:46:22 +00004103 QualType Type = OriginalVD->getType();
4104 if (Type->isArrayType()) {
4105 // Initialize firstprivate array.
4106 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4107 // Perform simple memcpy.
4108 CGF.EmitAggregateAssign(PrivateLValue.getAddress(),
4109 SharedRefLValue.getAddress(), Type);
4110 } else {
4111 // Initialize firstprivate array using element-by-element
Simon Pilgrim2c518802017-03-30 14:13:19 +00004112 // initialization.
Alexey Bataevf93095a2016-05-05 08:46:22 +00004113 CGF.EmitOMPAggregateAssign(
4114 PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type,
4115 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4116 Address SrcElement) {
4117 // Clean up any temporaries needed by the initialization.
4118 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4119 InitScope.addPrivate(
4120 Elem, [SrcElement]() -> Address { return SrcElement; });
4121 (void)InitScope.Privatize();
4122 // Emit initialization for single element.
4123 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4124 CGF, &CapturesInfo);
4125 CGF.EmitAnyExprToMem(Init, DestElement,
4126 Init->getType().getQualifiers(),
4127 /*IsInitializer=*/false);
4128 });
4129 }
4130 } else {
4131 CodeGenFunction::OMPPrivateScope InitScope(CGF);
4132 InitScope.addPrivate(Elem, [SharedRefLValue]() -> Address {
4133 return SharedRefLValue.getAddress();
4134 });
4135 (void)InitScope.Privatize();
4136 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4137 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4138 /*capturedByInit=*/false);
4139 }
4140 } else
4141 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4142 }
Alexey Bataevf93095a2016-05-05 08:46:22 +00004143 ++FI;
4144 }
Alexey Bataevf93095a2016-05-05 08:46:22 +00004145}
4146
4147/// Check if duplication function is required for taskloops.
4148static bool checkInitIsRequired(CodeGenFunction &CGF,
4149 ArrayRef<PrivateDataTy> Privates) {
4150 bool InitRequired = false;
4151 for (auto &&Pair : Privates) {
4152 auto *VD = Pair.second.PrivateCopy;
4153 auto *Init = VD->getAnyInitializer();
4154 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4155 !CGF.isTrivialInitializer(Init));
4156 }
4157 return InitRequired;
4158}
4159
4160
4161/// Emit task_dup function (for initialization of
4162/// private/firstprivate/lastprivate vars and last_iter flag)
4163/// \code
4164/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4165/// lastpriv) {
4166/// // setup lastprivate flag
4167/// task_dst->last = lastpriv;
4168/// // could be constructor calls here...
4169/// }
4170/// \endcode
4171static llvm::Value *
4172emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
4173 const OMPExecutableDirective &D,
4174 QualType KmpTaskTWithPrivatesPtrQTy,
4175 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4176 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
4177 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
4178 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
4179 auto &C = CGM.getContext();
4180 FunctionArgList Args;
Alexey Bataev56223232017-06-09 13:40:18 +00004181 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4182 KmpTaskTWithPrivatesPtrQTy,
4183 ImplicitParamDecl::Other);
4184 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4185 KmpTaskTWithPrivatesPtrQTy,
4186 ImplicitParamDecl::Other);
4187 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
4188 ImplicitParamDecl::Other);
Alexey Bataevf93095a2016-05-05 08:46:22 +00004189 Args.push_back(&DstArg);
4190 Args.push_back(&SrcArg);
4191 Args.push_back(&LastprivArg);
4192 auto &TaskDupFnInfo =
4193 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4194 auto *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
4195 auto *TaskDup =
4196 llvm::Function::Create(TaskDupTy, llvm::GlobalValue::InternalLinkage,
4197 ".omp_task_dup.", &CGM.getModule());
4198 CGM.SetInternalFunctionAttributes(/*D=*/nullptr, TaskDup, TaskDupFnInfo);
4199 CodeGenFunction CGF(CGM);
4200 CGF.disableDebugInfo();
4201 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args);
4202
4203 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4204 CGF.GetAddrOfLocalVar(&DstArg),
4205 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4206 // task_dst->liter = lastpriv;
4207 if (WithLastIter) {
4208 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4209 LValue Base = CGF.EmitLValueForField(
4210 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4211 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4212 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
4213 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
4214 CGF.EmitStoreOfScalar(Lastpriv, LILVal);
4215 }
4216
4217 // Emit initial values for private copies (if any).
4218 assert(!Privates.empty());
4219 Address KmpTaskSharedsPtr = Address::invalid();
4220 if (!Data.FirstprivateVars.empty()) {
4221 LValue TDBase = CGF.EmitLoadOfPointerLValue(
4222 CGF.GetAddrOfLocalVar(&SrcArg),
4223 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4224 LValue Base = CGF.EmitLValueForField(
4225 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4226 KmpTaskSharedsPtr = Address(
4227 CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
4228 Base, *std::next(KmpTaskTQTyRD->field_begin(),
4229 KmpTaskTShareds)),
4230 Loc),
4231 CGF.getNaturalTypeAlignment(SharedsTy));
4232 }
Alexey Bataev8a831592016-05-10 10:36:51 +00004233 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4234 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
Alexey Bataevf93095a2016-05-05 08:46:22 +00004235 CGF.FinishFunction();
4236 return TaskDup;
4237}
4238
Alexey Bataev8a831592016-05-10 10:36:51 +00004239/// Checks if destructor function is required to be generated.
4240/// \return true if cleanups are required, false otherwise.
4241static bool
4242checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
4243 bool NeedsCleanup = false;
4244 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4245 auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4246 for (auto *FD : PrivateRD->fields()) {
4247 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4248 if (NeedsCleanup)
4249 break;
4250 }
4251 return NeedsCleanup;
4252}
4253
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004254CGOpenMPRuntime::TaskResultTy
4255CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4256 const OMPExecutableDirective &D,
4257 llvm::Value *TaskFunction, QualType SharedsTy,
4258 Address Shareds, const OMPTaskDataTy &Data) {
Alexey Bataev36c1eb92015-04-30 06:51:57 +00004259 auto &C = CGM.getContext();
Alexey Bataev7292c292016-04-25 12:22:29 +00004260 llvm::SmallVector<PrivateDataTy, 4> Privates;
Alexey Bataev3ae88e22015-05-22 08:56:35 +00004261 // Aggregate privates and sort them by the alignment.
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004262 auto I = Data.PrivateCopies.begin();
4263 for (auto *E : Data.PrivateVars) {
Alexey Bataev36c1eb92015-04-30 06:51:57 +00004264 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4265 Privates.push_back(std::make_pair(
Alexey Bataevc71a4092015-09-11 10:29:41 +00004266 C.getDeclAlign(VD),
Alexey Bataev9e034042015-05-05 04:05:12 +00004267 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4268 /*PrivateElemInit=*/nullptr)));
Alexey Bataev36c1eb92015-04-30 06:51:57 +00004269 ++I;
4270 }
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004271 I = Data.FirstprivateCopies.begin();
4272 auto IElemInitRef = Data.FirstprivateInits.begin();
4273 for (auto *E : Data.FirstprivateVars) {
Alexey Bataev9e034042015-05-05 04:05:12 +00004274 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4275 Privates.push_back(std::make_pair(
Alexey Bataevc71a4092015-09-11 10:29:41 +00004276 C.getDeclAlign(VD),
Alexey Bataev9e034042015-05-05 04:05:12 +00004277 PrivateHelpersTy(
4278 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4279 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl()))));
Richard Trieucc3949d2016-02-18 22:34:54 +00004280 ++I;
4281 ++IElemInitRef;
Alexey Bataev9e034042015-05-05 04:05:12 +00004282 }
Alexey Bataevf93095a2016-05-05 08:46:22 +00004283 I = Data.LastprivateCopies.begin();
4284 for (auto *E : Data.LastprivateVars) {
4285 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4286 Privates.push_back(std::make_pair(
4287 C.getDeclAlign(VD),
4288 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4289 /*PrivateElemInit=*/nullptr)));
4290 ++I;
4291 }
Mandeep Singh Grangb14fb6a22017-11-28 20:41:13 +00004292 std::stable_sort(Privates.begin(), Privates.end(), stable_sort_comparator);
Alexey Bataev62b63b12015-03-10 07:28:44 +00004293 auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4294 // Build type kmp_routine_entry_t (if not built yet).
4295 emitKmpRoutineEntryT(KmpInt32Ty);
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00004296 // Build type kmp_task_t (if not built yet).
Alexey Bataeve213f3e2017-10-11 15:29:40 +00004297 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
4298 if (SavedKmpTaskloopTQTy.isNull()) {
4299 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4300 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4301 }
4302 KmpTaskTQTy = SavedKmpTaskloopTQTy;
Alexey Bataev3a03a7f2017-10-11 15:56:38 +00004303 } else {
Alexey Bataeve213f3e2017-10-11 15:29:40 +00004304 assert(D.getDirectiveKind() == OMPD_task &&
4305 "Expected taskloop or task directive");
4306 if (SavedKmpTaskTQTy.isNull()) {
4307 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4308 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4309 }
4310 KmpTaskTQTy = SavedKmpTaskTQTy;
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00004311 }
4312 auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
Alexey Bataev62b63b12015-03-10 07:28:44 +00004313 // Build particular struct kmp_task_t for the given task.
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00004314 auto *KmpTaskTWithPrivatesQTyRD =
4315 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
4316 auto KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
4317 QualType KmpTaskTWithPrivatesPtrQTy =
4318 C.getPointerType(KmpTaskTWithPrivatesQTy);
4319 auto *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
4320 auto *KmpTaskTWithPrivatesPtrTy = KmpTaskTWithPrivatesTy->getPointerTo();
Alexey Bataev1189bd02016-01-26 12:20:39 +00004321 auto *KmpTaskTWithPrivatesTySize = CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
Alexey Bataev62b63b12015-03-10 07:28:44 +00004322 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
4323
Alexey Bataev3ae88e22015-05-22 08:56:35 +00004324 // Emit initial values for private copies (if any).
4325 llvm::Value *TaskPrivatesMap = nullptr;
4326 auto *TaskPrivatesMapTy =
Reid Klecknere258c442017-03-16 18:55:46 +00004327 std::next(cast<llvm::Function>(TaskFunction)->arg_begin(), 3)->getType();
Alexey Bataev3ae88e22015-05-22 08:56:35 +00004328 if (!Privates.empty()) {
4329 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
Alexey Bataevf93095a2016-05-05 08:46:22 +00004330 TaskPrivatesMap = emitTaskPrivateMappingFunction(
4331 CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
4332 FI->getType(), Privates);
Alexey Bataev3ae88e22015-05-22 08:56:35 +00004333 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4334 TaskPrivatesMap, TaskPrivatesMapTy);
4335 } else {
4336 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4337 cast<llvm::PointerType>(TaskPrivatesMapTy));
4338 }
Alexey Bataev62b63b12015-03-10 07:28:44 +00004339 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
4340 // kmp_task_t *tt);
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00004341 auto *TaskEntry = emitProxyTaskFunction(
Alexey Bataev7292c292016-04-25 12:22:29 +00004342 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4343 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4344 TaskPrivatesMap);
Alexey Bataev62b63b12015-03-10 07:28:44 +00004345
4346 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
4347 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
4348 // kmp_routine_entry_t *task_entry);
4349 // Task flags. Format is taken from
4350 // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h,
4351 // description of kmp_tasking_flags struct.
Alexey Bataev1e1e2862016-05-10 12:21:02 +00004352 enum {
4353 TiedFlag = 0x1,
4354 FinalFlag = 0x2,
4355 DestructorsFlag = 0x8,
4356 PriorityFlag = 0x20
4357 };
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004358 unsigned Flags = Data.Tied ? TiedFlag : 0;
Alexey Bataev8a831592016-05-10 10:36:51 +00004359 bool NeedsCleanup = false;
4360 if (!Privates.empty()) {
4361 NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
4362 if (NeedsCleanup)
4363 Flags = Flags | DestructorsFlag;
4364 }
Alexey Bataev1e1e2862016-05-10 12:21:02 +00004365 if (Data.Priority.getInt())
4366 Flags = Flags | PriorityFlag;
Alexey Bataev62b63b12015-03-10 07:28:44 +00004367 auto *TaskFlags =
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004368 Data.Final.getPointer()
4369 ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
Alexey Bataev62b63b12015-03-10 07:28:44 +00004370 CGF.Builder.getInt32(FinalFlag),
4371 CGF.Builder.getInt32(/*C=*/0))
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004372 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
Alexey Bataev62b63b12015-03-10 07:28:44 +00004373 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
Alexey Bataev40e36f12015-11-24 13:01:44 +00004374 auto *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004375 llvm::Value *AllocArgs[] = {emitUpdateLocation(CGF, Loc),
4376 getThreadID(CGF, Loc), TaskFlags,
4377 KmpTaskTWithPrivatesTySize, SharedsSize,
4378 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4379 TaskEntry, KmpRoutineEntryPtrTy)};
Alexey Bataev62b63b12015-03-10 07:28:44 +00004380 auto *NewTask = CGF.EmitRuntimeCall(
4381 createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00004382 auto *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4383 NewTask, KmpTaskTWithPrivatesPtrTy);
4384 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
4385 KmpTaskTWithPrivatesQTy);
4386 LValue TDBase =
4387 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
Alexey Bataev62b63b12015-03-10 07:28:44 +00004388 // Fill the data in the resulting kmp_task_t record.
4389 // Copy shareds if there are any.
John McCall7f416cc2015-09-08 08:05:57 +00004390 Address KmpTaskSharedsPtr = Address::invalid();
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00004391 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
Alexey Bataev2377fe92015-09-10 08:12:02 +00004392 KmpTaskSharedsPtr =
4393 Address(CGF.EmitLoadOfScalar(
4394 CGF.EmitLValueForField(
4395 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
4396 KmpTaskTShareds)),
4397 Loc),
4398 CGF.getNaturalTypeAlignment(SharedsTy));
Alexey Bataev36c1eb92015-04-30 06:51:57 +00004399 CGF.EmitAggregateCopy(KmpTaskSharedsPtr, Shareds, SharedsTy);
Alexey Bataev8fc69dc2015-05-18 07:54:53 +00004400 }
Alexey Bataev36c1eb92015-04-30 06:51:57 +00004401 // Emit initial values for private copies (if any).
Alexey Bataevf93095a2016-05-05 08:46:22 +00004402 TaskResultTy Result;
Alexey Bataev36c1eb92015-04-30 06:51:57 +00004403 if (!Privates.empty()) {
Alexey Bataev8a831592016-05-10 10:36:51 +00004404 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
4405 SharedsTy, SharedsPtrTy, Data, Privates,
4406 /*ForDup=*/false);
Alexey Bataevf93095a2016-05-05 08:46:22 +00004407 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
4408 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4409 Result.TaskDupFn = emitTaskDupFunction(
4410 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4411 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4412 /*WithLastIter=*/!Data.LastprivateVars.empty());
Alexey Bataev36c1eb92015-04-30 06:51:57 +00004413 }
4414 }
Alexey Bataevad537bb2016-05-30 09:06:50 +00004415 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4416 enum { Priority = 0, Destructors = 1 };
Alexey Bataev62b63b12015-03-10 07:28:44 +00004417 // Provide pointer to function with destructors for privates.
Alexey Bataevad537bb2016-05-30 09:06:50 +00004418 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4419 auto *KmpCmplrdataUD = (*FI)->getType()->getAsUnionType()->getDecl();
4420 if (NeedsCleanup) {
4421 llvm::Value *DestructorFn = emitDestructorsFunction(
4422 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4423 KmpTaskTWithPrivatesQTy);
4424 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
4425 LValue DestructorsLV = CGF.EmitLValueForField(
4426 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
4427 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4428 DestructorFn, KmpRoutineEntryPtrTy),
4429 DestructorsLV);
4430 }
4431 // Set priority.
4432 if (Data.Priority.getInt()) {
4433 LValue Data2LV = CGF.EmitLValueForField(
4434 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4435 LValue PriorityLV = CGF.EmitLValueForField(
4436 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
4437 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
4438 }
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004439 Result.NewTask = NewTask;
4440 Result.TaskEntry = TaskEntry;
4441 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4442 Result.TDBase = TDBase;
4443 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4444 return Result;
Alexey Bataev7292c292016-04-25 12:22:29 +00004445}
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004446
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004447void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
4448 const OMPExecutableDirective &D,
4449 llvm::Value *TaskFunction,
4450 QualType SharedsTy, Address Shareds,
4451 const Expr *IfCond,
4452 const OMPTaskDataTy &Data) {
Alexey Bataev7292c292016-04-25 12:22:29 +00004453 if (!CGF.HaveInsertPoint())
4454 return;
4455
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004456 TaskResultTy Result =
4457 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4458 llvm::Value *NewTask = Result.NewTask;
4459 llvm::Value *TaskEntry = Result.TaskEntry;
4460 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
4461 LValue TDBase = Result.TDBase;
4462 RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
Alexey Bataev7292c292016-04-25 12:22:29 +00004463 auto &C = CGM.getContext();
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004464 // Process list of dependences.
John McCall7f416cc2015-09-08 08:05:57 +00004465 Address DependenciesArray = Address::invalid();
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004466 unsigned NumDependencies = Data.Dependences.size();
John McCall7f416cc2015-09-08 08:05:57 +00004467 if (NumDependencies) {
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004468 // Dependence kind for RTL.
Alexey Bataev92e82f92015-11-23 13:33:42 +00004469 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3 };
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004470 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
4471 RecordDecl *KmpDependInfoRD;
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004472 QualType FlagsTy =
4473 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004474 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4475 if (KmpDependInfoTy.isNull()) {
4476 KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
4477 KmpDependInfoRD->startDefinition();
4478 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
4479 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
4480 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
4481 KmpDependInfoRD->completeDefinition();
4482 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004483 } else
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004484 KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
John McCall7f416cc2015-09-08 08:05:57 +00004485 CharUnits DependencySize = C.getTypeSizeInChars(KmpDependInfoTy);
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004486 // Define type kmp_depend_info[<Dependences.size()>];
4487 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
John McCall7f416cc2015-09-08 08:05:57 +00004488 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004489 ArrayType::Normal, /*IndexTypeQuals=*/0);
4490 // kmp_depend_info[<Dependences.size()>] deps;
Alexey Bataev48591dd2016-04-20 04:01:36 +00004491 DependenciesArray =
4492 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
John McCall7f416cc2015-09-08 08:05:57 +00004493 for (unsigned i = 0; i < NumDependencies; ++i) {
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004494 const Expr *E = Data.Dependences[i].second;
John McCall7f416cc2015-09-08 08:05:57 +00004495 auto Addr = CGF.EmitLValue(E);
Alexey Bataevd6fdc8b2015-08-31 07:32:19 +00004496 llvm::Value *Size;
4497 QualType Ty = E->getType();
Alexey Bataevd6fdc8b2015-08-31 07:32:19 +00004498 if (auto *ASE = dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
4499 LValue UpAddrLVal =
4500 CGF.EmitOMPArraySectionExpr(ASE, /*LowerBound=*/false);
4501 llvm::Value *UpAddr =
John McCall7f416cc2015-09-08 08:05:57 +00004502 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1);
Alexey Bataevd6fdc8b2015-08-31 07:32:19 +00004503 llvm::Value *LowIntPtr =
John McCall7f416cc2015-09-08 08:05:57 +00004504 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
Alexey Bataevd6fdc8b2015-08-31 07:32:19 +00004505 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
4506 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004507 } else
Alexey Bataev1189bd02016-01-26 12:20:39 +00004508 Size = CGF.getTypeSize(Ty);
John McCall7f416cc2015-09-08 08:05:57 +00004509 auto Base = CGF.MakeAddrLValue(
4510 CGF.Builder.CreateConstArrayGEP(DependenciesArray, i, DependencySize),
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004511 KmpDependInfoTy);
4512 // deps[i].base_addr = &<Dependences[i].second>;
4513 auto BaseAddrLVal = CGF.EmitLValueForField(
4514 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
John McCall7f416cc2015-09-08 08:05:57 +00004515 CGF.EmitStoreOfScalar(
4516 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy),
4517 BaseAddrLVal);
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004518 // deps[i].len = sizeof(<Dependences[i].second>);
4519 auto LenLVal = CGF.EmitLValueForField(
4520 Base, *std::next(KmpDependInfoRD->field_begin(), Len));
4521 CGF.EmitStoreOfScalar(Size, LenLVal);
4522 // deps[i].flags = <Dependences[i].first>;
4523 RTLDependenceKindTy DepKind;
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004524 switch (Data.Dependences[i].first) {
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004525 case OMPC_DEPEND_in:
4526 DepKind = DepIn;
4527 break;
Alexey Bataev92e82f92015-11-23 13:33:42 +00004528 // Out and InOut dependencies must use the same code.
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004529 case OMPC_DEPEND_out:
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004530 case OMPC_DEPEND_inout:
4531 DepKind = DepInOut;
4532 break;
Alexey Bataeveb482352015-12-18 05:05:56 +00004533 case OMPC_DEPEND_source:
Alexey Bataeva636c7f2015-12-23 10:27:45 +00004534 case OMPC_DEPEND_sink:
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004535 case OMPC_DEPEND_unknown:
4536 llvm_unreachable("Unknown task dependence type");
4537 }
4538 auto FlagsLVal = CGF.EmitLValueForField(
4539 Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
4540 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
4541 FlagsLVal);
4542 }
John McCall7f416cc2015-09-08 08:05:57 +00004543 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4544 CGF.Builder.CreateStructGEP(DependenciesArray, 0, CharUnits::Zero()),
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004545 CGF.VoidPtrTy);
4546 }
4547
Alexey Bataev62b63b12015-03-10 07:28:44 +00004548 // NOTE: routine and part_id fields are intialized by __kmpc_omp_task_alloc()
4549 // libcall.
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004550 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
4551 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
4552 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
4553 // list is not empty
Alexey Bataev1d677132015-04-22 13:57:31 +00004554 auto *ThreadID = getThreadID(CGF, Loc);
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004555 auto *UpLoc = emitUpdateLocation(CGF, Loc);
John McCall7f416cc2015-09-08 08:05:57 +00004556 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4557 llvm::Value *DepTaskArgs[7];
4558 if (NumDependencies) {
4559 DepTaskArgs[0] = UpLoc;
4560 DepTaskArgs[1] = ThreadID;
4561 DepTaskArgs[2] = NewTask;
4562 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
4563 DepTaskArgs[4] = DependenciesArray.getPointer();
4564 DepTaskArgs[5] = CGF.Builder.getInt32(0);
4565 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4566 }
Malcolm Parsonsc6e45832017-01-13 18:55:32 +00004567 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
4568 &TaskArgs,
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004569 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004570 if (!Data.Tied) {
Alexey Bataev48591dd2016-04-20 04:01:36 +00004571 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4572 auto PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
4573 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
4574 }
John McCall7f416cc2015-09-08 08:05:57 +00004575 if (NumDependencies) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004576 CGF.EmitRuntimeCall(
Alexey Bataev48591dd2016-04-20 04:01:36 +00004577 createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
John McCall7f416cc2015-09-08 08:05:57 +00004578 } else {
Alexey Bataev48591dd2016-04-20 04:01:36 +00004579 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
John McCall7f416cc2015-09-08 08:05:57 +00004580 TaskArgs);
4581 }
Alexey Bataev48591dd2016-04-20 04:01:36 +00004582 // Check if parent region is untied and build return for untied task;
4583 if (auto *Region =
4584 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
4585 Region->emitUntiedSwitch(CGF);
Alexey Bataev1d677132015-04-22 13:57:31 +00004586 };
John McCall7f416cc2015-09-08 08:05:57 +00004587
4588 llvm::Value *DepWaitTaskArgs[6];
4589 if (NumDependencies) {
4590 DepWaitTaskArgs[0] = UpLoc;
4591 DepWaitTaskArgs[1] = ThreadID;
4592 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
4593 DepWaitTaskArgs[3] = DependenciesArray.getPointer();
4594 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
4595 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4596 }
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004597 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
Alexey Bataev3c595a62017-08-14 15:01:03 +00004598 NumDependencies, &DepWaitTaskArgs,
4599 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004600 auto &RT = CGF.CGM.getOpenMPRuntime();
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004601 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
4602 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
4603 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
4604 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
4605 // is specified.
John McCall7f416cc2015-09-08 08:05:57 +00004606 if (NumDependencies)
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004607 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004608 DepWaitTaskArgs);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004609 // Call proxy_task_entry(gtid, new_task);
Alexey Bataev3c595a62017-08-14 15:01:03 +00004610 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4611 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004612 Action.Enter(CGF);
4613 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
Alexey Bataev3c595a62017-08-14 15:01:03 +00004614 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
Alexey Bataev2c7eee52017-08-04 19:10:54 +00004615 OutlinedFnArgs);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004616 };
4617
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004618 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
4619 // kmp_task_t *new_task);
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004620 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
4621 // kmp_task_t *new_task);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004622 RegionCodeGenTy RCG(CodeGen);
4623 CommonActionTy Action(
4624 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
4625 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
4626 RCG.setAction(Action);
4627 RCG(CGF);
Alexey Bataev1d2353d2015-06-24 11:01:36 +00004628 };
John McCall7f416cc2015-09-08 08:05:57 +00004629
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004630 if (IfCond)
Alexey Bataev1d677132015-04-22 13:57:31 +00004631 emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004632 else {
4633 RegionCodeGenTy ThenRCG(ThenCodeGen);
4634 ThenRCG(CGF);
Alexey Bataevf539faa2016-03-28 12:58:34 +00004635 }
Alexey Bataev62b63b12015-03-10 07:28:44 +00004636}
4637
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004638void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
4639 const OMPLoopDirective &D,
4640 llvm::Value *TaskFunction,
4641 QualType SharedsTy, Address Shareds,
4642 const Expr *IfCond,
4643 const OMPTaskDataTy &Data) {
Alexey Bataev7292c292016-04-25 12:22:29 +00004644 if (!CGF.HaveInsertPoint())
4645 return;
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004646 TaskResultTy Result =
4647 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
Alexey Bataev7292c292016-04-25 12:22:29 +00004648 // NOTE: routine and part_id fields are intialized by __kmpc_omp_task_alloc()
4649 // libcall.
4650 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
4651 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
4652 // sched, kmp_uint64 grainsize, void *task_dup);
4653 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4654 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4655 llvm::Value *IfVal;
4656 if (IfCond) {
4657 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
4658 /*isSigned=*/true);
4659 } else
4660 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
4661
4662 LValue LBLVal = CGF.EmitLValueForField(
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004663 Result.TDBase,
4664 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
Alexey Bataev7292c292016-04-25 12:22:29 +00004665 auto *LBVar =
4666 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
4667 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
4668 /*IsInitializer=*/true);
4669 LValue UBLVal = CGF.EmitLValueForField(
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004670 Result.TDBase,
4671 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
Alexey Bataev7292c292016-04-25 12:22:29 +00004672 auto *UBVar =
4673 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
4674 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
4675 /*IsInitializer=*/true);
4676 LValue StLVal = CGF.EmitLValueForField(
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004677 Result.TDBase,
4678 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
Alexey Bataev7292c292016-04-25 12:22:29 +00004679 auto *StVar =
4680 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
4681 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
4682 /*IsInitializer=*/true);
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00004683 // Store reductions address.
4684 LValue RedLVal = CGF.EmitLValueForField(
4685 Result.TDBase,
4686 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
4687 if (Data.Reductions)
4688 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
4689 else {
4690 CGF.EmitNullInitialization(RedLVal.getAddress(),
4691 CGF.getContext().VoidPtrTy);
4692 }
Alexey Bataev2b19a6f2016-04-28 09:15:06 +00004693 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
Alexey Bataev7292c292016-04-25 12:22:29 +00004694 llvm::Value *TaskArgs[] = {
Alexey Bataev33446032017-07-12 18:09:32 +00004695 UpLoc,
4696 ThreadID,
4697 Result.NewTask,
4698 IfVal,
4699 LBLVal.getPointer(),
4700 UBLVal.getPointer(),
4701 CGF.EmitLoadOfScalar(StLVal, SourceLocation()),
4702 llvm::ConstantInt::getNullValue(
4703 CGF.IntTy), // Always 0 because taskgroup emitted by the compiler
Alexey Bataev2b19a6f2016-04-28 09:15:06 +00004704 llvm::ConstantInt::getSigned(
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004705 CGF.IntTy, Data.Schedule.getPointer()
4706 ? Data.Schedule.getInt() ? NumTasks : Grainsize
Alexey Bataev2b19a6f2016-04-28 09:15:06 +00004707 : NoSchedule),
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004708 Data.Schedule.getPointer()
4709 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
Alexey Bataev2b19a6f2016-04-28 09:15:06 +00004710 /*isSigned=*/false)
4711 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
Alexey Bataev33446032017-07-12 18:09:32 +00004712 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4713 Result.TaskDupFn, CGF.VoidPtrTy)
4714 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
Alexey Bataev7292c292016-04-25 12:22:29 +00004715 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
4716}
4717
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004718/// \brief Emit reduction operation for each element of array (required for
4719/// array sections) LHS op = RHS.
4720/// \param Type Type of array.
4721/// \param LHSVar Variable on the left side of the reduction operation
4722/// (references element of array in original variable).
4723/// \param RHSVar Variable on the right side of the reduction operation
4724/// (references element of array in original variable).
4725/// \param RedOpGen Generator of reduction operation with use of LHSVar and
4726/// RHSVar.
Benjamin Kramere003ca22015-10-28 13:54:16 +00004727static void EmitOMPAggregateReduction(
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004728 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
4729 const VarDecl *RHSVar,
4730 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
4731 const Expr *, const Expr *)> &RedOpGen,
4732 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
4733 const Expr *UpExpr = nullptr) {
4734 // Perform element-by-element initialization.
4735 QualType ElementTy;
4736 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
4737 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
4738
4739 // Drill down to the base element type on both arrays.
4740 auto ArrayTy = Type->getAsArrayTypeUnsafe();
4741 auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
4742
4743 auto RHSBegin = RHSAddr.getPointer();
4744 auto LHSBegin = LHSAddr.getPointer();
4745 // Cast from pointer to array type to pointer to single element.
4746 auto LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
4747 // The basic structure here is a while-do loop.
4748 auto BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
4749 auto DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
4750 auto IsEmpty =
4751 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
4752 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4753
4754 // Enter the loop body, making that address the current address.
4755 auto EntryBB = CGF.Builder.GetInsertBlock();
4756 CGF.EmitBlock(BodyBB);
4757
4758 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
4759
4760 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
4761 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
4762 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
4763 Address RHSElementCurrent =
4764 Address(RHSElementPHI,
4765 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
4766
4767 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
4768 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
4769 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
4770 Address LHSElementCurrent =
4771 Address(LHSElementPHI,
4772 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
4773
4774 // Emit copy.
4775 CodeGenFunction::OMPPrivateScope Scope(CGF);
4776 Scope.addPrivate(LHSVar, [=]() -> Address { return LHSElementCurrent; });
4777 Scope.addPrivate(RHSVar, [=]() -> Address { return RHSElementCurrent; });
4778 Scope.Privatize();
4779 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4780 Scope.ForceCleanup();
4781
4782 // Shift the address forward by one element.
4783 auto LHSElementNext = CGF.Builder.CreateConstGEP1_32(
4784 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
4785 auto RHSElementNext = CGF.Builder.CreateConstGEP1_32(
4786 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
4787 // Check whether we've reached the end.
4788 auto Done =
4789 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
4790 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
4791 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
4792 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
4793
4794 // Done.
4795 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
4796}
4797
Alexey Bataeva839ddd2016-03-17 10:19:46 +00004798/// Emit reduction combiner. If the combiner is a simple expression emit it as
4799/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
4800/// UDR combiner function.
4801static void emitReductionCombiner(CodeGenFunction &CGF,
4802 const Expr *ReductionOp) {
4803 if (auto *CE = dyn_cast<CallExpr>(ReductionOp))
4804 if (auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
4805 if (auto *DRE =
4806 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
4807 if (auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
4808 std::pair<llvm::Function *, llvm::Function *> Reduction =
4809 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
4810 RValue Func = RValue::get(Reduction.first);
4811 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
4812 CGF.EmitIgnoredExpr(ReductionOp);
4813 return;
4814 }
4815 CGF.EmitIgnoredExpr(ReductionOp);
4816}
4817
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00004818llvm::Value *CGOpenMPRuntime::emitReductionFunction(
4819 CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates,
4820 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
4821 ArrayRef<const Expr *> ReductionOps) {
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004822 auto &C = CGM.getContext();
4823
4824 // void reduction_func(void *LHSArg, void *RHSArg);
4825 FunctionArgList Args;
Alexey Bataev56223232017-06-09 13:40:18 +00004826 ImplicitParamDecl LHSArg(C, C.VoidPtrTy, ImplicitParamDecl::Other);
4827 ImplicitParamDecl RHSArg(C, C.VoidPtrTy, ImplicitParamDecl::Other);
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004828 Args.push_back(&LHSArg);
4829 Args.push_back(&RHSArg);
John McCallc56a8b32016-03-11 04:30:31 +00004830 auto &CGFI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004831 auto *Fn = llvm::Function::Create(
4832 CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
4833 ".omp.reduction.reduction_func", &CGM.getModule());
Akira Hatanaka44a59f82015-10-28 02:30:47 +00004834 CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, CGFI);
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004835 CodeGenFunction CGF(CGM);
4836 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args);
4837
4838 // Dst = (void*[n])(LHSArg);
4839 // Src = (void*[n])(RHSArg);
John McCall7f416cc2015-09-08 08:05:57 +00004840 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4841 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
4842 ArgsType), CGF.getPointerAlign());
4843 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4844 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
4845 ArgsType), CGF.getPointerAlign());
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004846
4847 // ...
4848 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
4849 // ...
4850 CodeGenFunction::OMPPrivateScope Scope(CGF);
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004851 auto IPriv = Privates.begin();
4852 unsigned Idx = 0;
4853 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
John McCall7f416cc2015-09-08 08:05:57 +00004854 auto RHSVar = cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
4855 Scope.addPrivate(RHSVar, [&]() -> Address {
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004856 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
John McCall7f416cc2015-09-08 08:05:57 +00004857 });
4858 auto LHSVar = cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
4859 Scope.addPrivate(LHSVar, [&]() -> Address {
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004860 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
John McCall7f416cc2015-09-08 08:05:57 +00004861 });
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004862 QualType PrivTy = (*IPriv)->getType();
Alexey Bataev1189bd02016-01-26 12:20:39 +00004863 if (PrivTy->isVariablyModifiedType()) {
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004864 // Get array size and emit VLA type.
4865 ++Idx;
4866 Address Elem =
4867 CGF.Builder.CreateConstArrayGEP(LHS, Idx, CGF.getPointerSize());
4868 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
Alexey Bataev1189bd02016-01-26 12:20:39 +00004869 auto *VLA = CGF.getContext().getAsVariableArrayType(PrivTy);
4870 auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004871 CodeGenFunction::OpaqueValueMapping OpaqueMap(
Alexey Bataev1189bd02016-01-26 12:20:39 +00004872 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004873 CGF.EmitVariablyModifiedType(PrivTy);
4874 }
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004875 }
4876 Scope.Privatize();
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004877 IPriv = Privates.begin();
4878 auto ILHS = LHSExprs.begin();
4879 auto IRHS = RHSExprs.begin();
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004880 for (auto *E : ReductionOps) {
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004881 if ((*IPriv)->getType()->isArrayType()) {
4882 // Emit reduction for array section.
4883 auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
4884 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
Alexey Bataeva839ddd2016-03-17 10:19:46 +00004885 EmitOMPAggregateReduction(
4886 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4887 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
4888 emitReductionCombiner(CGF, E);
4889 });
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004890 } else
4891 // Emit reduction for array subscript or single variable.
Alexey Bataeva839ddd2016-03-17 10:19:46 +00004892 emitReductionCombiner(CGF, E);
Richard Trieucc3949d2016-02-18 22:34:54 +00004893 ++IPriv;
4894 ++ILHS;
4895 ++IRHS;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004896 }
4897 Scope.ForceCleanup();
4898 CGF.FinishFunction();
4899 return Fn;
4900}
4901
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00004902void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
4903 const Expr *ReductionOp,
4904 const Expr *PrivateRef,
4905 const DeclRefExpr *LHS,
4906 const DeclRefExpr *RHS) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004907 if (PrivateRef->getType()->isArrayType()) {
4908 // Emit reduction for array section.
4909 auto *LHSVar = cast<VarDecl>(LHS->getDecl());
4910 auto *RHSVar = cast<VarDecl>(RHS->getDecl());
4911 EmitOMPAggregateReduction(
4912 CGF, PrivateRef->getType(), LHSVar, RHSVar,
4913 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
4914 emitReductionCombiner(CGF, ReductionOp);
4915 });
4916 } else
4917 // Emit reduction for array subscript or single variable.
4918 emitReductionCombiner(CGF, ReductionOp);
4919}
4920
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004921void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004922 ArrayRef<const Expr *> Privates,
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004923 ArrayRef<const Expr *> LHSExprs,
4924 ArrayRef<const Expr *> RHSExprs,
4925 ArrayRef<const Expr *> ReductionOps,
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00004926 ReductionOptionsTy Options) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00004927 if (!CGF.HaveInsertPoint())
4928 return;
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00004929
4930 bool WithNowait = Options.WithNowait;
4931 bool SimpleReduction = Options.SimpleReduction;
4932
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004933 // Next code should be emitted for reduction:
4934 //
4935 // static kmp_critical_name lock = { 0 };
4936 //
4937 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
4938 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
4939 // ...
4940 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
4941 // *(Type<n>-1*)rhs[<n>-1]);
4942 // }
4943 //
4944 // ...
4945 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
4946 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
4947 // RedList, reduce_func, &<lock>)) {
4948 // case 1:
4949 // ...
4950 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
4951 // ...
4952 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
4953 // break;
4954 // case 2:
4955 // ...
4956 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
4957 // ...
Alexey Bataev69a47792015-05-07 03:54:03 +00004958 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004959 // break;
4960 // default:;
4961 // }
Alexey Bataev89e7e8e2015-06-17 06:21:39 +00004962 //
4963 // if SimpleReduction is true, only the next code is generated:
4964 // ...
4965 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
4966 // ...
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004967
4968 auto &C = CGM.getContext();
4969
Alexey Bataev89e7e8e2015-06-17 06:21:39 +00004970 if (SimpleReduction) {
4971 CodeGenFunction::RunCleanupsScope Scope(CGF);
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004972 auto IPriv = Privates.begin();
4973 auto ILHS = LHSExprs.begin();
4974 auto IRHS = RHSExprs.begin();
Alexey Bataev89e7e8e2015-06-17 06:21:39 +00004975 for (auto *E : ReductionOps) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004976 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
4977 cast<DeclRefExpr>(*IRHS));
Richard Trieucc3949d2016-02-18 22:34:54 +00004978 ++IPriv;
4979 ++ILHS;
4980 ++IRHS;
Alexey Bataev89e7e8e2015-06-17 06:21:39 +00004981 }
4982 return;
4983 }
4984
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004985 // 1. Build a list of reduction variables.
4986 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004987 auto Size = RHSExprs.size();
4988 for (auto *E : Privates) {
Alexey Bataev1189bd02016-01-26 12:20:39 +00004989 if (E->getType()->isVariablyModifiedType())
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004990 // Reserve place for array size.
4991 ++Size;
4992 }
4993 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004994 QualType ReductionArrayTy =
4995 C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
4996 /*IndexTypeQuals=*/0);
John McCall7f416cc2015-09-08 08:05:57 +00004997 Address ReductionList =
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004998 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
Alexey Bataevf24e7b12015-10-08 09:10:53 +00004999 auto IPriv = Privates.begin();
5000 unsigned Idx = 0;
5001 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
John McCall7f416cc2015-09-08 08:05:57 +00005002 Address Elem =
Alexey Bataevf24e7b12015-10-08 09:10:53 +00005003 CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, CGF.getPointerSize());
John McCall7f416cc2015-09-08 08:05:57 +00005004 CGF.Builder.CreateStore(
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005005 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
John McCall7f416cc2015-09-08 08:05:57 +00005006 CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy),
5007 Elem);
Alexey Bataev1189bd02016-01-26 12:20:39 +00005008 if ((*IPriv)->getType()->isVariablyModifiedType()) {
Alexey Bataevf24e7b12015-10-08 09:10:53 +00005009 // Store array size.
5010 ++Idx;
5011 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx,
5012 CGF.getPointerSize());
Alexey Bataev1189bd02016-01-26 12:20:39 +00005013 llvm::Value *Size = CGF.Builder.CreateIntCast(
5014 CGF.getVLASize(
5015 CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5016 .first,
5017 CGF.SizeTy, /*isSigned=*/false);
5018 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5019 Elem);
Alexey Bataevf24e7b12015-10-08 09:10:53 +00005020 }
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005021 }
5022
5023 // 2. Emit reduce_func().
5024 auto *ReductionFn = emitReductionFunction(
Alexey Bataevf24e7b12015-10-08 09:10:53 +00005025 CGM, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
5026 LHSExprs, RHSExprs, ReductionOps);
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005027
5028 // 3. Create static kmp_critical_name lock = { 0 };
5029 auto *Lock = getCriticalRegionLock(".reduction");
5030
5031 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5032 // RedList, reduce_func, &<lock>);
Alexey Bataev50b3c952016-02-19 10:38:26 +00005033 auto *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005034 auto *ThreadId = getThreadID(CGF, Loc);
Alexey Bataev1189bd02016-01-26 12:20:39 +00005035 auto *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
Samuel Antao4c8035b2016-12-12 18:00:20 +00005036 auto *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5037 ReductionList.getPointer(), CGF.VoidPtrTy);
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005038 llvm::Value *Args[] = {
5039 IdentTLoc, // ident_t *<loc>
5040 ThreadId, // i32 <gtid>
5041 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5042 ReductionArrayTySize, // size_type sizeof(RedList)
5043 RL, // void *RedList
5044 ReductionFn, // void (*) (void *, void *) <reduce_func>
5045 Lock // kmp_critical_name *&<lock>
5046 };
5047 auto Res = CGF.EmitRuntimeCall(
5048 createRuntimeFunction(WithNowait ? OMPRTL__kmpc_reduce_nowait
5049 : OMPRTL__kmpc_reduce),
5050 Args);
5051
5052 // 5. Build switch(res)
5053 auto *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5054 auto *SwInst = CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5055
5056 // 6. Build case 1:
5057 // ...
5058 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5059 // ...
5060 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5061 // break;
5062 auto *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5063 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5064 CGF.EmitBlock(Case1BB);
5065
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005066 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5067 llvm::Value *EndArgs[] = {
5068 IdentTLoc, // ident_t *<loc>
5069 ThreadId, // i32 <gtid>
5070 Lock // kmp_critical_name *&<lock>
5071 };
5072 auto &&CodeGen = [&Privates, &LHSExprs, &RHSExprs, &ReductionOps](
5073 CodeGenFunction &CGF, PrePostActionTy &Action) {
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00005074 auto &RT = CGF.CGM.getOpenMPRuntime();
Alexey Bataevf24e7b12015-10-08 09:10:53 +00005075 auto IPriv = Privates.begin();
5076 auto ILHS = LHSExprs.begin();
5077 auto IRHS = RHSExprs.begin();
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005078 for (auto *E : ReductionOps) {
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00005079 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5080 cast<DeclRefExpr>(*IRHS));
Richard Trieucc3949d2016-02-18 22:34:54 +00005081 ++IPriv;
5082 ++ILHS;
5083 ++IRHS;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005084 }
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005085 };
5086 RegionCodeGenTy RCG(CodeGen);
5087 CommonActionTy Action(
5088 nullptr, llvm::None,
5089 createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait
5090 : OMPRTL__kmpc_end_reduce),
5091 EndArgs);
5092 RCG.setAction(Action);
5093 RCG(CGF);
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005094
5095 CGF.EmitBranch(DefaultBB);
5096
5097 // 7. Build case 2:
5098 // ...
5099 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5100 // ...
5101 // break;
5102 auto *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5103 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5104 CGF.EmitBlock(Case2BB);
5105
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005106 auto &&AtomicCodeGen = [Loc, &Privates, &LHSExprs, &RHSExprs, &ReductionOps](
5107 CodeGenFunction &CGF, PrePostActionTy &Action) {
Alexey Bataevf24e7b12015-10-08 09:10:53 +00005108 auto ILHS = LHSExprs.begin();
5109 auto IRHS = RHSExprs.begin();
5110 auto IPriv = Privates.begin();
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005111 for (auto *E : ReductionOps) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005112 const Expr *XExpr = nullptr;
5113 const Expr *EExpr = nullptr;
5114 const Expr *UpExpr = nullptr;
5115 BinaryOperatorKind BO = BO_Comma;
5116 if (auto *BO = dyn_cast<BinaryOperator>(E)) {
5117 if (BO->getOpcode() == BO_Assign) {
5118 XExpr = BO->getLHS();
5119 UpExpr = BO->getRHS();
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005120 }
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005121 }
5122 // Try to emit update expression as a simple atomic.
5123 auto *RHSExpr = UpExpr;
5124 if (RHSExpr) {
5125 // Analyze RHS part of the whole expression.
5126 if (auto *ACO = dyn_cast<AbstractConditionalOperator>(
5127 RHSExpr->IgnoreParenImpCasts())) {
5128 // If this is a conditional operator, analyze its condition for
5129 // min/max reduction operator.
5130 RHSExpr = ACO->getCond();
Alexey Bataev69a47792015-05-07 03:54:03 +00005131 }
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005132 if (auto *BORHS =
5133 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
5134 EExpr = BORHS->getRHS();
5135 BO = BORHS->getOpcode();
Alexey Bataevf24e7b12015-10-08 09:10:53 +00005136 }
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005137 }
5138 if (XExpr) {
5139 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
Malcolm Parsonsc6e45832017-01-13 18:55:32 +00005140 auto &&AtomicRedGen = [BO, VD,
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005141 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5142 const Expr *EExpr, const Expr *UpExpr) {
5143 LValue X = CGF.EmitLValue(XExpr);
5144 RValue E;
5145 if (EExpr)
5146 E = CGF.EmitAnyExpr(EExpr);
5147 CGF.EmitOMPAtomicSimpleUpdateExpr(
JF Bastien92f4ef12016-04-06 17:26:42 +00005148 X, E, BO, /*IsXLHSInRHSPart=*/true,
5149 llvm::AtomicOrdering::Monotonic, Loc,
Malcolm Parsonsc6e45832017-01-13 18:55:32 +00005150 [&CGF, UpExpr, VD, Loc](RValue XRValue) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005151 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5152 PrivateScope.addPrivate(
5153 VD, [&CGF, VD, XRValue, Loc]() -> Address {
5154 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5155 CGF.emitOMPSimpleStore(
5156 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5157 VD->getType().getNonReferenceType(), Loc);
5158 return LHSTemp;
5159 });
5160 (void)PrivateScope.Privatize();
5161 return CGF.EmitAnyExpr(UpExpr);
5162 });
5163 };
5164 if ((*IPriv)->getType()->isArrayType()) {
5165 // Emit atomic reduction for array section.
5166 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5167 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5168 AtomicRedGen, XExpr, EExpr, UpExpr);
5169 } else
5170 // Emit atomic reduction for array subscript or single variable.
5171 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5172 } else {
5173 // Emit as a critical region.
5174 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5175 const Expr *, const Expr *) {
5176 auto &RT = CGF.CGM.getOpenMPRuntime();
5177 RT.emitCriticalRegion(
5178 CGF, ".atomic_reduction",
5179 [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5180 Action.Enter(CGF);
5181 emitReductionCombiner(CGF, E);
5182 },
5183 Loc);
5184 };
5185 if ((*IPriv)->getType()->isArrayType()) {
5186 auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5187 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5188 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5189 CritRedGen);
5190 } else
5191 CritRedGen(CGF, nullptr, nullptr, nullptr);
5192 }
Richard Trieucc3949d2016-02-18 22:34:54 +00005193 ++ILHS;
5194 ++IRHS;
5195 ++IPriv;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005196 }
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005197 };
5198 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5199 if (!WithNowait) {
5200 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5201 llvm::Value *EndArgs[] = {
5202 IdentTLoc, // ident_t *<loc>
5203 ThreadId, // i32 <gtid>
5204 Lock // kmp_critical_name *&<lock>
5205 };
5206 CommonActionTy Action(nullptr, llvm::None,
5207 createRuntimeFunction(OMPRTL__kmpc_end_reduce),
5208 EndArgs);
5209 AtomicRCG.setAction(Action);
5210 AtomicRCG(CGF);
5211 } else
5212 AtomicRCG(CGF);
Alexey Bataev794ba0d2015-04-10 10:43:45 +00005213
5214 CGF.EmitBranch(DefaultBB);
5215 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
5216}
5217
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00005218/// Generates unique name for artificial threadprivate variables.
5219/// Format is: <Prefix> "." <Loc_raw_encoding> "_" <N>
5220static std::string generateUniqueName(StringRef Prefix, SourceLocation Loc,
5221 unsigned N) {
5222 SmallString<256> Buffer;
5223 llvm::raw_svector_ostream Out(Buffer);
5224 Out << Prefix << "." << Loc.getRawEncoding() << "_" << N;
5225 return Out.str();
5226}
5227
5228/// Emits reduction initializer function:
5229/// \code
5230/// void @.red_init(void* %arg) {
5231/// %0 = bitcast void* %arg to <type>*
5232/// store <type> <init>, <type>* %0
5233/// ret void
5234/// }
5235/// \endcode
5236static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
5237 SourceLocation Loc,
5238 ReductionCodeGen &RCG, unsigned N) {
5239 auto &C = CGM.getContext();
5240 FunctionArgList Args;
5241 ImplicitParamDecl Param(C, C.VoidPtrTy, ImplicitParamDecl::Other);
5242 Args.emplace_back(&Param);
5243 auto &FnInfo =
5244 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5245 auto *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
5246 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5247 ".red_init.", &CGM.getModule());
5248 CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, FnInfo);
5249 CodeGenFunction CGF(CGM);
5250 CGF.disableDebugInfo();
5251 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args);
5252 Address PrivateAddr = CGF.EmitLoadOfPointer(
5253 CGF.GetAddrOfLocalVar(&Param),
5254 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5255 llvm::Value *Size = nullptr;
5256 // If the size of the reduction item is non-constant, load it from global
5257 // threadprivate variable.
5258 if (RCG.getSizes(N).second) {
5259 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5260 CGF, CGM.getContext().getSizeType(),
5261 generateUniqueName("reduction_size", Loc, N));
5262 Size =
5263 CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5264 CGM.getContext().getSizeType(), SourceLocation());
5265 }
5266 RCG.emitAggregateType(CGF, N, Size);
5267 LValue SharedLVal;
5268 // If initializer uses initializer from declare reduction construct, emit a
5269 // pointer to the address of the original reduction item (reuired by reduction
5270 // initializer)
5271 if (RCG.usesReductionInitializer(N)) {
5272 Address SharedAddr =
5273 CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5274 CGF, CGM.getContext().VoidPtrTy,
5275 generateUniqueName("reduction", Loc, N));
5276 SharedLVal = CGF.MakeAddrLValue(SharedAddr, CGM.getContext().VoidPtrTy);
5277 } else {
5278 SharedLVal = CGF.MakeNaturalAlignAddrLValue(
5279 llvm::ConstantPointerNull::get(CGM.VoidPtrTy),
5280 CGM.getContext().VoidPtrTy);
5281 }
5282 // Emit the initializer:
5283 // %0 = bitcast void* %arg to <type>*
5284 // store <type> <init>, <type>* %0
5285 RCG.emitInitialization(CGF, N, PrivateAddr, SharedLVal,
5286 [](CodeGenFunction &) { return false; });
5287 CGF.FinishFunction();
5288 return Fn;
5289}
5290
5291/// Emits reduction combiner function:
5292/// \code
5293/// void @.red_comb(void* %arg0, void* %arg1) {
5294/// %lhs = bitcast void* %arg0 to <type>*
5295/// %rhs = bitcast void* %arg1 to <type>*
5296/// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)
5297/// store <type> %2, <type>* %lhs
5298/// ret void
5299/// }
5300/// \endcode
5301static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
5302 SourceLocation Loc,
5303 ReductionCodeGen &RCG, unsigned N,
5304 const Expr *ReductionOp,
5305 const Expr *LHS, const Expr *RHS,
5306 const Expr *PrivateRef) {
5307 auto &C = CGM.getContext();
5308 auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
5309 auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
5310 FunctionArgList Args;
5311 ImplicitParamDecl ParamInOut(C, C.VoidPtrTy, ImplicitParamDecl::Other);
5312 ImplicitParamDecl ParamIn(C, C.VoidPtrTy, ImplicitParamDecl::Other);
5313 Args.emplace_back(&ParamInOut);
5314 Args.emplace_back(&ParamIn);
5315 auto &FnInfo =
5316 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5317 auto *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
5318 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5319 ".red_comb.", &CGM.getModule());
5320 CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, FnInfo);
5321 CodeGenFunction CGF(CGM);
5322 CGF.disableDebugInfo();
5323 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args);
5324 llvm::Value *Size = nullptr;
5325 // If the size of the reduction item is non-constant, load it from global
5326 // threadprivate variable.
5327 if (RCG.getSizes(N).second) {
5328 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5329 CGF, CGM.getContext().getSizeType(),
5330 generateUniqueName("reduction_size", Loc, N));
5331 Size =
5332 CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5333 CGM.getContext().getSizeType(), SourceLocation());
5334 }
5335 RCG.emitAggregateType(CGF, N, Size);
5336 // Remap lhs and rhs variables to the addresses of the function arguments.
5337 // %lhs = bitcast void* %arg0 to <type>*
5338 // %rhs = bitcast void* %arg1 to <type>*
5339 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5340 PrivateScope.addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() -> Address {
5341 // Pull out the pointer to the variable.
5342 Address PtrAddr = CGF.EmitLoadOfPointer(
5343 CGF.GetAddrOfLocalVar(&ParamInOut),
5344 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5345 return CGF.Builder.CreateElementBitCast(
5346 PtrAddr, CGF.ConvertTypeForMem(LHSVD->getType()));
5347 });
5348 PrivateScope.addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() -> Address {
5349 // Pull out the pointer to the variable.
5350 Address PtrAddr = CGF.EmitLoadOfPointer(
5351 CGF.GetAddrOfLocalVar(&ParamIn),
5352 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5353 return CGF.Builder.CreateElementBitCast(
5354 PtrAddr, CGF.ConvertTypeForMem(RHSVD->getType()));
5355 });
5356 PrivateScope.Privatize();
5357 // Emit the combiner body:
5358 // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
5359 // store <type> %2, <type>* %lhs
5360 CGM.getOpenMPRuntime().emitSingleReductionCombiner(
5361 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
5362 cast<DeclRefExpr>(RHS));
5363 CGF.FinishFunction();
5364 return Fn;
5365}
5366
5367/// Emits reduction finalizer function:
5368/// \code
5369/// void @.red_fini(void* %arg) {
5370/// %0 = bitcast void* %arg to <type>*
5371/// <destroy>(<type>* %0)
5372/// ret void
5373/// }
5374/// \endcode
5375static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
5376 SourceLocation Loc,
5377 ReductionCodeGen &RCG, unsigned N) {
5378 if (!RCG.needCleanups(N))
5379 return nullptr;
5380 auto &C = CGM.getContext();
5381 FunctionArgList Args;
5382 ImplicitParamDecl Param(C, C.VoidPtrTy, ImplicitParamDecl::Other);
5383 Args.emplace_back(&Param);
5384 auto &FnInfo =
5385 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5386 auto *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
5387 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5388 ".red_fini.", &CGM.getModule());
5389 CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, FnInfo);
5390 CodeGenFunction CGF(CGM);
5391 CGF.disableDebugInfo();
5392 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args);
5393 Address PrivateAddr = CGF.EmitLoadOfPointer(
5394 CGF.GetAddrOfLocalVar(&Param),
5395 C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5396 llvm::Value *Size = nullptr;
5397 // If the size of the reduction item is non-constant, load it from global
5398 // threadprivate variable.
5399 if (RCG.getSizes(N).second) {
5400 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5401 CGF, CGM.getContext().getSizeType(),
5402 generateUniqueName("reduction_size", Loc, N));
5403 Size =
5404 CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5405 CGM.getContext().getSizeType(), SourceLocation());
5406 }
5407 RCG.emitAggregateType(CGF, N, Size);
5408 // Emit the finalizer body:
5409 // <destroy>(<type>* %0)
5410 RCG.emitCleanups(CGF, N, PrivateAddr);
5411 CGF.FinishFunction();
5412 return Fn;
5413}
5414
5415llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
5416 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
5417 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
5418 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
5419 return nullptr;
5420
5421 // Build typedef struct:
5422 // kmp_task_red_input {
5423 // void *reduce_shar; // shared reduction item
5424 // size_t reduce_size; // size of data item
5425 // void *reduce_init; // data initialization routine
5426 // void *reduce_fini; // data finalization routine
5427 // void *reduce_comb; // data combiner routine
5428 // kmp_task_red_flags_t flags; // flags for additional info from compiler
5429 // } kmp_task_red_input_t;
5430 ASTContext &C = CGM.getContext();
5431 auto *RD = C.buildImplicitRecord("kmp_task_red_input_t");
5432 RD->startDefinition();
5433 const FieldDecl *SharedFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
5434 const FieldDecl *SizeFD = addFieldToRecordDecl(C, RD, C.getSizeType());
5435 const FieldDecl *InitFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
5436 const FieldDecl *FiniFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
5437 const FieldDecl *CombFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
5438 const FieldDecl *FlagsFD = addFieldToRecordDecl(
5439 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
5440 RD->completeDefinition();
5441 QualType RDType = C.getRecordType(RD);
5442 unsigned Size = Data.ReductionVars.size();
5443 llvm::APInt ArraySize(/*numBits=*/64, Size);
5444 QualType ArrayRDType = C.getConstantArrayType(
5445 RDType, ArraySize, ArrayType::Normal, /*IndexTypeQuals=*/0);
5446 // kmp_task_red_input_t .rd_input.[Size];
5447 Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
5448 ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionCopies,
5449 Data.ReductionOps);
5450 for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
5451 // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];
5452 llvm::Value *Idxs[] = {llvm::ConstantInt::get(CGM.SizeTy, /*V=*/0),
5453 llvm::ConstantInt::get(CGM.SizeTy, Cnt)};
5454 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
5455 TaskRedInput.getPointer(), Idxs,
5456 /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
5457 ".rd_input.gep.");
5458 LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType);
5459 // ElemLVal.reduce_shar = &Shareds[Cnt];
5460 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD);
5461 RCG.emitSharedLValue(CGF, Cnt);
5462 llvm::Value *CastedShared =
5463 CGF.EmitCastToVoidPtr(RCG.getSharedLValue(Cnt).getPointer());
5464 CGF.EmitStoreOfScalar(CastedShared, SharedLVal);
5465 RCG.emitAggregateType(CGF, Cnt);
5466 llvm::Value *SizeValInChars;
5467 llvm::Value *SizeVal;
5468 std::tie(SizeValInChars, SizeVal) = RCG.getSizes(Cnt);
5469 // We use delayed creation/initialization for VLAs, array sections and
5470 // custom reduction initializations. It is required because runtime does not
5471 // provide the way to pass the sizes of VLAs/array sections to
5472 // initializer/combiner/finalizer functions and does not pass the pointer to
5473 // original reduction item to the initializer. Instead threadprivate global
5474 // variables are used to store these values and use them in the functions.
5475 bool DelayedCreation = !!SizeVal;
5476 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy,
5477 /*isSigned=*/false);
5478 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD);
5479 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal);
5480 // ElemLVal.reduce_init = init;
5481 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD);
5482 llvm::Value *InitAddr =
5483 CGF.EmitCastToVoidPtr(emitReduceInitFunction(CGM, Loc, RCG, Cnt));
5484 CGF.EmitStoreOfScalar(InitAddr, InitLVal);
5485 DelayedCreation = DelayedCreation || RCG.usesReductionInitializer(Cnt);
5486 // ElemLVal.reduce_fini = fini;
5487 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD);
5488 llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, Cnt);
5489 llvm::Value *FiniAddr = Fini
5490 ? CGF.EmitCastToVoidPtr(Fini)
5491 : llvm::ConstantPointerNull::get(CGM.VoidPtrTy);
5492 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal);
5493 // ElemLVal.reduce_comb = comb;
5494 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD);
5495 llvm::Value *CombAddr = CGF.EmitCastToVoidPtr(emitReduceCombFunction(
5496 CGM, Loc, RCG, Cnt, Data.ReductionOps[Cnt], LHSExprs[Cnt],
5497 RHSExprs[Cnt], Data.ReductionCopies[Cnt]));
5498 CGF.EmitStoreOfScalar(CombAddr, CombLVal);
5499 // ElemLVal.flags = 0;
5500 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD);
5501 if (DelayedCreation) {
5502 CGF.EmitStoreOfScalar(
5503 llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1, /*IsSigned=*/true),
5504 FlagsLVal);
5505 } else
5506 CGF.EmitNullInitialization(FlagsLVal.getAddress(), FlagsLVal.getType());
5507 }
5508 // Build call void *__kmpc_task_reduction_init(int gtid, int num_data, void
5509 // *data);
5510 llvm::Value *Args[] = {
5511 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
5512 /*isSigned=*/true),
5513 llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
5514 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
5515 CGM.VoidPtrTy)};
5516 return CGF.EmitRuntimeCall(
5517 createRuntimeFunction(OMPRTL__kmpc_task_reduction_init), Args);
5518}
5519
5520void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
5521 SourceLocation Loc,
5522 ReductionCodeGen &RCG,
5523 unsigned N) {
5524 auto Sizes = RCG.getSizes(N);
5525 // Emit threadprivate global variable if the type is non-constant
5526 // (Sizes.second = nullptr).
5527 if (Sizes.second) {
5528 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy,
5529 /*isSigned=*/false);
5530 Address SizeAddr = getAddrOfArtificialThreadPrivate(
5531 CGF, CGM.getContext().getSizeType(),
5532 generateUniqueName("reduction_size", Loc, N));
5533 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false);
5534 }
5535 // Store address of the original reduction item if custom initializer is used.
5536 if (RCG.usesReductionInitializer(N)) {
5537 Address SharedAddr = getAddrOfArtificialThreadPrivate(
5538 CGF, CGM.getContext().VoidPtrTy,
5539 generateUniqueName("reduction", Loc, N));
5540 CGF.Builder.CreateStore(
5541 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5542 RCG.getSharedLValue(N).getPointer(), CGM.VoidPtrTy),
5543 SharedAddr, /*IsVolatile=*/false);
5544 }
5545}
5546
5547Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
5548 SourceLocation Loc,
5549 llvm::Value *ReductionsPtr,
5550 LValue SharedLVal) {
5551 // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
5552 // *d);
5553 llvm::Value *Args[] = {
5554 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
5555 /*isSigned=*/true),
5556 ReductionsPtr,
5557 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(SharedLVal.getPointer(),
5558 CGM.VoidPtrTy)};
5559 return Address(
5560 CGF.EmitRuntimeCall(
5561 createRuntimeFunction(OMPRTL__kmpc_task_reduction_get_th_data), Args),
5562 SharedLVal.getAlignment());
5563}
5564
Alexey Bataev8b8e2022015-04-27 05:22:09 +00005565void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
5566 SourceLocation Loc) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00005567 if (!CGF.HaveInsertPoint())
5568 return;
Alexey Bataev8b8e2022015-04-27 05:22:09 +00005569 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
5570 // global_tid);
5571 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
5572 // Ignore return result until untied tasks are supported.
5573 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args);
Alexey Bataev48591dd2016-04-20 04:01:36 +00005574 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5575 Region->emitUntiedSwitch(CGF);
Alexey Bataev8b8e2022015-04-27 05:22:09 +00005576}
5577
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00005578void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
Alexey Bataev81c7ea02015-07-03 09:56:58 +00005579 OpenMPDirectiveKind InnerKind,
Alexey Bataev25e5b442015-09-15 12:52:43 +00005580 const RegionCodeGenTy &CodeGen,
5581 bool HasCancel) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00005582 if (!CGF.HaveInsertPoint())
5583 return;
Alexey Bataev25e5b442015-09-15 12:52:43 +00005584 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00005585 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
Alexey Bataev8cbe0a62015-02-26 10:27:34 +00005586}
5587
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00005588namespace {
5589enum RTCancelKind {
5590 CancelNoreq = 0,
5591 CancelParallel = 1,
5592 CancelLoop = 2,
5593 CancelSections = 3,
5594 CancelTaskgroup = 4
5595};
Eugene Zelenko0a4f3f42016-02-10 19:11:58 +00005596} // anonymous namespace
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00005597
5598static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
5599 RTCancelKind CancelKind = CancelNoreq;
Alexey Bataev0f34da12015-07-02 04:17:07 +00005600 if (CancelRegion == OMPD_parallel)
5601 CancelKind = CancelParallel;
5602 else if (CancelRegion == OMPD_for)
5603 CancelKind = CancelLoop;
5604 else if (CancelRegion == OMPD_sections)
5605 CancelKind = CancelSections;
5606 else {
5607 assert(CancelRegion == OMPD_taskgroup);
5608 CancelKind = CancelTaskgroup;
5609 }
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00005610 return CancelKind;
5611}
5612
5613void CGOpenMPRuntime::emitCancellationPointCall(
5614 CodeGenFunction &CGF, SourceLocation Loc,
5615 OpenMPDirectiveKind CancelRegion) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00005616 if (!CGF.HaveInsertPoint())
5617 return;
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00005618 // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
5619 // global_tid, kmp_int32 cncl_kind);
Alexey Bataev81c7ea02015-07-03 09:56:58 +00005620 if (auto *OMPRegionInfo =
5621 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
Jonas Hahnfeldb07931f2017-02-17 18:32:58 +00005622 // For 'cancellation point taskgroup', the task region info may not have a
5623 // cancel. This may instead happen in another adjacent task.
5624 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00005625 llvm::Value *Args[] = {
5626 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
5627 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
Alexey Bataev81c7ea02015-07-03 09:56:58 +00005628 // Ignore return result until untied tasks are supported.
5629 auto *Result = CGF.EmitRuntimeCall(
5630 createRuntimeFunction(OMPRTL__kmpc_cancellationpoint), Args);
5631 // if (__kmpc_cancellationpoint()) {
Alexey Bataev81c7ea02015-07-03 09:56:58 +00005632 // exit from construct;
5633 // }
5634 auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
5635 auto *ContBB = CGF.createBasicBlock(".cancel.continue");
5636 auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
5637 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5638 CGF.EmitBlock(ExitBB);
Alexey Bataev81c7ea02015-07-03 09:56:58 +00005639 // exit from construct;
Alexey Bataev25e5b442015-09-15 12:52:43 +00005640 auto CancelDest =
5641 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
Alexey Bataev81c7ea02015-07-03 09:56:58 +00005642 CGF.EmitBranchThroughCleanup(CancelDest);
5643 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
5644 }
Alexey Bataev0f34da12015-07-02 04:17:07 +00005645 }
Alexey Bataev0f34da12015-07-02 04:17:07 +00005646}
5647
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00005648void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
Alexey Bataev87933c72015-09-18 08:07:34 +00005649 const Expr *IfCond,
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00005650 OpenMPDirectiveKind CancelRegion) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00005651 if (!CGF.HaveInsertPoint())
5652 return;
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00005653 // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
5654 // kmp_int32 cncl_kind);
5655 if (auto *OMPRegionInfo =
5656 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005657 auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](CodeGenFunction &CGF,
5658 PrePostActionTy &) {
5659 auto &RT = CGF.CGM.getOpenMPRuntime();
Alexey Bataev87933c72015-09-18 08:07:34 +00005660 llvm::Value *Args[] = {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005661 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
Alexey Bataev87933c72015-09-18 08:07:34 +00005662 CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
5663 // Ignore return result until untied tasks are supported.
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005664 auto *Result = CGF.EmitRuntimeCall(
5665 RT.createRuntimeFunction(OMPRTL__kmpc_cancel), Args);
Alexey Bataev87933c72015-09-18 08:07:34 +00005666 // if (__kmpc_cancel()) {
Alexey Bataev87933c72015-09-18 08:07:34 +00005667 // exit from construct;
5668 // }
5669 auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
5670 auto *ContBB = CGF.createBasicBlock(".cancel.continue");
5671 auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
5672 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
5673 CGF.EmitBlock(ExitBB);
Alexey Bataev87933c72015-09-18 08:07:34 +00005674 // exit from construct;
5675 auto CancelDest =
5676 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
5677 CGF.EmitBranchThroughCleanup(CancelDest);
5678 CGF.EmitBlock(ContBB, /*IsFinished=*/true);
5679 };
5680 if (IfCond)
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005681 emitOMPIfClause(CGF, IfCond, ThenGen,
5682 [](CodeGenFunction &, PrePostActionTy &) {});
5683 else {
5684 RegionCodeGenTy ThenRCG(ThenGen);
5685 ThenRCG(CGF);
5686 }
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00005687 }
5688}
Samuel Antaobed3c462015-10-02 16:14:20 +00005689
Samuel Antaoee8fb302016-01-06 13:42:12 +00005690/// \brief Obtain information that uniquely identifies a target entry. This
Samuel Antao2de62b02016-02-13 23:35:10 +00005691/// consists of the file and device IDs as well as line number associated with
5692/// the relevant entry source location.
Samuel Antaoee8fb302016-01-06 13:42:12 +00005693static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
5694 unsigned &DeviceID, unsigned &FileID,
Samuel Antao2de62b02016-02-13 23:35:10 +00005695 unsigned &LineNum) {
Samuel Antaoee8fb302016-01-06 13:42:12 +00005696
5697 auto &SM = C.getSourceManager();
5698
5699 // The loc should be always valid and have a file ID (the user cannot use
5700 // #pragma directives in macros)
5701
5702 assert(Loc.isValid() && "Source location is expected to be always valid.");
5703 assert(Loc.isFileID() && "Source location is expected to refer to a file.");
5704
5705 PresumedLoc PLoc = SM.getPresumedLoc(Loc);
5706 assert(PLoc.isValid() && "Source location is expected to be always valid.");
5707
5708 llvm::sys::fs::UniqueID ID;
5709 if (llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
5710 llvm_unreachable("Source file with target region no longer exists!");
5711
5712 DeviceID = ID.getDevice();
5713 FileID = ID.getFile();
5714 LineNum = PLoc.getLine();
Samuel Antaoee8fb302016-01-06 13:42:12 +00005715}
5716
5717void CGOpenMPRuntime::emitTargetOutlinedFunction(
5718 const OMPExecutableDirective &D, StringRef ParentName,
5719 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
Alexey Bataev14fa1c62016-03-29 05:34:15 +00005720 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
Samuel Antaoee8fb302016-01-06 13:42:12 +00005721 assert(!ParentName.empty() && "Invalid target region parent name!");
5722
Arpith Chacko Jacob5c309e42016-03-22 01:48:56 +00005723 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
5724 IsOffloadEntry, CodeGen);
5725}
5726
5727void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
5728 const OMPExecutableDirective &D, StringRef ParentName,
5729 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
5730 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
Samuel Antao2de62b02016-02-13 23:35:10 +00005731 // Create a unique name for the entry function using the source location
5732 // information of the current target region. The name will be something like:
Samuel Antaoee8fb302016-01-06 13:42:12 +00005733 //
Samuel Antao2de62b02016-02-13 23:35:10 +00005734 // __omp_offloading_DD_FFFF_PP_lBB
Samuel Antaoee8fb302016-01-06 13:42:12 +00005735 //
5736 // where DD_FFFF is an ID unique to the file (device and file IDs), PP is the
Samuel Antao2de62b02016-02-13 23:35:10 +00005737 // mangled name of the function that encloses the target region and BB is the
5738 // line number of the target region.
Samuel Antaoee8fb302016-01-06 13:42:12 +00005739
5740 unsigned DeviceID;
5741 unsigned FileID;
5742 unsigned Line;
Samuel Antaoee8fb302016-01-06 13:42:12 +00005743 getTargetEntryUniqueInfo(CGM.getContext(), D.getLocStart(), DeviceID, FileID,
Samuel Antao2de62b02016-02-13 23:35:10 +00005744 Line);
Samuel Antaoee8fb302016-01-06 13:42:12 +00005745 SmallString<64> EntryFnName;
5746 {
5747 llvm::raw_svector_ostream OS(EntryFnName);
Samuel Antao2de62b02016-02-13 23:35:10 +00005748 OS << "__omp_offloading" << llvm::format("_%x", DeviceID)
5749 << llvm::format("_%x_", FileID) << ParentName << "_l" << Line;
Samuel Antaoee8fb302016-01-06 13:42:12 +00005750 }
5751
Arpith Chacko Jacob5c309e42016-03-22 01:48:56 +00005752 const CapturedStmt &CS = *cast<CapturedStmt>(D.getAssociatedStmt());
5753
Samuel Antaobed3c462015-10-02 16:14:20 +00005754 CodeGenFunction CGF(CGM, true);
Samuel Antaoee8fb302016-01-06 13:42:12 +00005755 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
Samuel Antaobed3c462015-10-02 16:14:20 +00005756 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
Samuel Antaoee8fb302016-01-06 13:42:12 +00005757
Samuel Antao6d004262016-06-16 18:39:34 +00005758 OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS);
Samuel Antaoee8fb302016-01-06 13:42:12 +00005759
5760 // If this target outline function is not an offload entry, we don't need to
5761 // register it.
5762 if (!IsOffloadEntry)
5763 return;
5764
5765 // The target region ID is used by the runtime library to identify the current
5766 // target region, so it only has to be unique and not necessarily point to
5767 // anything. It could be the pointer to the outlined function that implements
5768 // the target region, but we aren't using that so that the compiler doesn't
5769 // need to keep that, and could therefore inline the host function if proven
5770 // worthwhile during optimization. In the other hand, if emitting code for the
5771 // device, the ID has to be the function address so that it can retrieved from
5772 // the offloading entry and launched by the runtime library. We also mark the
5773 // outlined function to have external linkage in case we are emitting code for
5774 // the device, because these functions will be entry points to the device.
5775
5776 if (CGM.getLangOpts().OpenMPIsDevice) {
5777 OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn, CGM.Int8PtrTy);
5778 OutlinedFn->setLinkage(llvm::GlobalValue::ExternalLinkage);
5779 } else
5780 OutlinedFnID = new llvm::GlobalVariable(
5781 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
5782 llvm::GlobalValue::PrivateLinkage,
5783 llvm::Constant::getNullValue(CGM.Int8Ty), ".omp_offload.region_id");
5784
5785 // Register the information for the entry associated with this target region.
5786 OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
Samuel Antaof83efdb2017-01-05 16:02:49 +00005787 DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID,
5788 /*Flags=*/0);
Samuel Antaobed3c462015-10-02 16:14:20 +00005789}
5790
Carlo Bertolli6eee9062016-04-29 01:37:30 +00005791/// discard all CompoundStmts intervening between two constructs
5792static const Stmt *ignoreCompoundStmts(const Stmt *Body) {
5793 while (auto *CS = dyn_cast_or_null<CompoundStmt>(Body))
5794 Body = CS->body_front();
5795
5796 return Body;
5797}
5798
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00005799/// Emit the number of teams for a target directive. Inspect the num_teams
5800/// clause associated with a teams construct combined or closely nested
5801/// with the target directive.
5802///
5803/// Emit a team of size one for directives such as 'target parallel' that
5804/// have no associated teams construct.
5805///
5806/// Otherwise, return nullptr.
Samuel Antaob68e2db2016-03-03 16:20:23 +00005807static llvm::Value *
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00005808emitNumTeamsForTargetDirective(CGOpenMPRuntime &OMPRuntime,
5809 CodeGenFunction &CGF,
5810 const OMPExecutableDirective &D) {
Samuel Antaob68e2db2016-03-03 16:20:23 +00005811
5812 assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the "
5813 "teams directive expected to be "
5814 "emitted only for the host!");
5815
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00005816 auto &Bld = CGF.Builder;
5817
5818 // If the target directive is combined with a teams directive:
5819 // Return the value in the num_teams clause, if any.
5820 // Otherwise, return 0 to denote the runtime default.
5821 if (isOpenMPTeamsDirective(D.getDirectiveKind())) {
5822 if (const auto *NumTeamsClause = D.getSingleClause<OMPNumTeamsClause>()) {
5823 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
5824 auto NumTeams = CGF.EmitScalarExpr(NumTeamsClause->getNumTeams(),
5825 /*IgnoreResultAssign*/ true);
5826 return Bld.CreateIntCast(NumTeams, CGF.Int32Ty,
5827 /*IsSigned=*/true);
5828 }
5829
5830 // The default value is 0.
5831 return Bld.getInt32(0);
5832 }
5833
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00005834 // If the target directive is combined with a parallel directive but not a
5835 // teams directive, start one team.
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00005836 if (isOpenMPParallelDirective(D.getDirectiveKind()))
5837 return Bld.getInt32(1);
Samuel Antaob68e2db2016-03-03 16:20:23 +00005838
5839 // If the current target region has a teams region enclosed, we need to get
5840 // the number of teams to pass to the runtime function call. This is done
5841 // by generating the expression in a inlined region. This is required because
5842 // the expression is captured in the enclosing target environment when the
5843 // teams directive is not combined with target.
5844
5845 const CapturedStmt &CS = *cast<CapturedStmt>(D.getAssociatedStmt());
5846
Alexey Bataev50a1c782017-12-01 21:31:08 +00005847 if (auto *TeamsDir = dyn_cast_or_null<OMPExecutableDirective>(
Carlo Bertolli6eee9062016-04-29 01:37:30 +00005848 ignoreCompoundStmts(CS.getCapturedStmt()))) {
Alexey Bataev50a1c782017-12-01 21:31:08 +00005849 if (isOpenMPTeamsDirective(TeamsDir->getDirectiveKind())) {
5850 if (auto *NTE = TeamsDir->getSingleClause<OMPNumTeamsClause>()) {
5851 CGOpenMPInnerExprInfo CGInfo(CGF, CS);
5852 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
5853 llvm::Value *NumTeams = CGF.EmitScalarExpr(NTE->getNumTeams());
5854 return Bld.CreateIntCast(NumTeams, CGF.Int32Ty,
5855 /*IsSigned=*/true);
5856 }
Samuel Antaob68e2db2016-03-03 16:20:23 +00005857
Alexey Bataev50a1c782017-12-01 21:31:08 +00005858 // If we have an enclosed teams directive but no num_teams clause we use
5859 // the default value 0.
5860 return Bld.getInt32(0);
5861 }
Samuel Antaob68e2db2016-03-03 16:20:23 +00005862 }
5863
5864 // No teams associated with the directive.
5865 return nullptr;
5866}
5867
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00005868/// Emit the number of threads for a target directive. Inspect the
5869/// thread_limit clause associated with a teams construct combined or closely
5870/// nested with the target directive.
5871///
5872/// Emit the num_threads clause for directives such as 'target parallel' that
5873/// have no associated teams construct.
5874///
5875/// Otherwise, return nullptr.
Samuel Antaob68e2db2016-03-03 16:20:23 +00005876static llvm::Value *
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00005877emitNumThreadsForTargetDirective(CGOpenMPRuntime &OMPRuntime,
5878 CodeGenFunction &CGF,
5879 const OMPExecutableDirective &D) {
Samuel Antaob68e2db2016-03-03 16:20:23 +00005880
5881 assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the "
5882 "teams directive expected to be "
5883 "emitted only for the host!");
5884
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00005885 auto &Bld = CGF.Builder;
5886
5887 //
5888 // If the target directive is combined with a teams directive:
5889 // Return the value in the thread_limit clause, if any.
5890 //
5891 // If the target directive is combined with a parallel directive:
5892 // Return the value in the num_threads clause, if any.
5893 //
5894 // If both clauses are set, select the minimum of the two.
5895 //
5896 // If neither teams or parallel combined directives set the number of threads
5897 // in a team, return 0 to denote the runtime default.
5898 //
5899 // If this is not a teams directive return nullptr.
5900
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00005901 if (isOpenMPTeamsDirective(D.getDirectiveKind()) ||
5902 isOpenMPParallelDirective(D.getDirectiveKind())) {
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00005903 llvm::Value *DefaultThreadLimitVal = Bld.getInt32(0);
5904 llvm::Value *NumThreadsVal = nullptr;
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00005905 llvm::Value *ThreadLimitVal = nullptr;
5906
5907 if (const auto *ThreadLimitClause =
5908 D.getSingleClause<OMPThreadLimitClause>()) {
5909 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
5910 auto ThreadLimit = CGF.EmitScalarExpr(ThreadLimitClause->getThreadLimit(),
5911 /*IgnoreResultAssign*/ true);
5912 ThreadLimitVal = Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty,
5913 /*IsSigned=*/true);
5914 }
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00005915
5916 if (const auto *NumThreadsClause =
5917 D.getSingleClause<OMPNumThreadsClause>()) {
5918 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
5919 llvm::Value *NumThreads =
5920 CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
5921 /*IgnoreResultAssign*/ true);
5922 NumThreadsVal =
5923 Bld.CreateIntCast(NumThreads, CGF.Int32Ty, /*IsSigned=*/true);
5924 }
5925
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00005926 // Select the lesser of thread_limit and num_threads.
5927 if (NumThreadsVal)
5928 ThreadLimitVal = ThreadLimitVal
5929 ? Bld.CreateSelect(Bld.CreateICmpSLT(NumThreadsVal,
5930 ThreadLimitVal),
5931 NumThreadsVal, ThreadLimitVal)
5932 : NumThreadsVal;
Samuel Antaob68e2db2016-03-03 16:20:23 +00005933
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00005934 // Set default value passed to the runtime if either teams or a target
5935 // parallel type directive is found but no clause is specified.
5936 if (!ThreadLimitVal)
5937 ThreadLimitVal = DefaultThreadLimitVal;
5938
5939 return ThreadLimitVal;
5940 }
Arpith Chacko Jacob86f9e462017-01-25 01:45:59 +00005941
Samuel Antaob68e2db2016-03-03 16:20:23 +00005942 // If the current target region has a teams region enclosed, we need to get
5943 // the thread limit to pass to the runtime function call. This is done
5944 // by generating the expression in a inlined region. This is required because
5945 // the expression is captured in the enclosing target environment when the
5946 // teams directive is not combined with target.
5947
5948 const CapturedStmt &CS = *cast<CapturedStmt>(D.getAssociatedStmt());
5949
Alexey Bataev50a1c782017-12-01 21:31:08 +00005950 if (auto *TeamsDir = dyn_cast_or_null<OMPExecutableDirective>(
Carlo Bertolli6eee9062016-04-29 01:37:30 +00005951 ignoreCompoundStmts(CS.getCapturedStmt()))) {
Alexey Bataev50a1c782017-12-01 21:31:08 +00005952 if (isOpenMPTeamsDirective(TeamsDir->getDirectiveKind())) {
5953 if (auto *TLE = TeamsDir->getSingleClause<OMPThreadLimitClause>()) {
5954 CGOpenMPInnerExprInfo CGInfo(CGF, CS);
5955 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
5956 llvm::Value *ThreadLimit = CGF.EmitScalarExpr(TLE->getThreadLimit());
5957 return CGF.Builder.CreateIntCast(ThreadLimit, CGF.Int32Ty,
5958 /*IsSigned=*/true);
5959 }
Samuel Antaob68e2db2016-03-03 16:20:23 +00005960
Alexey Bataev50a1c782017-12-01 21:31:08 +00005961 // If we have an enclosed teams directive but no thread_limit clause we
5962 // use the default value 0.
5963 return CGF.Builder.getInt32(0);
5964 }
Samuel Antaob68e2db2016-03-03 16:20:23 +00005965 }
5966
5967 // No teams associated with the directive.
5968 return nullptr;
5969}
5970
Samuel Antao86ace552016-04-27 22:40:57 +00005971namespace {
5972// \brief Utility to handle information from clauses associated with a given
5973// construct that use mappable expressions (e.g. 'map' clause, 'to' clause).
5974// It provides a convenient interface to obtain the information and generate
5975// code for that information.
5976class MappableExprsHandler {
5977public:
5978 /// \brief Values for bit flags used to specify the mapping type for
5979 /// offloading.
5980 enum OpenMPOffloadMappingFlags {
Samuel Antao86ace552016-04-27 22:40:57 +00005981 /// \brief Allocate memory on the device and move data from host to device.
5982 OMP_MAP_TO = 0x01,
5983 /// \brief Allocate memory on the device and move data from device to host.
5984 OMP_MAP_FROM = 0x02,
5985 /// \brief Always perform the requested mapping action on the element, even
5986 /// if it was already mapped before.
5987 OMP_MAP_ALWAYS = 0x04,
Samuel Antao86ace552016-04-27 22:40:57 +00005988 /// \brief Delete the element from the device environment, ignoring the
5989 /// current reference count associated with the element.
Samuel Antao6782e942016-05-26 16:48:10 +00005990 OMP_MAP_DELETE = 0x08,
George Rokos065755d2017-11-07 18:27:04 +00005991 /// \brief The element being mapped is a pointer-pointee pair; both the
5992 /// pointer and the pointee should be mapped.
5993 OMP_MAP_PTR_AND_OBJ = 0x10,
5994 /// \brief This flags signals that the base address of an entry should be
5995 /// passed to the target kernel as an argument.
5996 OMP_MAP_TARGET_PARAM = 0x20,
Samuel Antaocc10b852016-07-28 14:23:26 +00005997 /// \brief Signal that the runtime library has to return the device pointer
George Rokos065755d2017-11-07 18:27:04 +00005998 /// in the current position for the data being mapped. Used when we have the
5999 /// use_device_ptr clause.
6000 OMP_MAP_RETURN_PARAM = 0x40,
Samuel Antaod486f842016-05-26 16:53:38 +00006001 /// \brief This flag signals that the reference being passed is a pointer to
6002 /// private data.
George Rokos065755d2017-11-07 18:27:04 +00006003 OMP_MAP_PRIVATE = 0x80,
Samuel Antao86ace552016-04-27 22:40:57 +00006004 /// \brief Pass the element to the device by value.
George Rokos065755d2017-11-07 18:27:04 +00006005 OMP_MAP_LITERAL = 0x100,
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006006 /// Implicit map
6007 OMP_MAP_IMPLICIT = 0x200,
Samuel Antao86ace552016-04-27 22:40:57 +00006008 };
6009
Samuel Antaocc10b852016-07-28 14:23:26 +00006010 /// Class that associates information with a base pointer to be passed to the
6011 /// runtime library.
6012 class BasePointerInfo {
6013 /// The base pointer.
6014 llvm::Value *Ptr = nullptr;
6015 /// The base declaration that refers to this device pointer, or null if
6016 /// there is none.
6017 const ValueDecl *DevPtrDecl = nullptr;
6018
6019 public:
6020 BasePointerInfo(llvm::Value *Ptr, const ValueDecl *DevPtrDecl = nullptr)
6021 : Ptr(Ptr), DevPtrDecl(DevPtrDecl) {}
6022 llvm::Value *operator*() const { return Ptr; }
6023 const ValueDecl *getDevicePtrDecl() const { return DevPtrDecl; }
6024 void setDevicePtrDecl(const ValueDecl *D) { DevPtrDecl = D; }
6025 };
6026
6027 typedef SmallVector<BasePointerInfo, 16> MapBaseValuesArrayTy;
Samuel Antao86ace552016-04-27 22:40:57 +00006028 typedef SmallVector<llvm::Value *, 16> MapValuesArrayTy;
George Rokos63bc9d62017-11-21 18:25:12 +00006029 typedef SmallVector<uint64_t, 16> MapFlagsArrayTy;
Samuel Antao86ace552016-04-27 22:40:57 +00006030
6031private:
6032 /// \brief Directive from where the map clauses were extracted.
Samuel Antao44bcdb32016-07-28 15:31:29 +00006033 const OMPExecutableDirective &CurDir;
Samuel Antao86ace552016-04-27 22:40:57 +00006034
6035 /// \brief Function the directive is being generated for.
6036 CodeGenFunction &CGF;
6037
Samuel Antaod486f842016-05-26 16:53:38 +00006038 /// \brief Set of all first private variables in the current directive.
6039 llvm::SmallPtrSet<const VarDecl *, 8> FirstPrivateDecls;
6040
Samuel Antao6890b092016-07-28 14:25:09 +00006041 /// Map between device pointer declarations and their expression components.
6042 /// The key value for declarations in 'this' is null.
6043 llvm::DenseMap<
6044 const ValueDecl *,
6045 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
6046 DevPointersMap;
6047
Samuel Antao86ace552016-04-27 22:40:57 +00006048 llvm::Value *getExprTypeSize(const Expr *E) const {
6049 auto ExprTy = E->getType().getCanonicalType();
6050
6051 // Reference types are ignored for mapping purposes.
6052 if (auto *RefTy = ExprTy->getAs<ReferenceType>())
6053 ExprTy = RefTy->getPointeeType().getCanonicalType();
6054
6055 // Given that an array section is considered a built-in type, we need to
6056 // do the calculation based on the length of the section instead of relying
6057 // on CGF.getTypeSize(E->getType()).
6058 if (const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) {
6059 QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(
6060 OAE->getBase()->IgnoreParenImpCasts())
6061 .getCanonicalType();
6062
6063 // If there is no length associated with the expression, that means we
6064 // are using the whole length of the base.
6065 if (!OAE->getLength() && OAE->getColonLoc().isValid())
6066 return CGF.getTypeSize(BaseTy);
6067
6068 llvm::Value *ElemSize;
6069 if (auto *PTy = BaseTy->getAs<PointerType>())
6070 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType());
6071 else {
6072 auto *ATy = cast<ArrayType>(BaseTy.getTypePtr());
6073 assert(ATy && "Expecting array type if not a pointer type.");
6074 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType());
6075 }
6076
6077 // If we don't have a length at this point, that is because we have an
6078 // array section with a single element.
6079 if (!OAE->getLength())
6080 return ElemSize;
6081
6082 auto *LengthVal = CGF.EmitScalarExpr(OAE->getLength());
6083 LengthVal =
6084 CGF.Builder.CreateIntCast(LengthVal, CGF.SizeTy, /*isSigned=*/false);
6085 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
6086 }
6087 return CGF.getTypeSize(ExprTy);
6088 }
6089
6090 /// \brief Return the corresponding bits for a given map clause modifier. Add
6091 /// a flag marking the map as a pointer if requested. Add a flag marking the
Samuel Antao6782e942016-05-26 16:48:10 +00006092 /// map as the first one of a series of maps that relate to the same map
6093 /// expression.
George Rokos63bc9d62017-11-21 18:25:12 +00006094 uint64_t getMapTypeBits(OpenMPMapClauseKind MapType,
Samuel Antao86ace552016-04-27 22:40:57 +00006095 OpenMPMapClauseKind MapTypeModifier, bool AddPtrFlag,
George Rokos065755d2017-11-07 18:27:04 +00006096 bool AddIsTargetParamFlag) const {
George Rokos63bc9d62017-11-21 18:25:12 +00006097 uint64_t Bits = 0u;
Samuel Antao86ace552016-04-27 22:40:57 +00006098 switch (MapType) {
6099 case OMPC_MAP_alloc:
Samuel Antao6782e942016-05-26 16:48:10 +00006100 case OMPC_MAP_release:
6101 // alloc and release is the default behavior in the runtime library, i.e.
6102 // if we don't pass any bits alloc/release that is what the runtime is
6103 // going to do. Therefore, we don't need to signal anything for these two
6104 // type modifiers.
Samuel Antao86ace552016-04-27 22:40:57 +00006105 break;
6106 case OMPC_MAP_to:
6107 Bits = OMP_MAP_TO;
6108 break;
6109 case OMPC_MAP_from:
6110 Bits = OMP_MAP_FROM;
6111 break;
6112 case OMPC_MAP_tofrom:
6113 Bits = OMP_MAP_TO | OMP_MAP_FROM;
6114 break;
6115 case OMPC_MAP_delete:
6116 Bits = OMP_MAP_DELETE;
6117 break;
Samuel Antao86ace552016-04-27 22:40:57 +00006118 default:
6119 llvm_unreachable("Unexpected map type!");
6120 break;
6121 }
6122 if (AddPtrFlag)
George Rokos065755d2017-11-07 18:27:04 +00006123 Bits |= OMP_MAP_PTR_AND_OBJ;
6124 if (AddIsTargetParamFlag)
6125 Bits |= OMP_MAP_TARGET_PARAM;
Samuel Antao86ace552016-04-27 22:40:57 +00006126 if (MapTypeModifier == OMPC_MAP_always)
6127 Bits |= OMP_MAP_ALWAYS;
6128 return Bits;
6129 }
6130
6131 /// \brief Return true if the provided expression is a final array section. A
6132 /// final array section, is one whose length can't be proved to be one.
6133 bool isFinalArraySectionExpression(const Expr *E) const {
6134 auto *OASE = dyn_cast<OMPArraySectionExpr>(E);
6135
6136 // It is not an array section and therefore not a unity-size one.
6137 if (!OASE)
6138 return false;
6139
6140 // An array section with no colon always refer to a single element.
6141 if (OASE->getColonLoc().isInvalid())
6142 return false;
6143
6144 auto *Length = OASE->getLength();
6145
6146 // If we don't have a length we have to check if the array has size 1
6147 // for this dimension. Also, we should always expect a length if the
6148 // base type is pointer.
6149 if (!Length) {
6150 auto BaseQTy = OMPArraySectionExpr::getBaseOriginalType(
6151 OASE->getBase()->IgnoreParenImpCasts())
6152 .getCanonicalType();
6153 if (auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
6154 return ATy->getSize().getSExtValue() != 1;
6155 // If we don't have a constant dimension length, we have to consider
6156 // the current section as having any size, so it is not necessarily
6157 // unitary. If it happen to be unity size, that's user fault.
6158 return true;
6159 }
6160
6161 // Check if the length evaluates to 1.
6162 llvm::APSInt ConstLength;
6163 if (!Length->EvaluateAsInt(ConstLength, CGF.getContext()))
6164 return true; // Can have more that size 1.
6165
6166 return ConstLength.getSExtValue() != 1;
6167 }
6168
6169 /// \brief Generate the base pointers, section pointers, sizes and map type
6170 /// bits for the provided map type, map modifier, and expression components.
6171 /// \a IsFirstComponent should be set to true if the provided set of
6172 /// components is the first associated with a capture.
6173 void generateInfoForComponentList(
6174 OpenMPMapClauseKind MapType, OpenMPMapClauseKind MapTypeModifier,
6175 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
Samuel Antaocc10b852016-07-28 14:23:26 +00006176 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
Samuel Antao86ace552016-04-27 22:40:57 +00006177 MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006178 bool IsFirstComponentList, bool IsImplicit) const {
Samuel Antao86ace552016-04-27 22:40:57 +00006179
6180 // The following summarizes what has to be generated for each map and the
6181 // types bellow. The generated information is expressed in this order:
6182 // base pointer, section pointer, size, flags
6183 // (to add to the ones that come from the map type and modifier).
6184 //
6185 // double d;
6186 // int i[100];
6187 // float *p;
6188 //
6189 // struct S1 {
6190 // int i;
6191 // float f[50];
6192 // }
6193 // struct S2 {
6194 // int i;
6195 // float f[50];
6196 // S1 s;
6197 // double *p;
6198 // struct S2 *ps;
6199 // }
6200 // S2 s;
6201 // S2 *ps;
6202 //
6203 // map(d)
6204 // &d, &d, sizeof(double), noflags
6205 //
6206 // map(i)
6207 // &i, &i, 100*sizeof(int), noflags
6208 //
6209 // map(i[1:23])
6210 // &i(=&i[0]), &i[1], 23*sizeof(int), noflags
6211 //
6212 // map(p)
6213 // &p, &p, sizeof(float*), noflags
6214 //
6215 // map(p[1:24])
6216 // p, &p[1], 24*sizeof(float), noflags
6217 //
6218 // map(s)
6219 // &s, &s, sizeof(S2), noflags
6220 //
6221 // map(s.i)
6222 // &s, &(s.i), sizeof(int), noflags
6223 //
6224 // map(s.s.f)
6225 // &s, &(s.i.f), 50*sizeof(int), noflags
6226 //
6227 // map(s.p)
6228 // &s, &(s.p), sizeof(double*), noflags
6229 //
6230 // map(s.p[:22], s.a s.b)
6231 // &s, &(s.p), sizeof(double*), noflags
George Rokos065755d2017-11-07 18:27:04 +00006232 // &(s.p), &(s.p[0]), 22*sizeof(double), ptr_flag
Samuel Antao86ace552016-04-27 22:40:57 +00006233 //
6234 // map(s.ps)
6235 // &s, &(s.ps), sizeof(S2*), noflags
6236 //
6237 // map(s.ps->s.i)
6238 // &s, &(s.ps), sizeof(S2*), noflags
George Rokos065755d2017-11-07 18:27:04 +00006239 // &(s.ps), &(s.ps->s.i), sizeof(int), ptr_flag
Samuel Antao86ace552016-04-27 22:40:57 +00006240 //
6241 // map(s.ps->ps)
6242 // &s, &(s.ps), sizeof(S2*), noflags
George Rokos065755d2017-11-07 18:27:04 +00006243 // &(s.ps), &(s.ps->ps), sizeof(S2*), ptr_flag
Samuel Antao86ace552016-04-27 22:40:57 +00006244 //
6245 // map(s.ps->ps->ps)
6246 // &s, &(s.ps), sizeof(S2*), noflags
George Rokos065755d2017-11-07 18:27:04 +00006247 // &(s.ps), &(s.ps->ps), sizeof(S2*), ptr_flag
6248 // &(s.ps->ps), &(s.ps->ps->ps), sizeof(S2*), ptr_flag
Samuel Antao86ace552016-04-27 22:40:57 +00006249 //
6250 // map(s.ps->ps->s.f[:22])
6251 // &s, &(s.ps), sizeof(S2*), noflags
George Rokos065755d2017-11-07 18:27:04 +00006252 // &(s.ps), &(s.ps->ps), sizeof(S2*), ptr_flag
6253 // &(s.ps->ps), &(s.ps->ps->s.f[0]), 22*sizeof(float), ptr_flag
Samuel Antao86ace552016-04-27 22:40:57 +00006254 //
6255 // map(ps)
6256 // &ps, &ps, sizeof(S2*), noflags
6257 //
6258 // map(ps->i)
6259 // ps, &(ps->i), sizeof(int), noflags
6260 //
6261 // map(ps->s.f)
6262 // ps, &(ps->s.f[0]), 50*sizeof(float), noflags
6263 //
6264 // map(ps->p)
6265 // ps, &(ps->p), sizeof(double*), noflags
6266 //
6267 // map(ps->p[:22])
6268 // ps, &(ps->p), sizeof(double*), noflags
George Rokos065755d2017-11-07 18:27:04 +00006269 // &(ps->p), &(ps->p[0]), 22*sizeof(double), ptr_flag
Samuel Antao86ace552016-04-27 22:40:57 +00006270 //
6271 // map(ps->ps)
6272 // ps, &(ps->ps), sizeof(S2*), noflags
6273 //
6274 // map(ps->ps->s.i)
6275 // ps, &(ps->ps), sizeof(S2*), noflags
George Rokos065755d2017-11-07 18:27:04 +00006276 // &(ps->ps), &(ps->ps->s.i), sizeof(int), ptr_flag
Samuel Antao86ace552016-04-27 22:40:57 +00006277 //
6278 // map(ps->ps->ps)
6279 // ps, &(ps->ps), sizeof(S2*), noflags
George Rokos065755d2017-11-07 18:27:04 +00006280 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), ptr_flag
Samuel Antao86ace552016-04-27 22:40:57 +00006281 //
6282 // map(ps->ps->ps->ps)
6283 // ps, &(ps->ps), sizeof(S2*), noflags
George Rokos065755d2017-11-07 18:27:04 +00006284 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), ptr_flag
6285 // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(S2*), ptr_flag
Samuel Antao86ace552016-04-27 22:40:57 +00006286 //
6287 // map(ps->ps->ps->s.f[:22])
6288 // ps, &(ps->ps), sizeof(S2*), noflags
George Rokos065755d2017-11-07 18:27:04 +00006289 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), ptr_flag
6290 // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), 22*sizeof(float), ptr_flag
Samuel Antao86ace552016-04-27 22:40:57 +00006291
6292 // Track if the map information being generated is the first for a capture.
6293 bool IsCaptureFirstInfo = IsFirstComponentList;
6294
6295 // Scan the components from the base to the complete expression.
6296 auto CI = Components.rbegin();
6297 auto CE = Components.rend();
6298 auto I = CI;
6299
6300 // Track if the map information being generated is the first for a list of
6301 // components.
6302 bool IsExpressionFirstInfo = true;
6303 llvm::Value *BP = nullptr;
6304
6305 if (auto *ME = dyn_cast<MemberExpr>(I->getAssociatedExpression())) {
6306 // The base is the 'this' pointer. The content of the pointer is going
6307 // to be the base of the field being mapped.
6308 BP = CGF.EmitScalarExpr(ME->getBase());
6309 } else {
6310 // The base is the reference to the variable.
6311 // BP = &Var.
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006312 BP = CGF.EmitOMPSharedLValue(I->getAssociatedExpression()).getPointer();
Samuel Antao86ace552016-04-27 22:40:57 +00006313
6314 // If the variable is a pointer and is being dereferenced (i.e. is not
Nico Webera6916892016-06-10 18:53:04 +00006315 // the last component), the base has to be the pointer itself, not its
Samuel Antao403ffd42016-07-27 22:49:49 +00006316 // reference. References are ignored for mapping purposes.
6317 QualType Ty =
6318 I->getAssociatedDeclaration()->getType().getNonReferenceType();
6319 if (Ty->isAnyPointerType() && std::next(I) != CE) {
6320 auto PtrAddr = CGF.MakeNaturalAlignAddrLValue(BP, Ty);
Samuel Antao86ace552016-04-27 22:40:57 +00006321 BP = CGF.EmitLoadOfPointerLValue(PtrAddr.getAddress(),
Samuel Antao403ffd42016-07-27 22:49:49 +00006322 Ty->castAs<PointerType>())
Samuel Antao86ace552016-04-27 22:40:57 +00006323 .getPointer();
6324
6325 // We do not need to generate individual map information for the
6326 // pointer, it can be associated with the combined storage.
6327 ++I;
6328 }
6329 }
6330
George Rokos63bc9d62017-11-21 18:25:12 +00006331 uint64_t DefaultFlags = IsImplicit ? OMP_MAP_IMPLICIT : 0;
Samuel Antao86ace552016-04-27 22:40:57 +00006332 for (; I != CE; ++I) {
6333 auto Next = std::next(I);
6334
6335 // We need to generate the addresses and sizes if this is the last
6336 // component, if the component is a pointer or if it is an array section
6337 // whose length can't be proved to be one. If this is a pointer, it
6338 // becomes the base address for the following components.
6339
6340 // A final array section, is one whose length can't be proved to be one.
6341 bool IsFinalArraySection =
6342 isFinalArraySectionExpression(I->getAssociatedExpression());
6343
6344 // Get information on whether the element is a pointer. Have to do a
6345 // special treatment for array sections given that they are built-in
6346 // types.
6347 const auto *OASE =
6348 dyn_cast<OMPArraySectionExpr>(I->getAssociatedExpression());
6349 bool IsPointer =
6350 (OASE &&
6351 OMPArraySectionExpr::getBaseOriginalType(OASE)
6352 .getCanonicalType()
6353 ->isAnyPointerType()) ||
6354 I->getAssociatedExpression()->getType()->isAnyPointerType();
6355
6356 if (Next == CE || IsPointer || IsFinalArraySection) {
6357
6358 // If this is not the last component, we expect the pointer to be
6359 // associated with an array expression or member expression.
6360 assert((Next == CE ||
6361 isa<MemberExpr>(Next->getAssociatedExpression()) ||
6362 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
6363 isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) &&
6364 "Unexpected expression");
6365
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006366 llvm::Value *LB =
6367 CGF.EmitOMPSharedLValue(I->getAssociatedExpression()).getPointer();
Samuel Antao86ace552016-04-27 22:40:57 +00006368 auto *Size = getExprTypeSize(I->getAssociatedExpression());
6369
Samuel Antao03a3cec2016-07-27 22:52:16 +00006370 // If we have a member expression and the current component is a
6371 // reference, we have to map the reference too. Whenever we have a
6372 // reference, the section that reference refers to is going to be a
6373 // load instruction from the storage assigned to the reference.
6374 if (isa<MemberExpr>(I->getAssociatedExpression()) &&
6375 I->getAssociatedDeclaration()->getType()->isReferenceType()) {
6376 auto *LI = cast<llvm::LoadInst>(LB);
6377 auto *RefAddr = LI->getPointerOperand();
6378
6379 BasePointers.push_back(BP);
6380 Pointers.push_back(RefAddr);
6381 Sizes.push_back(CGF.getTypeSize(CGF.getContext().VoidPtrTy));
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006382 Types.push_back(DefaultFlags |
6383 getMapTypeBits(
6384 /*MapType*/ OMPC_MAP_alloc,
6385 /*MapTypeModifier=*/OMPC_MAP_unknown,
6386 !IsExpressionFirstInfo, IsCaptureFirstInfo));
Samuel Antao03a3cec2016-07-27 22:52:16 +00006387 IsExpressionFirstInfo = false;
6388 IsCaptureFirstInfo = false;
6389 // The reference will be the next base address.
6390 BP = RefAddr;
6391 }
6392
6393 BasePointers.push_back(BP);
Samuel Antao86ace552016-04-27 22:40:57 +00006394 Pointers.push_back(LB);
6395 Sizes.push_back(Size);
Samuel Antao03a3cec2016-07-27 22:52:16 +00006396
Samuel Antao6782e942016-05-26 16:48:10 +00006397 // We need to add a pointer flag for each map that comes from the
6398 // same expression except for the first one. We also need to signal
6399 // this map is the first one that relates with the current capture
6400 // (there is a set of entries for each capture).
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006401 Types.push_back(DefaultFlags | getMapTypeBits(MapType, MapTypeModifier,
6402 !IsExpressionFirstInfo,
6403 IsCaptureFirstInfo));
Samuel Antao86ace552016-04-27 22:40:57 +00006404
6405 // If we have a final array section, we are done with this expression.
6406 if (IsFinalArraySection)
6407 break;
6408
6409 // The pointer becomes the base for the next element.
6410 if (Next != CE)
6411 BP = LB;
6412
6413 IsExpressionFirstInfo = false;
6414 IsCaptureFirstInfo = false;
Samuel Antao86ace552016-04-27 22:40:57 +00006415 }
6416 }
6417 }
6418
Samuel Antaod486f842016-05-26 16:53:38 +00006419 /// \brief Return the adjusted map modifiers if the declaration a capture
6420 /// refers to appears in a first-private clause. This is expected to be used
6421 /// only with directives that start with 'target'.
6422 unsigned adjustMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap,
6423 unsigned CurrentModifiers) {
6424 assert(Cap.capturesVariable() && "Expected capture by reference only!");
6425
6426 // A first private variable captured by reference will use only the
6427 // 'private ptr' and 'map to' flag. Return the right flags if the captured
6428 // declaration is known as first-private in this handler.
6429 if (FirstPrivateDecls.count(Cap.getCapturedVar()))
George Rokos065755d2017-11-07 18:27:04 +00006430 return MappableExprsHandler::OMP_MAP_PRIVATE |
Samuel Antaod486f842016-05-26 16:53:38 +00006431 MappableExprsHandler::OMP_MAP_TO;
6432
6433 // We didn't modify anything.
6434 return CurrentModifiers;
6435 }
6436
Samuel Antao86ace552016-04-27 22:40:57 +00006437public:
6438 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
Samuel Antao44bcdb32016-07-28 15:31:29 +00006439 : CurDir(Dir), CGF(CGF) {
Samuel Antaod486f842016-05-26 16:53:38 +00006440 // Extract firstprivate clause information.
6441 for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
6442 for (const auto *D : C->varlists())
6443 FirstPrivateDecls.insert(
6444 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
Samuel Antao6890b092016-07-28 14:25:09 +00006445 // Extract device pointer clause information.
6446 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
6447 for (auto L : C->component_lists())
6448 DevPointersMap[L.first].push_back(L.second);
Samuel Antaod486f842016-05-26 16:53:38 +00006449 }
Samuel Antao86ace552016-04-27 22:40:57 +00006450
6451 /// \brief Generate all the base pointers, section pointers, sizes and map
Samuel Antaocc10b852016-07-28 14:23:26 +00006452 /// types for the extracted mappable expressions. Also, for each item that
6453 /// relates with a device pointer, a pair of the relevant declaration and
6454 /// index where it occurs is appended to the device pointers info array.
6455 void generateAllInfo(MapBaseValuesArrayTy &BasePointers,
Samuel Antao86ace552016-04-27 22:40:57 +00006456 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
6457 MapFlagsArrayTy &Types) const {
6458 BasePointers.clear();
6459 Pointers.clear();
6460 Sizes.clear();
6461 Types.clear();
6462
6463 struct MapInfo {
Samuel Antaocc10b852016-07-28 14:23:26 +00006464 /// Kind that defines how a device pointer has to be returned.
6465 enum ReturnPointerKind {
6466 // Don't have to return any pointer.
6467 RPK_None,
6468 // Pointer is the base of the declaration.
6469 RPK_Base,
6470 // Pointer is a member of the base declaration - 'this'
6471 RPK_Member,
6472 // Pointer is a reference and a member of the base declaration - 'this'
6473 RPK_MemberReference,
6474 };
Samuel Antao86ace552016-04-27 22:40:57 +00006475 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006476 OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
6477 OpenMPMapClauseKind MapTypeModifier = OMPC_MAP_unknown;
6478 ReturnPointerKind ReturnDevicePointer = RPK_None;
6479 bool IsImplicit = false;
Hans Wennborgbc1b58d2016-07-30 00:41:37 +00006480
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006481 MapInfo() = default;
Samuel Antaocc10b852016-07-28 14:23:26 +00006482 MapInfo(
6483 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
6484 OpenMPMapClauseKind MapType, OpenMPMapClauseKind MapTypeModifier,
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006485 ReturnPointerKind ReturnDevicePointer, bool IsImplicit)
Samuel Antaocc10b852016-07-28 14:23:26 +00006486 : Components(Components), MapType(MapType),
6487 MapTypeModifier(MapTypeModifier),
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006488 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit) {}
Samuel Antao86ace552016-04-27 22:40:57 +00006489 };
6490
6491 // We have to process the component lists that relate with the same
6492 // declaration in a single chunk so that we can generate the map flags
6493 // correctly. Therefore, we organize all lists in a map.
Alexey Bataev5d1c3f62017-06-27 15:46:42 +00006494 llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
Samuel Antao8d2d7302016-05-26 18:30:22 +00006495
6496 // Helper function to fill the information map for the different supported
6497 // clauses.
Samuel Antaocc10b852016-07-28 14:23:26 +00006498 auto &&InfoGen = [&Info](
6499 const ValueDecl *D,
6500 OMPClauseMappableExprCommon::MappableExprComponentListRef L,
6501 OpenMPMapClauseKind MapType, OpenMPMapClauseKind MapModifier,
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006502 MapInfo::ReturnPointerKind ReturnDevicePointer, bool IsImplicit) {
Samuel Antaocc10b852016-07-28 14:23:26 +00006503 const ValueDecl *VD =
6504 D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr;
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006505 Info[VD].emplace_back(L, MapType, MapModifier, ReturnDevicePointer,
6506 IsImplicit);
Samuel Antaocc10b852016-07-28 14:23:26 +00006507 };
Samuel Antao8d2d7302016-05-26 18:30:22 +00006508
Paul Robinson78fb1322016-08-01 22:12:46 +00006509 // FIXME: MSVC 2013 seems to require this-> to find member CurDir.
Paul Robinson15c84002016-07-29 20:46:16 +00006510 for (auto *C : this->CurDir.getClausesOfKind<OMPMapClause>())
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006511 for (auto L : C->component_lists()) {
Samuel Antaocf3f83e2016-07-28 14:47:35 +00006512 InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifier(),
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006513 MapInfo::RPK_None, C->isImplicit());
6514 }
Paul Robinson15c84002016-07-29 20:46:16 +00006515 for (auto *C : this->CurDir.getClausesOfKind<OMPToClause>())
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006516 for (auto L : C->component_lists()) {
Samuel Antaocf3f83e2016-07-28 14:47:35 +00006517 InfoGen(L.first, L.second, OMPC_MAP_to, OMPC_MAP_unknown,
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006518 MapInfo::RPK_None, C->isImplicit());
6519 }
Paul Robinson15c84002016-07-29 20:46:16 +00006520 for (auto *C : this->CurDir.getClausesOfKind<OMPFromClause>())
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006521 for (auto L : C->component_lists()) {
Samuel Antaocf3f83e2016-07-28 14:47:35 +00006522 InfoGen(L.first, L.second, OMPC_MAP_from, OMPC_MAP_unknown,
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006523 MapInfo::RPK_None, C->isImplicit());
6524 }
Samuel Antao86ace552016-04-27 22:40:57 +00006525
Samuel Antaocc10b852016-07-28 14:23:26 +00006526 // Look at the use_device_ptr clause information and mark the existing map
6527 // entries as such. If there is no map information for an entry in the
6528 // use_device_ptr list, we create one with map type 'alloc' and zero size
6529 // section. It is the user fault if that was not mapped before.
Paul Robinson78fb1322016-08-01 22:12:46 +00006530 // FIXME: MSVC 2013 seems to require this-> to find member CurDir.
Paul Robinson15c84002016-07-29 20:46:16 +00006531 for (auto *C : this->CurDir.getClausesOfKind<OMPUseDevicePtrClause>())
Samuel Antaocc10b852016-07-28 14:23:26 +00006532 for (auto L : C->component_lists()) {
6533 assert(!L.second.empty() && "Not expecting empty list of components!");
6534 const ValueDecl *VD = L.second.back().getAssociatedDeclaration();
6535 VD = cast<ValueDecl>(VD->getCanonicalDecl());
6536 auto *IE = L.second.back().getAssociatedExpression();
6537 // If the first component is a member expression, we have to look into
6538 // 'this', which maps to null in the map of map information. Otherwise
6539 // look directly for the information.
6540 auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);
6541
6542 // We potentially have map information for this declaration already.
6543 // Look for the first set of components that refer to it.
6544 if (It != Info.end()) {
6545 auto CI = std::find_if(
6546 It->second.begin(), It->second.end(), [VD](const MapInfo &MI) {
6547 return MI.Components.back().getAssociatedDeclaration() == VD;
6548 });
6549 // If we found a map entry, signal that the pointer has to be returned
6550 // and move on to the next declaration.
6551 if (CI != It->second.end()) {
6552 CI->ReturnDevicePointer = isa<MemberExpr>(IE)
6553 ? (VD->getType()->isReferenceType()
6554 ? MapInfo::RPK_MemberReference
6555 : MapInfo::RPK_Member)
6556 : MapInfo::RPK_Base;
6557 continue;
6558 }
6559 }
6560
6561 // We didn't find any match in our map information - generate a zero
6562 // size array section.
Paul Robinson78fb1322016-08-01 22:12:46 +00006563 // FIXME: MSVC 2013 seems to require this-> to find member CGF.
Samuel Antaocc10b852016-07-28 14:23:26 +00006564 llvm::Value *Ptr =
Paul Robinson15c84002016-07-29 20:46:16 +00006565 this->CGF
6566 .EmitLoadOfLValue(this->CGF.EmitLValue(IE), SourceLocation())
Samuel Antaocc10b852016-07-28 14:23:26 +00006567 .getScalarVal();
6568 BasePointers.push_back({Ptr, VD});
6569 Pointers.push_back(Ptr);
Paul Robinson15c84002016-07-29 20:46:16 +00006570 Sizes.push_back(llvm::Constant::getNullValue(this->CGF.SizeTy));
George Rokos065755d2017-11-07 18:27:04 +00006571 Types.push_back(OMP_MAP_RETURN_PARAM | OMP_MAP_TARGET_PARAM);
Samuel Antaocc10b852016-07-28 14:23:26 +00006572 }
6573
Samuel Antao86ace552016-04-27 22:40:57 +00006574 for (auto &M : Info) {
6575 // We need to know when we generate information for the first component
6576 // associated with a capture, because the mapping flags depend on it.
6577 bool IsFirstComponentList = true;
6578 for (MapInfo &L : M.second) {
6579 assert(!L.Components.empty() &&
6580 "Not expecting declaration with no component lists.");
Samuel Antaocc10b852016-07-28 14:23:26 +00006581
6582 // Remember the current base pointer index.
6583 unsigned CurrentBasePointersIdx = BasePointers.size();
Paul Robinson78fb1322016-08-01 22:12:46 +00006584 // FIXME: MSVC 2013 seems to require this-> to find the member method.
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006585 this->generateInfoForComponentList(
6586 L.MapType, L.MapTypeModifier, L.Components, BasePointers, Pointers,
6587 Sizes, Types, IsFirstComponentList, L.IsImplicit);
Samuel Antaocc10b852016-07-28 14:23:26 +00006588
6589 // If this entry relates with a device pointer, set the relevant
6590 // declaration and add the 'return pointer' flag.
6591 if (IsFirstComponentList &&
6592 L.ReturnDevicePointer != MapInfo::RPK_None) {
6593 // If the pointer is not the base of the map, we need to skip the
6594 // base. If it is a reference in a member field, we also need to skip
6595 // the map of the reference.
6596 if (L.ReturnDevicePointer != MapInfo::RPK_Base) {
6597 ++CurrentBasePointersIdx;
6598 if (L.ReturnDevicePointer == MapInfo::RPK_MemberReference)
6599 ++CurrentBasePointersIdx;
6600 }
6601 assert(BasePointers.size() > CurrentBasePointersIdx &&
6602 "Unexpected number of mapped base pointers.");
6603
6604 auto *RelevantVD = L.Components.back().getAssociatedDeclaration();
6605 assert(RelevantVD &&
6606 "No relevant declaration related with device pointer??");
6607
6608 BasePointers[CurrentBasePointersIdx].setDevicePtrDecl(RelevantVD);
George Rokos065755d2017-11-07 18:27:04 +00006609 Types[CurrentBasePointersIdx] |= OMP_MAP_RETURN_PARAM;
Samuel Antaocc10b852016-07-28 14:23:26 +00006610 }
Samuel Antao86ace552016-04-27 22:40:57 +00006611 IsFirstComponentList = false;
6612 }
6613 }
6614 }
6615
6616 /// \brief Generate the base pointers, section pointers, sizes and map types
6617 /// associated to a given capture.
6618 void generateInfoForCapture(const CapturedStmt::Capture *Cap,
Samuel Antao6890b092016-07-28 14:25:09 +00006619 llvm::Value *Arg,
Samuel Antaocc10b852016-07-28 14:23:26 +00006620 MapBaseValuesArrayTy &BasePointers,
Samuel Antao86ace552016-04-27 22:40:57 +00006621 MapValuesArrayTy &Pointers,
6622 MapValuesArrayTy &Sizes,
6623 MapFlagsArrayTy &Types) const {
6624 assert(!Cap->capturesVariableArrayType() &&
6625 "Not expecting to generate map info for a variable array type!");
6626
6627 BasePointers.clear();
6628 Pointers.clear();
6629 Sizes.clear();
6630 Types.clear();
6631
Samuel Antao6890b092016-07-28 14:25:09 +00006632 // We need to know when we generating information for the first component
6633 // associated with a capture, because the mapping flags depend on it.
6634 bool IsFirstComponentList = true;
6635
Samuel Antao86ace552016-04-27 22:40:57 +00006636 const ValueDecl *VD =
6637 Cap->capturesThis()
6638 ? nullptr
6639 : cast<ValueDecl>(Cap->getCapturedVar()->getCanonicalDecl());
6640
Samuel Antao6890b092016-07-28 14:25:09 +00006641 // If this declaration appears in a is_device_ptr clause we just have to
6642 // pass the pointer by value. If it is a reference to a declaration, we just
6643 // pass its value, otherwise, if it is a member expression, we need to map
6644 // 'to' the field.
6645 if (!VD) {
6646 auto It = DevPointersMap.find(VD);
6647 if (It != DevPointersMap.end()) {
6648 for (auto L : It->second) {
6649 generateInfoForComponentList(
6650 /*MapType=*/OMPC_MAP_to, /*MapTypeModifier=*/OMPC_MAP_unknown, L,
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006651 BasePointers, Pointers, Sizes, Types, IsFirstComponentList,
6652 /*IsImplicit=*/false);
Samuel Antao6890b092016-07-28 14:25:09 +00006653 IsFirstComponentList = false;
6654 }
6655 return;
6656 }
6657 } else if (DevPointersMap.count(VD)) {
6658 BasePointers.push_back({Arg, VD});
6659 Pointers.push_back(Arg);
6660 Sizes.push_back(CGF.getTypeSize(CGF.getContext().VoidPtrTy));
George Rokos065755d2017-11-07 18:27:04 +00006661 Types.push_back(OMP_MAP_LITERAL | OMP_MAP_TARGET_PARAM);
Samuel Antao6890b092016-07-28 14:25:09 +00006662 return;
6663 }
6664
Paul Robinson78fb1322016-08-01 22:12:46 +00006665 // FIXME: MSVC 2013 seems to require this-> to find member CurDir.
Paul Robinson15c84002016-07-29 20:46:16 +00006666 for (auto *C : this->CurDir.getClausesOfKind<OMPMapClause>())
Samuel Antao86ace552016-04-27 22:40:57 +00006667 for (auto L : C->decl_component_lists(VD)) {
6668 assert(L.first == VD &&
6669 "We got information for the wrong declaration??");
6670 assert(!L.second.empty() &&
6671 "Not expecting declaration with no component lists.");
Alexey Bataevf47c4b42017-09-26 13:47:31 +00006672 generateInfoForComponentList(
6673 C->getMapType(), C->getMapTypeModifier(), L.second, BasePointers,
6674 Pointers, Sizes, Types, IsFirstComponentList, C->isImplicit());
Samuel Antao86ace552016-04-27 22:40:57 +00006675 IsFirstComponentList = false;
6676 }
6677
6678 return;
6679 }
Samuel Antaod486f842016-05-26 16:53:38 +00006680
6681 /// \brief Generate the default map information for a given capture \a CI,
6682 /// record field declaration \a RI and captured value \a CV.
Samuel Antaocc10b852016-07-28 14:23:26 +00006683 void generateDefaultMapInfo(const CapturedStmt::Capture &CI,
6684 const FieldDecl &RI, llvm::Value *CV,
6685 MapBaseValuesArrayTy &CurBasePointers,
6686 MapValuesArrayTy &CurPointers,
6687 MapValuesArrayTy &CurSizes,
6688 MapFlagsArrayTy &CurMapTypes) {
Samuel Antaod486f842016-05-26 16:53:38 +00006689
6690 // Do the default mapping.
6691 if (CI.capturesThis()) {
6692 CurBasePointers.push_back(CV);
6693 CurPointers.push_back(CV);
6694 const PointerType *PtrTy = cast<PointerType>(RI.getType().getTypePtr());
6695 CurSizes.push_back(CGF.getTypeSize(PtrTy->getPointeeType()));
6696 // Default map type.
Samuel Antaocc10b852016-07-28 14:23:26 +00006697 CurMapTypes.push_back(OMP_MAP_TO | OMP_MAP_FROM);
Samuel Antaod486f842016-05-26 16:53:38 +00006698 } else if (CI.capturesVariableByCopy()) {
Samuel Antao6d004262016-06-16 18:39:34 +00006699 CurBasePointers.push_back(CV);
6700 CurPointers.push_back(CV);
Samuel Antaod486f842016-05-26 16:53:38 +00006701 if (!RI.getType()->isAnyPointerType()) {
Samuel Antao6d004262016-06-16 18:39:34 +00006702 // We have to signal to the runtime captures passed by value that are
6703 // not pointers.
George Rokos065755d2017-11-07 18:27:04 +00006704 CurMapTypes.push_back(OMP_MAP_LITERAL);
Samuel Antaod486f842016-05-26 16:53:38 +00006705 CurSizes.push_back(CGF.getTypeSize(RI.getType()));
6706 } else {
6707 // Pointers are implicitly mapped with a zero size and no flags
6708 // (other than first map that is added for all implicit maps).
6709 CurMapTypes.push_back(0u);
Samuel Antaod486f842016-05-26 16:53:38 +00006710 CurSizes.push_back(llvm::Constant::getNullValue(CGF.SizeTy));
6711 }
6712 } else {
6713 assert(CI.capturesVariable() && "Expected captured reference.");
6714 CurBasePointers.push_back(CV);
6715 CurPointers.push_back(CV);
6716
6717 const ReferenceType *PtrTy =
6718 cast<ReferenceType>(RI.getType().getTypePtr());
6719 QualType ElementType = PtrTy->getPointeeType();
6720 CurSizes.push_back(CGF.getTypeSize(ElementType));
6721 // The default map type for a scalar/complex type is 'to' because by
6722 // default the value doesn't have to be retrieved. For an aggregate
6723 // type, the default is 'tofrom'.
6724 CurMapTypes.push_back(ElementType->isAggregateType()
Samuel Antaocc10b852016-07-28 14:23:26 +00006725 ? (OMP_MAP_TO | OMP_MAP_FROM)
6726 : OMP_MAP_TO);
Samuel Antaod486f842016-05-26 16:53:38 +00006727
6728 // If we have a capture by reference we may need to add the private
6729 // pointer flag if the base declaration shows in some first-private
6730 // clause.
6731 CurMapTypes.back() =
6732 adjustMapModifiersForPrivateClauses(CI, CurMapTypes.back());
6733 }
George Rokos065755d2017-11-07 18:27:04 +00006734 // Every default map produces a single argument which is a target parameter.
6735 CurMapTypes.back() |= OMP_MAP_TARGET_PARAM;
Samuel Antaod486f842016-05-26 16:53:38 +00006736 }
Samuel Antao86ace552016-04-27 22:40:57 +00006737};
Samuel Antaodf158d52016-04-27 22:58:19 +00006738
6739enum OpenMPOffloadingReservedDeviceIDs {
6740 /// \brief Device ID if the device was not defined, runtime should get it
6741 /// from environment variables in the spec.
6742 OMP_DEVICEID_UNDEF = -1,
6743};
6744} // anonymous namespace
6745
6746/// \brief Emit the arrays used to pass the captures and map information to the
6747/// offloading runtime library. If there is no map or capture information,
6748/// return nullptr by reference.
6749static void
Samuel Antaocc10b852016-07-28 14:23:26 +00006750emitOffloadingArrays(CodeGenFunction &CGF,
6751 MappableExprsHandler::MapBaseValuesArrayTy &BasePointers,
Samuel Antaodf158d52016-04-27 22:58:19 +00006752 MappableExprsHandler::MapValuesArrayTy &Pointers,
6753 MappableExprsHandler::MapValuesArrayTy &Sizes,
Samuel Antaocc10b852016-07-28 14:23:26 +00006754 MappableExprsHandler::MapFlagsArrayTy &MapTypes,
6755 CGOpenMPRuntime::TargetDataInfo &Info) {
Samuel Antaodf158d52016-04-27 22:58:19 +00006756 auto &CGM = CGF.CGM;
6757 auto &Ctx = CGF.getContext();
6758
Samuel Antaocc10b852016-07-28 14:23:26 +00006759 // Reset the array information.
6760 Info.clearArrayInfo();
6761 Info.NumberOfPtrs = BasePointers.size();
Samuel Antaodf158d52016-04-27 22:58:19 +00006762
Samuel Antaocc10b852016-07-28 14:23:26 +00006763 if (Info.NumberOfPtrs) {
Samuel Antaodf158d52016-04-27 22:58:19 +00006764 // Detect if we have any capture size requiring runtime evaluation of the
6765 // size so that a constant array could be eventually used.
6766 bool hasRuntimeEvaluationCaptureSize = false;
6767 for (auto *S : Sizes)
6768 if (!isa<llvm::Constant>(S)) {
6769 hasRuntimeEvaluationCaptureSize = true;
6770 break;
6771 }
6772
Samuel Antaocc10b852016-07-28 14:23:26 +00006773 llvm::APInt PointerNumAP(32, Info.NumberOfPtrs, /*isSigned=*/true);
Samuel Antaodf158d52016-04-27 22:58:19 +00006774 QualType PointerArrayType =
6775 Ctx.getConstantArrayType(Ctx.VoidPtrTy, PointerNumAP, ArrayType::Normal,
6776 /*IndexTypeQuals=*/0);
6777
Samuel Antaocc10b852016-07-28 14:23:26 +00006778 Info.BasePointersArray =
Samuel Antaodf158d52016-04-27 22:58:19 +00006779 CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer();
Samuel Antaocc10b852016-07-28 14:23:26 +00006780 Info.PointersArray =
Samuel Antaodf158d52016-04-27 22:58:19 +00006781 CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer();
6782
6783 // If we don't have any VLA types or other types that require runtime
6784 // evaluation, we can use a constant array for the map sizes, otherwise we
6785 // need to fill up the arrays as we do for the pointers.
6786 if (hasRuntimeEvaluationCaptureSize) {
6787 QualType SizeArrayType = Ctx.getConstantArrayType(
6788 Ctx.getSizeType(), PointerNumAP, ArrayType::Normal,
6789 /*IndexTypeQuals=*/0);
Samuel Antaocc10b852016-07-28 14:23:26 +00006790 Info.SizesArray =
Samuel Antaodf158d52016-04-27 22:58:19 +00006791 CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer();
6792 } else {
6793 // We expect all the sizes to be constant, so we collect them to create
6794 // a constant array.
6795 SmallVector<llvm::Constant *, 16> ConstSizes;
6796 for (auto S : Sizes)
6797 ConstSizes.push_back(cast<llvm::Constant>(S));
6798
6799 auto *SizesArrayInit = llvm::ConstantArray::get(
6800 llvm::ArrayType::get(CGM.SizeTy, ConstSizes.size()), ConstSizes);
6801 auto *SizesArrayGbl = new llvm::GlobalVariable(
6802 CGM.getModule(), SizesArrayInit->getType(),
6803 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
6804 SizesArrayInit, ".offload_sizes");
Peter Collingbournebcf909d2016-06-14 21:02:05 +00006805 SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
Samuel Antaocc10b852016-07-28 14:23:26 +00006806 Info.SizesArray = SizesArrayGbl;
Samuel Antaodf158d52016-04-27 22:58:19 +00006807 }
6808
6809 // The map types are always constant so we don't need to generate code to
6810 // fill arrays. Instead, we create an array constant.
6811 llvm::Constant *MapTypesArrayInit =
6812 llvm::ConstantDataArray::get(CGF.Builder.getContext(), MapTypes);
6813 auto *MapTypesArrayGbl = new llvm::GlobalVariable(
6814 CGM.getModule(), MapTypesArrayInit->getType(),
6815 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
6816 MapTypesArrayInit, ".offload_maptypes");
Peter Collingbournebcf909d2016-06-14 21:02:05 +00006817 MapTypesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
Samuel Antaocc10b852016-07-28 14:23:26 +00006818 Info.MapTypesArray = MapTypesArrayGbl;
Samuel Antaodf158d52016-04-27 22:58:19 +00006819
Samuel Antaocc10b852016-07-28 14:23:26 +00006820 for (unsigned i = 0; i < Info.NumberOfPtrs; ++i) {
6821 llvm::Value *BPVal = *BasePointers[i];
Samuel Antaodf158d52016-04-27 22:58:19 +00006822 llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32(
Samuel Antaocc10b852016-07-28 14:23:26 +00006823 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
6824 Info.BasePointersArray, 0, i);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +00006825 BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6826 BP, BPVal->getType()->getPointerTo(/*AddrSpace=*/0));
Samuel Antaodf158d52016-04-27 22:58:19 +00006827 Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
6828 CGF.Builder.CreateStore(BPVal, BPAddr);
6829
Samuel Antaocc10b852016-07-28 14:23:26 +00006830 if (Info.requiresDevicePointerInfo())
6831 if (auto *DevVD = BasePointers[i].getDevicePtrDecl())
6832 Info.CaptureDeviceAddrMap.insert(std::make_pair(DevVD, BPAddr));
6833
Samuel Antaodf158d52016-04-27 22:58:19 +00006834 llvm::Value *PVal = Pointers[i];
Samuel Antaodf158d52016-04-27 22:58:19 +00006835 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32(
Samuel Antaocc10b852016-07-28 14:23:26 +00006836 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
6837 Info.PointersArray, 0, i);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +00006838 P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6839 P, PVal->getType()->getPointerTo(/*AddrSpace=*/0));
Samuel Antaodf158d52016-04-27 22:58:19 +00006840 Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
6841 CGF.Builder.CreateStore(PVal, PAddr);
6842
6843 if (hasRuntimeEvaluationCaptureSize) {
6844 llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32(
Samuel Antaocc10b852016-07-28 14:23:26 +00006845 llvm::ArrayType::get(CGM.SizeTy, Info.NumberOfPtrs),
6846 Info.SizesArray,
Samuel Antaodf158d52016-04-27 22:58:19 +00006847 /*Idx0=*/0,
6848 /*Idx1=*/i);
6849 Address SAddr(S, Ctx.getTypeAlignInChars(Ctx.getSizeType()));
6850 CGF.Builder.CreateStore(
6851 CGF.Builder.CreateIntCast(Sizes[i], CGM.SizeTy, /*isSigned=*/true),
6852 SAddr);
6853 }
6854 }
6855 }
6856}
6857/// \brief Emit the arguments to be passed to the runtime library based on the
6858/// arrays of pointers, sizes and map types.
6859static void emitOffloadingArraysArgument(
6860 CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg,
6861 llvm::Value *&PointersArrayArg, llvm::Value *&SizesArrayArg,
Samuel Antaocc10b852016-07-28 14:23:26 +00006862 llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info) {
Samuel Antaodf158d52016-04-27 22:58:19 +00006863 auto &CGM = CGF.CGM;
Samuel Antaocc10b852016-07-28 14:23:26 +00006864 if (Info.NumberOfPtrs) {
Samuel Antaodf158d52016-04-27 22:58:19 +00006865 BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
Samuel Antaocc10b852016-07-28 14:23:26 +00006866 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
6867 Info.BasePointersArray,
Samuel Antaodf158d52016-04-27 22:58:19 +00006868 /*Idx0=*/0, /*Idx1=*/0);
6869 PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
Samuel Antaocc10b852016-07-28 14:23:26 +00006870 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
6871 Info.PointersArray,
Samuel Antaodf158d52016-04-27 22:58:19 +00006872 /*Idx0=*/0,
6873 /*Idx1=*/0);
6874 SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
Samuel Antaocc10b852016-07-28 14:23:26 +00006875 llvm::ArrayType::get(CGM.SizeTy, Info.NumberOfPtrs), Info.SizesArray,
Samuel Antaodf158d52016-04-27 22:58:19 +00006876 /*Idx0=*/0, /*Idx1=*/0);
6877 MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
George Rokos63bc9d62017-11-21 18:25:12 +00006878 llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs),
Samuel Antaocc10b852016-07-28 14:23:26 +00006879 Info.MapTypesArray,
Samuel Antaodf158d52016-04-27 22:58:19 +00006880 /*Idx0=*/0,
6881 /*Idx1=*/0);
6882 } else {
6883 BasePointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
6884 PointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
6885 SizesArrayArg = llvm::ConstantPointerNull::get(CGM.SizeTy->getPointerTo());
6886 MapTypesArrayArg =
George Rokos63bc9d62017-11-21 18:25:12 +00006887 llvm::ConstantPointerNull::get(CGM.Int64Ty->getPointerTo());
Samuel Antaodf158d52016-04-27 22:58:19 +00006888 }
Samuel Antao86ace552016-04-27 22:40:57 +00006889}
6890
Samuel Antaobed3c462015-10-02 16:14:20 +00006891void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF,
6892 const OMPExecutableDirective &D,
6893 llvm::Value *OutlinedFn,
Samuel Antaoee8fb302016-01-06 13:42:12 +00006894 llvm::Value *OutlinedFnID,
Samuel Antaobed3c462015-10-02 16:14:20 +00006895 const Expr *IfCond, const Expr *Device,
6896 ArrayRef<llvm::Value *> CapturedVars) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00006897 if (!CGF.HaveInsertPoint())
6898 return;
Samuel Antaobed3c462015-10-02 16:14:20 +00006899
Samuel Antaoee8fb302016-01-06 13:42:12 +00006900 assert(OutlinedFn && "Invalid outlined function!");
6901
Samuel Antao86ace552016-04-27 22:40:57 +00006902 // Fill up the arrays with all the captured variables.
6903 MappableExprsHandler::MapValuesArrayTy KernelArgs;
Samuel Antaocc10b852016-07-28 14:23:26 +00006904 MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
Samuel Antao86ace552016-04-27 22:40:57 +00006905 MappableExprsHandler::MapValuesArrayTy Pointers;
6906 MappableExprsHandler::MapValuesArrayTy Sizes;
6907 MappableExprsHandler::MapFlagsArrayTy MapTypes;
Samuel Antaobed3c462015-10-02 16:14:20 +00006908
Samuel Antaocc10b852016-07-28 14:23:26 +00006909 MappableExprsHandler::MapBaseValuesArrayTy CurBasePointers;
Samuel Antao86ace552016-04-27 22:40:57 +00006910 MappableExprsHandler::MapValuesArrayTy CurPointers;
6911 MappableExprsHandler::MapValuesArrayTy CurSizes;
6912 MappableExprsHandler::MapFlagsArrayTy CurMapTypes;
6913
Samuel Antaod486f842016-05-26 16:53:38 +00006914 // Get mappable expression information.
6915 MappableExprsHandler MEHandler(D, CGF);
Samuel Antaobed3c462015-10-02 16:14:20 +00006916
6917 const CapturedStmt &CS = *cast<CapturedStmt>(D.getAssociatedStmt());
6918 auto RI = CS.getCapturedRecordDecl()->field_begin();
Samuel Antaobed3c462015-10-02 16:14:20 +00006919 auto CV = CapturedVars.begin();
6920 for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),
6921 CE = CS.capture_end();
6922 CI != CE; ++CI, ++RI, ++CV) {
Samuel Antao86ace552016-04-27 22:40:57 +00006923 CurBasePointers.clear();
6924 CurPointers.clear();
6925 CurSizes.clear();
6926 CurMapTypes.clear();
6927
6928 // VLA sizes are passed to the outlined region by copy and do not have map
6929 // information associated.
Samuel Antaobed3c462015-10-02 16:14:20 +00006930 if (CI->capturesVariableArrayType()) {
Samuel Antao86ace552016-04-27 22:40:57 +00006931 CurBasePointers.push_back(*CV);
6932 CurPointers.push_back(*CV);
6933 CurSizes.push_back(CGF.getTypeSize(RI->getType()));
Samuel Antao4af1b7b2015-12-02 17:44:43 +00006934 // Copy to the device as an argument. No need to retrieve it.
George Rokos065755d2017-11-07 18:27:04 +00006935 CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_LITERAL |
6936 MappableExprsHandler::OMP_MAP_TARGET_PARAM);
Samuel Antaobed3c462015-10-02 16:14:20 +00006937 } else {
Samuel Antao86ace552016-04-27 22:40:57 +00006938 // If we have any information in the map clause, we use it, otherwise we
6939 // just do a default mapping.
Samuel Antao6890b092016-07-28 14:25:09 +00006940 MEHandler.generateInfoForCapture(CI, *CV, CurBasePointers, CurPointers,
Samuel Antao86ace552016-04-27 22:40:57 +00006941 CurSizes, CurMapTypes);
Samuel Antaod486f842016-05-26 16:53:38 +00006942 if (CurBasePointers.empty())
6943 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurBasePointers,
6944 CurPointers, CurSizes, CurMapTypes);
Samuel Antaobed3c462015-10-02 16:14:20 +00006945 }
Samuel Antao86ace552016-04-27 22:40:57 +00006946 // We expect to have at least an element of information for this capture.
6947 assert(!CurBasePointers.empty() && "Non-existing map pointer for capture!");
6948 assert(CurBasePointers.size() == CurPointers.size() &&
6949 CurBasePointers.size() == CurSizes.size() &&
6950 CurBasePointers.size() == CurMapTypes.size() &&
6951 "Inconsistent map information sizes!");
Samuel Antaobed3c462015-10-02 16:14:20 +00006952
Samuel Antao86ace552016-04-27 22:40:57 +00006953 // The kernel args are always the first elements of the base pointers
6954 // associated with a capture.
Samuel Antaocc10b852016-07-28 14:23:26 +00006955 KernelArgs.push_back(*CurBasePointers.front());
Samuel Antao86ace552016-04-27 22:40:57 +00006956 // We need to append the results of this capture to what we already have.
6957 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
6958 Pointers.append(CurPointers.begin(), CurPointers.end());
6959 Sizes.append(CurSizes.begin(), CurSizes.end());
6960 MapTypes.append(CurMapTypes.begin(), CurMapTypes.end());
Samuel Antaobed3c462015-10-02 16:14:20 +00006961 }
6962
Samuel Antaobed3c462015-10-02 16:14:20 +00006963 // Fill up the pointer arrays and transfer execution to the device.
Alexey Bataev2a007e02017-10-02 14:20:58 +00006964 auto &&ThenGen = [this, &BasePointers, &Pointers, &Sizes, &MapTypes, Device,
6965 OutlinedFn, OutlinedFnID, &D,
6966 &KernelArgs](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00006967 auto &RT = CGF.CGM.getOpenMPRuntime();
Samuel Antaodf158d52016-04-27 22:58:19 +00006968 // Emit the offloading arrays.
Samuel Antaocc10b852016-07-28 14:23:26 +00006969 TargetDataInfo Info;
6970 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
6971 emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
6972 Info.PointersArray, Info.SizesArray,
6973 Info.MapTypesArray, Info);
Samuel Antaobed3c462015-10-02 16:14:20 +00006974
6975 // On top of the arrays that were filled up, the target offloading call
6976 // takes as arguments the device id as well as the host pointer. The host
6977 // pointer is used by the runtime library to identify the current target
6978 // region, so it only has to be unique and not necessarily point to
6979 // anything. It could be the pointer to the outlined function that
6980 // implements the target region, but we aren't using that so that the
6981 // compiler doesn't need to keep that, and could therefore inline the host
6982 // function if proven worthwhile during optimization.
6983
Samuel Antaoee8fb302016-01-06 13:42:12 +00006984 // From this point on, we need to have an ID of the target region defined.
6985 assert(OutlinedFnID && "Invalid outlined function ID!");
Samuel Antaobed3c462015-10-02 16:14:20 +00006986
6987 // Emit device ID if any.
6988 llvm::Value *DeviceID;
George Rokos63bc9d62017-11-21 18:25:12 +00006989 if (Device) {
Samuel Antaobed3c462015-10-02 16:14:20 +00006990 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
George Rokos63bc9d62017-11-21 18:25:12 +00006991 CGF.Int64Ty, /*isSigned=*/true);
6992 } else {
6993 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
6994 }
Samuel Antaobed3c462015-10-02 16:14:20 +00006995
Samuel Antaodf158d52016-04-27 22:58:19 +00006996 // Emit the number of elements in the offloading arrays.
6997 llvm::Value *PointerNum = CGF.Builder.getInt32(BasePointers.size());
6998
Samuel Antaob68e2db2016-03-03 16:20:23 +00006999 // Return value of the runtime offloading call.
7000 llvm::Value *Return;
7001
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00007002 auto *NumTeams = emitNumTeamsForTargetDirective(RT, CGF, D);
7003 auto *NumThreads = emitNumThreadsForTargetDirective(RT, CGF, D);
Samuel Antaob68e2db2016-03-03 16:20:23 +00007004
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00007005 // The target region is an outlined function launched by the runtime
7006 // via calls __tgt_target() or __tgt_target_teams().
7007 //
7008 // __tgt_target() launches a target region with one team and one thread,
7009 // executing a serial region. This master thread may in turn launch
7010 // more threads within its team upon encountering a parallel region,
7011 // however, no additional teams can be launched on the device.
7012 //
7013 // __tgt_target_teams() launches a target region with one or more teams,
7014 // each with one or more threads. This call is required for target
7015 // constructs such as:
7016 // 'target teams'
7017 // 'target' / 'teams'
7018 // 'target teams distribute parallel for'
7019 // 'target parallel'
7020 // and so on.
7021 //
7022 // Note that on the host and CPU targets, the runtime implementation of
7023 // these calls simply call the outlined function without forking threads.
7024 // The outlined functions themselves have runtime calls to
7025 // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
7026 // the compiler in emitTeamsCall() and emitParallelCall().
7027 //
7028 // In contrast, on the NVPTX target, the implementation of
7029 // __tgt_target_teams() launches a GPU kernel with the requested number
7030 // of teams and threads so no additional calls to the runtime are required.
Samuel Antaob68e2db2016-03-03 16:20:23 +00007031 if (NumTeams) {
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00007032 // If we have NumTeams defined this means that we have an enclosed teams
7033 // region. Therefore we also expect to have NumThreads defined. These two
7034 // values should be defined in the presence of a teams directive,
7035 // regardless of having any clauses associated. If the user is using teams
7036 // but no clauses, these two values will be the default that should be
7037 // passed to the runtime library - a 32-bit integer with the value zero.
7038 assert(NumThreads && "Thread limit expression should be available along "
7039 "with number of teams.");
Samuel Antaob68e2db2016-03-03 16:20:23 +00007040 llvm::Value *OffloadingArgs[] = {
Samuel Antaocc10b852016-07-28 14:23:26 +00007041 DeviceID, OutlinedFnID,
7042 PointerNum, Info.BasePointersArray,
7043 Info.PointersArray, Info.SizesArray,
7044 Info.MapTypesArray, NumTeams,
Arpith Chacko Jacob33c849a2017-01-25 00:57:16 +00007045 NumThreads};
Samuel Antaob68e2db2016-03-03 16:20:23 +00007046 Return = CGF.EmitRuntimeCall(
Alexey Bataev14fa1c62016-03-29 05:34:15 +00007047 RT.createRuntimeFunction(OMPRTL__tgt_target_teams), OffloadingArgs);
Samuel Antaob68e2db2016-03-03 16:20:23 +00007048 } else {
7049 llvm::Value *OffloadingArgs[] = {
Samuel Antaocc10b852016-07-28 14:23:26 +00007050 DeviceID, OutlinedFnID,
7051 PointerNum, Info.BasePointersArray,
7052 Info.PointersArray, Info.SizesArray,
7053 Info.MapTypesArray};
Alexey Bataev14fa1c62016-03-29 05:34:15 +00007054 Return = CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target),
Samuel Antaob68e2db2016-03-03 16:20:23 +00007055 OffloadingArgs);
7056 }
Samuel Antaobed3c462015-10-02 16:14:20 +00007057
Alexey Bataev2a007e02017-10-02 14:20:58 +00007058 // Check the error code and execute the host version if required.
7059 llvm::BasicBlock *OffloadFailedBlock =
7060 CGF.createBasicBlock("omp_offload.failed");
7061 llvm::BasicBlock *OffloadContBlock =
7062 CGF.createBasicBlock("omp_offload.cont");
7063 llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return);
7064 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
7065
7066 CGF.EmitBlock(OffloadFailedBlock);
7067 emitOutlinedFunctionCall(CGF, D.getLocStart(), OutlinedFn, KernelArgs);
7068 CGF.EmitBranch(OffloadContBlock);
7069
7070 CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true);
Samuel Antaobed3c462015-10-02 16:14:20 +00007071 };
7072
Samuel Antaoee8fb302016-01-06 13:42:12 +00007073 // Notify that the host version must be executed.
Alexey Bataev2a007e02017-10-02 14:20:58 +00007074 auto &&ElseGen = [this, &D, OutlinedFn, &KernelArgs](CodeGenFunction &CGF,
7075 PrePostActionTy &) {
7076 emitOutlinedFunctionCall(CGF, D.getLocStart(), OutlinedFn,
7077 KernelArgs);
Samuel Antaoee8fb302016-01-06 13:42:12 +00007078 };
7079
7080 // If we have a target function ID it means that we need to support
7081 // offloading, otherwise, just execute on the host. We need to execute on host
7082 // regardless of the conditional in the if clause if, e.g., the user do not
7083 // specify target triples.
7084 if (OutlinedFnID) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00007085 if (IfCond)
Samuel Antaoee8fb302016-01-06 13:42:12 +00007086 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00007087 else {
7088 RegionCodeGenTy ThenRCG(ThenGen);
7089 ThenRCG(CGF);
Alexey Bataevf539faa2016-03-28 12:58:34 +00007090 }
7091 } else {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00007092 RegionCodeGenTy ElseRCG(ElseGen);
7093 ElseRCG(CGF);
Alexey Bataevf539faa2016-03-28 12:58:34 +00007094 }
Samuel Antaobed3c462015-10-02 16:14:20 +00007095}
Samuel Antaoee8fb302016-01-06 13:42:12 +00007096
7097void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
7098 StringRef ParentName) {
7099 if (!S)
7100 return;
7101
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00007102 // Codegen OMP target directives that offload compute to the device.
7103 bool requiresDeviceCodegen =
7104 isa<OMPExecutableDirective>(S) &&
7105 isOpenMPTargetExecutionDirective(
7106 cast<OMPExecutableDirective>(S)->getDirectiveKind());
Samuel Antaoee8fb302016-01-06 13:42:12 +00007107
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00007108 if (requiresDeviceCodegen) {
7109 auto &E = *cast<OMPExecutableDirective>(S);
Samuel Antaoee8fb302016-01-06 13:42:12 +00007110 unsigned DeviceID;
7111 unsigned FileID;
7112 unsigned Line;
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00007113 getTargetEntryUniqueInfo(CGM.getContext(), E.getLocStart(), DeviceID,
Samuel Antao2de62b02016-02-13 23:35:10 +00007114 FileID, Line);
Samuel Antaoee8fb302016-01-06 13:42:12 +00007115
7116 // Is this a target region that should not be emitted as an entry point? If
7117 // so just signal we are done with this target region.
Samuel Antao2de62b02016-02-13 23:35:10 +00007118 if (!OffloadEntriesInfoManager.hasTargetRegionEntryInfo(DeviceID, FileID,
7119 ParentName, Line))
Samuel Antaoee8fb302016-01-06 13:42:12 +00007120 return;
7121
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00007122 switch (S->getStmtClass()) {
7123 case Stmt::OMPTargetDirectiveClass:
7124 CodeGenFunction::EmitOMPTargetDeviceFunction(
7125 CGM, ParentName, cast<OMPTargetDirective>(*S));
7126 break;
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00007127 case Stmt::OMPTargetParallelDirectiveClass:
7128 CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
7129 CGM, ParentName, cast<OMPTargetParallelDirective>(*S));
7130 break;
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00007131 case Stmt::OMPTargetTeamsDirectiveClass:
7132 CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
7133 CGM, ParentName, cast<OMPTargetTeamsDirective>(*S));
7134 break;
Alexey Bataevdfa430f2017-12-08 15:03:50 +00007135 case Stmt::OMPTargetTeamsDistributeDirectiveClass:
7136 CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
7137 CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(*S));
7138 break;
Alexey Bataevfb0ebec2017-11-08 20:16:14 +00007139 case Stmt::OMPTargetParallelForDirectiveClass:
7140 CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
7141 CGM, ParentName, cast<OMPTargetParallelForDirective>(*S));
7142 break;
Alexey Bataev5d7edca2017-11-09 17:32:15 +00007143 case Stmt::OMPTargetParallelForSimdDirectiveClass:
7144 CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
7145 CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(*S));
7146 break;
Alexey Bataevf8365372017-11-17 17:57:25 +00007147 case Stmt::OMPTargetSimdDirectiveClass:
7148 CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
7149 CGM, ParentName, cast<OMPTargetSimdDirective>(*S));
7150 break;
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00007151 default:
7152 llvm_unreachable("Unknown target directive for OpenMP device codegen.");
7153 }
Samuel Antaoee8fb302016-01-06 13:42:12 +00007154 return;
7155 }
7156
7157 if (const OMPExecutableDirective *E = dyn_cast<OMPExecutableDirective>(S)) {
Samuel Antaoe49645c2016-05-08 06:43:56 +00007158 if (!E->hasAssociatedStmt())
Samuel Antaoee8fb302016-01-06 13:42:12 +00007159 return;
7160
7161 scanForTargetRegionsFunctions(
7162 cast<CapturedStmt>(E->getAssociatedStmt())->getCapturedStmt(),
7163 ParentName);
7164 return;
7165 }
7166
7167 // If this is a lambda function, look into its body.
7168 if (auto *L = dyn_cast<LambdaExpr>(S))
7169 S = L->getBody();
7170
7171 // Keep looking for target regions recursively.
7172 for (auto *II : S->children())
7173 scanForTargetRegionsFunctions(II, ParentName);
Samuel Antaoee8fb302016-01-06 13:42:12 +00007174}
7175
7176bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {
7177 auto &FD = *cast<FunctionDecl>(GD.getDecl());
7178
7179 // If emitting code for the host, we do not process FD here. Instead we do
7180 // the normal code generation.
7181 if (!CGM.getLangOpts().OpenMPIsDevice)
7182 return false;
7183
7184 // Try to detect target regions in the function.
7185 scanForTargetRegionsFunctions(FD.getBody(), CGM.getMangledName(GD));
7186
Samuel Antao4b75b872016-12-12 19:26:31 +00007187 // We should not emit any function other that the ones created during the
Samuel Antaoee8fb302016-01-06 13:42:12 +00007188 // scanning. Therefore, we signal that this function is completely dealt
7189 // with.
7190 return true;
7191}
7192
7193bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
7194 if (!CGM.getLangOpts().OpenMPIsDevice)
7195 return false;
7196
7197 // Check if there are Ctors/Dtors in this declaration and look for target
7198 // regions in it. We use the complete variant to produce the kernel name
7199 // mangling.
7200 QualType RDTy = cast<VarDecl>(GD.getDecl())->getType();
7201 if (auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
7202 for (auto *Ctor : RD->ctors()) {
7203 StringRef ParentName =
7204 CGM.getMangledName(GlobalDecl(Ctor, Ctor_Complete));
7205 scanForTargetRegionsFunctions(Ctor->getBody(), ParentName);
7206 }
7207 auto *Dtor = RD->getDestructor();
7208 if (Dtor) {
7209 StringRef ParentName =
7210 CGM.getMangledName(GlobalDecl(Dtor, Dtor_Complete));
7211 scanForTargetRegionsFunctions(Dtor->getBody(), ParentName);
7212 }
7213 }
7214
Gheorghe-Teodor Bercea47633db2017-06-13 15:35:27 +00007215 // If we are in target mode, we do not emit any global (declare target is not
Samuel Antaoee8fb302016-01-06 13:42:12 +00007216 // implemented yet). Therefore we signal that GD was processed in this case.
7217 return true;
7218}
7219
7220bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {
7221 auto *VD = GD.getDecl();
7222 if (isa<FunctionDecl>(VD))
7223 return emitTargetFunctions(GD);
7224
7225 return emitTargetGlobalVariable(GD);
7226}
7227
7228llvm::Function *CGOpenMPRuntime::emitRegistrationFunction() {
7229 // If we have offloading in the current module, we need to emit the entries
7230 // now and register the offloading descriptor.
7231 createOffloadEntriesAndInfoMetadata();
7232
7233 // Create and register the offloading binary descriptors. This is the main
7234 // entity that captures all the information about offloading in the current
7235 // compilation unit.
7236 return createOffloadingBinaryDescriptorRegistration();
7237}
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00007238
7239void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
7240 const OMPExecutableDirective &D,
7241 SourceLocation Loc,
7242 llvm::Value *OutlinedFn,
7243 ArrayRef<llvm::Value *> CapturedVars) {
7244 if (!CGF.HaveInsertPoint())
7245 return;
7246
7247 auto *RTLoc = emitUpdateLocation(CGF, Loc);
7248 CodeGenFunction::RunCleanupsScope Scope(CGF);
7249
7250 // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);
7251 llvm::Value *Args[] = {
7252 RTLoc,
7253 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
7254 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())};
7255 llvm::SmallVector<llvm::Value *, 16> RealArgs;
7256 RealArgs.append(std::begin(Args), std::end(Args));
7257 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
7258
7259 auto RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_teams);
7260 CGF.EmitRuntimeCall(RTLFn, RealArgs);
7261}
7262
7263void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
Carlo Bertollic6872252016-04-04 15:55:02 +00007264 const Expr *NumTeams,
7265 const Expr *ThreadLimit,
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00007266 SourceLocation Loc) {
7267 if (!CGF.HaveInsertPoint())
7268 return;
7269
7270 auto *RTLoc = emitUpdateLocation(CGF, Loc);
7271
Carlo Bertollic6872252016-04-04 15:55:02 +00007272 llvm::Value *NumTeamsVal =
7273 (NumTeams)
7274 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams),
7275 CGF.CGM.Int32Ty, /* isSigned = */ true)
7276 : CGF.Builder.getInt32(0);
7277
7278 llvm::Value *ThreadLimitVal =
7279 (ThreadLimit)
7280 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
7281 CGF.CGM.Int32Ty, /* isSigned = */ true)
7282 : CGF.Builder.getInt32(0);
7283
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00007284 // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
Carlo Bertollic6872252016-04-04 15:55:02 +00007285 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
7286 ThreadLimitVal};
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00007287 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_teams),
7288 PushNumTeamsArgs);
7289}
Samuel Antaodf158d52016-04-27 22:58:19 +00007290
Samuel Antaocc10b852016-07-28 14:23:26 +00007291void CGOpenMPRuntime::emitTargetDataCalls(
7292 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
7293 const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) {
Samuel Antaodf158d52016-04-27 22:58:19 +00007294 if (!CGF.HaveInsertPoint())
7295 return;
7296
Samuel Antaocc10b852016-07-28 14:23:26 +00007297 // Action used to replace the default codegen action and turn privatization
7298 // off.
7299 PrePostActionTy NoPrivAction;
Samuel Antaodf158d52016-04-27 22:58:19 +00007300
7301 // Generate the code for the opening of the data environment. Capture all the
7302 // arguments of the runtime call by reference because they are used in the
7303 // closing of the region.
Malcolm Parsonsc6e45832017-01-13 18:55:32 +00007304 auto &&BeginThenGen = [&D, Device, &Info, &CodeGen](CodeGenFunction &CGF,
7305 PrePostActionTy &) {
Samuel Antaodf158d52016-04-27 22:58:19 +00007306 // Fill up the arrays with all the mapped variables.
Samuel Antaocc10b852016-07-28 14:23:26 +00007307 MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
Samuel Antaodf158d52016-04-27 22:58:19 +00007308 MappableExprsHandler::MapValuesArrayTy Pointers;
7309 MappableExprsHandler::MapValuesArrayTy Sizes;
7310 MappableExprsHandler::MapFlagsArrayTy MapTypes;
7311
7312 // Get map clause information.
7313 MappableExprsHandler MCHandler(D, CGF);
7314 MCHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
Samuel Antaodf158d52016-04-27 22:58:19 +00007315
7316 // Fill up the arrays and create the arguments.
Samuel Antaocc10b852016-07-28 14:23:26 +00007317 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
Samuel Antaodf158d52016-04-27 22:58:19 +00007318
7319 llvm::Value *BasePointersArrayArg = nullptr;
7320 llvm::Value *PointersArrayArg = nullptr;
7321 llvm::Value *SizesArrayArg = nullptr;
7322 llvm::Value *MapTypesArrayArg = nullptr;
7323 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
Samuel Antaocc10b852016-07-28 14:23:26 +00007324 SizesArrayArg, MapTypesArrayArg, Info);
Samuel Antaodf158d52016-04-27 22:58:19 +00007325
7326 // Emit device ID if any.
7327 llvm::Value *DeviceID = nullptr;
George Rokos63bc9d62017-11-21 18:25:12 +00007328 if (Device) {
Samuel Antaodf158d52016-04-27 22:58:19 +00007329 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
George Rokos63bc9d62017-11-21 18:25:12 +00007330 CGF.Int64Ty, /*isSigned=*/true);
7331 } else {
7332 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
7333 }
Samuel Antaodf158d52016-04-27 22:58:19 +00007334
7335 // Emit the number of elements in the offloading arrays.
Samuel Antaocc10b852016-07-28 14:23:26 +00007336 auto *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
Samuel Antaodf158d52016-04-27 22:58:19 +00007337
7338 llvm::Value *OffloadingArgs[] = {
7339 DeviceID, PointerNum, BasePointersArrayArg,
7340 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
7341 auto &RT = CGF.CGM.getOpenMPRuntime();
7342 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target_data_begin),
7343 OffloadingArgs);
Samuel Antaocc10b852016-07-28 14:23:26 +00007344
7345 // If device pointer privatization is required, emit the body of the region
7346 // here. It will have to be duplicated: with and without privatization.
7347 if (!Info.CaptureDeviceAddrMap.empty())
7348 CodeGen(CGF);
Samuel Antaodf158d52016-04-27 22:58:19 +00007349 };
7350
7351 // Generate code for the closing of the data region.
Malcolm Parsonsc6e45832017-01-13 18:55:32 +00007352 auto &&EndThenGen = [Device, &Info](CodeGenFunction &CGF, PrePostActionTy &) {
Samuel Antaocc10b852016-07-28 14:23:26 +00007353 assert(Info.isValid() && "Invalid data environment closing arguments.");
Samuel Antaodf158d52016-04-27 22:58:19 +00007354
7355 llvm::Value *BasePointersArrayArg = nullptr;
7356 llvm::Value *PointersArrayArg = nullptr;
7357 llvm::Value *SizesArrayArg = nullptr;
7358 llvm::Value *MapTypesArrayArg = nullptr;
7359 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
Samuel Antaocc10b852016-07-28 14:23:26 +00007360 SizesArrayArg, MapTypesArrayArg, Info);
Samuel Antaodf158d52016-04-27 22:58:19 +00007361
7362 // Emit device ID if any.
7363 llvm::Value *DeviceID = nullptr;
George Rokos63bc9d62017-11-21 18:25:12 +00007364 if (Device) {
Samuel Antaodf158d52016-04-27 22:58:19 +00007365 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
George Rokos63bc9d62017-11-21 18:25:12 +00007366 CGF.Int64Ty, /*isSigned=*/true);
7367 } else {
7368 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
7369 }
Samuel Antaodf158d52016-04-27 22:58:19 +00007370
7371 // Emit the number of elements in the offloading arrays.
Samuel Antaocc10b852016-07-28 14:23:26 +00007372 auto *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
Samuel Antaodf158d52016-04-27 22:58:19 +00007373
7374 llvm::Value *OffloadingArgs[] = {
7375 DeviceID, PointerNum, BasePointersArrayArg,
7376 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
7377 auto &RT = CGF.CGM.getOpenMPRuntime();
7378 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target_data_end),
7379 OffloadingArgs);
7380 };
7381
Samuel Antaocc10b852016-07-28 14:23:26 +00007382 // If we need device pointer privatization, we need to emit the body of the
7383 // region with no privatization in the 'else' branch of the conditional.
7384 // Otherwise, we don't have to do anything.
7385 auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF,
7386 PrePostActionTy &) {
7387 if (!Info.CaptureDeviceAddrMap.empty()) {
7388 CodeGen.setAction(NoPrivAction);
7389 CodeGen(CGF);
7390 }
7391 };
7392
7393 // We don't have to do anything to close the region if the if clause evaluates
7394 // to false.
7395 auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {};
Samuel Antaodf158d52016-04-27 22:58:19 +00007396
7397 if (IfCond) {
Samuel Antaocc10b852016-07-28 14:23:26 +00007398 emitOMPIfClause(CGF, IfCond, BeginThenGen, BeginElseGen);
Samuel Antaodf158d52016-04-27 22:58:19 +00007399 } else {
Samuel Antaocc10b852016-07-28 14:23:26 +00007400 RegionCodeGenTy RCG(BeginThenGen);
7401 RCG(CGF);
Samuel Antaodf158d52016-04-27 22:58:19 +00007402 }
7403
Samuel Antaocc10b852016-07-28 14:23:26 +00007404 // If we don't require privatization of device pointers, we emit the body in
7405 // between the runtime calls. This avoids duplicating the body code.
7406 if (Info.CaptureDeviceAddrMap.empty()) {
7407 CodeGen.setAction(NoPrivAction);
7408 CodeGen(CGF);
7409 }
Samuel Antaodf158d52016-04-27 22:58:19 +00007410
7411 if (IfCond) {
Samuel Antaocc10b852016-07-28 14:23:26 +00007412 emitOMPIfClause(CGF, IfCond, EndThenGen, EndElseGen);
Samuel Antaodf158d52016-04-27 22:58:19 +00007413 } else {
Samuel Antaocc10b852016-07-28 14:23:26 +00007414 RegionCodeGenTy RCG(EndThenGen);
7415 RCG(CGF);
Samuel Antaodf158d52016-04-27 22:58:19 +00007416 }
7417}
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007418
Samuel Antao8d2d7302016-05-26 18:30:22 +00007419void CGOpenMPRuntime::emitTargetDataStandAloneCall(
Samuel Antao8dd66282016-04-27 23:14:30 +00007420 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
7421 const Expr *Device) {
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007422 if (!CGF.HaveInsertPoint())
7423 return;
7424
Samuel Antao8dd66282016-04-27 23:14:30 +00007425 assert((isa<OMPTargetEnterDataDirective>(D) ||
Samuel Antao8d2d7302016-05-26 18:30:22 +00007426 isa<OMPTargetExitDataDirective>(D) ||
7427 isa<OMPTargetUpdateDirective>(D)) &&
7428 "Expecting either target enter, exit data, or update directives.");
Samuel Antao8dd66282016-04-27 23:14:30 +00007429
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007430 // Generate the code for the opening of the data environment.
Malcolm Parsonsc6e45832017-01-13 18:55:32 +00007431 auto &&ThenGen = [&D, Device](CodeGenFunction &CGF, PrePostActionTy &) {
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007432 // Fill up the arrays with all the mapped variables.
Samuel Antaocc10b852016-07-28 14:23:26 +00007433 MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007434 MappableExprsHandler::MapValuesArrayTy Pointers;
7435 MappableExprsHandler::MapValuesArrayTy Sizes;
7436 MappableExprsHandler::MapFlagsArrayTy MapTypes;
7437
7438 // Get map clause information.
Samuel Antao8d2d7302016-05-26 18:30:22 +00007439 MappableExprsHandler MEHandler(D, CGF);
7440 MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007441
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007442 // Fill up the arrays and create the arguments.
Samuel Antaocc10b852016-07-28 14:23:26 +00007443 TargetDataInfo Info;
7444 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
7445 emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
7446 Info.PointersArray, Info.SizesArray,
7447 Info.MapTypesArray, Info);
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007448
7449 // Emit device ID if any.
7450 llvm::Value *DeviceID = nullptr;
George Rokos63bc9d62017-11-21 18:25:12 +00007451 if (Device) {
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007452 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
George Rokos63bc9d62017-11-21 18:25:12 +00007453 CGF.Int64Ty, /*isSigned=*/true);
7454 } else {
7455 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
7456 }
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007457
7458 // Emit the number of elements in the offloading arrays.
7459 auto *PointerNum = CGF.Builder.getInt32(BasePointers.size());
7460
7461 llvm::Value *OffloadingArgs[] = {
Samuel Antaocc10b852016-07-28 14:23:26 +00007462 DeviceID, PointerNum, Info.BasePointersArray,
7463 Info.PointersArray, Info.SizesArray, Info.MapTypesArray};
Samuel Antao8d2d7302016-05-26 18:30:22 +00007464
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007465 auto &RT = CGF.CGM.getOpenMPRuntime();
Samuel Antao8d2d7302016-05-26 18:30:22 +00007466 // Select the right runtime function call for each expected standalone
7467 // directive.
7468 OpenMPRTLFunction RTLFn;
7469 switch (D.getDirectiveKind()) {
7470 default:
7471 llvm_unreachable("Unexpected standalone target data directive.");
7472 break;
7473 case OMPD_target_enter_data:
7474 RTLFn = OMPRTL__tgt_target_data_begin;
7475 break;
7476 case OMPD_target_exit_data:
7477 RTLFn = OMPRTL__tgt_target_data_end;
7478 break;
7479 case OMPD_target_update:
7480 RTLFn = OMPRTL__tgt_target_data_update;
7481 break;
7482 }
7483 CGF.EmitRuntimeCall(RT.createRuntimeFunction(RTLFn), OffloadingArgs);
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00007484 };
7485
7486 // In the event we get an if clause, we don't have to take any action on the
7487 // else side.
7488 auto &&ElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {};
7489
7490 if (IfCond) {
7491 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
7492 } else {
7493 RegionCodeGenTy ThenGenRCG(ThenGen);
7494 ThenGenRCG(CGF);
7495 }
7496}
Alexey Bataevc7a82b42016-05-06 09:40:08 +00007497
7498namespace {
7499 /// Kind of parameter in a function with 'declare simd' directive.
7500 enum ParamKindTy { LinearWithVarStride, Linear, Uniform, Vector };
7501 /// Attribute set of the parameter.
7502 struct ParamAttrTy {
7503 ParamKindTy Kind = Vector;
7504 llvm::APSInt StrideOrArg;
7505 llvm::APSInt Alignment;
7506 };
7507} // namespace
7508
7509static unsigned evaluateCDTSize(const FunctionDecl *FD,
7510 ArrayRef<ParamAttrTy> ParamAttrs) {
7511 // Every vector variant of a SIMD-enabled function has a vector length (VLEN).
7512 // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument
7513 // of that clause. The VLEN value must be power of 2.
7514 // In other case the notion of the function`s "characteristic data type" (CDT)
7515 // is used to compute the vector length.
7516 // CDT is defined in the following order:
7517 // a) For non-void function, the CDT is the return type.
7518 // b) If the function has any non-uniform, non-linear parameters, then the
7519 // CDT is the type of the first such parameter.
7520 // c) If the CDT determined by a) or b) above is struct, union, or class
7521 // type which is pass-by-value (except for the type that maps to the
7522 // built-in complex data type), the characteristic data type is int.
7523 // d) If none of the above three cases is applicable, the CDT is int.
7524 // The VLEN is then determined based on the CDT and the size of vector
7525 // register of that ISA for which current vector version is generated. The
7526 // VLEN is computed using the formula below:
7527 // VLEN = sizeof(vector_register) / sizeof(CDT),
7528 // where vector register size specified in section 3.2.1 Registers and the
7529 // Stack Frame of original AMD64 ABI document.
7530 QualType RetType = FD->getReturnType();
7531 if (RetType.isNull())
7532 return 0;
7533 ASTContext &C = FD->getASTContext();
7534 QualType CDT;
7535 if (!RetType.isNull() && !RetType->isVoidType())
7536 CDT = RetType;
7537 else {
7538 unsigned Offset = 0;
7539 if (auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
7540 if (ParamAttrs[Offset].Kind == Vector)
7541 CDT = C.getPointerType(C.getRecordType(MD->getParent()));
7542 ++Offset;
7543 }
7544 if (CDT.isNull()) {
7545 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
7546 if (ParamAttrs[I + Offset].Kind == Vector) {
7547 CDT = FD->getParamDecl(I)->getType();
7548 break;
7549 }
7550 }
7551 }
7552 }
7553 if (CDT.isNull())
7554 CDT = C.IntTy;
7555 CDT = CDT->getCanonicalTypeUnqualified();
7556 if (CDT->isRecordType() || CDT->isUnionType())
7557 CDT = C.IntTy;
7558 return C.getTypeSize(CDT);
7559}
7560
7561static void
7562emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn,
Benjamin Kramer81cb4b72016-11-24 16:01:20 +00007563 const llvm::APSInt &VLENVal,
Alexey Bataevc7a82b42016-05-06 09:40:08 +00007564 ArrayRef<ParamAttrTy> ParamAttrs,
7565 OMPDeclareSimdDeclAttr::BranchStateTy State) {
7566 struct ISADataTy {
7567 char ISA;
7568 unsigned VecRegSize;
7569 };
7570 ISADataTy ISAData[] = {
7571 {
7572 'b', 128
7573 }, // SSE
7574 {
7575 'c', 256
7576 }, // AVX
7577 {
7578 'd', 256
7579 }, // AVX2
7580 {
7581 'e', 512
7582 }, // AVX512
7583 };
7584 llvm::SmallVector<char, 2> Masked;
7585 switch (State) {
7586 case OMPDeclareSimdDeclAttr::BS_Undefined:
7587 Masked.push_back('N');
7588 Masked.push_back('M');
7589 break;
7590 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
7591 Masked.push_back('N');
7592 break;
7593 case OMPDeclareSimdDeclAttr::BS_Inbranch:
7594 Masked.push_back('M');
7595 break;
7596 }
7597 for (auto Mask : Masked) {
7598 for (auto &Data : ISAData) {
7599 SmallString<256> Buffer;
7600 llvm::raw_svector_ostream Out(Buffer);
7601 Out << "_ZGV" << Data.ISA << Mask;
7602 if (!VLENVal) {
7603 Out << llvm::APSInt::getUnsigned(Data.VecRegSize /
7604 evaluateCDTSize(FD, ParamAttrs));
7605 } else
7606 Out << VLENVal;
7607 for (auto &ParamAttr : ParamAttrs) {
7608 switch (ParamAttr.Kind){
7609 case LinearWithVarStride:
7610 Out << 's' << ParamAttr.StrideOrArg;
7611 break;
7612 case Linear:
7613 Out << 'l';
7614 if (!!ParamAttr.StrideOrArg)
7615 Out << ParamAttr.StrideOrArg;
7616 break;
7617 case Uniform:
7618 Out << 'u';
7619 break;
7620 case Vector:
7621 Out << 'v';
7622 break;
7623 }
7624 if (!!ParamAttr.Alignment)
7625 Out << 'a' << ParamAttr.Alignment;
7626 }
7627 Out << '_' << Fn->getName();
7628 Fn->addFnAttr(Out.str());
7629 }
7630 }
7631}
7632
7633void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
7634 llvm::Function *Fn) {
7635 ASTContext &C = CGM.getContext();
7636 FD = FD->getCanonicalDecl();
7637 // Map params to their positions in function decl.
7638 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
7639 if (isa<CXXMethodDecl>(FD))
7640 ParamPositions.insert({FD, 0});
7641 unsigned ParamPos = ParamPositions.size();
David Majnemer59f77922016-06-24 04:05:48 +00007642 for (auto *P : FD->parameters()) {
Alexey Bataevc7a82b42016-05-06 09:40:08 +00007643 ParamPositions.insert({P->getCanonicalDecl(), ParamPos});
7644 ++ParamPos;
7645 }
7646 for (auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
7647 llvm::SmallVector<ParamAttrTy, 8> ParamAttrs(ParamPositions.size());
7648 // Mark uniform parameters.
7649 for (auto *E : Attr->uniforms()) {
7650 E = E->IgnoreParenImpCasts();
7651 unsigned Pos;
7652 if (isa<CXXThisExpr>(E))
7653 Pos = ParamPositions[FD];
7654 else {
7655 auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
7656 ->getCanonicalDecl();
7657 Pos = ParamPositions[PVD];
7658 }
7659 ParamAttrs[Pos].Kind = Uniform;
7660 }
7661 // Get alignment info.
7662 auto NI = Attr->alignments_begin();
7663 for (auto *E : Attr->aligneds()) {
7664 E = E->IgnoreParenImpCasts();
7665 unsigned Pos;
7666 QualType ParmTy;
7667 if (isa<CXXThisExpr>(E)) {
7668 Pos = ParamPositions[FD];
7669 ParmTy = E->getType();
7670 } else {
7671 auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
7672 ->getCanonicalDecl();
7673 Pos = ParamPositions[PVD];
7674 ParmTy = PVD->getType();
7675 }
7676 ParamAttrs[Pos].Alignment =
7677 (*NI) ? (*NI)->EvaluateKnownConstInt(C)
7678 : llvm::APSInt::getUnsigned(
7679 C.toCharUnitsFromBits(C.getOpenMPDefaultSimdAlign(ParmTy))
7680 .getQuantity());
7681 ++NI;
7682 }
7683 // Mark linear parameters.
7684 auto SI = Attr->steps_begin();
7685 auto MI = Attr->modifiers_begin();
7686 for (auto *E : Attr->linears()) {
7687 E = E->IgnoreParenImpCasts();
7688 unsigned Pos;
7689 if (isa<CXXThisExpr>(E))
7690 Pos = ParamPositions[FD];
7691 else {
7692 auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
7693 ->getCanonicalDecl();
7694 Pos = ParamPositions[PVD];
7695 }
7696 auto &ParamAttr = ParamAttrs[Pos];
7697 ParamAttr.Kind = Linear;
7698 if (*SI) {
7699 if (!(*SI)->EvaluateAsInt(ParamAttr.StrideOrArg, C,
7700 Expr::SE_AllowSideEffects)) {
7701 if (auto *DRE = cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
7702 if (auto *StridePVD = cast<ParmVarDecl>(DRE->getDecl())) {
7703 ParamAttr.Kind = LinearWithVarStride;
7704 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(
7705 ParamPositions[StridePVD->getCanonicalDecl()]);
7706 }
7707 }
7708 }
7709 }
7710 ++SI;
7711 ++MI;
7712 }
7713 llvm::APSInt VLENVal;
7714 if (const Expr *VLEN = Attr->getSimdlen())
7715 VLENVal = VLEN->EvaluateKnownConstInt(C);
7716 OMPDeclareSimdDeclAttr::BranchStateTy State = Attr->getBranchState();
7717 if (CGM.getTriple().getArch() == llvm::Triple::x86 ||
7718 CGM.getTriple().getArch() == llvm::Triple::x86_64)
7719 emitX86DeclareSimdFunction(FD, Fn, VLENVal, ParamAttrs, State);
7720 }
7721}
Alexey Bataev8b427062016-05-25 12:36:08 +00007722
7723namespace {
7724/// Cleanup action for doacross support.
7725class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
7726public:
7727 static const int DoacrossFinArgs = 2;
7728
7729private:
7730 llvm::Value *RTLFn;
7731 llvm::Value *Args[DoacrossFinArgs];
7732
7733public:
7734 DoacrossCleanupTy(llvm::Value *RTLFn, ArrayRef<llvm::Value *> CallArgs)
7735 : RTLFn(RTLFn) {
7736 assert(CallArgs.size() == DoacrossFinArgs);
7737 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
7738 }
7739 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
7740 if (!CGF.HaveInsertPoint())
7741 return;
7742 CGF.EmitRuntimeCall(RTLFn, Args);
7743 }
7744};
7745} // namespace
7746
7747void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
7748 const OMPLoopDirective &D) {
7749 if (!CGF.HaveInsertPoint())
7750 return;
7751
7752 ASTContext &C = CGM.getContext();
7753 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
7754 RecordDecl *RD;
7755 if (KmpDimTy.isNull()) {
7756 // Build struct kmp_dim { // loop bounds info casted to kmp_int64
7757 // kmp_int64 lo; // lower
7758 // kmp_int64 up; // upper
7759 // kmp_int64 st; // stride
7760 // };
7761 RD = C.buildImplicitRecord("kmp_dim");
7762 RD->startDefinition();
7763 addFieldToRecordDecl(C, RD, Int64Ty);
7764 addFieldToRecordDecl(C, RD, Int64Ty);
7765 addFieldToRecordDecl(C, RD, Int64Ty);
7766 RD->completeDefinition();
7767 KmpDimTy = C.getRecordType(RD);
7768 } else
7769 RD = cast<RecordDecl>(KmpDimTy->getAsTagDecl());
7770
7771 Address DimsAddr = CGF.CreateMemTemp(KmpDimTy, "dims");
7772 CGF.EmitNullInitialization(DimsAddr, KmpDimTy);
7773 enum { LowerFD = 0, UpperFD, StrideFD };
7774 // Fill dims with data.
7775 LValue DimsLVal = CGF.MakeAddrLValue(DimsAddr, KmpDimTy);
7776 // dims.upper = num_iterations;
7777 LValue UpperLVal =
7778 CGF.EmitLValueForField(DimsLVal, *std::next(RD->field_begin(), UpperFD));
7779 llvm::Value *NumIterVal = CGF.EmitScalarConversion(
7780 CGF.EmitScalarExpr(D.getNumIterations()), D.getNumIterations()->getType(),
7781 Int64Ty, D.getNumIterations()->getExprLoc());
7782 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal);
7783 // dims.stride = 1;
7784 LValue StrideLVal =
7785 CGF.EmitLValueForField(DimsLVal, *std::next(RD->field_begin(), StrideFD));
7786 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1),
7787 StrideLVal);
7788
7789 // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
7790 // kmp_int32 num_dims, struct kmp_dim * dims);
7791 llvm::Value *Args[] = {emitUpdateLocation(CGF, D.getLocStart()),
7792 getThreadID(CGF, D.getLocStart()),
7793 llvm::ConstantInt::getSigned(CGM.Int32Ty, 1),
7794 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
7795 DimsAddr.getPointer(), CGM.VoidPtrTy)};
7796
7797 llvm::Value *RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_init);
7798 CGF.EmitRuntimeCall(RTLFn, Args);
7799 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
7800 emitUpdateLocation(CGF, D.getLocEnd()), getThreadID(CGF, D.getLocEnd())};
7801 llvm::Value *FiniRTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_fini);
7802 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
7803 llvm::makeArrayRef(FiniArgs));
7804}
7805
7806void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
7807 const OMPDependClause *C) {
7808 QualType Int64Ty =
7809 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
7810 const Expr *CounterVal = C->getCounterValue();
7811 assert(CounterVal);
7812 llvm::Value *CntVal = CGF.EmitScalarConversion(CGF.EmitScalarExpr(CounterVal),
7813 CounterVal->getType(), Int64Ty,
7814 CounterVal->getExprLoc());
7815 Address CntAddr = CGF.CreateMemTemp(Int64Ty, ".cnt.addr");
7816 CGF.EmitStoreOfScalar(CntVal, CntAddr, /*Volatile=*/false, Int64Ty);
7817 llvm::Value *Args[] = {emitUpdateLocation(CGF, C->getLocStart()),
7818 getThreadID(CGF, C->getLocStart()),
7819 CntAddr.getPointer()};
7820 llvm::Value *RTLFn;
7821 if (C->getDependencyKind() == OMPC_DEPEND_source)
7822 RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_post);
7823 else {
7824 assert(C->getDependencyKind() == OMPC_DEPEND_sink);
7825 RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_wait);
7826 }
7827 CGF.EmitRuntimeCall(RTLFn, Args);
7828}
7829
Alexey Bataev3c595a62017-08-14 15:01:03 +00007830void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, llvm::Value *Callee,
7831 ArrayRef<llvm::Value *> Args,
7832 SourceLocation Loc) const {
7833 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
7834
7835 if (auto *Fn = dyn_cast<llvm::Function>(Callee)) {
Alexey Bataev2c7eee52017-08-04 19:10:54 +00007836 if (Fn->doesNotThrow()) {
Alexey Bataev3c595a62017-08-14 15:01:03 +00007837 CGF.EmitNounwindRuntimeCall(Fn, Args);
Alexey Bataev2c7eee52017-08-04 19:10:54 +00007838 return;
7839 }
7840 }
Alexey Bataev3c595a62017-08-14 15:01:03 +00007841 CGF.EmitRuntimeCall(Callee, Args);
7842}
7843
7844void CGOpenMPRuntime::emitOutlinedFunctionCall(
7845 CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn,
7846 ArrayRef<llvm::Value *> Args) const {
7847 assert(Loc.isValid() && "Outlined function call location must be valid.");
7848 emitCall(CGF, OutlinedFn, Args, Loc);
Alexey Bataev2c7eee52017-08-04 19:10:54 +00007849}
Alexey Bataev3b8d5582017-08-08 18:04:06 +00007850
7851Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
7852 const VarDecl *NativeParam,
7853 const VarDecl *TargetParam) const {
7854 return CGF.GetAddrOfLocalVar(NativeParam);
7855}