Revert "[OPENMP] Allow runtime insert its own code inside OpenMP regions."
This reverts commit 3ee791165100607178073f14531a0dc90c622b36.
llvm-svn: 264570
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 75103db..810a2db 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -252,7 +252,7 @@
StringRef HelperName;
};
-static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
+static void EmptyCodeGen(CodeGenFunction &) {
llvm_unreachable("No codegen for expressions");
}
/// \brief API for generation of expressions captured in a innermost OpenMP
@@ -564,33 +564,8 @@
OMPRTL__tgt_unregister_lib,
};
-/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
-/// region.
-class CleanupTy final : public EHScopeStack::Cleanup {
- PrePostActionTy *Action;
-
-public:
- explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
- void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
- if (!CGF.HaveInsertPoint())
- return;
- Action->Exit(CGF);
- }
-};
-
} // anonymous namespace
-void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
- CodeGenFunction::RunCleanupsScope Scope(CGF);
- if (PrePostAction) {
- CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
- Callback(CodeGen, CGF, *PrePostAction);
- } else {
- PrePostActionTy Action;
- Callback(CodeGen, CGF, Action);
- }
-}
-
LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
return CGF.EmitLoadOfPointerLValue(
CGF.GetAddrOfLocalVar(getThreadIDVariable()),
@@ -606,7 +581,10 @@
// The point of exit cannot be a branch out of the structured block.
// longjmp() and throw() must not violate the entry/exit criteria.
CGF.EHStack.pushTerminate();
- CodeGen(CGF);
+ {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ CodeGen(CGF);
+ }
CGF.EHStack.popTerminate();
}
@@ -1666,10 +1644,12 @@
// the condition and the dead arm of the if/else.
bool CondConstant;
if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
- if (CondConstant)
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ if (CondConstant) {
ThenGen(CGF);
- else
+ } else {
ElseGen(CGF);
+ }
return;
}
@@ -1682,16 +1662,26 @@
// Emit the 'then' code.
CGF.EmitBlock(ThenBlock);
- ThenGen(CGF);
+ {
+ CodeGenFunction::RunCleanupsScope ThenScope(CGF);
+ ThenGen(CGF);
+ }
CGF.EmitBranch(ContBlock);
// Emit the 'else' code if present.
- // There is no need to emit line number for unconditional branch.
- (void)ApplyDebugLocation::CreateEmpty(CGF);
- CGF.EmitBlock(ElseBlock);
- ElseGen(CGF);
- // There is no need to emit line number for unconditional branch.
- (void)ApplyDebugLocation::CreateEmpty(CGF);
- CGF.EmitBranch(ContBlock);
+ {
+ // There is no need to emit line number for unconditional branch.
+ auto NL = ApplyDebugLocation::CreateEmpty(CGF);
+ CGF.EmitBlock(ElseBlock);
+ }
+ {
+ CodeGenFunction::RunCleanupsScope ThenScope(CGF);
+ ElseGen(CGF);
+ }
+ {
+ // There is no need to emit line number for unconditional branch.
+ auto NL = ApplyDebugLocation::CreateEmpty(CGF);
+ CGF.EmitBranch(ContBlock);
+ }
// Emit the continuation block for code after the if.
CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
}
@@ -1703,8 +1693,8 @@
if (!CGF.HaveInsertPoint())
return;
auto *RTLoc = emitUpdateLocation(CGF, Loc);
- RegionCodeGenTy ThenGen = [this, OutlinedFn, CapturedVars,
- RTLoc](CodeGenFunction &CGF, PrePostActionTy &) {
+ auto &&ThenGen = [this, OutlinedFn, CapturedVars,
+ RTLoc](CodeGenFunction &CGF) {
// Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
llvm::Value *Args[] = {
RTLoc,
@@ -1717,8 +1707,8 @@
auto RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_call);
CGF.EmitRuntimeCall(RTLFn, RealArgs);
};
- RegionCodeGenTy ElseGen = [this, OutlinedFn, CapturedVars, RTLoc,
- Loc](CodeGenFunction &CGF, PrePostActionTy &) {
+ auto &&ElseGen = [this, OutlinedFn, CapturedVars, RTLoc,
+ Loc](CodeGenFunction &CGF) {
auto ThreadID = getThreadID(CGF, Loc);
// Build calls:
// __kmpc_serialized_parallel(&Loc, GTid);
@@ -1743,10 +1733,12 @@
CGF.EmitRuntimeCall(
createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel), EndArgs);
};
- if (IfCond)
+ if (IfCond) {
emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
- else
+ } else {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
ThenGen(CGF);
+ }
}
// If we're inside an (outlined) parallel region, use the region info's
@@ -1798,39 +1790,21 @@
}
namespace {
-/// Common pre(post)-action for different OpenMP constructs.
-class CommonActionTy final : public PrePostActionTy {
- llvm::Value *EnterCallee;
- ArrayRef<llvm::Value *> EnterArgs;
- llvm::Value *ExitCallee;
- ArrayRef<llvm::Value *> ExitArgs;
- bool Conditional;
- llvm::BasicBlock *ContBlock = nullptr;
+template <size_t N> class CallEndCleanup final : public EHScopeStack::Cleanup {
+ llvm::Value *Callee;
+ llvm::Value *Args[N];
public:
- CommonActionTy(llvm::Value *EnterCallee, ArrayRef<llvm::Value *> EnterArgs,
- llvm::Value *ExitCallee, ArrayRef<llvm::Value *> ExitArgs,
- bool Conditional = false)
- : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
- ExitArgs(ExitArgs), Conditional(Conditional) {}
- void Enter(CodeGenFunction &CGF) override {
- llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
- if (Conditional) {
- llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
- auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
- ContBlock = CGF.createBasicBlock("omp_if.end");
- // Generate the branch (If-stmt)
- CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
- CGF.EmitBlock(ThenBlock);
- }
+ CallEndCleanup(llvm::Value *Callee, ArrayRef<llvm::Value *> CleanupArgs)
+ : Callee(Callee) {
+ assert(CleanupArgs.size() == N);
+ std::copy(CleanupArgs.begin(), CleanupArgs.end(), std::begin(Args));
}
- void Done(CodeGenFunction &CGF) {
- // Emit the rest of blocks/branches
- CGF.EmitBranch(ContBlock);
- CGF.EmitBlock(ContBlock, true);
- }
- void Exit(CodeGenFunction &CGF) override {
- CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
+
+ void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
+ if (!CGF.HaveInsertPoint())
+ return;
+ CGF.EmitRuntimeCall(Callee, Args);
}
};
} // anonymous namespace
@@ -1845,22 +1819,45 @@
// Prepare arguments and build a call to __kmpc_critical
if (!CGF.HaveInsertPoint())
return;
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
getCriticalRegionLock(CriticalName)};
- llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
- std::end(Args));
if (Hint) {
- EnterArgs.push_back(CGF.Builder.CreateIntCast(
- CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
- }
- CommonActionTy Action(
- createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
- : OMPRTL__kmpc_critical),
- EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
- CriticalOpGen.setAction(Action);
+ llvm::SmallVector<llvm::Value *, 8> ArgsWithHint(std::begin(Args),
+ std::end(Args));
+ auto *HintVal = CGF.EmitScalarExpr(Hint);
+ ArgsWithHint.push_back(
+ CGF.Builder.CreateIntCast(HintVal, CGM.IntPtrTy, /*isSigned=*/false));
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical_with_hint),
+ ArgsWithHint);
+ } else
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical), Args);
+ // Build a call to __kmpc_end_critical
+ CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_critical),
+ llvm::makeArrayRef(Args));
emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
}
+static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond,
+ OpenMPDirectiveKind Kind, SourceLocation Loc,
+ const RegionCodeGenTy &BodyOpGen) {
+ llvm::Value *CallBool = CGF.EmitScalarConversion(
+ IfCond,
+ CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true),
+ CGF.getContext().BoolTy, Loc);
+
+ auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
+ auto *ContBlock = CGF.createBasicBlock("omp_if.end");
+ // Generate the branch (If-stmt)
+ CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
+ CGF.EmitBlock(ThenBlock);
+ CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, Kind, BodyOpGen);
+ // Emit the rest of bblocks/branches
+ CGF.EmitBranch(ContBlock);
+ CGF.EmitBlock(ContBlock, true);
+}
+
void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
const RegionCodeGenTy &MasterOpGen,
SourceLocation Loc) {
@@ -1872,12 +1869,18 @@
// }
// Prepare arguments and build a call to __kmpc_master
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
- CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
- createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
- /*Conditional=*/true);
- MasterOpGen.setAction(Action);
- emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
- Action.Done(CGF);
+ auto *IsMaster =
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_master), Args);
+ typedef CallEndCleanup<std::extent<decltype(Args)>::value>
+ MasterCallEndCleanup;
+ emitIfStmt(
+ CGF, IsMaster, OMPD_master, Loc, [&](CodeGenFunction &CGF) -> void {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ CGF.EHStack.pushCleanup<MasterCallEndCleanup>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_master),
+ llvm::makeArrayRef(Args));
+ MasterOpGen(CGF);
+ });
}
void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
@@ -1900,12 +1903,16 @@
// TaskgroupOpGen();
// __kmpc_end_taskgroup(ident_t *, gtid);
// Prepare arguments and build a call to __kmpc_taskgroup
- llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
- CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
- createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
- Args);
- TaskgroupOpGen.setAction(Action);
- emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
+ {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args);
+ // Build a call to __kmpc_end_taskgroup
+ CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
+ llvm::makeArrayRef(Args));
+ emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
+ }
}
/// Given an array of pointers to variables, project the address of a
@@ -2001,16 +2008,22 @@
}
// Prepare arguments and build a call to __kmpc_single
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
- CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
- createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
- /*Conditional=*/true);
- SingleOpGen.setAction(Action);
- emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
- if (DidIt.isValid()) {
- // did_it = 1;
- CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
- }
- Action.Done(CGF);
+ auto *IsSingle =
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_single), Args);
+ typedef CallEndCleanup<std::extent<decltype(Args)>::value>
+ SingleCallEndCleanup;
+ emitIfStmt(
+ CGF, IsSingle, OMPD_single, Loc, [&](CodeGenFunction &CGF) -> void {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ CGF.EHStack.pushCleanup<SingleCallEndCleanup>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_single),
+ llvm::makeArrayRef(Args));
+ SingleOpGen(CGF);
+ if (DidIt.isValid()) {
+ // did_it = 1;
+ CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
+ }
+ });
// call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
// <copy_func>, did_it);
if (DidIt.isValid()) {
@@ -2060,14 +2073,14 @@
// OrderedOpGen();
// __kmpc_end_ordered(ident_t *, gtid);
// Prepare arguments and build a call to __kmpc_ordered
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
if (IsThreads) {
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
- CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
- createRuntimeFunction(OMPRTL__kmpc_end_ordered),
- Args);
- OrderedOpGen.setAction(Action);
- emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
- return;
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_ordered), Args);
+ // Build a call to __kmpc_end_ordered
+ CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_ordered),
+ llvm::makeArrayRef(Args));
}
emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
}
@@ -2583,14 +2596,12 @@
IdentInfo, C.CharTy);
auto *UnRegFn = createOffloadingBinaryDescriptorFunction(
- CGM, ".omp_offloading.descriptor_unreg",
- [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ CGM, ".omp_offloading.descriptor_unreg", [&](CodeGenFunction &CGF) {
CGF.EmitCallOrInvoke(createRuntimeFunction(OMPRTL__tgt_unregister_lib),
Desc);
});
auto *RegFn = createOffloadingBinaryDescriptorFunction(
- CGM, ".omp_offloading.descriptor_reg",
- [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ CGM, ".omp_offloading.descriptor_reg", [&](CodeGenFunction &CGF) {
CGF.EmitCallOrInvoke(createRuntimeFunction(OMPRTL__tgt_register_lib),
Desc);
CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
@@ -3458,10 +3469,9 @@
DepTaskArgs[5] = CGF.Builder.getInt32(0);
DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
}
- RegionCodeGenTy ThenCodeGen = [this, NumDependencies, &TaskArgs,
- &DepTaskArgs](CodeGenFunction &CGF,
- PrePostActionTy &) {
- // TODO: add check for untied tasks.
+ auto &&ThenCodeGen = [this, NumDependencies,
+ &TaskArgs, &DepTaskArgs](CodeGenFunction &CGF) {
+ // TODO: add check for untied tasks.
if (NumDependencies) {
CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps),
DepTaskArgs);
@@ -3470,6 +3480,8 @@
TaskArgs);
}
};
+ typedef CallEndCleanup<std::extent<decltype(TaskArgs)>::value>
+ IfCallEndCleanup;
llvm::Value *DepWaitTaskArgs[6];
if (NumDependencies) {
@@ -3481,8 +3493,7 @@
DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
}
auto &&ElseCodeGen = [this, &TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
- NumDependencies, &DepWaitTaskArgs](CodeGenFunction &CGF,
- PrePostActionTy &) {
+ NumDependencies, &DepWaitTaskArgs](CodeGenFunction &CGF) {
CodeGenFunction::RunCleanupsScope LocalScope(CGF);
// Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
// kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
@@ -3491,29 +3502,28 @@
if (NumDependencies)
CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
DepWaitTaskArgs);
- // Call proxy_task_entry(gtid, new_task);
- RegionCodeGenTy CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy](
- CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
- CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs);
- };
-
// Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
// kmp_task_t *new_task);
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0),
+ TaskArgs);
// Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
// kmp_task_t *new_task);
- CommonActionTy Action(
- createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
- createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
- CodeGen.setAction(Action);
- CodeGen(CGF);
+ CGF.EHStack.pushCleanup<IfCallEndCleanup>(
+ NormalAndEHCleanup,
+ createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0),
+ llvm::makeArrayRef(TaskArgs));
+
+ // Call proxy_task_entry(gtid, new_task);
+ llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
+ CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs);
};
- if (IfCond)
+ if (IfCond) {
emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
- else
+ } else {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
ThenCodeGen(CGF);
+ }
}
/// \brief Emit reduction operation for each element of array (required for
@@ -3704,25 +3714,6 @@
return Fn;
}
-static void emitSingleReductionCombiner(CodeGenFunction &CGF,
- const Expr *ReductionOp,
- const Expr *PrivateRef,
- const DeclRefExpr *LHS,
- const DeclRefExpr *RHS) {
- if (PrivateRef->getType()->isArrayType()) {
- // Emit reduction for array section.
- auto *LHSVar = cast<VarDecl>(LHS->getDecl());
- auto *RHSVar = cast<VarDecl>(RHS->getDecl());
- EmitOMPAggregateReduction(
- CGF, PrivateRef->getType(), LHSVar, RHSVar,
- [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
- emitReductionCombiner(CGF, ReductionOp);
- });
- } else
- // Emit reduction for array subscript or single variable.
- emitReductionCombiner(CGF, ReductionOp);
-}
-
void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
ArrayRef<const Expr *> Privates,
ArrayRef<const Expr *> LHSExprs,
@@ -3774,8 +3765,15 @@
auto ILHS = LHSExprs.begin();
auto IRHS = RHSExprs.begin();
for (auto *E : ReductionOps) {
- emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
- cast<DeclRefExpr>(*IRHS));
+ if ((*IPriv)->getType()->isArrayType()) {
+ auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
+ auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
+ EmitOMPAggregateReduction(
+ CGF, (*IPriv)->getType(), LHSVar, RHSVar,
+ [=](CodeGenFunction &CGF, const Expr *, const Expr *,
+ const Expr *) { emitReductionCombiner(CGF, E); });
+ } else
+ emitReductionCombiner(CGF, E);
++IPriv;
++ILHS;
++IRHS;
@@ -3865,32 +3863,40 @@
SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
CGF.EmitBlock(Case1BB);
- // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
- llvm::Value *EndArgs[] = {
- IdentTLoc, // ident_t *<loc>
- ThreadId, // i32 <gtid>
- Lock // kmp_critical_name *&<lock>
- };
- RegionCodeGenTy CodeGen = [&Privates, &LHSExprs, &RHSExprs, &ReductionOps](
- CodeGenFunction &CGF, PrePostActionTy &Action) {
+ {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
+ llvm::Value *EndArgs[] = {
+ IdentTLoc, // ident_t *<loc>
+ ThreadId, // i32 <gtid>
+ Lock // kmp_critical_name *&<lock>
+ };
+ CGF.EHStack
+ .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>(
+ NormalAndEHCleanup,
+ createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait
+ : OMPRTL__kmpc_end_reduce),
+ llvm::makeArrayRef(EndArgs));
auto IPriv = Privates.begin();
auto ILHS = LHSExprs.begin();
auto IRHS = RHSExprs.begin();
for (auto *E : ReductionOps) {
- emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
- cast<DeclRefExpr>(*IRHS));
+ if ((*IPriv)->getType()->isArrayType()) {
+ // Emit reduction for array section.
+ auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
+ auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
+ EmitOMPAggregateReduction(
+ CGF, (*IPriv)->getType(), LHSVar, RHSVar,
+ [=](CodeGenFunction &CGF, const Expr *, const Expr *,
+ const Expr *) { emitReductionCombiner(CGF, E); });
+ } else
+ // Emit reduction for array subscript or single variable.
+ emitReductionCombiner(CGF, E);
++IPriv;
++ILHS;
++IRHS;
}
- };
- CommonActionTy Action(
- nullptr, llvm::None,
- createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait
- : OMPRTL__kmpc_end_reduce),
- EndArgs);
- CodeGen.setAction(Action);
- CodeGen(CGF);
+ }
CGF.EmitBranch(DefaultBB);
@@ -3903,111 +3909,106 @@
SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
CGF.EmitBlock(Case2BB);
- RegionCodeGenTy AtomicCodeGen = [this, Loc, &Privates, &LHSExprs, &RHSExprs,
- &ReductionOps](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
+ {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ if (!WithNowait) {
+ // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
+ llvm::Value *EndArgs[] = {
+ IdentTLoc, // ident_t *<loc>
+ ThreadId, // i32 <gtid>
+ Lock // kmp_critical_name *&<lock>
+ };
+ CGF.EHStack
+ .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>(
+ NormalAndEHCleanup,
+ createRuntimeFunction(OMPRTL__kmpc_end_reduce),
+ llvm::makeArrayRef(EndArgs));
+ }
auto ILHS = LHSExprs.begin();
auto IRHS = RHSExprs.begin();
auto IPriv = Privates.begin();
for (auto *E : ReductionOps) {
- const Expr *XExpr = nullptr;
- const Expr *EExpr = nullptr;
- const Expr *UpExpr = nullptr;
- BinaryOperatorKind BO = BO_Comma;
- if (auto *BO = dyn_cast<BinaryOperator>(E)) {
- if (BO->getOpcode() == BO_Assign) {
- XExpr = BO->getLHS();
- UpExpr = BO->getRHS();
+ const Expr *XExpr = nullptr;
+ const Expr *EExpr = nullptr;
+ const Expr *UpExpr = nullptr;
+ BinaryOperatorKind BO = BO_Comma;
+ if (auto *BO = dyn_cast<BinaryOperator>(E)) {
+ if (BO->getOpcode() == BO_Assign) {
+ XExpr = BO->getLHS();
+ UpExpr = BO->getRHS();
+ }
}
- }
- // Try to emit update expression as a simple atomic.
- auto *RHSExpr = UpExpr;
- if (RHSExpr) {
- // Analyze RHS part of the whole expression.
- if (auto *ACO = dyn_cast<AbstractConditionalOperator>(
- RHSExpr->IgnoreParenImpCasts())) {
- // If this is a conditional operator, analyze its condition for
- // min/max reduction operator.
- RHSExpr = ACO->getCond();
+ // Try to emit update expression as a simple atomic.
+ auto *RHSExpr = UpExpr;
+ if (RHSExpr) {
+ // Analyze RHS part of the whole expression.
+ if (auto *ACO = dyn_cast<AbstractConditionalOperator>(
+ RHSExpr->IgnoreParenImpCasts())) {
+ // If this is a conditional operator, analyze its condition for
+ // min/max reduction operator.
+ RHSExpr = ACO->getCond();
+ }
+ if (auto *BORHS =
+ dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
+ EExpr = BORHS->getRHS();
+ BO = BORHS->getOpcode();
+ }
}
- if (auto *BORHS =
- dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
- EExpr = BORHS->getRHS();
- BO = BORHS->getOpcode();
+ if (XExpr) {
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
+ auto &&AtomicRedGen = [this, BO, VD, IPriv,
+ Loc](CodeGenFunction &CGF, const Expr *XExpr,
+ const Expr *EExpr, const Expr *UpExpr) {
+ LValue X = CGF.EmitLValue(XExpr);
+ RValue E;
+ if (EExpr)
+ E = CGF.EmitAnyExpr(EExpr);
+ CGF.EmitOMPAtomicSimpleUpdateExpr(
+ X, E, BO, /*IsXLHSInRHSPart=*/true, llvm::Monotonic, Loc,
+ [&CGF, UpExpr, VD, IPriv, Loc](RValue XRValue) {
+ CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
+ PrivateScope.addPrivate(
+ VD, [&CGF, VD, XRValue, Loc]() -> Address {
+ Address LHSTemp = CGF.CreateMemTemp(VD->getType());
+ CGF.emitOMPSimpleStore(
+ CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
+ VD->getType().getNonReferenceType(), Loc);
+ return LHSTemp;
+ });
+ (void)PrivateScope.Privatize();
+ return CGF.EmitAnyExpr(UpExpr);
+ });
+ };
+ if ((*IPriv)->getType()->isArrayType()) {
+ // Emit atomic reduction for array section.
+ auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
+ EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
+ AtomicRedGen, XExpr, EExpr, UpExpr);
+ } else
+ // Emit atomic reduction for array subscript or single variable.
+ AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
+ } else {
+ // Emit as a critical region.
+ auto &&CritRedGen = [this, E, Loc](CodeGenFunction &CGF, const Expr *,
+ const Expr *, const Expr *) {
+ emitCriticalRegion(
+ CGF, ".atomic_reduction",
+ [=](CodeGenFunction &CGF) { emitReductionCombiner(CGF, E); },
+ Loc);
+ };
+ if ((*IPriv)->getType()->isArrayType()) {
+ auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
+ auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
+ EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
+ CritRedGen);
+ } else
+ CritRedGen(CGF, nullptr, nullptr, nullptr);
}
- }
- if (XExpr) {
- auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
- auto &&AtomicRedGen = [this, BO, VD, IPriv,
- Loc](CodeGenFunction &CGF, const Expr *XExpr,
- const Expr *EExpr, const Expr *UpExpr) {
- LValue X = CGF.EmitLValue(XExpr);
- RValue E;
- if (EExpr)
- E = CGF.EmitAnyExpr(EExpr);
- CGF.EmitOMPAtomicSimpleUpdateExpr(
- X, E, BO, /*IsXLHSInRHSPart=*/true, llvm::Monotonic, Loc,
- [&CGF, UpExpr, VD, IPriv, Loc](RValue XRValue) {
- CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- PrivateScope.addPrivate(
- VD, [&CGF, VD, XRValue, Loc]() -> Address {
- Address LHSTemp = CGF.CreateMemTemp(VD->getType());
- CGF.emitOMPSimpleStore(
- CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
- VD->getType().getNonReferenceType(), Loc);
- return LHSTemp;
- });
- (void)PrivateScope.Privatize();
- return CGF.EmitAnyExpr(UpExpr);
- });
- };
- if ((*IPriv)->getType()->isArrayType()) {
- // Emit atomic reduction for array section.
- auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
- EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
- AtomicRedGen, XExpr, EExpr, UpExpr);
- } else
- // Emit atomic reduction for array subscript or single variable.
- AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
- } else {
- // Emit as a critical region.
- auto &&CritRedGen = [this, E, Loc](CodeGenFunction &CGF, const Expr *,
- const Expr *, const Expr *) {
- emitCriticalRegion(
- CGF, ".atomic_reduction",
- [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- emitReductionCombiner(CGF, E);
- },
- Loc);
- };
- if ((*IPriv)->getType()->isArrayType()) {
- auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
- auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
- EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
- CritRedGen);
- } else
- CritRedGen(CGF, nullptr, nullptr, nullptr);
- }
++ILHS;
++IRHS;
++IPriv;
}
- };
- if (!WithNowait) {
- // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
- llvm::Value *EndArgs[] = {
- IdentTLoc, // ident_t *<loc>
- ThreadId, // i32 <gtid>
- Lock // kmp_critical_name *&<lock>
- };
- CommonActionTy Action(nullptr, llvm::None,
- createRuntimeFunction(OMPRTL__kmpc_end_reduce),
- EndArgs);
- AtomicCodeGen.setAction(Action);
- AtomicCodeGen(CGF);
- } else
- AtomicCodeGen(CGF);
+ }
CGF.EmitBranch(DefaultBB);
CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
@@ -4104,8 +4105,8 @@
// kmp_int32 cncl_kind);
if (auto *OMPRegionInfo =
dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
- RegionCodeGenTy ThenGen = [this, Loc, CancelRegion, OMPRegionInfo](
- CodeGenFunction &CGF, PrePostActionTy &) {
+ auto &&ThenGen = [this, Loc, CancelRegion,
+ OMPRegionInfo](CodeGenFunction &CGF) {
llvm::Value *Args[] = {
emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
@@ -4130,8 +4131,7 @@
CGF.EmitBlock(ContBB, /*IsFinished=*/true);
};
if (IfCond)
- emitOMPIfClause(CGF, IfCond, ThenGen,
- [](CodeGenFunction &, PrePostActionTy &) {});
+ emitOMPIfClause(CGF, IfCond, ThenGen, [](CodeGenFunction &) {});
else
ThenGen(CGF);
}
@@ -4167,9 +4167,21 @@
void CGOpenMPRuntime::emitTargetOutlinedFunction(
const OMPExecutableDirective &D, StringRef ParentName,
llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
- bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
+ bool IsOffloadEntry) {
assert(!ParentName.empty() && "Invalid target region parent name!");
+ const CapturedStmt &CS = *cast<CapturedStmt>(D.getAssociatedStmt());
+
+ // Emit target region as a standalone region.
+ auto &&CodeGen = [&CS, &D](CodeGenFunction &CGF) {
+ CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
+ (void)CGF.EmitOMPFirstprivateClause(D, PrivateScope);
+ CGF.EmitOMPPrivateClause(D, PrivateScope);
+ (void)PrivateScope.Privatize();
+
+ CGF.EmitStmt(CS.getCapturedStmt());
+ };
+
emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
IsOffloadEntry, CodeGen);
}
@@ -4459,10 +4471,9 @@
OffloadError);
// Fill up the pointer arrays and transfer execution to the device.
- RegionCodeGenTy ThenGen = [this, &Ctx, &BasePointers, &Pointers, &Sizes,
- &MapTypes, hasVLACaptures, Device, OutlinedFnID,
- OffloadError, OffloadErrorQType,
- &D](CodeGenFunction &CGF, PrePostActionTy &) {
+ auto &&ThenGen = [this, &Ctx, &BasePointers, &Pointers, &Sizes, &MapTypes,
+ hasVLACaptures, Device, OutlinedFnID, OffloadError,
+ OffloadErrorQType, &D](CodeGenFunction &CGF) {
unsigned PointerNumVal = BasePointers.size();
llvm::Value *PointerNum = CGF.Builder.getInt32(PointerNumVal);
llvm::Value *BasePointersArray;
@@ -4635,8 +4646,8 @@
};
// Notify that the host version must be executed.
- RegionCodeGenTy ElseGen = [this, OffloadError, OffloadErrorQType](
- CodeGenFunction &CGF, PrePostActionTy &) {
+ auto &&ElseGen = [this, OffloadError,
+ OffloadErrorQType](CodeGenFunction &CGF) {
CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/-1u),
OffloadError);
};
@@ -4646,12 +4657,16 @@
// regardless of the conditional in the if clause if, e.g., the user do not
// specify target triples.
if (OutlinedFnID) {
- if (IfCond)
+ if (IfCond) {
emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
- else
+ } else {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
ThenGen(CGF);
- } else
+ }
+ } else {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
ElseGen(CGF);
+ }
// Check the error code and execute the host version if required.
auto OffloadFailedBlock = CGF.createBasicBlock("omp_offload.failed");
@@ -4693,10 +4708,8 @@
llvm::Function *Fn;
llvm::Constant *Addr;
- std::tie(Fn, Addr) =
- CodeGenFunction::EmitOMPTargetDirectiveOutlinedFunction(
- CGM, cast<OMPTargetDirective>(*E), ParentName,
- /*isOffloadEntry=*/true);
+ emitTargetOutlinedFunction(*E, ParentName, Fn, Addr,
+ /*isOffloadEntry=*/true);
assert(Fn && Addr && "Target region emission failed.");
return;
}