[OPENMP] Codegen for 'ordered' directive.
Add codegen for 'ordered' directive:
__kmpc_ordered(ident_t *, gtid);
<associated statement>;
__kmpc_end_ordered(ident_t *, gtid);
Also for 'for' directives with the dynamic scheduling and an 'ordered' clause added a call to '__kmpc_dispatch_fini_(4|8)[u]()' function after increment expression for loop control variable:
while(__kmpc_dispatch_next(&LB, &UB)) {
idx = LB;
while (idx <= UB) { BODY; ++idx;
__kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
} // inner loop
}
Differential Revision: http://reviews.llvm.org/D9070
llvm-svn: 235496
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 5988c78..f7caee1 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -710,6 +710,22 @@
CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
break;
}
+ case OMPRTL__kmpc_ordered: {
+ // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
+ break;
+ }
+ case OMPRTL__kmpc_end_ordered: {
+ // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
+ break;
+ }
}
return RTLFn;
}
@@ -762,6 +778,23 @@
return CGM.CreateRuntimeFunction(FnTy, Name);
}
+llvm::Constant *CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize,
+ bool IVSigned) {
+ assert((IVSize == 32 || IVSize == 64) &&
+ "IV size is not compatible with the omp runtime");
+ auto Name =
+ IVSize == 32
+ ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
+ : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
+ llvm::Type *TypeParams[] = {
+ getIdentTyPointerTy(), // loc
+ CGM.Int32Ty, // tid
+ };
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ return CGM.CreateRuntimeFunction(FnTy, Name);
+}
+
llvm::Constant *CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize,
bool IVSigned) {
assert((IVSize == 32 || IVSize == 64) &&
@@ -1246,6 +1279,25 @@
}
}
+void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
+ const RegionCodeGenTy &OrderedOpGen,
+ SourceLocation Loc) {
+ // __kmpc_ordered(ident_t *, gtid);
+ // OrderedOpGen();
+ // __kmpc_end_ordered(ident_t *, gtid);
+ // Prepare arguments and build a call to __kmpc_ordered
+ {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_ordered), Args);
+ // Build a call to __kmpc_end_ordered
+ CGF.EHStack.pushCleanup<CallEndCleanup>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_ordered),
+ llvm::makeArrayRef(Args));
+ emitInlinedDirective(CGF, OrderedOpGen);
+ }
+}
+
void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
OpenMPDirectiveKind Kind) {
// Build call __kmpc_cancel_barrier(loc, thread_id);
@@ -1378,12 +1430,8 @@
}
}
-void CGOpenMPRuntime::emitForFinish(CodeGenFunction &CGF, SourceLocation Loc,
- OpenMPScheduleClauseKind ScheduleKind) {
- assert((ScheduleKind == OMPC_SCHEDULE_static ||
- ScheduleKind == OMPC_SCHEDULE_unknown) &&
- "Non-static schedule kinds are not yet implemented");
- (void)ScheduleKind;
+void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
+ SourceLocation Loc) {
// Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
getThreadID(CGF, Loc)};
@@ -1391,6 +1439,16 @@
Args);
}
+void CGOpenMPRuntime::emitForOrderedDynamicIterationEnd(CodeGenFunction &CGF,
+ SourceLocation Loc,
+ unsigned IVSize,
+ bool IVSigned) {
+ // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
+ getThreadID(CGF, Loc)};
+ CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
+}
+
llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
SourceLocation Loc, unsigned IVSize,
bool IVSigned, llvm::Value *IL,
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index fa59930..f3c4379 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -118,6 +118,10 @@
// Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
// kmp_critical_name *lck);
OMPRTL__kmpc_end_reduce_nowait,
+ // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
+ OMPRTL__kmpc_ordered,
+ // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
+ OMPRTL__kmpc_end_ordered,
};
/// \brief Values for bit flags used in the ident_t to describe the fields.
@@ -252,6 +256,10 @@
/// size \a IVSize and sign \a IVSigned.
llvm::Constant *createDispatchNextFunction(unsigned IVSize, bool IVSigned);
+ /// \brief Returns __kmpc_dispatch_fini_* runtime function for the specified
+ /// size \a IVSize and sign \a IVSigned.
+ llvm::Constant *createDispatchFiniFunction(unsigned IVSize, bool IVSigned);
+
/// \brief If the specified mangled name is not in the module, create and
/// return threadprivate cache object. This object is a pointer's worth of
/// storage that's reserved for use by the OpenMP runtime.
@@ -378,6 +386,13 @@
ArrayRef<const Expr *> SrcExprs,
ArrayRef<const Expr *> AssignmentOps);
+ /// \brief Emit an ordered region.
+ /// \param OrderedOpGen Generator for the statement associated with the given
+ /// critical region.
+ virtual void emitOrderedRegion(CodeGenFunction &CGF,
+ const RegionCodeGenTy &OrderedOpGen,
+ SourceLocation Loc);
+
/// \brief Emit an implicit/explicit barrier for OpenMP threads.
/// \param Kind Directive for which this implicit barrier call must be
/// generated. Must be OMPD_barrier for explicit barrier generation.
@@ -429,14 +444,25 @@
llvm::Value *Chunk = nullptr);
/// \brief Call the appropriate runtime routine to notify that we finished
+ /// iteration of the ordered loop with the dynamic scheduling.
+ ///
+ /// \param CGF Reference to current CodeGenFunction.
+ /// \param Loc Clang source location.
+ /// \param IVSize Size of the iteration variable in bits.
+ /// \param IVSigned Sign of the interation variable.
+ ///
+ virtual void emitForOrderedDynamicIterationEnd(CodeGenFunction &CGF,
+ SourceLocation Loc,
+ unsigned IVSize,
+ bool IVSigned);
+
+ /// \brief Call the appropriate runtime routine to notify that we finished
/// all the work with current loop.
///
/// \param CGF Reference to current CodeGenFunction.
/// \param Loc Clang source location.
- /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
///
- virtual void emitForFinish(CodeGenFunction &CGF, SourceLocation Loc,
- OpenMPScheduleClauseKind ScheduleKind);
+ virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc);
/// Call __kmpc_dispatch_next(
/// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 1541ba68..9451802 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -576,7 +576,8 @@
void CodeGenFunction::EmitOMPInnerLoop(
const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
const Expr *IncExpr,
- const llvm::function_ref<void(CodeGenFunction &)> &BodyGen) {
+ const llvm::function_ref<void(CodeGenFunction &)> &BodyGen,
+ const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen) {
auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
auto Cnt = getPGORegionCounter(&S);
@@ -612,6 +613,7 @@
// Emit "IV = IV + 1" and a back-edge to the condition block.
EmitBlock(Continue.getBlock());
EmitIgnoredExpr(IncExpr);
+ PostIncGen(*this);
BreakContinueStack.pop_back();
EmitBranch(CondBlock);
LoopStack.pop();
@@ -799,7 +801,8 @@
[&S](CodeGenFunction &CGF) {
CGF.EmitOMPLoopBody(S);
CGF.EmitStopPoint(&S);
- });
+ },
+ [](CodeGenFunction &) {});
CGF.EmitOMPLoopBody(S, /* SeparateIter */ true);
}
CGF.EmitOMPSimdFinal(S);
@@ -818,7 +821,8 @@
[&S](CodeGenFunction &CGF) {
CGF.EmitOMPLoopBody(S);
CGF.EmitStopPoint(&S);
- });
+ },
+ [](CodeGenFunction &) {});
}
CGF.EmitOMPSimdFinal(S);
}
@@ -873,7 +877,9 @@
//
// while(__kmpc_dispatch_next(&LB, &UB)) {
// idx = LB;
- // while (idx <= UB) { BODY; ++idx; } // inner loop
+ // while (idx <= UB) { BODY; ++idx;
+ // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
+ // } // inner loop
// }
//
// OpenMP [2.7.1, Loop Construct, Description, table 2-1]
@@ -940,12 +946,22 @@
auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
- EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
- S.getCond(/*SeparateIter=*/false), S.getInc(),
- [&S](CodeGenFunction &CGF) {
- CGF.EmitOMPLoopBody(S);
- CGF.EmitStopPoint(&S);
- });
+ bool DynamicWithOrderedClause =
+ Dynamic && S.getSingleClause(OMPC_ordered) != nullptr;
+ SourceLocation Loc = S.getLocStart();
+ EmitOMPInnerLoop(
+ S, LoopScope.requiresCleanups(), S.getCond(/*SeparateIter=*/false),
+ S.getInc(),
+ [&S](CodeGenFunction &CGF) {
+ CGF.EmitOMPLoopBody(S);
+ CGF.EmitStopPoint(&S);
+ },
+ [DynamicWithOrderedClause, IVSize, IVSigned, Loc](CodeGenFunction &CGF) {
+ if (DynamicWithOrderedClause) {
+ CGF.CGM.getOpenMPRuntime().emitForOrderedDynamicIterationEnd(
+ CGF, Loc, IVSize, IVSigned);
+ }
+ });
EmitBlock(Continue.getBlock());
BreakContinueStack.pop_back();
@@ -961,9 +977,8 @@
EmitBlock(LoopExit.getBlock());
// Tell the runtime we are done.
- // FIXME: Also call fini for ordered loops with dynamic scheduling.
if (!Dynamic)
- RT.emitForFinish(*this, S.getLocStart(), ScheduleKind);
+ RT.emitForStaticFinish(*this, S.getLocEnd());
}
/// \brief Emit a helper variable and return corresponding lvalue.
@@ -1058,9 +1073,10 @@
[&S](CodeGenFunction &CGF) {
CGF.EmitOMPLoopBody(S);
CGF.EmitStopPoint(&S);
- });
+ },
+ [](CodeGenFunction &) {});
// Tell the runtime we are done.
- RT.emitForFinish(*this, S.getLocStart(), ScheduleKind);
+ RT.emitForStaticFinish(*this, S.getLocStart());
} else {
// Emit the outer loop, which requests its work chunk [LB..UB] from
// runtime and runs the inner loop to process it.
@@ -1177,10 +1193,10 @@
// IV = LB;
CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
// while (idx <= UB) { BODY; ++idx; }
- CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen);
+ CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
+ [](CodeGenFunction &) {});
// Tell the runtime we are done.
- CGF.CGM.getOpenMPRuntime().emitForFinish(CGF, S.getLocStart(),
- OMPC_SCHEDULE_static);
+ CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocStart());
};
CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, CodeGen);
@@ -1372,8 +1388,13 @@
}(), S.getLocStart());
}
-void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &) {
- llvm_unreachable("CodeGen for 'omp ordered' is not supported yet.");
+void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
+ LexicalScope Scope(*this, S.getSourceRange());
+ auto &&CodeGen = [&S](CodeGenFunction &CGF) {
+ CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+ CGF.EnsureInsertPoint();
+ };
+ CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart());
}
static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 4e7a7e2..efbe07c 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -2139,10 +2139,21 @@
void EmitOMPTargetDirective(const OMPTargetDirective &S);
void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
- void
- EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
- const Expr *IncExpr,
- const llvm::function_ref<void(CodeGenFunction &)> &BodyGen);
+ /// \brief Emit inner loop of the worksharing/simd construct.
+ ///
+ /// \param S Directive, for which the inner loop must be emitted.
+ /// \param RequiresCleanup true, if directive has some associated private
+ /// variables.
+ /// \param LoopCond Bollean condition for loop continuation.
+ /// \param IncExpr Increment expression for loop control variable.
+ /// \param BodyGen Generator for the inner body of the inner loop.
+ /// \param PostIncGen Genrator for post-increment code (required for ordered
+ /// loop directvies).
+ void EmitOMPInnerLoop(
+ const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
+ const Expr *IncExpr,
+ const llvm::function_ref<void(CodeGenFunction &)> &BodyGen,
+ const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen);
private: