Update aosp/master clang for rebase to r230699.
Change-Id: I6a546ab3d4ae37119eebb735e102cca4f80ab520
diff --git a/lib/CodeGen/CGStmtOpenMP.cpp b/lib/CodeGen/CGStmtOpenMP.cpp
index b160f17..daf5fcc 100644
--- a/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/lib/CodeGen/CGStmtOpenMP.cpp
@@ -14,9 +14,9 @@
#include "CGOpenMPRuntime.h"
#include "CodeGenFunction.h"
#include "CodeGenModule.h"
+#include "TargetInfo.h"
#include "clang/AST/Stmt.h"
#include "clang/AST/StmtOpenMP.h"
-#include "TargetInfo.h"
using namespace clang;
using namespace CodeGen;
@@ -57,13 +57,13 @@
// Emit the 'else' code if present.
{
// There is no need to emit line number for unconditional branch.
- SuppressDebugLocation SDL(CGF.Builder);
+ auto NL = ApplyDebugLocation::CreateEmpty(CGF);
CGF.EmitBlock(ElseBlock);
}
CodeGen(/*ThenBlock*/ false);
{
// There is no need to emit line number for unconditional branch.
- SuppressDebugLocation SDL(CGF.Builder);
+ auto NL = ApplyDebugLocation::CreateEmpty(CGF);
CGF.EmitBranch(ContBlock);
}
// Emit the continuation block for code after the if.
@@ -220,17 +220,17 @@
auto NumThreadsClause = cast<OMPNumThreadsClause>(C);
auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
/*IgnoreResultAssign*/ true);
- CGF.CGM.getOpenMPRuntime().EmitOMPNumThreadsClause(
+ CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
CGF, NumThreads, NumThreadsClause->getLocStart());
}
- CGF.CGM.getOpenMPRuntime().EmitOMPParallelCall(CGF, S.getLocStart(),
- OutlinedFn, CapturedStruct);
+ CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
+ CapturedStruct);
}
void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
- auto OutlinedFn = CGM.getOpenMPRuntime().EmitOpenMPOutlinedFunction(
+ auto OutlinedFn = CGM.getOpenMPRuntime().emitOutlinedFunction(
S, *CS->getCapturedDecl()->param_begin());
if (auto C = S.getSingleClause(/*K*/ OMPC_if)) {
auto Cond = cast<OMPIfClause>(C)->getCondition();
@@ -238,8 +238,8 @@
if (ThenBlock)
EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct);
else
- CGM.getOpenMPRuntime().EmitOMPSerialCall(*this, S.getLocStart(),
- OutlinedFn, CapturedStruct);
+ CGM.getOpenMPRuntime().emitSerialCall(*this, S.getLocStart(),
+ OutlinedFn, CapturedStruct);
});
} else
EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct);
@@ -417,6 +417,7 @@
}
}
+ InlinedOpenMPRegionRAII Region(*this, S);
RunCleanupsScope DirectiveScope(*this);
CGDebugInfo *DI = getDebugInfo();
@@ -470,8 +471,199 @@
DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd());
}
-void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &) {
- llvm_unreachable("CodeGen for 'omp for' is not supported yet.");
+void CodeGenFunction::EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind,
+ const OMPLoopDirective &S,
+ OMPPrivateScope &LoopScope,
+ llvm::Value *LB, llvm::Value *UB,
+ llvm::Value *ST, llvm::Value *IL,
+ llvm::Value *Chunk) {
+ auto &RT = CGM.getOpenMPRuntime();
+ assert(!RT.isStaticNonchunked(ScheduleKind, /* Chunked */ Chunk != nullptr) &&
+ "static non-chunked schedule does not need outer loop");
+ if (RT.isDynamic(ScheduleKind)) {
+ ErrorUnsupported(&S, "OpenMP loop with dynamic schedule");
+ return;
+ }
+
+ // Emit outer loop.
+ //
+ // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
+ // When schedule(static, chunk_size) is specified, iterations are divided into
+ // chunks of size chunk_size, and the chunks are assigned to the threads in
+ // the team in a round-robin fashion in the order of the thread number.
+ //
+ // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
+ // while (idx <= UB) { BODY; ++idx; } // inner loop
+ // LB = LB + ST;
+ // UB = UB + ST;
+ // }
+ //
+ const Expr *IVExpr = S.getIterationVariable();
+ const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
+ const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
+
+ RT.emitForInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, IL, LB,
+ UB, ST, Chunk);
+ auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
+
+ // Start the loop with a block that tests the condition.
+ auto CondBlock = createBasicBlock("omp.dispatch.cond");
+ EmitBlock(CondBlock);
+ LoopStack.push(CondBlock);
+
+ llvm::Value *BoolCondVal = nullptr;
+ // UB = min(UB, GlobalUB)
+ EmitIgnoredExpr(S.getEnsureUpperBound());
+ // IV = LB
+ EmitIgnoredExpr(S.getInit());
+ // IV < UB
+ BoolCondVal = EvaluateExprAsBool(S.getCond(false));
+
+ // If there are any cleanups between here and the loop-exit scope,
+ // create a block to stage a loop exit along.
+ auto ExitBlock = LoopExit.getBlock();
+ if (LoopScope.requiresCleanups())
+ ExitBlock = createBasicBlock("omp.dispatch.cleanup");
+
+ auto LoopBody = createBasicBlock("omp.dispatch.body");
+ Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
+ if (ExitBlock != LoopExit.getBlock()) {
+ EmitBlock(ExitBlock);
+ EmitBranchThroughCleanup(LoopExit);
+ }
+ EmitBlock(LoopBody);
+
+ // Create a block for the increment.
+ auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
+ BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
+
+ EmitOMPInnerLoop(S, LoopScope);
+
+ EmitBlock(Continue.getBlock());
+ BreakContinueStack.pop_back();
+ // Emit "LB = LB + Stride", "UB = UB + Stride".
+ EmitIgnoredExpr(S.getNextLowerBound());
+ EmitIgnoredExpr(S.getNextUpperBound());
+
+ EmitBranch(CondBlock);
+ LoopStack.pop();
+ // Emit the fall-through block.
+ EmitBlock(LoopExit.getBlock());
+
+ // Tell the runtime we are done.
+ RT.emitForFinish(*this, S.getLocStart(), ScheduleKind);
+}
+
+/// \brief Emit a helper variable and return corresponding lvalue.
+static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
+ const DeclRefExpr *Helper) {
+ auto VDecl = cast<VarDecl>(Helper->getDecl());
+ CGF.EmitVarDecl(*VDecl);
+ return CGF.EmitLValue(Helper);
+}
+
+void CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) {
+ // Emit the loop iteration variable.
+ auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
+ auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
+ EmitVarDecl(*IVDecl);
+
+ // Emit the iterations count variable.
+ // If it is not a variable, Sema decided to calculate iterations count on each
+ // iteration (e.g., it is foldable into a constant).
+ if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
+ EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
+ // Emit calculation of the iterations count.
+ EmitIgnoredExpr(S.getCalcLastIteration());
+ }
+
+ auto &RT = CGM.getOpenMPRuntime();
+
+ // Check pre-condition.
+ {
+ // Skip the entire loop if we don't meet the precondition.
+ RegionCounter Cnt = getPGORegionCounter(&S);
+ auto ThenBlock = createBasicBlock("omp.precond.then");
+ auto ContBlock = createBasicBlock("omp.precond.end");
+ EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount());
+ EmitBlock(ThenBlock);
+ Cnt.beginRegion(Builder);
+ // Emit 'then' code.
+ {
+ // Emit helper vars inits.
+ LValue LB =
+ EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getLowerBoundVariable()));
+ LValue UB =
+ EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getUpperBoundVariable()));
+ LValue ST =
+ EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
+ LValue IL =
+ EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
+
+ OMPPrivateScope LoopScope(*this);
+ EmitPrivateLoopCounters(*this, LoopScope, S.counters());
+
+ // Detect the loop schedule kind and chunk.
+ auto ScheduleKind = OMPC_SCHEDULE_unknown;
+ llvm::Value *Chunk = nullptr;
+ if (auto C = cast_or_null<OMPScheduleClause>(
+ S.getSingleClause(OMPC_schedule))) {
+ ScheduleKind = C->getScheduleKind();
+ if (auto Ch = C->getChunkSize()) {
+ Chunk = EmitScalarExpr(Ch);
+ Chunk = EmitScalarConversion(Chunk, Ch->getType(),
+ S.getIterationVariable()->getType());
+ }
+ }
+ const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
+ const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
+ if (RT.isStaticNonchunked(ScheduleKind,
+ /* Chunked */ Chunk != nullptr)) {
+ // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
+ // When no chunk_size is specified, the iteration space is divided into
+ // chunks that are approximately equal in size, and at most one chunk is
+ // distributed to each thread. Note that the size of the chunks is
+ // unspecified in this case.
+ RT.emitForInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned,
+ IL.getAddress(), LB.getAddress(), UB.getAddress(),
+ ST.getAddress());
+ // UB = min(UB, GlobalUB);
+ EmitIgnoredExpr(S.getEnsureUpperBound());
+ // IV = LB;
+ EmitIgnoredExpr(S.getInit());
+ // while (idx <= UB) { BODY; ++idx; }
+ EmitOMPInnerLoop(S, LoopScope);
+ // Tell the runtime we are done.
+ RT.emitForFinish(*this, S.getLocStart(), ScheduleKind);
+ } else {
+ // Emit the outer loop, which requests its work chunk [LB..UB] from
+ // runtime and runs the inner loop to process it.
+ EmitOMPForOuterLoop(ScheduleKind, S, LoopScope, LB.getAddress(),
+ UB.getAddress(), ST.getAddress(), IL.getAddress(),
+ Chunk);
+ }
+ }
+ // We're now done with the loop, so jump to the continuation block.
+ EmitBranch(ContBlock);
+ EmitBlock(ContBlock, true);
+ }
+}
+
+void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
+ InlinedOpenMPRegionRAII Region(*this, S);
+ RunCleanupsScope DirectiveScope(*this);
+
+ CGDebugInfo *DI = getDebugInfo();
+ if (DI)
+ DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin());
+
+ EmitOMPWorksharingLoop(S);
+
+ // Emit an implicit barrier at the end.
+ CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
+ /*IsExplicit*/ false);
+ if (DI)
+ DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd());
}
void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &) {
@@ -486,30 +678,32 @@
llvm_unreachable("CodeGen for 'omp section' is not supported yet.");
}
-void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &) {
- llvm_unreachable("CodeGen for 'omp single' is not supported yet.");
-}
-
-void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &) {
- llvm_unreachable("CodeGen for 'omp master' is not supported yet.");
-}
-
-void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
- // __kmpc_critical();
- // <captured_body>
- // __kmpc_end_critical();
- //
-
- auto Lock = CGM.getOpenMPRuntime().GetCriticalRegionLock(
- S.getDirectiveName().getAsString());
- CGM.getOpenMPRuntime().EmitOMPCriticalRegionStart(*this, Lock,
- S.getLocStart());
- {
+void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
+ CGM.getOpenMPRuntime().emitSingleRegion(*this, [&]() -> void {
+ InlinedOpenMPRegionRAII Region(*this, S);
RunCleanupsScope Scope(*this);
EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
EnsureInsertPoint();
- }
- CGM.getOpenMPRuntime().EmitOMPCriticalRegionEnd(*this, Lock, S.getLocEnd());
+ }, S.getLocStart());
+}
+
+void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
+ CGM.getOpenMPRuntime().emitMasterRegion(*this, [&]() -> void {
+ InlinedOpenMPRegionRAII Region(*this, S);
+ RunCleanupsScope Scope(*this);
+ EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+ EnsureInsertPoint();
+ }, S.getLocStart());
+}
+
+void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
+ CGM.getOpenMPRuntime().emitCriticalRegion(
+ *this, S.getDirectiveName().getAsString(), [&]() -> void {
+ InlinedOpenMPRegionRAII Region(*this, S);
+ RunCleanupsScope Scope(*this);
+ EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+ EnsureInsertPoint();
+ }, S.getLocStart());
}
void
@@ -531,12 +725,13 @@
llvm_unreachable("CodeGen for 'omp task' is not supported yet.");
}
-void CodeGenFunction::EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &) {
- llvm_unreachable("CodeGen for 'omp taskyield' is not supported yet.");
+void CodeGenFunction::EmitOMPTaskyieldDirective(
+ const OMPTaskyieldDirective &S) {
+ CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart());
}
-void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &) {
- llvm_unreachable("CodeGen for 'omp barrier' is not supported yet.");
+void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
+ CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart());
}
void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &) {
@@ -544,24 +739,142 @@
}
void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
- CGM.getOpenMPRuntime().EmitOMPFlush(
- *this, [&]() -> ArrayRef<const Expr *> {
- if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) {
- auto FlushClause = cast<OMPFlushClause>(C);
- return llvm::makeArrayRef(FlushClause->varlist_begin(),
- FlushClause->varlist_end());
- }
- return llvm::None;
- }(),
- S.getLocStart());
+ CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> {
+ if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) {
+ auto FlushClause = cast<OMPFlushClause>(C);
+ return llvm::makeArrayRef(FlushClause->varlist_begin(),
+ FlushClause->varlist_end());
+ }
+ return llvm::None;
+ }(), S.getLocStart());
}
void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &) {
llvm_unreachable("CodeGen for 'omp ordered' is not supported yet.");
}
-void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &) {
- llvm_unreachable("CodeGen for 'omp atomic' is not supported yet.");
+static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
+ QualType SrcType, QualType DestType) {
+ assert(CGF.hasScalarEvaluationKind(DestType) &&
+ "DestType must have scalar evaluation kind.");
+ assert(!Val.isAggregate() && "Must be a scalar or complex.");
+ return Val.isScalar()
+ ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType)
+ : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType,
+ DestType);
+}
+
+static CodeGenFunction::ComplexPairTy
+convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
+ QualType DestType) {
+ assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
+ "DestType must have complex evaluation kind.");
+ CodeGenFunction::ComplexPairTy ComplexVal;
+ if (Val.isScalar()) {
+ // Convert the input element to the element type of the complex.
+ auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
+ auto ScalarVal =
+ CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestElementType);
+ ComplexVal = CodeGenFunction::ComplexPairTy(
+ ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
+ } else {
+ assert(Val.isComplex() && "Must be a scalar or complex.");
+ auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
+ auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
+ ComplexVal.first = CGF.EmitScalarConversion(
+ Val.getComplexVal().first, SrcElementType, DestElementType);
+ ComplexVal.second = CGF.EmitScalarConversion(
+ Val.getComplexVal().second, SrcElementType, DestElementType);
+ }
+ return ComplexVal;
+}
+
+static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
+ const Expr *X, const Expr *V,
+ SourceLocation Loc) {
+ // v = x;
+ assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
+ assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
+ LValue XLValue = CGF.EmitLValue(X);
+ LValue VLValue = CGF.EmitLValue(V);
+ RValue Res = XLValue.isGlobalReg()
+ ? CGF.EmitLoadOfLValue(XLValue, Loc)
+ : CGF.EmitAtomicLoad(XLValue, Loc,
+ IsSeqCst ? llvm::SequentiallyConsistent
+ : llvm::Monotonic);
+ // OpenMP, 2.12.6, atomic Construct
+ // Any atomic construct with a seq_cst clause forces the atomically
+ // performed operation to include an implicit flush operation without a
+ // list.
+ if (IsSeqCst)
+ CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
+ switch (CGF.getEvaluationKind(V->getType())) {
+ case TEK_Scalar:
+ CGF.EmitStoreOfScalar(
+ convertToScalarValue(CGF, Res, X->getType(), V->getType()), VLValue);
+ break;
+ case TEK_Complex:
+ CGF.EmitStoreOfComplex(
+ convertToComplexValue(CGF, Res, X->getType(), V->getType()), VLValue,
+ /*isInit=*/false);
+ break;
+ case TEK_Aggregate:
+ llvm_unreachable("Must be a scalar or complex.");
+ }
+}
+
+static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
+ bool IsSeqCst, const Expr *X, const Expr *V,
+ const Expr *, SourceLocation Loc) {
+ switch (Kind) {
+ case OMPC_read:
+ EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
+ break;
+ case OMPC_write:
+ case OMPC_update:
+ case OMPC_capture:
+ llvm_unreachable("CodeGen for 'omp atomic clause' is not supported yet.");
+ case OMPC_if:
+ case OMPC_final:
+ case OMPC_num_threads:
+ case OMPC_private:
+ case OMPC_firstprivate:
+ case OMPC_lastprivate:
+ case OMPC_reduction:
+ case OMPC_safelen:
+ case OMPC_collapse:
+ case OMPC_default:
+ case OMPC_seq_cst:
+ case OMPC_shared:
+ case OMPC_linear:
+ case OMPC_aligned:
+ case OMPC_copyin:
+ case OMPC_copyprivate:
+ case OMPC_flush:
+ case OMPC_proc_bind:
+ case OMPC_schedule:
+ case OMPC_ordered:
+ case OMPC_nowait:
+ case OMPC_untied:
+ case OMPC_threadprivate:
+ case OMPC_mergeable:
+ case OMPC_unknown:
+ llvm_unreachable("Clause is not allowed in 'omp atomic'.");
+ }
+}
+
+void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
+ bool IsSeqCst = S.getSingleClause(/*K=*/OMPC_seq_cst);
+ OpenMPClauseKind Kind = OMPC_unknown;
+ for (auto *C : S.clauses()) {
+ // Find first clause (skip seq_cst clause, if it is first).
+ if (C->getClauseKind() != OMPC_seq_cst) {
+ Kind = C->getClauseKind();
+ break;
+ }
+ }
+ EmitOMPAtomicExpr(*this, Kind, IsSeqCst, S.getX(), S.getV(), S.getExpr(),
+ S.getLocStart());
}
void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) {