[OPENMP] Fixed codegen for arrays in 'copyprivate' clause.
Fixed a bug with codegen of variables with array types specified in 'copyprivate' clause of 'single' directive.
Differential Revision: http://reviews.llvm.org/D8914
llvm-svn: 234856
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index dd14314..8624195 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1106,8 +1106,9 @@
}
static llvm::Value *emitCopyprivateCopyFunction(
- CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef<const Expr *> SrcExprs,
- ArrayRef<const Expr *> DstExprs, ArrayRef<const Expr *> AssignmentOps) {
+ CodeGenModule &CGM, llvm::Type *ArgsType,
+ ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
+ ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps) {
auto &C = CGM.getContext();
// void copy_func(void *LHSArg, void *RHSArg);
FunctionArgList Args;
@@ -1126,7 +1127,7 @@
CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn);
CodeGenFunction CGF(CGM);
CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args);
- // Dst = (void*[n])(LHSArg);
+ // Dest = (void*[n])(LHSArg);
// Src = (void*[n])(RHSArg);
auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg),
@@ -1140,32 +1141,22 @@
// *(Type1*)Dst[1] = *(Type1*)Src[1];
// ...
// *(Typen*)Dst[n] = *(Typen*)Src[n];
- CodeGenFunction::OMPPrivateScope Scope(CGF);
for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
- Scope.addPrivate(
- cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl()),
- [&]() -> llvm::Value *{
- return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateAlignedLoad(
- CGF.Builder.CreateStructGEP(nullptr, RHS, I),
- CGM.PointerAlignInBytes),
- CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType())));
- });
- Scope.addPrivate(
- cast<VarDecl>(cast<DeclRefExpr>(DstExprs[I])->getDecl()),
- [&]() -> llvm::Value *{
- return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateAlignedLoad(
- CGF.Builder.CreateStructGEP(nullptr, LHS, I),
- CGM.PointerAlignInBytes),
- CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType())));
- });
+ auto *DestAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateAlignedLoad(
+ CGF.Builder.CreateStructGEP(nullptr, LHS, I),
+ CGM.PointerAlignInBytes),
+ CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType())));
+ auto *SrcAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateAlignedLoad(
+ CGF.Builder.CreateStructGEP(nullptr, RHS, I),
+ CGM.PointerAlignInBytes),
+ CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType())));
+ CGF.EmitOMPCopy(CGF, CopyprivateVars[I]->getType(), DestAddr, SrcAddr,
+ cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl()),
+ cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl()),
+ AssignmentOps[I]);
}
- Scope.Privatize();
- for (auto *E : AssignmentOps) {
- CGF.EmitIgnoredExpr(E);
- }
- Scope.ForceCleanup();
CGF.FinishFunction();
return Fn;
}
@@ -1235,7 +1226,7 @@
// threads in the corresponding parallel region.
auto *CpyFn = emitCopyprivateCopyFunction(
CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
- SrcExprs, DstExprs, AssignmentOps);
+ CopyprivateVars, SrcExprs, DstExprs, AssignmentOps);
auto *BufSize = CGF.Builder.getInt32(
C.getTypeSizeInChars(CopyprivateArrayTy).getQuantity());
auto *CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index f632617..fa59930 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -374,8 +374,8 @@
const RegionCodeGenTy &SingleOpGen,
SourceLocation Loc,
ArrayRef<const Expr *> CopyprivateVars,
+ ArrayRef<const Expr *> DestExprs,
ArrayRef<const Expr *> SrcExprs,
- ArrayRef<const Expr *> DstExprs,
ArrayRef<const Expr *> AssignmentOps);
/// \brief Emit an implicit/explicit barrier for OpenMP threads.
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index fd475ba..9a25f2e 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -69,72 +69,93 @@
CGF.EmitBlock(ContBlock, /*IsFinished*/ true);
}
-void CodeGenFunction::EmitOMPAggregateAssign(LValue OriginalAddr,
- llvm::Value *PrivateAddr,
- const Expr *AssignExpr,
- QualType OriginalType,
- const VarDecl *VDInit) {
- EmitBlock(createBasicBlock(".omp.assign.begin."));
- if (!isa<CXXConstructExpr>(AssignExpr) || isTrivialInitializer(AssignExpr)) {
- // Perform simple memcpy.
- EmitAggregateAssign(PrivateAddr, OriginalAddr.getAddress(),
- AssignExpr->getType());
- } else {
- // Perform element-by-element initialization.
- QualType ElementTy;
- auto SrcBegin = OriginalAddr.getAddress();
- auto DestBegin = PrivateAddr;
- auto ArrayTy = OriginalType->getAsArrayTypeUnsafe();
- auto SrcNumElements = emitArrayLength(ArrayTy, ElementTy, SrcBegin);
- auto DestNumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin);
- auto SrcEnd = Builder.CreateGEP(SrcBegin, SrcNumElements);
- auto DestEnd = Builder.CreateGEP(DestBegin, DestNumElements);
- // The basic structure here is a do-while loop, because we don't
- // need to check for the zero-element case.
- auto BodyBB = createBasicBlock("omp.arraycpy.body");
- auto DoneBB = createBasicBlock("omp.arraycpy.done");
- auto IsEmpty =
- Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
- Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
+void CodeGenFunction::EmitOMPAggregateAssign(
+ llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType,
+ const llvm::function_ref<void(llvm::Value *, llvm::Value *)> &CopyGen) {
+ // Perform element-by-element initialization.
+ QualType ElementTy;
+ auto SrcBegin = SrcAddr;
+ auto DestBegin = DestAddr;
+ auto ArrayTy = OriginalType->getAsArrayTypeUnsafe();
+ auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin);
+ // Cast from pointer to array type to pointer to single element.
+ SrcBegin = Builder.CreatePointerBitCastOrAddrSpaceCast(SrcBegin,
+ DestBegin->getType());
+ auto DestEnd = Builder.CreateGEP(DestBegin, NumElements);
+ // The basic structure here is a while-do loop.
+ auto BodyBB = createBasicBlock("omp.arraycpy.body");
+ auto DoneBB = createBasicBlock("omp.arraycpy.done");
+ auto IsEmpty =
+ Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
+ Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
- // Enter the loop body, making that address the current address.
- auto EntryBB = Builder.GetInsertBlock();
- EmitBlock(BodyBB);
- auto SrcElementPast = Builder.CreatePHI(SrcBegin->getType(), 2,
- "omp.arraycpy.srcElementPast");
- SrcElementPast->addIncoming(SrcEnd, EntryBB);
- auto DestElementPast = Builder.CreatePHI(DestBegin->getType(), 2,
- "omp.arraycpy.destElementPast");
- DestElementPast->addIncoming(DestEnd, EntryBB);
+ // Enter the loop body, making that address the current address.
+ auto EntryBB = Builder.GetInsertBlock();
+ EmitBlock(BodyBB);
+ auto SrcElementCurrent =
+ Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
+ SrcElementCurrent->addIncoming(SrcBegin, EntryBB);
+ auto DestElementCurrent = Builder.CreatePHI(DestBegin->getType(), 2,
+ "omp.arraycpy.destElementPast");
+ DestElementCurrent->addIncoming(DestBegin, EntryBB);
- // Shift the address back by one element.
- auto NegativeOne = llvm::ConstantInt::get(SizeTy, -1, true);
- auto DestElement = Builder.CreateGEP(DestElementPast, NegativeOne,
- "omp.arraycpy.dest.element");
- auto SrcElement = Builder.CreateGEP(SrcElementPast, NegativeOne,
- "omp.arraycpy.src.element");
- {
- // Create RunCleanScope to cleanup possible temps.
- CodeGenFunction::RunCleanupsScope Init(*this);
- // Emit initialization for single element.
- LocalDeclMap[VDInit] = SrcElement;
- EmitAnyExprToMem(AssignExpr, DestElement,
- AssignExpr->getType().getQualifiers(),
- /*IsInitializer*/ false);
- LocalDeclMap.erase(VDInit);
+ // Emit copy.
+ CopyGen(DestElementCurrent, SrcElementCurrent);
+
+ // Shift the address forward by one element.
+ auto DestElementNext = Builder.CreateConstGEP1_32(
+ DestElementCurrent, /*Idx0=*/1, "omp.arraycpy.dest.element");
+ auto SrcElementNext = Builder.CreateConstGEP1_32(
+ SrcElementCurrent, /*Idx0=*/1, "omp.arraycpy.src.element");
+ // Check whether we've reached the end.
+ auto Done =
+ Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
+ Builder.CreateCondBr(Done, DoneBB, BodyBB);
+ DestElementCurrent->addIncoming(DestElementNext, Builder.GetInsertBlock());
+ SrcElementCurrent->addIncoming(SrcElementNext, Builder.GetInsertBlock());
+
+ // Done.
+ EmitBlock(DoneBB, /*IsFinished=*/true);
+}
+
+void CodeGenFunction::EmitOMPCopy(CodeGenFunction &CGF,
+ QualType OriginalType, llvm::Value *DestAddr,
+ llvm::Value *SrcAddr, const VarDecl *DestVD,
+ const VarDecl *SrcVD, const Expr *Copy) {
+ if (OriginalType->isArrayType()) {
+ auto *BO = dyn_cast<BinaryOperator>(Copy);
+ if (BO && BO->getOpcode() == BO_Assign) {
+ // Perform simple memcpy for simple copying.
+ CGF.EmitAggregateAssign(DestAddr, SrcAddr, OriginalType);
+ } else {
+ // For arrays with complex element types perform element by element
+ // copying.
+ CGF.EmitOMPAggregateAssign(
+ DestAddr, SrcAddr, OriginalType,
+ [&CGF, Copy, SrcVD, DestVD](llvm::Value *DestElement,
+ llvm::Value *SrcElement) {
+ // Working with the single array element, so have to remap
+ // destination and source variables to corresponding array
+ // elements.
+ CodeGenFunction::OMPPrivateScope Remap(CGF);
+ Remap.addPrivate(DestVD, [DestElement]() -> llvm::Value *{
+ return DestElement;
+ });
+ Remap.addPrivate(
+ SrcVD, [SrcElement]() -> llvm::Value *{ return SrcElement; });
+ (void)Remap.Privatize();
+ CGF.EmitIgnoredExpr(Copy);
+ });
}
-
- // Check whether we've reached the end.
- auto Done =
- Builder.CreateICmpEQ(DestElement, DestBegin, "omp.arraycpy.done");
- Builder.CreateCondBr(Done, DoneBB, BodyBB);
- DestElementPast->addIncoming(DestElement, Builder.GetInsertBlock());
- SrcElementPast->addIncoming(SrcElement, Builder.GetInsertBlock());
-
- // Done.
- EmitBlock(DoneBB, true);
+ } else {
+ // Remap pseudo source variable to private copy.
+ CodeGenFunction::OMPPrivateScope Remap(CGF);
+ Remap.addPrivate(SrcVD, [SrcAddr]() -> llvm::Value *{ return SrcAddr; });
+ Remap.addPrivate(DestVD, [DestAddr]() -> llvm::Value *{ return DestAddr; });
+ (void)Remap.Privatize();
+ // Emit copying of the whole variable.
+ CGF.EmitIgnoredExpr(Copy);
}
- EmitBlock(createBasicBlock(".omp.assign.end."));
}
void CodeGenFunction::EmitOMPFirstprivateClause(
@@ -158,18 +179,37 @@
LValue Base = MakeNaturalAlignAddrLValue(
CapturedStmtInfo->getContextValue(),
getContext().getTagDeclType(FD->getParent()));
- auto OriginalAddr = EmitLValueForField(Base, FD);
+ auto *OriginalAddr = EmitLValueForField(Base, FD).getAddress();
auto VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
- IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * {
+ IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{
auto Emission = EmitAutoVarAlloca(*VD);
// Emit initialization of aggregate firstprivate vars.
- EmitOMPAggregateAssign(OriginalAddr, Emission.getAllocatedAddress(),
- VD->getInit(), (*IRef)->getType(), VDInit);
+ auto *Init = VD->getInit();
+ if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
+ // Perform simple memcpy.
+ EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr,
+ (*IRef)->getType());
+ } else {
+ EmitOMPAggregateAssign(
+ Emission.getAllocatedAddress(), OriginalAddr,
+ (*IRef)->getType(),
+ [this, VDInit, Init](llvm::Value *DestElement,
+ llvm::Value *SrcElement) {
+ // Clean up any temporaries needed by the initialization.
+ RunCleanupsScope InitScope(*this);
+ // Emit initialization for single element.
+ LocalDeclMap[VDInit] = SrcElement;
+ EmitAnyExprToMem(Init, DestElement,
+ Init->getType().getQualifiers(),
+ /*IsInitializer*/ false);
+ LocalDeclMap.erase(VDInit);
+ });
+ }
EmitAutoVarCleanups(Emission);
return Emission.getAllocatedAddress();
});
} else
- IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * {
+ IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{
// Emit private VarDecl with copy init.
EmitDecl(*VD);
return GetAddrOfLocalVar(VD);
@@ -994,8 +1034,8 @@
void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
llvm::SmallVector<const Expr *, 8> CopyprivateVars;
+ llvm::SmallVector<const Expr *, 8> DestExprs;
llvm::SmallVector<const Expr *, 8> SrcExprs;
- llvm::SmallVector<const Expr *, 8> DstExprs;
llvm::SmallVector<const Expr *, 8> AssignmentOps;
// Check if there are any 'copyprivate' clauses associated with this
// 'single'
@@ -1010,9 +1050,9 @@
for (CopyprivateIter I(S.clauses(), CopyprivateFilter); I; ++I) {
auto *C = cast<OMPCopyprivateClause>(*I);
CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
+ DestExprs.append(C->destination_exprs().begin(),
+ C->destination_exprs().end());
SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
- DstExprs.append(C->destination_exprs().begin(),
- C->destination_exprs().end());
AssignmentOps.append(C->assignment_ops().begin(),
C->assignment_ops().end());
}
@@ -1023,7 +1063,7 @@
CGF.EnsureInsertPoint();
};
CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
- CopyprivateVars, SrcExprs, DstExprs,
+ CopyprivateVars, DestExprs, SrcExprs,
AssignmentOps);
// Emit an implicit barrier at the end.
if (!S.getSingleClause(OMPC_nowait)) {
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6ba3db0..9116b71 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -2026,9 +2026,33 @@
llvm::Function *GenerateCapturedStmtFunctionEpilog(const CapturedStmt &S);
llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
llvm::Value *GenerateCapturedStmtArgument(const CapturedStmt &S);
- void EmitOMPAggregateAssign(LValue OriginalAddr, llvm::Value *PrivateAddr,
- const Expr *AssignExpr, QualType Type,
- const VarDecl *VDInit);
+ /// \brief Perform element by element copying of arrays with type \a
+ /// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
+ /// generated by \a CopyGen.
+ ///
+ /// \param DestAddr Address of the destination array.
+ /// \param SrcAddr Address of the source array.
+ /// \param OriginalType Type of destination and source arrays.
+ /// \param CopyGen Copying procedure that copies value of single array element
+ /// to another single array element.
+ void EmitOMPAggregateAssign(
+ llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType,
+ const llvm::function_ref<void(llvm::Value *, llvm::Value *)> &CopyGen);
+ /// \brief Emit proper copying of data from one variable to another.
+ ///
+ /// \param OriginalType Original type of the copied variables.
+ /// \param DestAddr Destination address.
+ /// \param SrcAddr Source address.
+ /// \param DestVD Destination variable used in \a CopyExpr (for arrays, has
+ /// type of the base array element).
+ /// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of
+ /// the base array element).
+ /// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
+ /// DestVD.
+ void EmitOMPCopy(CodeGenFunction &CGF, QualType OriginalType,
+ llvm::Value *DestAddr, llvm::Value *SrcAddr,
+ const VarDecl *DestVD, const VarDecl *SrcVD,
+ const Expr *Copy);
/// \brief Emit atomic update code for constructs: \a X = \a X \a BO \a E or
/// \a X = \a E \a BO \a E.
///