Compute and preserve alignment more faithfully in IR-generation.
Introduce an Address type to bundle a pointer value with an
alignment. Introduce APIs on CGBuilderTy to work with Address
values. Change core APIs on CGF/CGM to traffic in Address where
appropriate. Require alignments to be non-zero. Update a ton
of code to compute and propagate alignment information.
As part of this, I've promoted CGBuiltin's EmitPointerWithAlignment
helper function to CGF and made use of it in a number of places in
the expression emitter.
The end result is that we should now be significantly more correct
when performing operations on objects that are locally known to
be under-aligned. Since alignment is not reliably tracked in the
type system, there are inherent limits to this, but at least we
are no longer confused by standard operations like derived-to-base
conversions and array-to-pointer decay. I've also fixed a large
number of bugs where we were applying the complete-object alignment
to a pointer instead of the non-virtual alignment, although most of
these were hidden by the very conservative approach we took with
member alignment.
Also, because IRGen now reliably asserts on zero alignments, we
should no longer be subject to an absurd but frustrating recurring
bug where an incomplete type would report a zero alignment and then
we'd naively do a alignmentAtOffset on it and emit code using an
alignment equal to the largest power-of-two factor of the offset.
We should also now be emitting much more aggressive alignment
attributes in the presence of over-alignment. In particular,
field access now uses alignmentAtOffset instead of min.
Several times in this patch, I had to change the existing
code-generation pattern in order to more effectively use
the Address APIs. For the most part, this seems to be a strict
improvement, like doing pointer arithmetic with GEPs instead of
ptrtoint. That said, I've tried very hard to not change semantics,
but it is likely that I've failed in a few places, for which I
apologize.
ABIArgInfo now always carries the assumed alignment of indirect and
indirect byval arguments. In order to cut down on what was already
a dauntingly large patch, I changed the code to never set align
attributes in the IR on non-byval indirect arguments. That is,
we still generate code which assumes that indirect arguments have
the given alignment, but we don't express this information to the
backend except where it's semantically required (i.e. on byvals).
This is likely a minor regression for those targets that did provide
this information, but it'll be trivial to add it back in a later
patch.
I partially punted on applying this work to CGBuiltin. Please
do not add more uses of the CreateDefaultAligned{Load,Store}
APIs; they will be going away eventually.
llvm-svn: 246985
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index b28ffbf..0c6a6d7 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -69,6 +69,45 @@
return RAA_Default;
}
+ bool isThisCompleteObject(GlobalDecl GD) const override {
+ // The Itanium ABI has separate complete-object vs. base-object
+ // variants of both constructors and destructors.
+ if (isa<CXXDestructorDecl>(GD.getDecl())) {
+ switch (GD.getDtorType()) {
+ case Dtor_Complete:
+ case Dtor_Deleting:
+ return true;
+
+ case Dtor_Base:
+ return false;
+
+ case Dtor_Comdat:
+ llvm_unreachable("emitting dtor comdat as function?");
+ }
+ llvm_unreachable("bad dtor kind");
+ }
+ if (isa<CXXConstructorDecl>(GD.getDecl())) {
+ switch (GD.getCtorType()) {
+ case Ctor_Complete:
+ return true;
+
+ case Ctor_Base:
+ return false;
+
+ case Ctor_CopyingClosure:
+ case Ctor_DefaultClosure:
+ llvm_unreachable("closure ctors in Itanium ABI?");
+
+ case Ctor_Comdat:
+ llvm_unreachable("emitting ctor comdat as function?");
+ }
+ llvm_unreachable("bad dtor kind");
+ }
+
+ // No other kinds.
+ return false;
+ }
+
bool isZeroInitializable(const MemberPointerType *MPT) override;
llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT) override;
@@ -76,13 +115,14 @@
llvm::Value *
EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
const Expr *E,
- llvm::Value *&This,
+ Address This,
+ llvm::Value *&ThisPtrForCall,
llvm::Value *MemFnPtr,
const MemberPointerType *MPT) override;
llvm::Value *
EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
- llvm::Value *Base,
+ Address Base,
llvm::Value *MemPtr,
const MemberPointerType *MPT) override;
@@ -111,9 +151,22 @@
const MemberPointerType *MPT) override;
void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE,
- llvm::Value *Ptr, QualType ElementType,
+ Address Ptr, QualType ElementType,
const CXXDestructorDecl *Dtor) override;
+ /// Itanium says that an _Unwind_Exception has to be "double-word"
+ /// aligned (and thus the end of it is also so-aligned), meaning 16
+ /// bytes. Of course, that was written for the actual Itanium,
+ /// which is a 64-bit platform. Classically, the ABI doesn't really
+ /// specify the alignment on other platforms, but in practice
+ /// libUnwind declares the struct with __attribute__((aligned)), so
+ /// we assume that alignment here. (It's generally 16 bytes, but
+ /// some targets overwrite it.)
+ CharUnits getAlignmentOfExnObject() {
+ auto align = CGM.getContext().getTargetDefaultAlignForAttributeAligned();
+ return CGM.getContext().toCharUnitsFromBits(align);
+ }
+
void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override;
void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override;
@@ -135,25 +188,25 @@
bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override;
void EmitBadTypeidCall(CodeGenFunction &CGF) override;
llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy,
- llvm::Value *ThisPtr,
+ Address ThisPtr,
llvm::Type *StdTypeInfoPtrTy) override;
bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
QualType SrcRecordTy) override;
- llvm::Value *EmitDynamicCastCall(CodeGenFunction &CGF, llvm::Value *Value,
+ llvm::Value *EmitDynamicCastCall(CodeGenFunction &CGF, Address Value,
QualType SrcRecordTy, QualType DestTy,
QualType DestRecordTy,
llvm::BasicBlock *CastEnd) override;
- llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, llvm::Value *Value,
+ llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, Address Value,
QualType SrcRecordTy,
QualType DestTy) override;
bool EmitBadCastCall(CodeGenFunction &CGF) override;
llvm::Value *
- GetVirtualBaseClassOffset(CodeGenFunction &CGF, llvm::Value *This,
+ GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This,
const CXXRecordDecl *ClassDecl,
const CXXRecordDecl *BaseClassDecl) override;
@@ -185,7 +238,7 @@
void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
CXXDtorType Type, bool ForVirtualBase,
- bool Delegating, llvm::Value *This) override;
+ bool Delegating, Address This) override;
void emitVTableDefinitions(CodeGenVTables &CGVT,
const CXXRecordDecl *RD) override;
@@ -203,14 +256,13 @@
CharUnits VPtrOffset) override;
llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD,
- llvm::Value *This,
- llvm::Type *Ty,
+ Address This, llvm::Type *Ty,
SourceLocation Loc) override;
llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF,
const CXXDestructorDecl *Dtor,
CXXDtorType DtorType,
- llvm::Value *This,
+ Address This,
const CXXMemberCallExpr *CE) override;
void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override;
@@ -225,10 +277,10 @@
Thunk->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
}
- llvm::Value *performThisAdjustment(CodeGenFunction &CGF, llvm::Value *This,
+ llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This,
const ThisAdjustment &TA) override;
- llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret,
+ llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
const ReturnAdjustment &RA) override;
size_t getSrcArgforCopyCtor(const CXXConstructorDecl *,
@@ -242,13 +294,13 @@
{ return "__cxa_deleted_virtual"; }
CharUnits getArrayCookieSizeImpl(QualType elementType) override;
- llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *NewPtr,
- llvm::Value *NumElements,
- const CXXNewExpr *expr,
- QualType ElementType) override;
+ Address InitializeArrayCookie(CodeGenFunction &CGF,
+ Address NewPtr,
+ llvm::Value *NumElements,
+ const CXXNewExpr *expr,
+ QualType ElementType) override;
llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF,
- llvm::Value *allocPtr,
+ Address allocPtr,
CharUnits cookieSize) override;
void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
@@ -338,12 +390,12 @@
QualType ResTy) override;
CharUnits getArrayCookieSizeImpl(QualType elementType) override;
- llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *NewPtr,
- llvm::Value *NumElements,
- const CXXNewExpr *expr,
- QualType ElementType) override;
- llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, llvm::Value *allocPtr,
+ Address InitializeArrayCookie(CodeGenFunction &CGF,
+ Address NewPtr,
+ llvm::Value *NumElements,
+ const CXXNewExpr *expr,
+ QualType ElementType) override;
+ llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, Address allocPtr,
CharUnits cookieSize) override;
};
@@ -439,7 +491,8 @@
/// If the member is non-virtual, memptr.ptr is the address of
/// the function to call.
llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
- CodeGenFunction &CGF, const Expr *E, llvm::Value *&This,
+ CodeGenFunction &CGF, const Expr *E, Address ThisAddr,
+ llvm::Value *&ThisPtrForCall,
llvm::Value *MemFnPtr, const MemberPointerType *MPT) {
CGBuilderTy &Builder = CGF.Builder;
@@ -468,9 +521,11 @@
// Apply the adjustment and cast back to the original struct type
// for consistency.
+ llvm::Value *This = ThisAddr.getPointer();
llvm::Value *Ptr = Builder.CreateBitCast(This, Builder.getInt8PtrTy());
Ptr = Builder.CreateInBoundsGEP(Ptr, Adj);
This = Builder.CreateBitCast(Ptr, This->getType(), "this.adjusted");
+ ThisPtrForCall = This;
// Load the function pointer.
llvm::Value *FnAsInt = Builder.CreateExtractValue(MemFnPtr, 0, "memptr.ptr");
@@ -492,7 +547,11 @@
// Cast the adjusted this to a pointer to vtable pointer and load.
llvm::Type *VTableTy = Builder.getInt8PtrTy();
- llvm::Value *VTable = CGF.GetVTablePtr(This, VTableTy);
+ CharUnits VTablePtrAlign =
+ CGF.CGM.getDynamicOffsetAlignment(ThisAddr.getAlignment(), RD,
+ CGF.getPointerAlign());
+ llvm::Value *VTable =
+ CGF.GetVTablePtr(Address(This, VTablePtrAlign), VTableTy);
// Apply the offset.
llvm::Value *VTableOffset = FnAsInt;
@@ -502,7 +561,9 @@
// Load the virtual function to call.
VTable = Builder.CreateBitCast(VTable, FTy->getPointerTo()->getPointerTo());
- llvm::Value *VirtualFn = Builder.CreateLoad(VTable, "memptr.virtualfn");
+ llvm::Value *VirtualFn =
+ Builder.CreateAlignedLoad(VTable, CGF.getPointerAlign(),
+ "memptr.virtualfn");
CGF.EmitBranch(FnEnd);
// In the non-virtual path, the function pointer is actually a
@@ -522,24 +583,23 @@
/// Compute an l-value by applying the given pointer-to-member to a
/// base object.
llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(
- CodeGenFunction &CGF, const Expr *E, llvm::Value *Base, llvm::Value *MemPtr,
+ CodeGenFunction &CGF, const Expr *E, Address Base, llvm::Value *MemPtr,
const MemberPointerType *MPT) {
assert(MemPtr->getType() == CGM.PtrDiffTy);
CGBuilderTy &Builder = CGF.Builder;
- unsigned AS = Base->getType()->getPointerAddressSpace();
-
// Cast to char*.
- Base = Builder.CreateBitCast(Base, Builder.getInt8Ty()->getPointerTo(AS));
+ Base = Builder.CreateElementBitCast(Base, CGF.Int8Ty);
// Apply the offset, which we assume is non-null.
- llvm::Value *Addr = Builder.CreateInBoundsGEP(Base, MemPtr, "memptr.offset");
+ llvm::Value *Addr =
+ Builder.CreateInBoundsGEP(Base.getPointer(), MemPtr, "memptr.offset");
// Cast the address to the appropriate pointer type, adopting the
// address space of the base pointer.
- llvm::Type *PType
- = CGF.ConvertTypeForMem(MPT->getPointeeType())->getPointerTo(AS);
+ llvm::Type *PType = CGF.ConvertTypeForMem(MPT->getPointeeType())
+ ->getPointerTo(Base.getAddressSpace());
return Builder.CreateBitCast(Addr, PType);
}
@@ -893,7 +953,8 @@
// FIXME: Use canCopyArgument() when it is fixed to handle lazily declared
// special members.
if (RD->hasNonTrivialDestructor() || RD->hasNonTrivialCopyConstructor()) {
- FI.getReturnInfo() = ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ auto Align = CGM.getContext().getTypeAlignInChars(FI.getReturnType());
+ FI.getReturnInfo() = ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
return true;
}
return false;
@@ -909,7 +970,7 @@
/// at entry -2 in the vtable.
void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
const CXXDeleteExpr *DE,
- llvm::Value *Ptr,
+ Address Ptr,
QualType ElementType,
const CXXDestructorDecl *Dtor) {
bool UseGlobalDelete = DE->isGlobalDelete();
@@ -923,11 +984,12 @@
// Track back to entry -2 and pull out the offset there.
llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
VTable, -2, "complete-offset.ptr");
- llvm::LoadInst *Offset = CGF.Builder.CreateLoad(OffsetPtr);
- Offset->setAlignment(CGF.PointerAlignInBytes);
+ llvm::Value *Offset =
+ CGF.Builder.CreateAlignedLoad(OffsetPtr, CGF.getPointerAlign());
// Apply the offset.
- llvm::Value *CompletePtr = CGF.Builder.CreateBitCast(Ptr, CGF.Int8PtrTy);
+ llvm::Value *CompletePtr =
+ CGF.Builder.CreateBitCast(Ptr.getPointer(), CGF.Int8PtrTy);
CompletePtr = CGF.Builder.CreateInBoundsGEP(CompletePtr, Offset);
// If we're supposed to call the global delete, make sure we do so
@@ -989,7 +1051,8 @@
llvm::CallInst *ExceptionPtr = CGF.EmitNounwindRuntimeCall(
AllocExceptionFn, llvm::ConstantInt::get(SizeTy, TypeSize), "exception");
- CGF.EmitAnyExprToExn(E->getSubExpr(), ExceptionPtr);
+ CharUnits ExnAlign = getAlignmentOfExnObject();
+ CGF.EmitAnyExprToExn(E->getSubExpr(), Address(ExceptionPtr, ExnAlign));
// Now throw the exception.
llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType,
@@ -1113,14 +1176,14 @@
llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF,
QualType SrcRecordTy,
- llvm::Value *ThisPtr,
+ Address ThisPtr,
llvm::Type *StdTypeInfoPtrTy) {
llvm::Value *Value =
CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo());
// Load the type info.
Value = CGF.Builder.CreateConstInBoundsGEP1_64(Value, -1ULL);
- return CGF.Builder.CreateLoad(Value);
+ return CGF.Builder.CreateAlignedLoad(Value, CGF.getPointerAlign());
}
bool ItaniumCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
@@ -1129,7 +1192,7 @@
}
llvm::Value *ItaniumCXXABI::EmitDynamicCastCall(
- CodeGenFunction &CGF, llvm::Value *Value, QualType SrcRecordTy,
+ CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) {
llvm::Type *PtrDiffLTy =
CGF.ConvertType(CGF.getContext().getPointerDiffType());
@@ -1148,6 +1211,7 @@
computeOffsetHint(CGF.getContext(), SrcDecl, DestDecl).getQuantity());
// Emit the call to __dynamic_cast.
+ llvm::Value *Value = ThisAddr.getPointer();
Value = CGF.EmitCastToVoidPtr(Value);
llvm::Value *args[] = {Value, SrcRTTI, DestRTTI, OffsetHint};
@@ -1171,7 +1235,7 @@
}
llvm::Value *ItaniumCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF,
- llvm::Value *Value,
+ Address ThisAddr,
QualType SrcRecordTy,
QualType DestTy) {
llvm::Type *PtrDiffLTy =
@@ -1179,14 +1243,17 @@
llvm::Type *DestLTy = CGF.ConvertType(DestTy);
// Get the vtable pointer.
- llvm::Value *VTable = CGF.GetVTablePtr(Value, PtrDiffLTy->getPointerTo());
+ llvm::Value *VTable = CGF.GetVTablePtr(ThisAddr, PtrDiffLTy->getPointerTo());
// Get the offset-to-top from the vtable.
llvm::Value *OffsetToTop =
CGF.Builder.CreateConstInBoundsGEP1_64(VTable, -2ULL);
- OffsetToTop = CGF.Builder.CreateLoad(OffsetToTop, "offset.to.top");
+ OffsetToTop =
+ CGF.Builder.CreateAlignedLoad(OffsetToTop, CGF.getPointerAlign(),
+ "offset.to.top");
// Finally, add the offset to the pointer.
+ llvm::Value *Value = ThisAddr.getPointer();
Value = CGF.EmitCastToVoidPtr(Value);
Value = CGF.Builder.CreateInBoundsGEP(Value, OffsetToTop);
@@ -1202,7 +1269,7 @@
llvm::Value *
ItaniumCXXABI::GetVirtualBaseClassOffset(CodeGenFunction &CGF,
- llvm::Value *This,
+ Address This,
const CXXRecordDecl *ClassDecl,
const CXXRecordDecl *BaseClassDecl) {
llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy);
@@ -1217,7 +1284,8 @@
CGM.PtrDiffTy->getPointerTo());
llvm::Value *VBaseOffset =
- CGF.Builder.CreateLoad(VBaseOffsetPtr, "vbase.offset");
+ CGF.Builder.CreateAlignedLoad(VBaseOffsetPtr, CGF.getPointerAlign(),
+ "vbase.offset");
return VBaseOffset;
}
@@ -1328,7 +1396,7 @@
void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
const CXXDestructorDecl *DD,
CXXDtorType Type, bool ForVirtualBase,
- bool Delegating, llvm::Value *This) {
+ bool Delegating, Address This) {
GlobalDecl GD(DD, Type);
llvm::Value *VTT = CGF.GetVTTParameter(GD, ForVirtualBase, Delegating);
QualType VTTTy = getContext().getPointerType(getContext().VoidPtrTy);
@@ -1340,8 +1408,8 @@
if (!Callee)
Callee = CGM.getAddrOfCXXStructor(DD, getFromDtorType(Type));
- CGF.EmitCXXMemberOrOperatorCall(DD, Callee, ReturnValueSlot(), This, VTT,
- VTTTy, nullptr);
+ CGF.EmitCXXMemberOrOperatorCall(DD, Callee, ReturnValueSlot(),
+ This.getPointer(), VTT, VTTTy, nullptr);
}
void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
@@ -1409,7 +1477,7 @@
VTT = CGF.Builder.CreateConstInBoundsGEP1_64(VTT, VirtualPointerIndex);
// And load the address point from the VTT.
- VTableAddressPoint = CGF.Builder.CreateLoad(VTT);
+ VTableAddressPoint = CGF.Builder.CreateAlignedLoad(VTT, CGF.getPointerAlign());
} else {
llvm::Constant *VTable =
CGM.getCXXABI().getAddrOfVTable(VTableClass, CharUnits());
@@ -1473,7 +1541,7 @@
llvm::Value *ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
GlobalDecl GD,
- llvm::Value *This,
+ Address This,
llvm::Type *Ty,
SourceLocation Loc) {
GD = GD.getCanonicalDecl();
@@ -1487,12 +1555,12 @@
uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD);
llvm::Value *VFuncPtr =
CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn");
- return CGF.Builder.CreateLoad(VFuncPtr);
+ return CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign());
}
llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall(
CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType,
- llvm::Value *This, const CXXMemberCallExpr *CE) {
+ Address This, const CXXMemberCallExpr *CE) {
assert(CE == nullptr || CE->arg_begin() == CE->arg_end());
assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete);
@@ -1503,8 +1571,9 @@
getVirtualFunctionPointer(CGF, GlobalDecl(Dtor, DtorType), This, Ty,
CE ? CE->getLocStart() : SourceLocation());
- CGF.EmitCXXMemberOrOperatorCall(Dtor, Callee, ReturnValueSlot(), This,
- /*ImplicitParam=*/nullptr, QualType(), CE);
+ CGF.EmitCXXMemberOrOperatorCall(Dtor, Callee, ReturnValueSlot(),
+ This.getPointer(), /*ImplicitParam=*/nullptr,
+ QualType(), CE);
return nullptr;
}
@@ -1528,29 +1597,28 @@
return !hasAnyUsedVirtualInlineFunction(RD);
}
static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF,
- llvm::Value *Ptr,
+ Address InitialPtr,
int64_t NonVirtualAdjustment,
int64_t VirtualAdjustment,
bool IsReturnAdjustment) {
if (!NonVirtualAdjustment && !VirtualAdjustment)
- return Ptr;
+ return InitialPtr.getPointer();
- llvm::Type *Int8PtrTy = CGF.Int8PtrTy;
- llvm::Value *V = CGF.Builder.CreateBitCast(Ptr, Int8PtrTy);
+ Address V = CGF.Builder.CreateElementBitCast(InitialPtr, CGF.Int8Ty);
+ // In a base-to-derived cast, the non-virtual adjustment is applied first.
if (NonVirtualAdjustment && !IsReturnAdjustment) {
- // Perform the non-virtual adjustment for a base-to-derived cast.
- V = CGF.Builder.CreateConstInBoundsGEP1_64(V, NonVirtualAdjustment);
+ V = CGF.Builder.CreateConstInBoundsByteGEP(V,
+ CharUnits::fromQuantity(NonVirtualAdjustment));
}
+ // Perform the virtual adjustment if we have one.
+ llvm::Value *ResultPtr;
if (VirtualAdjustment) {
llvm::Type *PtrDiffTy =
CGF.ConvertType(CGF.getContext().getPointerDiffType());
- // Perform the virtual adjustment.
- llvm::Value *VTablePtrPtr =
- CGF.Builder.CreateBitCast(V, Int8PtrTy->getPointerTo());
-
+ Address VTablePtrPtr = CGF.Builder.CreateElementBitCast(V, CGF.Int8PtrTy);
llvm::Value *VTablePtr = CGF.Builder.CreateLoad(VTablePtrPtr);
llvm::Value *OffsetPtr =
@@ -1559,23 +1627,28 @@
OffsetPtr = CGF.Builder.CreateBitCast(OffsetPtr, PtrDiffTy->getPointerTo());
// Load the adjustment offset from the vtable.
- llvm::Value *Offset = CGF.Builder.CreateLoad(OffsetPtr);
+ llvm::Value *Offset =
+ CGF.Builder.CreateAlignedLoad(OffsetPtr, CGF.getPointerAlign());
// Adjust our pointer.
- V = CGF.Builder.CreateInBoundsGEP(V, Offset);
+ ResultPtr = CGF.Builder.CreateInBoundsGEP(V.getPointer(), Offset);
+ } else {
+ ResultPtr = V.getPointer();
}
+ // In a derived-to-base conversion, the non-virtual adjustment is
+ // applied second.
if (NonVirtualAdjustment && IsReturnAdjustment) {
- // Perform the non-virtual adjustment for a derived-to-base cast.
- V = CGF.Builder.CreateConstInBoundsGEP1_64(V, NonVirtualAdjustment);
+ ResultPtr = CGF.Builder.CreateConstInBoundsGEP1_64(ResultPtr,
+ NonVirtualAdjustment);
}
// Cast back to the original type.
- return CGF.Builder.CreateBitCast(V, Ptr->getType());
+ return CGF.Builder.CreateBitCast(ResultPtr, InitialPtr.getType());
}
llvm::Value *ItaniumCXXABI::performThisAdjustment(CodeGenFunction &CGF,
- llvm::Value *This,
+ Address This,
const ThisAdjustment &TA) {
return performTypeAdjustment(CGF, This, TA.NonVirtual,
TA.Virtual.Itanium.VCallOffsetOffset,
@@ -1583,7 +1656,7 @@
}
llvm::Value *
-ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret,
+ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
const ReturnAdjustment &RA) {
return performTypeAdjustment(CGF, Ret, RA.NonVirtual,
RA.Virtual.Itanium.VBaseOffsetOffset,
@@ -1596,8 +1669,7 @@
return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV, ResultType);
// Destructor thunks in the ARM ABI have indeterminate results.
- llvm::Type *T =
- cast<llvm::PointerType>(CGF.ReturnValue->getType())->getElementType();
+ llvm::Type *T = CGF.ReturnValue.getElementType();
RValue Undef = RValue::get(llvm::UndefValue::get(T));
return ItaniumCXXABI::EmitReturnFromThunk(CGF, Undef, ResultType);
}
@@ -1611,18 +1683,17 @@
CGM.getContext().getTypeAlignInChars(elementType));
}
-llvm::Value *ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *NewPtr,
- llvm::Value *NumElements,
- const CXXNewExpr *expr,
- QualType ElementType) {
+Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
+ Address NewPtr,
+ llvm::Value *NumElements,
+ const CXXNewExpr *expr,
+ QualType ElementType) {
assert(requiresArrayCookie(expr));
- unsigned AS = NewPtr->getType()->getPointerAddressSpace();
+ unsigned AS = NewPtr.getAddressSpace();
ASTContext &Ctx = getContext();
- QualType SizeTy = Ctx.getSizeType();
- CharUnits SizeSize = Ctx.getTypeSizeInChars(SizeTy);
+ CharUnits SizeSize = CGF.getSizeSize();
// The size of the cookie.
CharUnits CookieSize =
@@ -1630,49 +1701,45 @@
assert(CookieSize == getArrayCookieSizeImpl(ElementType));
// Compute an offset to the cookie.
- llvm::Value *CookiePtr = NewPtr;
+ Address CookiePtr = NewPtr;
CharUnits CookieOffset = CookieSize - SizeSize;
if (!CookieOffset.isZero())
- CookiePtr = CGF.Builder.CreateConstInBoundsGEP1_64(CookiePtr,
- CookieOffset.getQuantity());
+ CookiePtr = CGF.Builder.CreateConstInBoundsByteGEP(CookiePtr, CookieOffset);
// Write the number of elements into the appropriate slot.
- llvm::Type *NumElementsTy = CGF.ConvertType(SizeTy)->getPointerTo(AS);
- llvm::Value *NumElementsPtr =
- CGF.Builder.CreateBitCast(CookiePtr, NumElementsTy);
+ Address NumElementsPtr =
+ CGF.Builder.CreateElementBitCast(CookiePtr, CGF.SizeTy);
llvm::Instruction *SI = CGF.Builder.CreateStore(NumElements, NumElementsPtr);
+
+ // Handle the array cookie specially in ASan.
if (CGM.getLangOpts().Sanitize.has(SanitizerKind::Address) && AS == 0 &&
expr->getOperatorNew()->isReplaceableGlobalAllocationFunction()) {
// The store to the CookiePtr does not need to be instrumented.
CGM.getSanitizerMetadata()->disableSanitizerForInstruction(SI);
llvm::FunctionType *FTy =
- llvm::FunctionType::get(CGM.VoidTy, NumElementsTy, false);
+ llvm::FunctionType::get(CGM.VoidTy, NumElementsPtr.getType(), false);
llvm::Constant *F =
CGM.CreateRuntimeFunction(FTy, "__asan_poison_cxx_array_cookie");
- CGF.Builder.CreateCall(F, NumElementsPtr);
+ CGF.Builder.CreateCall(F, NumElementsPtr.getPointer());
}
// Finally, compute a pointer to the actual data buffer by skipping
// over the cookie completely.
- return CGF.Builder.CreateConstInBoundsGEP1_64(NewPtr,
- CookieSize.getQuantity());
+ return CGF.Builder.CreateConstInBoundsByteGEP(NewPtr, CookieSize);
}
llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
- llvm::Value *allocPtr,
+ Address allocPtr,
CharUnits cookieSize) {
// The element size is right-justified in the cookie.
- llvm::Value *numElementsPtr = allocPtr;
- CharUnits numElementsOffset =
- cookieSize - CharUnits::fromQuantity(CGF.SizeSizeInBytes);
+ Address numElementsPtr = allocPtr;
+ CharUnits numElementsOffset = cookieSize - CGF.getSizeSize();
if (!numElementsOffset.isZero())
numElementsPtr =
- CGF.Builder.CreateConstInBoundsGEP1_64(numElementsPtr,
- numElementsOffset.getQuantity());
+ CGF.Builder.CreateConstInBoundsByteGEP(numElementsPtr, numElementsOffset);
- unsigned AS = allocPtr->getType()->getPointerAddressSpace();
- numElementsPtr =
- CGF.Builder.CreateBitCast(numElementsPtr, CGF.SizeTy->getPointerTo(AS));
+ unsigned AS = allocPtr.getAddressSpace();
+ numElementsPtr = CGF.Builder.CreateElementBitCast(numElementsPtr, CGF.SizeTy);
if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address) || AS != 0)
return CGF.Builder.CreateLoad(numElementsPtr);
// In asan mode emit a function call instead of a regular load and let the
@@ -1684,7 +1751,7 @@
llvm::FunctionType::get(CGF.SizeTy, CGF.SizeTy->getPointerTo(0), false);
llvm::Constant *F =
CGM.CreateRuntimeFunction(FTy, "__asan_load_cxx_array_cookie");
- return CGF.Builder.CreateCall(F, numElementsPtr);
+ return CGF.Builder.CreateCall(F, numElementsPtr.getPointer());
}
CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) {
@@ -1700,47 +1767,41 @@
CGM.getContext().getTypeAlignInChars(elementType));
}
-llvm::Value *ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *newPtr,
- llvm::Value *numElements,
- const CXXNewExpr *expr,
- QualType elementType) {
+Address ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
+ Address newPtr,
+ llvm::Value *numElements,
+ const CXXNewExpr *expr,
+ QualType elementType) {
assert(requiresArrayCookie(expr));
- // NewPtr is a char*, but we generalize to arbitrary addrspaces.
- unsigned AS = newPtr->getType()->getPointerAddressSpace();
-
// The cookie is always at the start of the buffer.
- llvm::Value *cookie = newPtr;
+ Address cookie = newPtr;
// The first element is the element size.
- cookie = CGF.Builder.CreateBitCast(cookie, CGF.SizeTy->getPointerTo(AS));
+ cookie = CGF.Builder.CreateElementBitCast(cookie, CGF.SizeTy);
llvm::Value *elementSize = llvm::ConstantInt::get(CGF.SizeTy,
getContext().getTypeSizeInChars(elementType).getQuantity());
CGF.Builder.CreateStore(elementSize, cookie);
// The second element is the element count.
- cookie = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.SizeTy, cookie, 1);
+ cookie = CGF.Builder.CreateConstInBoundsGEP(cookie, 1, CGF.getSizeSize());
CGF.Builder.CreateStore(numElements, cookie);
// Finally, compute a pointer to the actual data buffer by skipping
// over the cookie completely.
CharUnits cookieSize = ARMCXXABI::getArrayCookieSizeImpl(elementType);
- return CGF.Builder.CreateConstInBoundsGEP1_64(newPtr,
- cookieSize.getQuantity());
+ return CGF.Builder.CreateConstInBoundsByteGEP(newPtr, cookieSize);
}
llvm::Value *ARMCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
- llvm::Value *allocPtr,
+ Address allocPtr,
CharUnits cookieSize) {
// The number of elements is at offset sizeof(size_t) relative to
// the allocated pointer.
- llvm::Value *numElementsPtr
- = CGF.Builder.CreateConstInBoundsGEP1_64(allocPtr, CGF.SizeSizeInBytes);
+ Address numElementsPtr
+ = CGF.Builder.CreateConstInBoundsByteGEP(allocPtr, CGF.getSizeSize());
- unsigned AS = allocPtr->getType()->getPointerAddressSpace();
- numElementsPtr =
- CGF.Builder.CreateBitCast(numElementsPtr, CGF.SizeTy->getPointerTo(AS));
+ numElementsPtr = CGF.Builder.CreateElementBitCast(numElementsPtr, CGF.SizeTy);
return CGF.Builder.CreateLoad(numElementsPtr);
}
@@ -1810,12 +1871,21 @@
bool useInt8GuardVariable = !threadsafe && var->hasInternalLinkage();
llvm::IntegerType *guardTy;
+ CharUnits guardAlignment;
if (useInt8GuardVariable) {
guardTy = CGF.Int8Ty;
+ guardAlignment = CharUnits::One();
} else {
// Guard variables are 64 bits in the generic ABI and size width on ARM
// (i.e. 32-bit on AArch32, 64-bit on AArch64).
- guardTy = (UseARMGuardVarABI ? CGF.SizeTy : CGF.Int64Ty);
+ if (UseARMGuardVarABI) {
+ guardTy = CGF.SizeTy;
+ guardAlignment = CGF.getSizeAlign();
+ } else {
+ guardTy = CGF.Int64Ty;
+ guardAlignment = CharUnits::fromQuantity(
+ CGM.getDataLayout().getABITypeAlignment(guardTy));
+ }
}
llvm::PointerType *guardPtrTy = guardTy->getPointerTo();
@@ -1839,6 +1909,7 @@
guard->setVisibility(var->getVisibility());
// If the variable is thread-local, so is its guard variable.
guard->setThreadLocalMode(var->getThreadLocalMode());
+ guard->setAlignment(guardAlignment.getQuantity());
// The ABI says: "It is suggested that it be emitted in the same COMDAT
// group as the associated data object." In practice, this doesn't work for
@@ -1855,6 +1926,8 @@
CGM.setStaticLocalDeclGuardAddress(&D, guard);
}
+ Address guardAddr = Address(guard, guardAlignment);
+
// Test whether the variable has completed initialization.
//
// Itanium C++ ABI 3.3.2:
@@ -1874,8 +1947,7 @@
// Load the first byte of the guard variable.
llvm::LoadInst *LI =
- Builder.CreateLoad(Builder.CreateBitCast(guard, CGM.Int8PtrTy));
- LI->setAlignment(1);
+ Builder.CreateLoad(Builder.CreateElementBitCast(guardAddr, CGM.Int8Ty));
// Itanium ABI:
// An implementation supporting thread-safety on multiprocessor
@@ -1945,9 +2017,10 @@
CGF.PopCleanupBlock();
// Call __cxa_guard_release. This cannot throw.
- CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy), guard);
+ CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy),
+ guardAddr.getPointer());
} else {
- Builder.CreateStore(llvm::ConstantInt::get(guardTy, 1), guard);
+ Builder.CreateStore(llvm::ConstantInt::get(guardTy, 1), guardAddr);
}
CGF.EmitBlock(EndBlock);
@@ -2090,8 +2163,13 @@
llvm::GlobalVariable::InternalLinkage,
llvm::ConstantInt::get(CGM.Int8Ty, 0), "__tls_guard");
Guard->setThreadLocal(true);
+
+ CharUnits GuardAlign = CharUnits::One();
+ Guard->setAlignment(GuardAlign.getQuantity());
+
CodeGenFunction(CGM)
- .GenerateCXXGlobalInitFunc(InitFunc, CXXThreadLocalInits, Guard);
+ .GenerateCXXGlobalInitFunc(InitFunc, CXXThreadLocalInits,
+ Address(Guard, GuardAlign));
}
for (auto &I : CXXThreadLocals) {
const VarDecl *VD = I.first;
@@ -2137,7 +2215,7 @@
llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Var);
llvm::LLVMContext &Context = CGM.getModule().getContext();
llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, "", Wrapper);
- CGBuilderTy Builder(Entry);
+ CGBuilderTy Builder(CGM, Entry);
if (InitIsInitFunc) {
if (Init)
Builder.CreateCall(Init);
@@ -2159,9 +2237,8 @@
// the referenced object.
llvm::Value *Val = Var;
if (VD->getType()->isReferenceType()) {
- llvm::LoadInst *LI = Builder.CreateLoad(Val);
- LI->setAlignment(CGM.getContext().getDeclAlign(VD).getQuantity());
- Val = LI;
+ CharUnits Align = CGM.getContext().getDeclAlign(VD);
+ Val = Builder.CreateAlignedLoad(Val, Align);
}
if (Val->getType() != Wrapper->getReturnType())
Val = Builder.CreatePointerBitCastOrAddrSpaceCast(
@@ -3418,7 +3495,7 @@
/// parameter during catch initialization.
static void InitCatchParam(CodeGenFunction &CGF,
const VarDecl &CatchParam,
- llvm::Value *ParamAddr,
+ Address ParamAddr,
SourceLocation Loc) {
// Load the exception from where the landing pad saved it.
llvm::Value *Exn = CGF.getExceptionFromSlot();
@@ -3472,12 +3549,13 @@
cast<llvm::PointerType>(LLVMCatchTy)->getElementType();
// Create the temporary and write the adjusted pointer into it.
- llvm::Value *ExnPtrTmp = CGF.CreateTempAlloca(PtrTy, "exn.byref.tmp");
+ Address ExnPtrTmp =
+ CGF.CreateTempAlloca(PtrTy, CGF.getPointerAlign(), "exn.byref.tmp");
llvm::Value *Casted = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
CGF.Builder.CreateStore(Casted, ExnPtrTmp);
// Bind the reference to the temporary.
- AdjustedExn = ExnPtrTmp;
+ AdjustedExn = ExnPtrTmp.getPointer();
}
}
@@ -3522,8 +3600,7 @@
llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
LValue srcLV = CGF.MakeNaturalAlignAddrLValue(Cast, CatchType);
- LValue destLV = CGF.MakeAddrLValue(ParamAddr, CatchType,
- CGF.getContext().getDeclAlign(&CatchParam));
+ LValue destLV = CGF.MakeAddrLValue(ParamAddr, CatchType);
switch (TEK) {
case TEK_Complex:
CGF.EmitStoreOfComplex(CGF.EmitLoadOfComplex(srcLV, Loc), destLV,
@@ -3541,6 +3618,8 @@
}
assert(isa<RecordType>(CatchType) && "unexpected catch type!");
+ auto catchRD = CatchType->getAsCXXRecordDecl();
+ CharUnits caughtExnAlignment = CGF.CGM.getClassPointerAlignment(catchRD);
llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
@@ -3549,7 +3628,8 @@
const Expr *copyExpr = CatchParam.getInit();
if (!copyExpr) {
llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, true);
- llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy);
+ Address adjustedExn(CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy),
+ caughtExnAlignment);
CGF.EmitAggregateCopy(ParamAddr, adjustedExn, CatchType);
return;
}
@@ -3560,7 +3640,8 @@
CGF.EmitNounwindRuntimeCall(getGetExceptionPtrFn(CGF.CGM), Exn);
// Cast that to the appropriate type.
- llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy);
+ Address adjustedExn(CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy),
+ caughtExnAlignment);
// The copy expression is defined in terms of an OpaqueValueExpr.
// Find it and map it to the adjusted expression.
@@ -3572,9 +3653,8 @@
CGF.EHStack.pushTerminate();
// Perform the copy construction.
- CharUnits Alignment = CGF.getContext().getDeclAlign(&CatchParam);
CGF.EmitAggExpr(copyExpr,
- AggValueSlot::forAddr(ParamAddr, Alignment, Qualifiers(),
+ AggValueSlot::forAddr(ParamAddr, Qualifiers(),
AggValueSlot::IsNotDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsNotAliased));
@@ -3658,7 +3738,7 @@
// Set up the function.
llvm::BasicBlock *entry =
llvm::BasicBlock::Create(CGM.getLLVMContext(), "", fn);
- CGBuilderTy builder(entry);
+ CGBuilderTy builder(CGM, entry);
// Pull the exception pointer out of the parameter list.
llvm::Value *exn = &*fn->arg_begin();