Decorating vptr load & stores with !invariant.group
Adding !invariant.group to vptr load/stores for devirtualization purposes.
For more goto:
http://lists.llvm.org/pipermail/cfe-dev/2015-July/044227.html
http://reviews.llvm.org/D12026
llvm-svn: 247725
diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index 4e457fd..7c9f3da 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -1246,7 +1246,7 @@
if (IsVolatile)
Load->setVolatile(true);
if (LVal.getTBAAInfo())
- CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo());
+ CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
return Load;
}
@@ -1769,7 +1769,7 @@
if (IsVolatile)
store->setVolatile(true);
if (dest.getTBAAInfo())
- CGM.DecorateInstruction(store, dest.getTBAAInfo());
+ CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
return;
}
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index 4c46d56..7ec9c15 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -25,6 +25,7 @@
#include "clang/CodeGen/CGFunctionInfo.h"
#include "clang/Frontend/CodeGenOptions.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Metadata.h"
using namespace clang;
using namespace CodeGen;
@@ -2087,7 +2088,8 @@
ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr,
Vptr.VTableClass, Vptr.NearestVBase);
- llvm::Value *VPtrValue = GetVTablePtr(This, VTableGlobal->getType());
+ llvm::Value *VPtrValue =
+ GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);
llvm::Value *Cmp =
Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");
Builder.CreateAssumption(Cmp);
@@ -2306,7 +2308,10 @@
VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy);
llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
- CGM.DecorateInstruction(Store, CGM.getTBAAInfoForVTablePtr());
+ CGM.DecorateInstructionWithTBAA(Store, CGM.getTBAAInfoForVTablePtr());
+ if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
+ CGM.getCodeGenOpts().StrictVTablePointers)
+ CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);
}
CodeGenFunction::VPtrsVector
@@ -2393,10 +2398,16 @@
}
llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
- llvm::Type *Ty) {
- Address VTablePtrSrc = Builder.CreateElementBitCast(This, Ty);
+ llvm::Type *VTableTy,
+ const CXXRecordDecl *RD) {
+ Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy);
llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");
- CGM.DecorateInstruction(VTable, CGM.getTBAAInfoForVTablePtr());
+ CGM.DecorateInstructionWithTBAA(VTable, CGM.getTBAAInfoForVTablePtr());
+
+ if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
+ CGM.getCodeGenOpts().StrictVTablePointers)
+ CGM.DecorateInstructionWithInvariantGroup(VTable, RD);
+
return VTable;
}
@@ -2481,7 +2492,8 @@
}
llvm::Value *VTable =
- GetVTablePtr(Address(Derived, getPointerAlign()), Int8PtrTy);
+ GetVTablePtr(Address(Derived, getPointerAlign()), Int8PtrTy, ClassDecl);
+
EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
if (MayBeNull) {
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index cda7766..c38dbdd 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -1277,7 +1277,8 @@
llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo,
TBAAOffset);
if (TBAAPath)
- CGM.DecorateInstruction(Load, TBAAPath, false/*ConvertTypeToTag*/);
+ CGM.DecorateInstructionWithTBAA(Load, TBAAPath,
+ false /*ConvertTypeToTag*/);
}
bool NeedsBoolCheck =
@@ -1391,7 +1392,8 @@
llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo,
TBAAOffset);
if (TBAAPath)
- CGM.DecorateInstruction(Store, TBAAPath, false/*ConvertTypeToTag*/);
+ CGM.DecorateInstructionWithTBAA(Store, TBAAPath,
+ false /*ConvertTypeToTag*/);
}
}
@@ -3115,7 +3117,7 @@
else
tbaa = CGM.getTBAAInfo(type);
if (tbaa)
- CGM.DecorateInstruction(load, tbaa);
+ CGM.DecorateInstructionWithTBAA(load, tbaa);
}
mayAlias = false;
diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp
index 1952282..7e0a6ab 100644
--- a/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/clang/lib/CodeGen/CGExprCXX.cpp
@@ -258,7 +258,7 @@
} else {
if (SanOpts.has(SanitizerKind::CFINVCall) &&
MD->getParent()->isDynamicClass()) {
- llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy);
+ llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy, MD->getParent());
EmitVTablePtrCheckForCall(MD, VTable, CFITCK_NVCall, CE->getLocStart());
}
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 76296de..cf04596 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -1380,7 +1380,8 @@
/// GetVTablePtr - Return the Value of the vtable pointer member pointed
/// to by This.
- llvm::Value *GetVTablePtr(Address This, llvm::Type *Ty);
+ llvm::Value *GetVTablePtr(Address This, llvm::Type *VTableTy,
+ const CXXRecordDecl *VTableClass);
enum CFITypeCheckKind {
CFITCK_VCall,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index d9bb24e..bb3bbf2 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -507,9 +507,9 @@
/// and struct-path aware TBAA, the tag has the same format:
/// base type, access type and offset.
/// When ConvertTypeToTag is true, we create a tag based on the scalar type.
-void CodeGenModule::DecorateInstruction(llvm::Instruction *Inst,
- llvm::MDNode *TBAAInfo,
- bool ConvertTypeToTag) {
+void CodeGenModule::DecorateInstructionWithTBAA(llvm::Instruction *Inst,
+ llvm::MDNode *TBAAInfo,
+ bool ConvertTypeToTag) {
if (ConvertTypeToTag && TBAA)
Inst->setMetadata(llvm::LLVMContext::MD_tbaa,
TBAA->getTBAAScalarTagInfo(TBAAInfo));
@@ -517,6 +517,16 @@
Inst->setMetadata(llvm::LLVMContext::MD_tbaa, TBAAInfo);
}
+void CodeGenModule::DecorateInstructionWithInvariantGroup(
+ llvm::Instruction *I, const CXXRecordDecl *RD) {
+ llvm::Metadata *MD = CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
+ auto *MetaDataNode = dyn_cast<llvm::MDNode>(MD);
+ // Check if we have to wrap MDString in MDNode.
+ if (!MetaDataNode)
+ MetaDataNode = llvm::MDNode::get(getLLVMContext(), MD);
+ I->setMetadata("invariant.group", MetaDataNode);
+}
+
void CodeGenModule::Error(SourceLocation loc, StringRef message) {
unsigned diagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error, "%0");
getDiags().Report(Context.getFullLoc(loc), diagID) << message;
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 754e3d1..e7ae20f 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -658,9 +658,13 @@
/// is the same as the type. For struct-path aware TBAA, the tag
/// is different from the type: base type, access type and offset.
/// When ConvertTypeToTag is true, we create a tag based on the scalar type.
- void DecorateInstruction(llvm::Instruction *Inst,
- llvm::MDNode *TBAAInfo,
- bool ConvertTypeToTag = true);
+ void DecorateInstructionWithTBAA(llvm::Instruction *Inst,
+ llvm::MDNode *TBAAInfo,
+ bool ConvertTypeToTag = true);
+
+ /// Adds !invariant.barrier !tag to instruction
+ void DecorateInstructionWithInvariantGroup(llvm::Instruction *I,
+ const CXXRecordDecl *RD);
/// Emit the given number of characters as a value of type size_t.
llvm::ConstantInt *getSize(CharUnits numChars);
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 7228a6d..da67b16 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -584,7 +584,7 @@
CGF.CGM.getDynamicOffsetAlignment(ThisAddr.getAlignment(), RD,
CGF.getPointerAlign());
llvm::Value *VTable =
- CGF.GetVTablePtr(Address(This, VTablePtrAlign), VTableTy);
+ CGF.GetVTablePtr(Address(This, VTablePtrAlign), VTableTy, RD);
// Apply the offset.
llvm::Value *VTableOffset = FnAsInt;
@@ -1012,7 +1012,10 @@
// to pass to the deallocation function.
// Grab the vtable pointer as an intptr_t*.
- llvm::Value *VTable = CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo());
+ auto *ClassDecl =
+ cast<CXXRecordDecl>(ElementType->getAs<RecordType>()->getDecl());
+ llvm::Value *VTable =
+ CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo(), ClassDecl);
// Track back to entry -2 and pull out the offset there.
llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
@@ -1211,8 +1214,10 @@
QualType SrcRecordTy,
Address ThisPtr,
llvm::Type *StdTypeInfoPtrTy) {
+ auto *ClassDecl =
+ cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl());
llvm::Value *Value =
- CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo());
+ CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo(), ClassDecl);
// Load the type info.
Value = CGF.Builder.CreateConstInBoundsGEP1_64(Value, -1ULL);
@@ -1275,8 +1280,11 @@
CGF.ConvertType(CGF.getContext().getPointerDiffType());
llvm::Type *DestLTy = CGF.ConvertType(DestTy);
+ auto *ClassDecl =
+ cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl());
// Get the vtable pointer.
- llvm::Value *VTable = CGF.GetVTablePtr(ThisAddr, PtrDiffLTy->getPointerTo());
+ llvm::Value *VTable = CGF.GetVTablePtr(ThisAddr, PtrDiffLTy->getPointerTo(),
+ ClassDecl);
// Get the offset-to-top from the vtable.
llvm::Value *OffsetToTop =
@@ -1305,7 +1313,7 @@
Address This,
const CXXRecordDecl *ClassDecl,
const CXXRecordDecl *BaseClassDecl) {
- llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy);
+ llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy, ClassDecl);
CharUnits VBaseOffsetOffset =
CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(ClassDecl,
BaseClassDecl);
@@ -1591,10 +1599,11 @@
SourceLocation Loc) {
GD = GD.getCanonicalDecl();
Ty = Ty->getPointerTo()->getPointerTo();
- llvm::Value *VTable = CGF.GetVTablePtr(This, Ty);
+ auto *MethodDecl = cast<CXXMethodDecl>(GD.getDecl());
+ llvm::Value *VTable = CGF.GetVTablePtr(This, Ty, MethodDecl->getParent());
if (CGF.SanOpts.has(SanitizerKind::CFIVCall))
- CGF.EmitVTablePtrCheckForCall(cast<CXXMethodDecl>(GD.getDecl()), VTable,
+ CGF.EmitVTablePtrCheckForCall(MethodDecl, VTable,
CodeGenFunction::CFITCK_VCall, Loc);
uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD);
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index 87080b0..a2f7165 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -1831,7 +1831,9 @@
Ty = Ty->getPointerTo()->getPointerTo();
Address VPtr =
adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true);
- llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty);
+
+ auto *MethodDecl = cast<CXXMethodDecl>(GD.getDecl());
+ llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty, MethodDecl->getParent());
MicrosoftVTableContext::MethodVFTableLocation ML =
CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD);
@@ -1958,7 +1960,8 @@
// Load the vfptr and then callee from the vftable. The callee should have
// adjusted 'this' so that the vfptr is at offset zero.
llvm::Value *VTable = CGF.GetVTablePtr(
- getThisAddress(CGF), ThunkTy->getPointerTo()->getPointerTo());
+ getThisAddress(CGF), ThunkTy->getPointerTo()->getPointerTo(), MD->getParent());
+
llvm::Value *VFuncPtr =
CGF.Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn");
llvm::Value *Callee =