Refactor call emission to package the function pointer together with
abstract information about the callee. NFC.
The goal here is to make it easier to recognize indirect calls and
trigger additional logic in certain cases. That logic will come in
a later patch; in the meantime, I felt that this was a significant
improvement to the code.
llvm-svn: 285258
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 955b86a..86aaad0 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -3519,21 +3519,22 @@
}
RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
- llvm::Value *Callee,
+ const CGCallee &Callee,
ReturnValueSlot ReturnValue,
const CallArgList &CallArgs,
- CGCalleeInfo CalleeInfo,
llvm::Instruction **callOrInvoke) {
// FIXME: We no longer need the types from CallArgs; lift up and simplify.
+ assert(Callee.isOrdinary());
+
// Handle struct-return functions by passing a pointer to the
// location that we would like to return into.
QualType RetTy = CallInfo.getReturnType();
const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
- llvm::FunctionType *IRFuncTy =
- cast<llvm::FunctionType>(
- cast<llvm::PointerType>(Callee->getType())->getElementType());
+ llvm::FunctionType *IRFuncTy = Callee.getFunctionType();
+
+ // 1. Set up the arguments.
// If we're using inalloca, insert the allocation after the stack save.
// FIXME: Do this earlier rather than hacking it in here!
@@ -3593,6 +3594,7 @@
Address swiftErrorTemp = Address::invalid();
Address swiftErrorArg = Address::invalid();
+ // Translate all of the arguments as necessary to match the IR lowering.
assert(CallInfo.arg_size() == CallArgs.size() &&
"Mismatch between function signature & arguments.");
unsigned ArgNo = 0;
@@ -3840,6 +3842,9 @@
}
}
+ llvm::Value *CalleePtr = Callee.getFunctionPointer();
+
+ // If we're using inalloca, set up that argument.
if (ArgMemory.isValid()) {
llvm::Value *Arg = ArgMemory.getPointer();
if (CallInfo.isVariadic()) {
@@ -3847,10 +3852,9 @@
// end up with a variadic prototype and an inalloca call site. In such
// cases, we can't do any parameter mismatch checks. Give up and bitcast
// the callee.
- unsigned CalleeAS =
- cast<llvm::PointerType>(Callee->getType())->getAddressSpace();
- Callee = Builder.CreateBitCast(
- Callee, getTypes().GetFunctionType(CallInfo)->getPointerTo(CalleeAS));
+ unsigned CalleeAS = CalleePtr->getType()->getPointerAddressSpace();
+ auto FnTy = getTypes().GetFunctionType(CallInfo)->getPointerTo(CalleeAS);
+ CalleePtr = Builder.CreateBitCast(CalleePtr, FnTy);
} else {
llvm::Type *LastParamTy =
IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1);
@@ -3874,39 +3878,57 @@
IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
}
+ // 2. Prepare the function pointer.
+
+ // If the callee is a bitcast of a non-variadic function to have a
+ // variadic function pointer type, check to see if we can remove the
+ // bitcast. This comes up with unprototyped functions.
+ //
+ // This makes the IR nicer, but more importantly it ensures that we
+ // can inline the function at -O0 if it is marked always_inline.
+ auto simplifyVariadicCallee = [](llvm::Value *Ptr) -> llvm::Value* {
+ llvm::FunctionType *CalleeFT =
+ cast<llvm::FunctionType>(Ptr->getType()->getPointerElementType());
+ if (!CalleeFT->isVarArg())
+ return Ptr;
+
+ llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Ptr);
+ if (!CE || CE->getOpcode() != llvm::Instruction::BitCast)
+ return Ptr;
+
+ llvm::Function *OrigFn = dyn_cast<llvm::Function>(CE->getOperand(0));
+ if (!OrigFn)
+ return Ptr;
+
+ llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
+
+ // If the original type is variadic, or if any of the component types
+ // disagree, we cannot remove the cast.
+ if (OrigFT->isVarArg() ||
+ OrigFT->getNumParams() != CalleeFT->getNumParams() ||
+ OrigFT->getReturnType() != CalleeFT->getReturnType())
+ return Ptr;
+
+ for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
+ if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
+ return Ptr;
+
+ return OrigFn;
+ };
+ CalleePtr = simplifyVariadicCallee(CalleePtr);
+
+ // 3. Perform the actual call.
+
+ // Deactivate any cleanups that we're supposed to do immediately before
+ // the call.
if (!CallArgs.getCleanupsToDeactivate().empty())
deactivateArgCleanupsBeforeCall(*this, CallArgs);
- // If the callee is a bitcast of a function to a varargs pointer to function
- // type, check to see if we can remove the bitcast. This handles some cases
- // with unprototyped functions.
- if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Callee))
- if (llvm::Function *CalleeF = dyn_cast<llvm::Function>(CE->getOperand(0))) {
- llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType());
- llvm::FunctionType *CurFT =
- cast<llvm::FunctionType>(CurPT->getElementType());
- llvm::FunctionType *ActualFT = CalleeF->getFunctionType();
-
- if (CE->getOpcode() == llvm::Instruction::BitCast &&
- ActualFT->getReturnType() == CurFT->getReturnType() &&
- ActualFT->getNumParams() == CurFT->getNumParams() &&
- ActualFT->getNumParams() == IRCallArgs.size() &&
- (CurFT->isVarArg() || !ActualFT->isVarArg())) {
- bool ArgsMatch = true;
- for (unsigned i = 0, e = ActualFT->getNumParams(); i != e; ++i)
- if (ActualFT->getParamType(i) != CurFT->getParamType(i)) {
- ArgsMatch = false;
- break;
- }
-
- // Strip the cast if we can get away with it. This is a nice cleanup,
- // but also allows us to inline the function at -O0 if it is marked
- // always_inline.
- if (ArgsMatch)
- Callee = CalleeF;
- }
- }
-
+ // Assert that the arguments we computed match up. The IR verifier
+ // will catch this, but this is a common enough source of problems
+ // during IRGen changes that it's way better for debugging to catch
+ // it ourselves here.
+#ifndef NDEBUG
assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg());
for (unsigned i = 0; i < IRCallArgs.size(); ++i) {
// Inalloca argument can have different type.
@@ -3916,75 +3938,106 @@
if (i < IRFuncTy->getNumParams())
assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i));
}
+#endif
+ // Compute the calling convention and attributes.
unsigned CallingConv;
CodeGen::AttributeListType AttributeList;
- CGM.ConstructAttributeList(Callee->getName(), CallInfo, CalleeInfo,
+ CGM.ConstructAttributeList(CalleePtr->getName(), CallInfo,
+ Callee.getAbstractInfo(),
AttributeList, CallingConv,
/*AttrOnCallSite=*/true);
llvm::AttributeSet Attrs = llvm::AttributeSet::get(getLLVMContext(),
AttributeList);
+ // Apply some call-site-specific attributes.
+ // TODO: work this into building the attribute set.
+
+ // Apply always_inline to all calls within flatten functions.
+ // FIXME: should this really take priority over __try, below?
+ if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
+ !(Callee.getAbstractInfo().getCalleeDecl() &&
+ Callee.getAbstractInfo().getCalleeDecl()->hasAttr<NoInlineAttr>())) {
+ Attrs =
+ Attrs.addAttribute(getLLVMContext(),
+ llvm::AttributeSet::FunctionIndex,
+ llvm::Attribute::AlwaysInline);
+ }
+
+ // Disable inlining inside SEH __try blocks.
+ if (isSEHTryScope()) {
+ Attrs =
+ Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
+ llvm::Attribute::NoInline);
+ }
+
+ // Decide whether to use a call or an invoke.
bool CannotThrow;
if (currentFunctionUsesSEHTry()) {
- // SEH cares about asynchronous exceptions, everything can "throw."
+ // SEH cares about asynchronous exceptions, so everything can "throw."
CannotThrow = false;
} else if (isCleanupPadScope() &&
EHPersonality::get(*this).isMSVCXXPersonality()) {
// The MSVC++ personality will implicitly terminate the program if an
- // exception is thrown. An unwind edge cannot be reached.
+ // exception is thrown during a cleanup outside of a try/catch.
+ // We don't need to model anything in IR to get this behavior.
CannotThrow = true;
} else {
- // Otherwise, nowunind callsites will never throw.
+ // Otherwise, nounwind call sites will never throw.
CannotThrow = Attrs.hasAttribute(llvm::AttributeSet::FunctionIndex,
llvm::Attribute::NoUnwind);
}
llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
SmallVector<llvm::OperandBundleDef, 1> BundleList;
- getBundlesForFunclet(Callee, CurrentFuncletPad, BundleList);
+ getBundlesForFunclet(CalleePtr, CurrentFuncletPad, BundleList);
+ // Emit the actual call/invoke instruction.
llvm::CallSite CS;
if (!InvokeDest) {
- CS = Builder.CreateCall(Callee, IRCallArgs, BundleList);
+ CS = Builder.CreateCall(CalleePtr, IRCallArgs, BundleList);
} else {
llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
- CS = Builder.CreateInvoke(Callee, Cont, InvokeDest, IRCallArgs,
+ CS = Builder.CreateInvoke(CalleePtr, Cont, InvokeDest, IRCallArgs,
BundleList);
EmitBlock(Cont);
}
+ llvm::Instruction *CI = CS.getInstruction();
if (callOrInvoke)
- *callOrInvoke = CS.getInstruction();
+ *callOrInvoke = CI;
- if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
- !CS.hasFnAttr(llvm::Attribute::NoInline))
- Attrs =
- Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
- llvm::Attribute::AlwaysInline);
-
- // Disable inlining inside SEH __try blocks.
- if (isSEHTryScope())
- Attrs =
- Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
- llvm::Attribute::NoInline);
-
+ // Apply the attributes and calling convention.
CS.setAttributes(Attrs);
CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
+ // Apply various metadata.
+
+ if (!CI->getType()->isVoidTy())
+ CI->setName("call");
+
// Insert instrumentation or attach profile metadata at indirect call sites.
// For more details, see the comment before the definition of
// IPVK_IndirectCallTarget in InstrProfData.inc.
if (!CS.getCalledFunction())
PGO.valueProfile(Builder, llvm::IPVK_IndirectCallTarget,
- CS.getInstruction(), Callee);
+ CI, CalleePtr);
// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
// optimizer it can aggressively ignore unwind edges.
if (CGM.getLangOpts().ObjCAutoRefCount)
- AddObjCARCExceptionMetadata(CS.getInstruction());
+ AddObjCARCExceptionMetadata(CI);
+
+ // Suppress tail calls if requested.
+ if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) {
+ const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl();
+ if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
+ Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
+ }
+
+ // 4. Finish the call.
// If the call doesn't return, finish the basic block and clear the
- // insertion point; this allows the rest of IRgen to discard
+ // insertion point; this allows the rest of IRGen to discard
// unreachable code.
if (CS.doesNotReturn()) {
if (UnusedReturnSize)
@@ -4003,18 +4056,14 @@
return GetUndefRValue(RetTy);
}
- llvm::Instruction *CI = CS.getInstruction();
- if (!CI->getType()->isVoidTy())
- CI->setName("call");
-
// Perform the swifterror writeback.
if (swiftErrorTemp.isValid()) {
llvm::Value *errorResult = Builder.CreateLoad(swiftErrorTemp);
Builder.CreateStore(errorResult, swiftErrorArg);
}
- // Emit any writebacks immediately. Arguably this should happen
- // after any return-value munging.
+ // Emit any call-associated writebacks immediately. Arguably this
+ // should happen after any return-value munging.
if (CallArgs.hasWritebacks())
emitWritebacks(*this, CallArgs);
@@ -4022,12 +4071,7 @@
// lexical order, so deactivate it and run it manually here.
CallArgs.freeArgumentMemory(*this);
- if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) {
- const Decl *TargetDecl = CalleeInfo.getCalleeDecl();
- if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
- Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
- }
-
+ // Extract the return value.
RValue Ret = [&] {
switch (RetAI.getKind()) {
case ABIArgInfo::CoerceAndExpand: {
@@ -4124,8 +4168,8 @@
llvm_unreachable("Unhandled ABIArgInfo::Kind");
} ();
- const Decl *TargetDecl = CalleeInfo.getCalleeDecl();
-
+ // Emit the assume_aligned check on the return value.
+ const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl();
if (Ret.isScalar() && TargetDecl) {
if (const auto *AA = TargetDecl->getAttr<AssumeAlignedAttr>()) {
llvm::Value *OffsetValue = nullptr;