Tail call optimization improvements:
Move platform independent code (lowering of possibly overwritten
arguments, check for tail call optimization eligibility) from
target X86ISelectionLowering.cpp to TargetLowering.h and
SelectionDAGISel.cpp.
Initial PowerPC tail call implementation:
Support ppc32 implemented and tested (passes my tests and
test-suite llvm-test).
Support ppc64 implemented and half tested (passes my tests).
On ppc tail call optimization is performed if
caller and callee are fastcc
call is a tail call (in tail call position, call followed by ret)
no variable argument lists or byval arguments
option -tailcallopt is enabled
Supported:
* non pic tail calls on linux/darwin
* module-local tail calls on linux(PIC/GOT)/darwin(PIC)
* inter-module tail calls on darwin(PIC)
If constraints are not met a normal call will be emitted.
A test checking the argument lowering behaviour on x86-64 was added.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@50477 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index 799ba00..8560dfe 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -788,19 +788,6 @@
#include "X86GenCallingConv.inc"
-/// GetPossiblePreceedingTailCall - Get preceeding X86ISD::TAILCALL node if it
-/// exists skip possible ISD:TokenFactor.
-static SDOperand GetPossiblePreceedingTailCall(SDOperand Chain) {
- if (Chain.getOpcode() == X86ISD::TAILCALL) {
- return Chain;
- } else if (Chain.getOpcode() == ISD::TokenFactor) {
- if (Chain.getNumOperands() &&
- Chain.getOperand(0).getOpcode() == X86ISD::TAILCALL)
- return Chain.getOperand(0);
- }
- return Chain;
-}
-
/// LowerRET - Lower an ISD::RET node.
SDOperand X86TargetLowering::LowerRET(SDOperand Op, SelectionDAG &DAG) {
assert((Op.getNumOperands() & 1) == 1 && "ISD::RET should have odd # args");
@@ -821,7 +808,7 @@
SDOperand Chain = Op.getOperand(0);
// Handle tail call return.
- Chain = GetPossiblePreceedingTailCall(Chain);
+ Chain = GetPossiblePreceedingTailCall(Chain, X86ISD::TAILCALL);
if (Chain.getOpcode() == X86ISD::TAILCALL) {
SDOperand TailCall = Chain;
SDOperand TargetAddress = TailCall.getOperand(1);
@@ -1057,27 +1044,6 @@
return None;
}
-/// IsPossiblyOverwrittenArgumentOfTailCall - Check if the operand could
-/// possibly be overwritten when lowering the outgoing arguments in a tail
-/// call. Currently the implementation of this call is very conservative and
-/// assumes all arguments sourcing from FORMAL_ARGUMENTS or a CopyFromReg with
-/// virtual registers would be overwritten by direct lowering.
-static bool IsPossiblyOverwrittenArgumentOfTailCall(SDOperand Op,
- MachineFrameInfo * MFI) {
- RegisterSDNode * OpReg = NULL;
- FrameIndexSDNode * FrameIdxNode = NULL;
- int FrameIdx = 0;
- if (Op.getOpcode() == ISD::FORMAL_ARGUMENTS ||
- (Op.getOpcode()== ISD::CopyFromReg &&
- (OpReg = dyn_cast<RegisterSDNode>(Op.getOperand(1))) &&
- (OpReg->getReg() >= TargetRegisterInfo::FirstVirtualRegister)) ||
- (Op.getOpcode() == ISD::LOAD &&
- (FrameIdxNode = dyn_cast<FrameIndexSDNode>(Op.getOperand(1))) &&
- (MFI->isFixedObjectIndex((FrameIdx = FrameIdxNode->getIndex()))) &&
- (MFI->getObjectOffset(FrameIdx) >= 0)))
- return true;
- return false;
-}
/// CallRequiresGOTInRegister - Check whether the call requires the GOT pointer
/// in a register before calling.
@@ -1087,7 +1053,6 @@
Subtarget->isPICStyleGOT();
}
-
/// CallRequiresFnAddressInReg - Check whether the call requires the function
/// address to be loaded in a register.
bool
@@ -1097,33 +1062,6 @@
Subtarget->isPICStyleGOT();
}
-/// CopyTailCallClobberedArgumentsToVRegs - Create virtual registers for all
-/// arguments to force loading and guarantee that arguments sourcing from
-/// incomming parameters are not overwriting each other.
-static SDOperand
-CopyTailCallClobberedArgumentsToVRegs(SDOperand Chain,
- SmallVector<std::pair<unsigned, SDOperand>, 8> &TailCallClobberedVRegs,
- SelectionDAG &DAG,
- MachineFunction &MF,
- const TargetLowering * TL) {
-
- SDOperand InFlag;
- for (unsigned i = 0, e = TailCallClobberedVRegs.size(); i != e; i++) {
- SDOperand Arg = TailCallClobberedVRegs[i].second;
- unsigned Idx = TailCallClobberedVRegs[i].first;
- unsigned VReg =
- MF.getRegInfo().
- createVirtualRegister(TL->getRegClassFor(Arg.getValueType()));
- Chain = DAG.getCopyToReg(Chain, VReg, Arg, InFlag);
- InFlag = Chain.getValue(1);
- Arg = DAG.getCopyFromReg(Chain, VReg, Arg.getValueType(), InFlag);
- TailCallClobberedVRegs[i] = std::make_pair(Idx, Arg);
- Chain = Arg.getValue(1);
- InFlag = Arg.getValue(2);
- }
- return Chain;
-}
-
/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
/// by "Src" to address "Dst" with size and alignment information specified by
/// the specific parameter attribute. The copy will be passed as a byval
@@ -1133,8 +1071,7 @@
ISD::ArgFlagsTy Flags, SelectionDAG &DAG) {
SDOperand SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
return DAG.getMemcpy(Chain, Dst, Src, SizeNode, Flags.getByValAlign(),
- /*AlwaysInline=*/true,
- NULL, 0, NULL, 0);
+ /*AlwaysInline=*/true, NULL, 0, NULL, 0);
}
SDOperand X86TargetLowering::LowerMemArgument(SDOperand Op, SelectionDAG &DAG,
@@ -1463,65 +1400,8 @@
return Chain;
}
-/// CopyTailCallByValClobberedRegToVirtReg - Copy arguments with register target
-/// which might be overwritten by later byval tail call lowering to a virtual
-/// register.
-bool
-X86TargetLowering::CopyTailCallByValClobberedRegToVirtReg(bool containsByValArg,
- SmallVector< std::pair<unsigned, unsigned>, 8> &TailCallByValClobberedVRegs,
- SmallVector<MVT::ValueType, 8> &TailCallByValClobberedVRegTypes,
- std::pair<unsigned, SDOperand> &RegToPass,
- SDOperand &OutChain,
- SDOperand &OutFlag,
- MachineFunction &MF,
- SelectionDAG & DAG) {
- if (!containsByValArg) return false;
-
- std::pair<unsigned, unsigned> ArgRegVReg;
- MVT::ValueType VT = RegToPass.second.getValueType();
-
- ArgRegVReg.first = RegToPass.first;
- ArgRegVReg.second = MF.getRegInfo().createVirtualRegister(getRegClassFor(VT));
-
- // Copy Argument to virtual register.
- OutChain = DAG.getCopyToReg(OutChain, ArgRegVReg.second,
- RegToPass.second, OutFlag);
- OutFlag = OutChain.getValue(1);
- // Remember virtual register and type.
- TailCallByValClobberedVRegs.push_back(ArgRegVReg);
- TailCallByValClobberedVRegTypes.push_back(VT);
- return true;
-}
-
-
-/// RestoreTailCallByValClobberedReg - Restore registers which were saved to
-/// virtual registers to prevent tail call byval lowering from overwriting
-/// parameter registers.
-static SDOperand
-RestoreTailCallByValClobberedRegs(SelectionDAG & DAG, SDOperand Chain,
- SmallVector< std::pair<unsigned, unsigned>, 8> &TailCallByValClobberedVRegs,
- SmallVector<MVT::ValueType, 8> &TailCallByValClobberedVRegTypes) {
- if (TailCallByValClobberedVRegs.size()==0) return Chain;
-
- SmallVector<SDOperand, 8> RegOpChains;
- for (unsigned i = 0, e=TailCallByValClobberedVRegs.size(); i != e; i++) {
- SDOperand InFlag;
- unsigned DestReg = TailCallByValClobberedVRegs[i].first;
- unsigned VirtReg = TailCallByValClobberedVRegs[i].second;
- MVT::ValueType VT = TailCallByValClobberedVRegTypes[i];
- SDOperand Tmp = DAG.getCopyFromReg(Chain, VirtReg, VT, InFlag);
- Chain = DAG.getCopyToReg(Chain, DestReg, Tmp, InFlag);
- RegOpChains.push_back(Chain);
- }
- if (!RegOpChains.empty())
- Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
- &RegOpChains[0], RegOpChains.size());
- return Chain;
-}
-
SDOperand X86TargetLowering::LowerCALL(SDOperand Op, SelectionDAG &DAG) {
MachineFunction &MF = DAG.getMachineFunction();
- MachineFrameInfo * MFI = MF.getFrameInfo();
SDOperand Chain = Op.getOperand(0);
unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue();
bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0;
@@ -1572,17 +1452,11 @@
FPDiff);
SmallVector<std::pair<unsigned, SDOperand>, 8> RegsToPass;
- SmallVector<std::pair<unsigned, SDOperand>, 8> TailCallClobberedVRegs;
-
SmallVector<SDOperand, 8> MemOpChains;
-
SDOperand StackPtr;
- bool containsTailCallByValArg = false;
- SmallVector<std::pair<unsigned, unsigned>, 8> TailCallByValClobberedVRegs;
- SmallVector<MVT::ValueType, 8> TailCallByValClobberedVRegTypes;
- // Walk the register/memloc assignments, inserting copies/loads. For tail
- // calls, remember all arguments for later special lowering.
+ // Walk the register/memloc assignments, inserting copies/loads. In the case
+ // of tail call optimization arguments are handle later.
for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
CCValAssign &VA = ArgLocs[i];
SDOperand Arg = Op.getOperand(5+2*VA.getValNo());
@@ -1638,10 +1512,6 @@
MemOpChains.push_back(LowerMemOpCallTo(Op, DAG, StackPtr, VA, Chain,
Arg));
- // Remember fact that this call contains byval arguments.
- containsTailCallByValArg |= IsTailCall && isByVal;
- } else if (IsPossiblyOverwrittenArgumentOfTailCall(Arg, MFI)) {
- TailCallClobberedVRegs.push_back(std::make_pair(i,Arg));
}
}
}
@@ -1653,21 +1523,14 @@
// Build a sequence of copy-to-reg nodes chained together with token chain
// and flag operands which copy the outgoing args into registers.
SDOperand InFlag;
- for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
- // Tail call byval lowering might overwrite argument registers so arguments
- // passed to be copied to a virtual register for
- // later processing.
- if (CopyTailCallByValClobberedRegToVirtReg(containsTailCallByValArg,
- TailCallByValClobberedVRegs,
- TailCallByValClobberedVRegTypes,
- RegsToPass[i], Chain, InFlag, MF,
- DAG))
- continue;
-
- Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second,
- InFlag);
- InFlag = Chain.getValue(1);
- }
+ // Tail call byval lowering might overwrite argument registers so in case of
+ // tail call optimization the copies to registers are lowered later.
+ if (!IsTailCall)
+ for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
+ Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second,
+ InFlag);
+ InFlag = Chain.getValue(1);
+ }
// ELF / PIC requires GOT in the EBX register before function calls via PLT
// GOT pointer.
@@ -1723,10 +1586,6 @@
int FI = 0;
// Do not flag preceeding copytoreg stuff together with the following stuff.
InFlag = SDOperand();
-
- Chain = CopyTailCallClobberedArgumentsToVRegs(Chain, TailCallClobberedVRegs,
- DAG, MF, this);
-
for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
CCValAssign &VA = ArgLocs[i];
if (!VA.isRegLoc()) {
@@ -1741,17 +1600,6 @@
FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset);
FIN = DAG.getFrameIndex(FI, getPointerTy());
- // Find virtual register for this argument.
- bool Found=false;
- for (unsigned idx=0, e= TailCallClobberedVRegs.size(); idx < e; idx++)
- if (TailCallClobberedVRegs[idx].first==i) {
- Arg = TailCallClobberedVRegs[idx].second;
- Found=true;
- break;
- }
- assert(IsPossiblyOverwrittenArgumentOfTailCall(Arg, MFI)==false ||
- (Found==true && "No corresponding Argument was found"));
-
if (Flags.isByVal()) {
// Copy relative to framepointer.
SDOperand Source = DAG.getIntPtrConstant(VA.getLocMemOffset());
@@ -1774,10 +1622,13 @@
Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
&MemOpChains2[0], MemOpChains2.size());
- // Restore byval lowering clobbered registers.
- Chain = RestoreTailCallByValClobberedRegs(DAG, Chain,
- TailCallByValClobberedVRegs,
- TailCallByValClobberedVRegTypes);
+ // Copy arguments to their registers.
+ for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
+ Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second,
+ InFlag);
+ InFlag = Chain.getValue(1);
+ }
+ InFlag =SDOperand();
// Store the return address to the appropriate stack slot.
Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit,
@@ -1955,15 +1806,7 @@
if (!PerformTailCallOpt)
return false;
- // Check whether CALL node immediatly preceeds the RET node and whether the
- // return uses the result of the node or is a void return.
- unsigned NumOps = Ret.getNumOperands();
- if ((NumOps == 1 &&
- (Ret.getOperand(0) == SDOperand(Call.Val,1) ||
- Ret.getOperand(0) == SDOperand(Call.Val,0))) ||
- (NumOps > 1 &&
- Ret.getOperand(0) == SDOperand(Call.Val,Call.Val->getNumValues()-1) &&
- Ret.getOperand(1) == SDOperand(Call.Val,0))) {
+ if (CheckTailCallReturnConstraints(Call, Ret)) {
MachineFunction &MF = DAG.getMachineFunction();
unsigned CallerCC = MF.getFunction()->getCallingConv();
unsigned CalleeCC = cast<ConstantSDNode>(Call.getOperand(1))->getValue();