[RISCV] Add custom CC_RISCV calling convention and improved call support
The TableGen-based calling convention definitions are inflexible, while
writing a function to implement the calling convention is very
straight-forward, and allows difficult cases to be handled more easily. With
this patch adds support for:
* Passing large scalars according to the RV32I calling convention
* Byval arguments
* Passing values on the stack when the argument registers are exhausted
The custom CC_RISCV calling convention is also used for returns.
This patch also documents the ABI lowering that a language frontend is
expected to perform. I would like to work to simplify these requirements over
time, but this will require further discussion within the LLVM community.
We add PendingArgFlags CCState, as a companion to PendingLocs.
The PendingLocs vector is used by a number of backends to handle arguments
that are split during legalisation. However CCValAssign doesn't keep track of
the original argument alignment. Therefore, add a PendingArgFlags vector which
can be used to keep track of the ISD::ArgFlagsTy for every value added to
PendingLocs.
Differential Revision: https://reviews.llvm.org/D39898
llvm-svn: 320359
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 5881753..7d32954 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -328,7 +328,242 @@
}
// Calling Convention Implementation.
-#include "RISCVGenCallingConv.inc"
+// The expectations for frontend ABI lowering vary from target to target.
+// Ideally, an LLVM frontend would be able to avoid worrying about many ABI
+// details, but this is a longer term goal. For now, we simply try to keep the
+// role of the frontend as simple and well-defined as possible. The rules can
+// be summarised as:
+// * Never split up large scalar arguments. We handle them here.
+// * If a hardfloat calling convention is being used, and the struct may be
+// passed in a pair of registers (fp+fp, int+fp), and both registers are
+// available, then pass as two separate arguments. If either the GPRs or FPRs
+// are exhausted, then pass according to the rule below.
+// * If a struct could never be passed in registers or directly in a stack
+// slot (as it is larger than 2*XLEN and the floating point rules don't
+// apply), then pass it using a pointer with the byval attribute.
+// * If a struct is less than 2*XLEN, then coerce to either a two-element
+// word-sized array or a 2*XLEN scalar (depending on alignment).
+// * The frontend can determine whether a struct is returned by reference or
+// not based on its size and fields. If it will be returned by reference, the
+// frontend must modify the prototype so a pointer with the sret annotation is
+// passed as the first argument. This is not necessary for large scalar
+// returns.
+// * Struct return values and varargs should be coerced to structs containing
+// register-size fields in the same situations they would be for fixed
+// arguments.
+
+static const MCPhysReg ArgGPRs[] = {
+ RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
+ RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
+};
+
+// Pass a 2*XLEN argument that has been split into two XLEN values through
+// registers or the stack as necessary.
+static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
+ ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
+ MVT ValVT2, MVT LocVT2,
+ ISD::ArgFlagsTy ArgFlags2) {
+ unsigned XLenInBytes = XLen / 8;
+ if (unsigned Reg = State.AllocateReg(ArgGPRs)) {
+ // At least one half can be passed via register.
+ State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
+ VA1.getLocVT(), CCValAssign::Full));
+ } else {
+ // Both halves must be passed on the stack, with proper alignment.
+ unsigned StackAlign = std::max(XLenInBytes, ArgFlags1.getOrigAlign());
+ State.addLoc(
+ CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
+ State.AllocateStack(XLenInBytes, StackAlign),
+ VA1.getLocVT(), CCValAssign::Full));
+ State.addLoc(CCValAssign::getMem(
+ ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
+ CCValAssign::Full));
+ return false;
+ }
+
+ if (unsigned Reg = State.AllocateReg(ArgGPRs)) {
+ // The second half can also be passed via register.
+ State.addLoc(
+ CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
+ } else {
+ // The second half is passed via the stack, without additional alignment.
+ State.addLoc(CCValAssign::getMem(
+ ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
+ CCValAssign::Full));
+ }
+
+ return false;
+}
+
+// Implements the RISC-V calling convention. Returns true upon failure.
+static bool CC_RISCV(const DataLayout &DL, unsigned ValNo, MVT ValVT, MVT LocVT,
+ CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+ CCState &State, bool IsFixed, bool IsRet) {
+ unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
+ assert(XLen == 32 || XLen == 64);
+ MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
+ assert(ValVT == XLenVT && "Unexpected ValVT");
+ assert(LocVT == XLenVT && "Unexpected LocVT");
+ assert(IsFixed && "Vararg support not yet implemented");
+
+ // Any return value split in to more than two values can't be returned
+ // directly.
+ if (IsRet && ValNo > 1)
+ return true;
+
+ SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
+ SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
+ State.getPendingArgFlags();
+
+ assert(PendingLocs.size() == PendingArgFlags.size() &&
+ "PendingLocs and PendingArgFlags out of sync");
+
+ // Split arguments might be passed indirectly, so keep track of the pending
+ // values.
+ if (ArgFlags.isSplit() || !PendingLocs.empty()) {
+ LocVT = XLenVT;
+ LocInfo = CCValAssign::Indirect;
+ PendingLocs.push_back(
+ CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
+ PendingArgFlags.push_back(ArgFlags);
+ if (!ArgFlags.isSplitEnd()) {
+ return false;
+ }
+ }
+
+ // If the split argument only had two elements, it should be passed directly
+ // in registers or on the stack.
+ if (ArgFlags.isSplitEnd() && PendingLocs.size() <= 2) {
+ assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
+ // Apply the normal calling convention rules to the first half of the
+ // split argument.
+ CCValAssign VA = PendingLocs[0];
+ ISD::ArgFlagsTy AF = PendingArgFlags[0];
+ PendingLocs.clear();
+ PendingArgFlags.clear();
+ return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
+ ArgFlags);
+ }
+
+ // Allocate to a register if possible, or else a stack slot.
+ unsigned Reg = State.AllocateReg(ArgGPRs);
+ unsigned StackOffset = Reg ? 0 : State.AllocateStack(XLen / 8, XLen / 8);
+
+ // If we reach this point and PendingLocs is non-empty, we must be at the
+ // end of a split argument that must be passed indirectly.
+ if (!PendingLocs.empty()) {
+ assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
+ assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
+
+ for (auto &It : PendingLocs) {
+ if (Reg)
+ It.convertToReg(Reg);
+ else
+ It.convertToMem(StackOffset);
+ State.addLoc(It);
+ }
+ PendingLocs.clear();
+ PendingArgFlags.clear();
+ return false;
+ }
+
+ assert(LocVT == XLenVT && "Expected an XLenVT at this stage");
+
+ if (Reg) {
+ State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+ } else {
+ State.addLoc(
+ CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
+ }
+ return false;
+}
+
+void RISCVTargetLowering::analyzeInputArgs(
+ MachineFunction &MF, CCState &CCInfo,
+ const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet) const {
+ unsigned NumArgs = Ins.size();
+
+ for (unsigned i = 0; i != NumArgs; ++i) {
+ MVT ArgVT = Ins[i].VT;
+ ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
+
+ if (CC_RISCV(MF.getDataLayout(), i, ArgVT, ArgVT, CCValAssign::Full,
+ ArgFlags, CCInfo, /*IsRet=*/true, IsRet)) {
+ DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
+ << EVT(ArgVT).getEVTString() << '\n');
+ llvm_unreachable(nullptr);
+ }
+ }
+}
+
+void RISCVTargetLowering::analyzeOutputArgs(
+ MachineFunction &MF, CCState &CCInfo,
+ const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet) const {
+ unsigned NumArgs = Outs.size();
+
+ for (unsigned i = 0; i != NumArgs; i++) {
+ MVT ArgVT = Outs[i].VT;
+ ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
+
+ if (CC_RISCV(MF.getDataLayout(), i, ArgVT, ArgVT, CCValAssign::Full,
+ ArgFlags, CCInfo, Outs[i].IsFixed, IsRet)) {
+ DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
+ << EVT(ArgVT).getEVTString() << "\n");
+ llvm_unreachable(nullptr);
+ }
+ }
+}
+
+// The caller is responsible for loading the full value if the argument is
+// passed with CCValAssign::Indirect.
+static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
+ const CCValAssign &VA, const SDLoc &DL) {
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineRegisterInfo &RegInfo = MF.getRegInfo();
+ EVT LocVT = VA.getLocVT();
+ SDValue Val;
+
+ unsigned VReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
+ RegInfo.addLiveIn(VA.getLocReg(), VReg);
+ Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
+
+ switch (VA.getLocInfo()) {
+ default:
+ llvm_unreachable("Unexpected CCValAssign::LocInfo");
+ case CCValAssign::Full:
+ case CCValAssign::Indirect:
+ return Val;
+ }
+}
+
+// The caller is responsible for loading the full value if the argument is
+// passed with CCValAssign::Indirect.
+static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
+ const CCValAssign &VA, const SDLoc &DL) {
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineFrameInfo &MFI = MF.getFrameInfo();
+ EVT LocVT = VA.getLocVT();
+ EVT ValVT = VA.getValVT();
+ EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
+ int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
+ VA.getLocMemOffset(), /*Immutable=*/true);
+ SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
+ SDValue Val;
+
+ ISD::LoadExtType ExtType;
+ switch (VA.getLocInfo()) {
+ default:
+ llvm_unreachable("Unexpected CCValAssign::LocInfo");
+ case CCValAssign::Full:
+ case CCValAssign::Indirect:
+ ExtType = ISD::NON_EXTLOAD;
+ break;
+ }
+ Val = DAG.getExtLoad(
+ ExtType, DL, LocVT, Chain, FIN,
+ MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
+ return Val;
+}
// Transform physical registers into virtual registers.
SDValue RISCVTargetLowering::LowerFormalArguments(
@@ -345,8 +580,8 @@
}
MachineFunction &MF = DAG.getMachineFunction();
- MachineRegisterInfo &RegInfo = MF.getRegInfo();
MVT XLenVT = Subtarget.getXLenVT();
+ EVT PtrVT = getPointerTy(DAG.getDataLayout());
if (IsVarArg)
report_fatal_error("VarArg not supported");
@@ -354,25 +589,37 @@
// Assign locations to all of the incoming arguments.
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
- CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV32);
+ analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false);
- for (auto &VA : ArgLocs) {
- if (!VA.isRegLoc())
- report_fatal_error("Defined with too many args");
+ for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
+ CCValAssign &VA = ArgLocs[i];
+ assert(VA.getLocVT() == XLenVT && "Unhandled argument type");
+ SDValue ArgValue;
+ if (VA.isRegLoc())
+ ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL);
+ else
+ ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
- // Arguments passed in registers.
- EVT RegVT = VA.getLocVT();
- if (RegVT != XLenVT) {
- DEBUG(dbgs() << "LowerFormalArguments Unhandled argument type: "
- << RegVT.getEVTString() << "\n");
- report_fatal_error("unhandled argument type");
+ if (VA.getLocInfo() == CCValAssign::Indirect) {
+ // If the original argument was split and passed by reference (e.g. i128
+ // on RV32), we need to load all parts of it here (using the same
+ // address).
+ InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
+ MachinePointerInfo()));
+ unsigned ArgIndex = Ins[i].OrigArgIndex;
+ assert(Ins[i].PartOffset == 0);
+ while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
+ CCValAssign &PartVA = ArgLocs[i + 1];
+ unsigned PartOffset = Ins[i + 1].PartOffset;
+ SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
+ DAG.getIntPtrConstant(PartOffset, DL));
+ InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
+ MachinePointerInfo()));
+ ++i;
+ }
+ continue;
}
- const unsigned VReg =
- RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
- RegInfo.addLiveIn(VA.getLocReg(), VReg);
- SDValue ArgIn = DAG.getCopyFromReg(Chain, DL, VReg, RegVT);
-
- InVals.push_back(ArgIn);
+ InVals.push_back(ArgValue);
}
return Chain;
}
@@ -392,6 +639,7 @@
CallingConv::ID CallConv = CLI.CallConv;
bool IsVarArg = CLI.IsVarArg;
EVT PtrVT = getPointerTy(DAG.getDataLayout());
+ MVT XLenVT = Subtarget.getXLenVT();
if (IsVarArg) {
report_fatal_error("LowerCall with varargs not implemented");
@@ -402,44 +650,105 @@
// Analyze the operands of the call, assigning locations to each operand.
SmallVector<CCValAssign, 16> ArgLocs;
CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
- ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV32);
+ analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false);
// Get a count of how many bytes are to be pushed on the stack.
unsigned NumBytes = ArgCCInfo.getNextStackOffset();
- for (auto &Arg : Outs) {
- if (!Arg.Flags.isByVal())
+ // Create local copies for byval args
+ SmallVector<SDValue, 8> ByValArgs;
+ for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
+ ISD::ArgFlagsTy Flags = Outs[i].Flags;
+ if (!Flags.isByVal())
continue;
- report_fatal_error("Passing arguments byval not yet implemented");
+
+ SDValue Arg = OutVals[i];
+ unsigned Size = Flags.getByValSize();
+ unsigned Align = Flags.getByValAlign();
+
+ int FI = MF.getFrameInfo().CreateStackObject(Size, Align, /*isSS=*/false);
+ SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
+ SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
+
+ Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Align,
+ /*IsVolatile=*/false,
+ /*AlwaysInline=*/false,
+ /*isTailCall=*/false, MachinePointerInfo(),
+ MachinePointerInfo());
+ ByValArgs.push_back(FIPtr);
}
Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
// Copy argument values to their designated locations.
SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
+ SmallVector<SDValue, 8> MemOpChains;
SDValue StackPtr;
- for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
- CCValAssign &VA = ArgLocs[I];
- SDValue ArgValue = OutVals[I];
+ for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
+ CCValAssign &VA = ArgLocs[i];
+ SDValue ArgValue = OutVals[i];
+ ISD::ArgFlagsTy Flags = Outs[i].Flags;
// Promote the value if needed.
- // For now, only handle fully promoted arguments.
+ // For now, only handle fully promoted and indirect arguments.
switch (VA.getLocInfo()) {
case CCValAssign::Full:
break;
+ case CCValAssign::Indirect: {
+ // Store the argument in a stack slot and pass its address.
+ SDValue SpillSlot = DAG.CreateStackTemporary(Outs[i].ArgVT);
+ int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
+ MemOpChains.push_back(
+ DAG.getStore(Chain, DL, ArgValue, SpillSlot,
+ MachinePointerInfo::getFixedStack(MF, FI)));
+ // If the original argument was split (e.g. i128), we need
+ // to store all parts of it here (and pass just one address).
+ unsigned ArgIndex = Outs[i].OrigArgIndex;
+ assert(Outs[i].PartOffset == 0);
+ while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
+ SDValue PartValue = OutVals[i + 1];
+ unsigned PartOffset = Outs[i + 1].PartOffset;
+ SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
+ DAG.getIntPtrConstant(PartOffset, DL));
+ MemOpChains.push_back(
+ DAG.getStore(Chain, DL, PartValue, Address,
+ MachinePointerInfo::getFixedStack(MF, FI)));
+ ++i;
+ }
+ ArgValue = SpillSlot;
+ break;
+ }
default:
llvm_unreachable("Unknown loc info!");
}
+ // Use local copy if it is a byval arg.
+ if (Flags.isByVal())
+ ArgValue = ByValArgs[j++];
+
if (VA.isRegLoc()) {
// Queue up the argument copies and emit them at the end.
RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
} else {
assert(VA.isMemLoc() && "Argument not register or memory");
- report_fatal_error("Passing arguments via the stack not yet implemented");
+
+ // Work out the address of the stack slot.
+ if (!StackPtr.getNode())
+ StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
+ SDValue Address =
+ DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
+ DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
+
+ // Emit the store.
+ MemOpChains.push_back(
+ DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
}
}
+ // Join the stores, which are independent of one another.
+ if (!MemOpChains.empty())
+ Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
+
SDValue Glue;
// Build a sequence of copy-to-reg nodes, chained and glued together.
@@ -489,7 +798,7 @@
// Assign locations to each value returned by this call.
SmallVector<CCValAssign, 16> RVLocs;
CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
- RetCCInfo.AnalyzeCallResult(Ins, RetCC_RISCV32);
+ analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true);
// Copy all of the result registers out of their specified physreg.
for (auto &VA : RVLocs) {
@@ -499,12 +808,28 @@
Chain = RetValue.getValue(1);
Glue = RetValue.getValue(2);
- InVals.push_back(Chain.getValue(0));
+ assert(VA.getLocInfo() == CCValAssign::Full && "Unknown loc info!");
+ InVals.push_back(RetValue);
}
return Chain;
}
+bool RISCVTargetLowering::CanLowerReturn(
+ CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
+ const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
+ SmallVector<CCValAssign, 16> RVLocs;
+ CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
+ for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
+ MVT VT = Outs[i].VT;
+ ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
+ if (CC_RISCV(MF.getDataLayout(), i, VT, VT, CCValAssign::Full, ArgFlags,
+ CCInfo, /*IsFixed=*/true, /*IsRet=*/true))
+ return false;
+ }
+ return true;
+}
+
SDValue
RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
bool IsVarArg,
@@ -522,17 +847,20 @@
CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
*DAG.getContext());
- CCInfo.AnalyzeReturn(Outs, RetCC_RISCV32);
+ analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true);
SDValue Flag;
SmallVector<SDValue, 4> RetOps(1, Chain);
// Copy the result values into the output registers.
for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
+ SDValue Val = OutVals[i];
CCValAssign &VA = RVLocs[i];
assert(VA.isRegLoc() && "Can only return in registers!");
+ assert(VA.getLocInfo() == CCValAssign::Full &&
+ "Unexpected CCValAssign::LocInfo");
- Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), OutVals[i], Flag);
+ Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Flag);
// Guarantee that all emitted copies are stuck together.
Flag = Chain.getValue(1);