Add support for passing and returning vectors in accordance with the x86 calling convention.
- Add TargetLowering::lowerArguments() as a new stage in TargetLowering.
- Add support for passing arguments/return values in XMM registers in the x86 target.
BUG=none
R=jvoung@chromium.org, stichnot@chromium.org
Review URL: https://codereview.chromium.org/372113005
diff --git a/src/IceCfg.cpp b/src/IceCfg.cpp
index 6bd9bf7..bc1bf63 100644
--- a/src/IceCfg.cpp
+++ b/src/IceCfg.cpp
@@ -110,6 +110,10 @@
}
}
+void Cfg::doArgLowering() {
+ getTarget()->lowerArguments();
+}
+
void Cfg::doAddressOpt() {
for (NodeList::iterator I = Nodes.begin(), E = Nodes.end(); I != E; ++I) {
(*I)->doAddressOpt();
diff --git a/src/IceCfg.h b/src/IceCfg.h
index dbf08c6..856b145 100644
--- a/src/IceCfg.h
+++ b/src/IceCfg.h
@@ -69,6 +69,7 @@
// Manage arguments to the function.
void addArg(Variable *Arg);
const VarList &getArgs() const { return Args; }
+ VarList &getArgs() { return Args; }
// Miscellaneous accessors.
TargetLowering *getTarget() const { return Target.get(); }
@@ -86,6 +87,7 @@
void placePhiStores();
void deletePhis();
void doAddressOpt();
+ void doArgLowering();
void genCode();
void genFrame();
void livenessLightweight();
diff --git a/src/IceDefs.h b/src/IceDefs.h
index 4c616f8..6aee09a 100644
--- a/src/IceDefs.h
+++ b/src/IceDefs.h
@@ -62,6 +62,7 @@
typedef std::list<Inst *> InstList;
typedef std::list<InstPhi *> PhiList;
typedef std::vector<Variable *> VarList;
+typedef std::vector<Operand *> OperandList;
typedef std::vector<CfgNode *> NodeList;
typedef std::vector<Constant *> ConstantList;
diff --git a/src/IceInstX8632.cpp b/src/IceInstX8632.cpp
index 63e2cd7..a3f0786 100644
--- a/src/IceInstX8632.cpp
+++ b/src/IceInstX8632.cpp
@@ -734,6 +734,14 @@
Str << "\n";
}
+void InstX8632Movp::dump(const Cfg *Func) const {
+ Ostream &Str = Func->getContext()->getStrDump();
+ Str << "movups." << getDest()->getType() << " ";
+ dumpDest(Func);
+ Str << ", ";
+ dumpSources(Func);
+}
+
void InstX8632Movq::emit(const Cfg *Func) const {
Ostream &Str = Func->getContext()->getStrEmit();
assert(getSrcSize() == 1);
@@ -746,14 +754,6 @@
Str << "\n";
}
-void InstX8632Movp::dump(const Cfg *Func) const {
- Ostream &Str = Func->getContext()->getStrDump();
- Str << "movups." << getDest()->getType() << " ";
- dumpDest(Func);
- Str << ", ";
- dumpSources(Func);
-}
-
void InstX8632Movq::dump(const Cfg *Func) const {
Ostream &Str = Func->getContext()->getStrDump();
Str << "movq." << getDest()->getType() << " ";
@@ -882,14 +882,19 @@
assert(getSrcSize() == 1);
Type Ty = getSrc(0)->getType();
Variable *Var = llvm::dyn_cast<Variable>(getSrc(0));
- if ((Ty == IceType_f32 || Ty == IceType_f64) && Var && Var->hasReg()) {
+ if ((isVectorType(Ty) || Ty == IceType_f32 || Ty == IceType_f64) && Var &&
+ Var->hasReg()) {
// The xmm registers can't be directly pushed, so we fake it by
// decrementing esp and then storing to [esp].
Str << "\tsub\tesp, " << typeWidthInBytes(Ty) << "\n";
if (!SuppressStackAdjustment)
Func->getTarget()->updateStackAdjustment(typeWidthInBytes(Ty));
- Str << "\tmov" << TypeX8632Attributes[Ty].SdSsString << "\t"
- << TypeX8632Attributes[Ty].WidthString << " [esp], ";
+ if (isVectorType(Ty)) {
+ Str << "\tmovups\txmmword ptr [esp], ";
+ } else {
+ Str << "\tmov" << TypeX8632Attributes[Ty].SdSsString << "\t"
+ << TypeX8632Attributes[Ty].WidthString << " [esp], ";
+ }
getSrc(0)->emit(Func);
Str << "\n";
} else if (Ty == IceType_f64 && (!Var || !Var->hasReg())) {
diff --git a/src/IceOperand.cpp b/src/IceOperand.cpp
index 02e85c7..d8a754b 100644
--- a/src/IceOperand.cpp
+++ b/src/IceOperand.cpp
@@ -159,14 +159,18 @@
setDefinition(Inst, Node);
}
-void Variable::setIsArg(Cfg *Func) {
- IsArgument = true;
- if (DefNode == NULL)
- return;
- CfgNode *Entry = Func->getEntryNode();
- if (DefNode == Entry)
- return;
- DefNode = NULL;
+void Variable::setIsArg(Cfg *Func, bool IsArg) {
+ if (IsArg) {
+ IsArgument = true;
+ if (DefNode == NULL)
+ return;
+ CfgNode *Entry = Func->getEntryNode();
+ if (DefNode == Entry)
+ return;
+ DefNode = NULL;
+ } else {
+ IsArgument = false;
+ }
}
IceString Variable::getName() const {
diff --git a/src/IceOperand.h b/src/IceOperand.h
index 018445a..bbd6688 100644
--- a/src/IceOperand.h
+++ b/src/IceOperand.h
@@ -339,7 +339,7 @@
void setUse(const Inst *Inst, const CfgNode *Node);
bool getIsArg() const { return IsArgument; }
- void setIsArg(Cfg *Func);
+ void setIsArg(Cfg *Func, bool IsArg = true);
int32_t getStackOffset() const { return StackOffset; }
void setStackOffset(int32_t Offset) { StackOffset = Offset; }
diff --git a/src/IceTargetLowering.h b/src/IceTargetLowering.h
index ddb66fa..ed5389c 100644
--- a/src/IceTargetLowering.h
+++ b/src/IceTargetLowering.h
@@ -148,6 +148,9 @@
virtual void emitVariable(const Variable *Var, const Cfg *Func) const = 0;
+ // Performs target-specific argument lowering.
+ virtual void lowerArguments() = 0;
+
virtual void addProlog(CfgNode *Node) = 0;
virtual void addEpilog(CfgNode *Node) = 0;
diff --git a/src/IceTargetLoweringX8632.cpp b/src/IceTargetLoweringX8632.cpp
index 62a525c..c2782f6 100644
--- a/src/IceTargetLoweringX8632.cpp
+++ b/src/IceTargetLoweringX8632.cpp
@@ -85,6 +85,9 @@
return TableIcmp32[Index].Mapping;
}
+// The maximum number of arguments to pass in XMM registers
+const unsigned X86_MAX_XMM_ARGS = 4;
+
// In some cases, there are x-macros tables for both high-level and
// low-level instructions/operands that use the same enum key value.
// The tables are kept separate to maintain a proper separation
@@ -247,6 +250,11 @@
Func->doAddressOpt();
T_doAddressOpt.printElapsedUs(Context, "doAddressOpt()");
+ // Argument lowering
+ Timer T_argLowering;
+ Func->doArgLowering();
+ T_argLowering.printElapsedUs(Context, "lowerArguments()");
+
// Target lowering. This requires liveness analysis for some parts
// of the lowering decisions, such as compare/branch fusing. If
// non-lightweight liveness analysis is used, the instructions need
@@ -258,6 +266,7 @@
if (Func->hasError())
return;
T_renumber1.printElapsedUs(Context, "renumberInstructions()");
+
// TODO: It should be sufficient to use the fastest liveness
// calculation, i.e. livenessLightweight(). However, for some
// reason that slows down the rest of the translation. Investigate.
@@ -267,6 +276,7 @@
return;
T_liveness1.printElapsedUs(Context, "liveness()");
Func->dump("After x86 address mode opt");
+
Timer T_genCode;
Func->genCode();
if (Func->hasError())
@@ -329,6 +339,10 @@
T_deletePhis.printElapsedUs(Context, "deletePhis()");
Func->dump("After Phi lowering");
+ Timer T_argLowering;
+ Func->doArgLowering();
+ T_argLowering.printElapsedUs(Context, "lowerArguments()");
+
Timer T_genCode;
Func->genCode();
if (Func->hasError())
@@ -412,34 +426,74 @@
Str << "]";
}
-// Helper function for addProlog(). Sets the frame offset for Arg,
-// updates InArgsSizeBytes according to Arg's width, and generates an
-// instruction to copy Arg into its assigned register if applicable.
-// For an I64 arg that has been split into Lo and Hi components, it
-// calls itself recursively on the components, taking care to handle
-// Lo first because of the little-endian architecture.
-void TargetX8632::setArgOffsetAndCopy(Variable *Arg, Variable *FramePtr,
- size_t BasicFrameOffset,
- size_t &InArgsSizeBytes) {
+void TargetX8632::lowerArguments() {
+ VarList &Args = Func->getArgs();
+ // The first four arguments of vector type, regardless of their
+ // position relative to the other arguments in the argument list, are
+ // passed in registers xmm0 - xmm3.
+ unsigned NumXmmArgs = 0;
+
+ Context.init(Func->getEntryNode());
+ Context.setInsertPoint(Context.getCur());
+
+ for (SizeT I = 0, E = Args.size(); I < E && NumXmmArgs < X86_MAX_XMM_ARGS;
+ ++I) {
+ Variable *Arg = Args[I];
+ Type Ty = Arg->getType();
+ if (!isVectorType(Ty))
+ continue;
+ // Replace Arg in the argument list with the home register. Then
+ // generate an instruction in the prolog to copy the home register
+ // to the assigned location of Arg.
+ int32_t RegNum = Reg_xmm0 + NumXmmArgs;
+ ++NumXmmArgs;
+ IceString Name = "home_reg:" + Arg->getName();
+ const CfgNode *DefNode = NULL;
+ Variable *RegisterArg = Func->makeVariable(Ty, DefNode, Name);
+ RegisterArg->setRegNum(RegNum);
+ RegisterArg->setIsArg(Func);
+ Arg->setIsArg(Func, false);
+
+ Args[I] = RegisterArg;
+ Context.insert(InstAssign::create(Func, Arg, RegisterArg));
+ }
+}
+
+// Helper function for addProlog().
+//
+// This assumes Arg is an argument passed on the stack. This sets the
+// frame offset for Arg and updates InArgsSizeBytes according to Arg's
+// width. For an I64 arg that has been split into Lo and Hi components,
+// it calls itself recursively on the components, taking care to handle
+// Lo first because of the little-endian architecture. Lastly, this
+// function generates an instruction to copy Arg into its assigned
+// register if applicable.
+void TargetX8632::finishArgumentLowering(Variable *Arg, Variable *FramePtr,
+ size_t BasicFrameOffset,
+ size_t &InArgsSizeBytes) {
Variable *Lo = Arg->getLo();
Variable *Hi = Arg->getHi();
Type Ty = Arg->getType();
if (Lo && Hi && Ty == IceType_i64) {
assert(Lo->getType() != IceType_i64); // don't want infinite recursion
assert(Hi->getType() != IceType_i64); // don't want infinite recursion
- setArgOffsetAndCopy(Lo, FramePtr, BasicFrameOffset, InArgsSizeBytes);
- setArgOffsetAndCopy(Hi, FramePtr, BasicFrameOffset, InArgsSizeBytes);
+ finishArgumentLowering(Lo, FramePtr, BasicFrameOffset, InArgsSizeBytes);
+ finishArgumentLowering(Hi, FramePtr, BasicFrameOffset, InArgsSizeBytes);
return;
}
Arg->setStackOffset(BasicFrameOffset + InArgsSizeBytes);
+ InArgsSizeBytes += typeWidthInBytesOnStack(Ty);
if (Arg->hasReg()) {
assert(Ty != IceType_i64);
OperandX8632Mem *Mem = OperandX8632Mem::create(
Func, Ty, FramePtr,
Ctx->getConstantInt(IceType_i32, Arg->getStackOffset()));
- _mov(Arg, Mem);
+ if (isVectorType(Arg->getType())) {
+ _movp(Arg, Mem);
+ } else {
+ _mov(Arg, Mem);
+ }
}
- InArgsSizeBytes += typeWidthInBytesOnStack(Ty);
}
Type TargetX8632::stackSlotType() { return IceType_i32; }
@@ -489,7 +543,8 @@
RegsUsed[Var->getRegNum()] = true;
continue;
}
- // An argument passed on the stack already has a stack slot.
+ // An argument either does not need a stack slot (if passed in a
+ // register) or already has one (if passed on the stack).
if (Var->getIsArg())
continue;
// An unreferenced variable doesn't need a stack slot.
@@ -547,23 +602,23 @@
resetStackAdjustment();
- // Fill in stack offsets for args, and copy args into registers for
- // those that were register-allocated. Args are pushed right to
+ // Fill in stack offsets for stack args, and copy args into registers
+ // for those that were register-allocated. Args are pushed right to
// left, so Arg[0] is closest to the stack/frame pointer.
- //
- // TODO: Make this right for different width args, calling
- // conventions, etc. For one thing, args passed in registers will
- // need to be copied/shuffled to their home registers (the
- // RegManager code may have some permutation logic to leverage),
- // and if they have no home register, home space will need to be
- // allocated on the stack to copy into.
Variable *FramePtr = getPhysicalRegister(getFrameOrStackReg());
size_t BasicFrameOffset = PreservedRegsSizeBytes + RetIpSizeBytes;
if (!IsEbpBasedFrame)
BasicFrameOffset += LocalsSizeBytes;
+
+ unsigned NumXmmArgs = 0;
for (SizeT i = 0; i < Args.size(); ++i) {
Variable *Arg = Args[i];
- setArgOffsetAndCopy(Arg, FramePtr, BasicFrameOffset, InArgsSizeBytes);
+ // Skip arguments passed in registers.
+ if (isVectorType(Arg->getType()) && NumXmmArgs < X86_MAX_XMM_ARGS) {
+ ++NumXmmArgs;
+ continue;
+ }
+ finishArgumentLowering(Arg, FramePtr, BasicFrameOffset, InArgsSizeBytes);
}
// Fill in stack offsets for locals.
@@ -1253,7 +1308,10 @@
const bool AllowOverlap = true;
// RI is either a physical register or an immediate.
Operand *RI = legalize(Src0, Legal_Reg | Legal_Imm, AllowOverlap);
- _mov(Dest, RI);
+ if (isVectorType(Dest->getType()))
+ _movp(Dest, RI);
+ else
+ _mov(Dest, RI);
}
}
@@ -1269,31 +1327,44 @@
}
void TargetX8632::lowerCall(const InstCall *Instr) {
- // Generate a sequence of push instructions, pushing right to left,
- // keeping track of stack offsets in case a push involves a stack
- // operand and we are using an esp-based frame.
+ // Classify each argument operand according to the location where the
+ // argument is passed.
+ OperandList XmmArgs;
+ OperandList StackArgs;
+ for (SizeT i = 0, NumArgs = Instr->getNumArgs(); i < NumArgs; ++i) {
+ Operand *Arg = Instr->getArg(i);
+ if (isVectorType(Arg->getType()) && XmmArgs.size() < X86_MAX_XMM_ARGS) {
+ XmmArgs.push_back(Arg);
+ } else {
+ StackArgs.push_back(Arg);
+ }
+ }
+ // For stack arguments, generate a sequence of push instructions,
+ // pushing right to left, keeping track of stack offsets in case a
+ // push involves a stack operand and we are using an esp-based frame.
uint32_t StackOffset = 0;
+ // TODO: Consolidate the stack adjustment for function calls by
+ // reserving enough space for the arguments only once.
+ //
// TODO: If for some reason the call instruction gets dead-code
// eliminated after lowering, we would need to ensure that the
// pre-call push instructions and the post-call esp adjustment get
// eliminated as well.
- for (SizeT NumArgs = Instr->getNumArgs(), i = 0; i < NumArgs; ++i) {
- Operand *Arg = legalize(Instr->getArg(NumArgs - i - 1));
+ for (OperandList::reverse_iterator I = StackArgs.rbegin(),
+ E = StackArgs.rend(); I != E; ++I) {
+ Operand *Arg = legalize(*I);
if (Arg->getType() == IceType_i64) {
_push(hiOperand(Arg));
_push(loOperand(Arg));
- } else if (Arg->getType() == IceType_f64) {
- // If the Arg turns out to be a memory operand, we need to push
- // 8 bytes, which requires two push instructions. This ends up
- // being somewhat clumsy in the current IR, so we use a
- // workaround. Force the operand into a (xmm) register, and
- // then push the register. An xmm register push is actually not
- // possible in x86, but the Push instruction emitter handles
- // this by decrementing the stack pointer and directly writing
- // the xmm register value.
- Variable *T = NULL;
- _mov(T, Arg);
- _push(T);
+ } else if (Arg->getType() == IceType_f64 || isVectorType(Arg->getType())) {
+ // If the Arg turns out to be a memory operand, more than one push
+ // instruction is required. This ends up being somewhat clumsy in
+ // the current IR, so we use a workaround. Force the operand into
+ // a (xmm) register, and then push the register. An xmm register
+ // push is actually not possible in x86, but the Push instruction
+ // emitter handles this by decrementing the stack pointer and
+ // directly writing the xmm register value.
+ _push(legalize(Arg, Legal_Reg));
} else {
// Otherwise PNaCl requires parameter types to be at least 32-bits.
assert(Arg->getType() == IceType_f32 || Arg->getType() == IceType_i32);
@@ -1301,11 +1372,28 @@
}
StackOffset += typeWidthInBytesOnStack(Arg->getType());
}
+ // Copy arguments to be passed in registers to the appropriate
+ // registers.
+ // TODO: Investigate the impact of lowering arguments passed in
+ // registers after lowering stack arguments as opposed to the other
+ // way around. Lowering register arguments after stack arguments may
+ // reduce register pressure. On the other hand, lowering register
+ // arguments first (before stack arguments) may result in more compact
+ // code, as the memory operand displacements may end up being smaller
+ // before any stack adjustment is done.
+ for (SizeT i = 0, NumXmmArgs = XmmArgs.size(); i < NumXmmArgs; ++i) {
+ Variable *Reg = legalizeToVar(XmmArgs[i], false, Reg_xmm0 + i);
+ // Generate a FakeUse of register arguments so that they do not get
+ // dead code eliminated as a result of the FakeKill of scratch
+ // registers after the call.
+ Context.insert(InstFakeUse::create(Func, Reg));
+ }
// Generate the call instruction. Assign its result to a temporary
// with high register allocation weight.
Variable *Dest = Instr->getDest();
- Variable *eax = NULL; // doubles as RegLo as necessary
- Variable *edx = NULL;
+ // ReturnReg doubles as ReturnRegLo as necessary.
+ Variable *ReturnReg = NULL;
+ Variable *ReturnRegHi = NULL;
if (Dest) {
switch (Dest->getType()) {
case IceType_NUM:
@@ -1317,16 +1405,16 @@
case IceType_i8:
case IceType_i16:
case IceType_i32:
- eax = makeReg(Dest->getType(), Reg_eax);
+ ReturnReg = makeReg(Dest->getType(), Reg_eax);
break;
case IceType_i64:
- eax = makeReg(IceType_i32, Reg_eax);
- edx = makeReg(IceType_i32, Reg_edx);
+ ReturnReg = makeReg(IceType_i32, Reg_eax);
+ ReturnRegHi = makeReg(IceType_i32, Reg_edx);
break;
case IceType_f32:
case IceType_f64:
- // Leave eax==edx==NULL, and capture the result with the fstp
- // instruction.
+ // Leave ReturnReg==ReturnRegHi==NULL, and capture the result with
+ // the fstp instruction.
break;
case IceType_v4i1:
case IceType_v8i1:
@@ -1334,24 +1422,18 @@
case IceType_v16i8:
case IceType_v8i16:
case IceType_v4i32:
- case IceType_v4f32: {
- // TODO(wala): Handle return values of vector type in the caller.
- IceString Ty;
- llvm::raw_string_ostream BaseOS(Ty);
- Ostream OS(&BaseOS);
- OS << Dest->getType();
- Func->setError("Unhandled dest type: " + BaseOS.str());
- return;
- }
+ case IceType_v4f32:
+ ReturnReg = makeReg(Dest->getType(), Reg_xmm0);
+ break;
}
}
// TODO(stichnot): LEAHACK: remove Legal_All (and use default) once
// a proper emitter is used.
Operand *CallTarget = legalize(Instr->getCallTarget(), Legal_All);
- Inst *NewCall = InstX8632Call::create(Func, eax, CallTarget);
+ Inst *NewCall = InstX8632Call::create(Func, ReturnReg, CallTarget);
Context.insert(NewCall);
- if (edx)
- Context.insert(InstFakeDef::create(Func, edx));
+ if (ReturnRegHi)
+ Context.insert(InstFakeDef::create(Func, ReturnRegHi));
// Add the appropriate offset to esp.
if (StackOffset) {
@@ -1368,34 +1450,42 @@
Context.insert(InstFakeKill::create(Func, KilledRegs, NewCall));
// Generate a FakeUse to keep the call live if necessary.
- if (Instr->hasSideEffects() && eax) {
- Inst *FakeUse = InstFakeUse::create(Func, eax);
+ if (Instr->hasSideEffects() && ReturnReg) {
+ Inst *FakeUse = InstFakeUse::create(Func, ReturnReg);
Context.insert(FakeUse);
}
+
+ if (!Dest)
+ return;
- // Generate Dest=eax assignment.
- if (Dest && eax) {
- if (edx) {
+ // Assign the result of the call to Dest.
+ if (ReturnReg) {
+ if (ReturnRegHi) {
+ assert(Dest->getType() == IceType_i64);
split64(Dest);
Variable *DestLo = Dest->getLo();
Variable *DestHi = Dest->getHi();
- DestLo->setPreferredRegister(eax, false);
- DestHi->setPreferredRegister(edx, false);
- _mov(DestLo, eax);
- _mov(DestHi, edx);
+ DestLo->setPreferredRegister(ReturnReg, false);
+ DestHi->setPreferredRegister(ReturnRegHi, false);
+ _mov(DestLo, ReturnReg);
+ _mov(DestHi, ReturnRegHi);
} else {
- Dest->setPreferredRegister(eax, false);
- _mov(Dest, eax);
+ assert(Dest->getType() == IceType_i32 || Dest->getType() == IceType_i16 ||
+ Dest->getType() == IceType_i8 || Dest->getType() == IceType_i1 ||
+ isVectorType(Dest->getType()));
+ Dest->setPreferredRegister(ReturnReg, false);
+ if (isVectorType(Dest->getType())) {
+ _movp(Dest, ReturnReg);
+ } else {
+ _mov(Dest, ReturnReg);
+ }
}
- }
-
- // Special treatment for an FP function which returns its result in
- // st(0).
- if (Dest &&
- (Dest->getType() == IceType_f32 || Dest->getType() == IceType_f64)) {
+ } else if (Dest->getType() == IceType_f32 || Dest->getType() == IceType_f64) {
+ // Special treatment for an FP function which returns its result in
+ // st(0).
_fstp(Dest);
- // If Dest ends up being a physical xmm register, the fstp emit
- // code will route st(0) through a temporary stack slot.
+ // If Dest ends up being a physical xmm register, the fstp emit code
+ // will route st(0) through a temporary stack slot.
}
}
diff --git a/src/IceTargetLoweringX8632.h b/src/IceTargetLoweringX8632.h
index f709704..b2871c0 100644
--- a/src/IceTargetLoweringX8632.h
+++ b/src/IceTargetLoweringX8632.h
@@ -46,6 +46,7 @@
return (typeWidthInBytes(Ty) + 3) & ~3;
}
virtual void emitVariable(const Variable *Var, const Cfg *Func) const;
+ virtual void lowerArguments();
virtual void addProlog(CfgNode *Node);
virtual void addEpilog(CfgNode *Node);
virtual void emitConstants() const;
@@ -56,8 +57,8 @@
// function calls using the 32-bit push instruction (though the
// latter could be done by directly writing to the stack).
void split64(Variable *Var);
- void setArgOffsetAndCopy(Variable *Arg, Variable *FramePtr,
- size_t BasicFrameOffset, size_t &InArgsSizeBytes);
+ void finishArgumentLowering(Variable *Arg, Variable *FramePtr,
+ size_t BasicFrameOffset, size_t &InArgsSizeBytes);
Operand *loOperand(Operand *Operand);
Operand *hiOperand(Operand *Operand);