Several changes:
1) Change the interface to TargetLowering::ExpandOperationResult to
take and return entire NODES that need a result expanded, not just
the value. This allows us to handle things like READCYCLECOUNTER,
which returns two values.
2) Implement (extremely limited) support in LegalizeDAG::ExpandOp for MERGE_VALUES.
3) Reimplement custom lowering in LegalizeDAGTypes in terms of the new
ExpandOperationResult. This makes the result simpler and fully
general.
4) Implement (fully general) expand support for MERGE_VALUES in LegalizeDAGTypes.
5) Implement ExpandOperationResult support for ARM f64->i64 bitconvert and ARM
i64 shifts, allowing them to work with LegalizeDAGTypes.
6) Implement ExpandOperationResult support for X86 READCYCLECOUNTER and FP_TO_SINT,
allowing them to work with LegalizeDAGTypes.
LegalizeDAGTypes now passes several more X86 codegen tests when enabled and when
type legalization in LegalizeDAG is ifdef'd out.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@44300 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 32af553..aa8ffaa 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -5187,10 +5187,21 @@
// Otherwise, try a larger type.
}
- // Okay, we found the operation and type to use. Truncate the result of the
- // extended FP_TO_*INT operation to the desired size.
- return DAG.getNode(ISD::TRUNCATE, DestVT,
- DAG.getNode(OpToUse, NewOutTy, LegalOp));
+
+ // Okay, we found the operation and type to use.
+ SDOperand Operation = DAG.getNode(OpToUse, NewOutTy, LegalOp);
+
+ // If the operation produces an invalid type, it must be custom lowered. Use
+ // the target lowering hooks to expand it. Just keep the low part of the
+ // expanded operation, we know that we're truncating anyway.
+ if (getTypeAction(NewOutTy) == Expand) {
+ Operation = SDOperand(TLI.ExpandOperationResult(Operation.Val, DAG), 0);
+ assert(Operation.Val && "Didn't return anything");
+ }
+
+ // Truncate the result of the extended FP_TO_*INT operation to the desired
+ // size.
+ return DAG.getNode(ISD::TRUNCATE, DestVT, Operation);
}
/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation.
@@ -5388,6 +5399,16 @@
Lo = Node->getOperand(0);
Hi = Node->getOperand(1);
break;
+
+ case ISD::MERGE_VALUES:
+ // FIXME: For now only expand i64,chain = MERGE_VALUES (x, y)
+ assert(Op.ResNo == 0 && Node->getNumValues() == 2 &&
+ Op.getValue(1).getValueType() == MVT::Other &&
+ "unhandled MERGE_VALUES");
+ ExpandOp(Op.getOperand(0), Lo, Hi);
+ // Remember that we legalized the chain.
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(Op.getOperand(1)));
+ break;
case ISD::SIGN_EXTEND_INREG:
ExpandOp(Node->getOperand(0), Lo, Hi);
@@ -5652,16 +5673,17 @@
break;
}
- case ISD::READCYCLECOUNTER:
+ case ISD::READCYCLECOUNTER: {
assert(TLI.getOperationAction(ISD::READCYCLECOUNTER, VT) ==
TargetLowering::Custom &&
"Must custom expand ReadCycleCounter");
- Lo = TLI.LowerOperation(Op, DAG);
- assert(Lo.Val && "Node must be custom expanded!");
- Hi = Lo.getValue(1);
+ SDOperand Tmp = TLI.LowerOperation(Op, DAG);
+ assert(Tmp.Val && "Node must be custom expanded!");
+ ExpandOp(Tmp.getValue(0), Lo, Hi);
AddLegalizedOperand(SDOperand(Node, 1), // Remember we legalized the chain.
- LegalizeOp(Lo.getValue(2)));
+ LegalizeOp(Tmp.getValue(1)));
break;
+ }
// These operators cannot be expanded directly, emit them as calls to
// library functions.
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAGTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAGTypes.cpp
index bb65a67..5be8253 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAGTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAGTypes.cpp
@@ -115,7 +115,8 @@
private:
void MarkNewNodes(SDNode *N);
- void ReplaceLegalValueWith(SDOperand From, SDOperand To);
+ void ReplaceValueWith(SDOperand From, SDOperand To);
+ void ReplaceNodeWith(SDNode *From, SDNode *To);
void RemapNode(SDOperand &N);
@@ -167,6 +168,7 @@
void ExpandResult_UNDEF (SDNode *N, SDOperand &Lo, SDOperand &Hi);
void ExpandResult_Constant (SDNode *N, SDOperand &Lo, SDOperand &Hi);
void ExpandResult_BUILD_PAIR (SDNode *N, SDOperand &Lo, SDOperand &Hi);
+ void ExpandResult_MERGE_VALUES(SDNode *N, SDOperand &Lo, SDOperand &Hi);
void ExpandResult_ANY_EXTEND (SDNode *N, SDOperand &Lo, SDOperand &Hi);
void ExpandResult_ZERO_EXTEND(SDNode *N, SDOperand &Lo, SDOperand &Hi);
void ExpandResult_SIGN_EXTEND(SDNode *N, SDOperand &Lo, SDOperand &Hi);
@@ -391,10 +393,10 @@
Worklist.push_back(N);
}
-/// ReplaceLegalValueWith - The specified value with a legal type was legalized
-/// to the specified other value. If they are different, update the DAG and
-/// NodeIDs replacing any uses of From to use To instead.
-void DAGTypeLegalizer::ReplaceLegalValueWith(SDOperand From, SDOperand To) {
+/// ReplaceValueWith - The specified value was legalized to the specified other
+/// value. If they are different, update the DAG and NodeIDs replacing any uses
+/// of From to use To instead.
+void DAGTypeLegalizer::ReplaceValueWith(SDOperand From, SDOperand To) {
if (From == To) return;
// If expansion produced new nodes, make sure they are properly marked.
@@ -410,8 +412,8 @@
ReplacedNodes[From] = To;
// Since we just made an unstructured update to the DAG, which could wreak
- // general havoc on anything that once used N and now uses Res, walk all users
- // of the result, updating their flags.
+ // general havoc on anything that once used From and now uses To, walk all
+ // users of the result, updating their flags.
for (SDNode::use_iterator I = To.Val->use_begin(), E = To.Val->use_end();
I != E; ++I) {
SDNode *User = *I;
@@ -425,14 +427,51 @@
}
}
+/// ReplaceNodeWith - Replace uses of the 'from' node's results with the 'to'
+/// node's results. The from and to node must define identical result types.
+void DAGTypeLegalizer::ReplaceNodeWith(SDNode *From, SDNode *To) {
+ if (From == To) return;
+ assert(From->getNumValues() == To->getNumValues() &&
+ "Node results don't match");
+
+ // If expansion produced new nodes, make sure they are properly marked.
+ if (To->getNodeId() == NewNode)
+ MarkNewNodes(To);
+
+ // Anything that used the old node should now use the new one. Note that this
+ // can potentially cause recursive merging.
+ DAG.ReplaceAllUsesWith(From, To);
+
+ // The old node may still be present in ExpandedNodes or PromotedNodes.
+ // Inform them about the replacement.
+ for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
+ assert(From->getValueType(i) == To->getValueType(i) &&
+ "Node results don't match");
+ ReplacedNodes[SDOperand(From, i)] = SDOperand(To, i);
+ }
+
+ // Since we just made an unstructured update to the DAG, which could wreak
+ // general havoc on anything that once used From and now uses To, walk all
+ // users of the result, updating their flags.
+ for (SDNode::use_iterator I = To->use_begin(), E = To->use_end();I != E; ++I){
+ SDNode *User = *I;
+ // If the node isn't already processed or in the worklist, mark it as new,
+ // then use MarkNewNodes to recompute its ID.
+ int NodeId = User->getNodeId();
+ if (NodeId != ReadyToProcess && NodeId != Processed) {
+ User->setNodeId(NewNode);
+ MarkNewNodes(User);
+ }
+ }
+}
+
+
/// RemapNode - If the specified value was already legalized to another value,
/// replace it by that value.
void DAGTypeLegalizer::RemapNode(SDOperand &N) {
- DenseMap<SDOperand, SDOperand>::iterator I = ReplacedNodes.find(N);
- if (I != ReplacedNodes.end()) {
- RemapNode(I->second);
+ for (DenseMap<SDOperand, SDOperand>::iterator I = ReplacedNodes.find(N);
+ I != ReplacedNodes.end(); I = ReplacedNodes.find(N))
N = I->second;
- }
}
void DAGTypeLegalizer::SetPromotedOp(SDOperand Op, SDOperand Result) {
@@ -712,7 +751,7 @@
// Legalized the chain result - switch anything that used the old chain to
// use the new one.
- ReplaceLegalValueWith(SDOperand(N, 1), Res.getValue(1));
+ ReplaceValueWith(SDOperand(N, 1), Res.getValue(1));
return Res;
}
@@ -798,15 +837,14 @@
SDOperand Lo, Hi;
Lo = Hi = SDOperand();
- // If this is a single-result node, see if the target wants to custom expand
- // it.
- if (N->getNumValues() == 1 &&
- TLI.getOperationAction(N->getOpcode(),
- N->getValueType(0)) == TargetLowering::Custom) {
+ // See if the target wants to custom expand this node.
+ if (TLI.getOperationAction(N->getOpcode(), N->getValueType(0)) ==
+ TargetLowering::Custom) {
// If the target wants to, allow it to lower this itself.
- std::pair<SDOperand,SDOperand> P = TLI.ExpandOperationResult(N, DAG);
- if (P.first.Val) {
- SetExpandedOp(SDOperand(N, ResNo), P.first, P.second);
+ if (SDNode *P = TLI.ExpandOperationResult(N, DAG)) {
+ // Everything that once used N now uses P. P had better not require
+ // custom expansion.
+ ReplaceNodeWith(N, P);
return;
}
}
@@ -817,12 +855,13 @@
cerr << "ExpandResult #" << ResNo << ": ";
N->dump(&DAG); cerr << "\n";
#endif
- assert(0 && "Do not know how to expand this operator!");
+ assert(0 && "Do not know how to expand the result of this operator!");
abort();
case ISD::UNDEF: ExpandResult_UNDEF(N, Lo, Hi); break;
case ISD::Constant: ExpandResult_Constant(N, Lo, Hi); break;
case ISD::BUILD_PAIR: ExpandResult_BUILD_PAIR(N, Lo, Hi); break;
+ case ISD::MERGE_VALUES: ExpandResult_MERGE_VALUES(N, Lo, Hi); break;
case ISD::ANY_EXTEND: ExpandResult_ANY_EXTEND(N, Lo, Hi); break;
case ISD::ZERO_EXTEND: ExpandResult_ZERO_EXTEND(N, Lo, Hi); break;
case ISD::SIGN_EXTEND: ExpandResult_SIGN_EXTEND(N, Lo, Hi); break;
@@ -846,7 +885,6 @@
case ISD::SHL:
case ISD::SRA:
case ISD::SRL: ExpandResult_Shift(N, Lo, Hi); break;
-
}
// If Lo/Hi is null, the sub-method took care of registering results etc.
@@ -875,6 +913,27 @@
Hi = N->getOperand(1);
}
+void DAGTypeLegalizer::ExpandResult_MERGE_VALUES(SDNode *N,
+ SDOperand &Lo, SDOperand &Hi) {
+ // A MERGE_VALUES node can produce any number of values. We know that the
+ // first illegal one needs to be expanded into Lo/Hi.
+ unsigned i;
+
+ // The string of legal results gets turns into the input operands, which have
+ // the same type.
+ for (i = 0; isTypeLegal(N->getValueType(i)); ++i)
+ ReplaceValueWith(SDOperand(N, i), SDOperand(N->getOperand(i)));
+
+ // The first illegal result must be the one that needs to be expanded.
+ GetExpandedOp(N->getOperand(i), Lo, Hi);
+
+ // Legalize the rest of the results into the input operands whether they are
+ // legal or not.
+ unsigned e = N->getNumValues();
+ for (++i; i != e; ++i)
+ ReplaceValueWith(SDOperand(N, i), SDOperand(N->getOperand(i)));
+}
+
void DAGTypeLegalizer::ExpandResult_ANY_EXTEND(SDNode *N,
SDOperand &Lo, SDOperand &Hi) {
MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
@@ -1096,7 +1155,7 @@
// Legalized the chain result - switch anything that used the old chain to
// use the new one.
- ReplaceLegalValueWith(SDOperand(N, 1), Ch);
+ ReplaceValueWith(SDOperand(N, 1), Ch);
}
void DAGTypeLegalizer::ExpandResult_Logical(SDNode *N,
@@ -1184,7 +1243,7 @@
// Legalized the flag result - switch anything that used the old flag to
// use the new one.
- ReplaceLegalValueWith(SDOperand(N, 1), Hi.getValue(1));
+ ReplaceValueWith(SDOperand(N, 1), Hi.getValue(1));
}
void DAGTypeLegalizer::ExpandResult_ADDSUBE(SDNode *N,
@@ -1203,7 +1262,7 @@
// Legalized the flag result - switch anything that used the old flag to
// use the new one.
- ReplaceLegalValueWith(SDOperand(N, 1), Hi.getValue(1));
+ ReplaceValueWith(SDOperand(N, 1), Hi.getValue(1));
}
void DAGTypeLegalizer::ExpandResult_MUL(SDNode *N,
@@ -1537,7 +1596,7 @@
assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
"Invalid operand expansion");
- ReplaceLegalValueWith(SDOperand(N, 0), Res);
+ ReplaceValueWith(SDOperand(N, 0), Res);
return false;
}
@@ -1759,7 +1818,7 @@
assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
"Invalid operand expansion");
- ReplaceLegalValueWith(SDOperand(N, 0), Res);
+ ReplaceValueWith(SDOperand(N, 0), Res);
return false;
}
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index ade0a28..270c9a9 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -1921,7 +1921,7 @@
if (Cases.size()>=2)
// Must recompute end() each iteration because it may be
// invalidated by erase if we hold on to it
- for (CaseItr I=Cases.begin(), J=next(Cases.begin()); J!=Cases.end(); ) {
+ for (CaseItr I=Cases.begin(), J=++(Cases.begin()); J!=Cases.end(); ) {
int64_t nextValue = cast<ConstantInt>(J->Low)->getSExtValue();
int64_t currentValue = cast<ConstantInt>(I->High)->getSExtValue();
MachineBasicBlock* nextBB = J->BB;
@@ -4144,14 +4144,6 @@
return SDOperand();
}
-std::pair<SDOperand,SDOperand>
-TargetLowering::ExpandOperationResult(SDNode *N, SelectionDAG &DAG) {
- assert(0 && "ExpandOperation not implemented for this target!");
- abort();
- return std::pair<SDOperand,SDOperand>();
-}
-
-
SDOperand TargetLowering::CustomPromoteOperation(SDOperand Op,
SelectionDAG &DAG) {
assert(0 && "CustomPromoteOperation not implemented for this target!");