blob: c43f96ea83bd8e5be96bfd8762613ee6fb1eea36 [file] [log] [blame]
Chris Lattnerc3aae252005-01-07 07:46:32 +00001//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
Misha Brukmanedf128a2005-04-21 22:36:52 +00002//
John Criswellb576c942003-10-20 19:43:21 +00003// The LLVM Compiler Infrastructure
4//
Chris Lattner4ee451d2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Misha Brukmanedf128a2005-04-21 22:36:52 +00007//
John Criswellb576c942003-10-20 19:43:21 +00008//===----------------------------------------------------------------------===//
Chris Lattner78ec3112003-08-11 14:57:33 +00009//
Chris Lattnerc3aae252005-01-07 07:46:32 +000010// This implements the SelectionDAG class.
Chris Lattner78ec3112003-08-11 14:57:33 +000011//
12//===----------------------------------------------------------------------===//
Bill Wendlinge36025e2009-12-21 19:34:59 +000013
Chris Lattner78ec3112003-08-11 14:57:33 +000014#include "llvm/CodeGen/SelectionDAG.h"
Evan Chenga8efe282010-03-14 19:56:39 +000015#include "SDNodeDbgValue.h"
Chandler Carruthd04a8d42012-12-03 16:50:05 +000016#include "SDNodeOrdering.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/StringExtras.h"
Chandler Carruthbe049292013-01-07 03:08:10 +000022#include "llvm/Analysis/TargetTransformInfo.h"
Chandler Carruthd04a8d42012-12-03 16:50:05 +000023#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/Assembly/Writer.h"
Chandler Carruthd04a8d42012-12-03 16:50:05 +000025#include "llvm/CodeGen/MachineBasicBlock.h"
26#include "llvm/CodeGen/MachineConstantPool.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/CodeGen/MachineModuleInfo.h"
Bill Wendling0bcbd1d2012-06-28 00:05:13 +000029#include "llvm/DebugInfo.h"
Chandler Carruth0b8c9a82013-01-02 11:36:10 +000030#include "llvm/IR/CallingConv.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/DataLayout.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/GlobalAlias.h"
36#include "llvm/IR/GlobalVariable.h"
37#include "llvm/IR/Intrinsics.h"
Bill Wendling6f287b22008-09-30 21:22:07 +000038#include "llvm/Support/CommandLine.h"
David Greene55d146e2010-01-05 01:24:36 +000039#include "llvm/Support/Debug.h"
Torok Edwinc25e7582009-07-11 20:10:48 +000040#include "llvm/Support/ErrorHandling.h"
Owen Andersonb4459082009-06-25 17:09:00 +000041#include "llvm/Support/ManagedStatic.h"
Chris Lattner944fac72008-08-23 22:23:09 +000042#include "llvm/Support/MathExtras.h"
Michael J. Spencer1f6efa32010-11-29 18:16:10 +000043#include "llvm/Support/Mutex.h"
Chandler Carruthd04a8d42012-12-03 16:50:05 +000044#include "llvm/Support/raw_ostream.h"
45#include "llvm/Target/TargetInstrInfo.h"
46#include "llvm/Target/TargetIntrinsicInfo.h"
47#include "llvm/Target/TargetLowering.h"
48#include "llvm/Target/TargetMachine.h"
49#include "llvm/Target/TargetOptions.h"
50#include "llvm/Target/TargetRegisterInfo.h"
51#include "llvm/Target/TargetSelectionDAGInfo.h"
Jeff Cohenfd161e92005-01-09 20:41:56 +000052#include <algorithm>
Jeff Cohen97af7512006-12-02 02:22:01 +000053#include <cmath>
Chris Lattnere25738c2004-06-02 04:28:06 +000054using namespace llvm;
Brian Gaeked0fde302003-11-11 22:41:34 +000055
Chris Lattner0b3e5252006-08-15 19:11:05 +000056/// makeVTList - Return an instance of the SDVTList struct initialized with the
57/// specified members.
Owen Andersone50ed302009-08-10 22:56:29 +000058static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
Chris Lattner0b3e5252006-08-15 19:11:05 +000059 SDVTList Res = {VTs, NumVTs};
60 return Res;
61}
62
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +000063// Default null implementations of the callbacks.
64void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
65void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
Chris Lattnerf8dc0612008-02-03 06:49:24 +000066
Jim Laskey58b968b2005-08-17 20:08:02 +000067//===----------------------------------------------------------------------===//
68// ConstantFPSDNode Class
69//===----------------------------------------------------------------------===//
70
71/// isExactlyValue - We don't rely on operator== working on double values, as
72/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
73/// As such, this method can be used to do an exact bit-for-bit comparison of
74/// two floating point values.
Dale Johannesene6c17422007-08-26 01:18:27 +000075bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
Dan Gohman4fbd7962008-09-12 18:08:03 +000076 return getValueAPF().bitwiseIsEqual(V);
Jim Laskey58b968b2005-08-17 20:08:02 +000077}
78
Owen Andersone50ed302009-08-10 22:56:29 +000079bool ConstantFPSDNode::isValueValidForType(EVT VT,
Dale Johannesenf04afdb2007-08-30 00:23:21 +000080 const APFloat& Val) {
Duncan Sands83ec4b62008-06-06 12:08:01 +000081 assert(VT.isFloatingPoint() && "Can only convert between FP types");
Scott Michelfdc40a02009-02-17 22:15:04 +000082
Dale Johannesenf04afdb2007-08-30 00:23:21 +000083 // convert modifies in place, so make a copy.
84 APFloat Val2 = APFloat(Val);
Dale Johannesen23a98552008-10-09 23:00:39 +000085 bool losesInfo;
Tim Northover0a29cb02013-01-22 09:46:31 +000086 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
87 APFloat::rmNearestTiesToEven,
Dale Johannesen23a98552008-10-09 23:00:39 +000088 &losesInfo);
89 return !losesInfo;
Dale Johannesenf04afdb2007-08-30 00:23:21 +000090}
91
Jim Laskey58b968b2005-08-17 20:08:02 +000092//===----------------------------------------------------------------------===//
Chris Lattner61d43992006-03-25 22:57:01 +000093// ISD Namespace
Jim Laskey58b968b2005-08-17 20:08:02 +000094//===----------------------------------------------------------------------===//
Chris Lattner5cdcc582005-01-09 20:52:51 +000095
Evan Chenga8df1662006-03-27 06:58:47 +000096/// isBuildVectorAllOnes - Return true if the specified node is a
Chris Lattner61d43992006-03-25 22:57:01 +000097/// BUILD_VECTOR where all of the elements are ~0 or undef.
Evan Chenga8df1662006-03-27 06:58:47 +000098bool ISD::isBuildVectorAllOnes(const SDNode *N) {
Chris Lattner547a16f2006-04-15 23:38:00 +000099 // Look through a bit convert.
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000100 if (N->getOpcode() == ISD::BITCAST)
Gabor Greifba36cb52008-08-28 21:40:38 +0000101 N = N->getOperand(0).getNode();
Scott Michelfdc40a02009-02-17 22:15:04 +0000102
Evan Chenga8df1662006-03-27 06:58:47 +0000103 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
Scott Michelfdc40a02009-02-17 22:15:04 +0000104
Chris Lattner61d43992006-03-25 22:57:01 +0000105 unsigned i = 0, e = N->getNumOperands();
Scott Michelfdc40a02009-02-17 22:15:04 +0000106
Chris Lattner61d43992006-03-25 22:57:01 +0000107 // Skip over all of the undef values.
108 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
109 ++i;
Scott Michelfdc40a02009-02-17 22:15:04 +0000110
Chris Lattner61d43992006-03-25 22:57:01 +0000111 // Do not accept an all-undef vector.
112 if (i == e) return false;
Scott Michelfdc40a02009-02-17 22:15:04 +0000113
Chris Lattner61d43992006-03-25 22:57:01 +0000114 // Do not accept build_vectors that aren't all constants or which have non-~0
Jim Grosbach331ff3b2012-03-21 17:48:04 +0000115 // elements. We have to be a bit careful here, as the type of the constant
116 // may not be the same as the type of the vector elements due to type
117 // legalization (the elements are promoted to a legal type for the target and
118 // a vector of a type may be legal when the base element type is not).
119 // We only want to check enough bits to cover the vector elements, because
120 // we care if the resultant vector is all ones, not whether the individual
121 // constants are.
Dan Gohman475871a2008-07-27 21:46:04 +0000122 SDValue NotZero = N->getOperand(i);
Jim Grosbach331ff3b2012-03-21 17:48:04 +0000123 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
Jakub Staszak554c6762012-09-30 21:24:57 +0000124 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
125 if (CN->getAPIntValue().countTrailingOnes() < EltSize)
Evan Chenga8df1662006-03-27 06:58:47 +0000126 return false;
Jakub Staszak554c6762012-09-30 21:24:57 +0000127 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
128 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
Dan Gohman6c231502008-02-29 01:47:35 +0000129 return false;
Evan Chenga8df1662006-03-27 06:58:47 +0000130 } else
Chris Lattner61d43992006-03-25 22:57:01 +0000131 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +0000132
Chris Lattner61d43992006-03-25 22:57:01 +0000133 // Okay, we have at least one ~0 value, check to see if the rest match or are
Jim Grosbach331ff3b2012-03-21 17:48:04 +0000134 // undefs. Even with the above element type twiddling, this should be OK, as
135 // the same type legalization should have applied to all the elements.
Chris Lattner61d43992006-03-25 22:57:01 +0000136 for (++i; i != e; ++i)
137 if (N->getOperand(i) != NotZero &&
138 N->getOperand(i).getOpcode() != ISD::UNDEF)
139 return false;
140 return true;
141}
142
143
Evan Cheng4a147842006-03-26 09:50:58 +0000144/// isBuildVectorAllZeros - Return true if the specified node is a
145/// BUILD_VECTOR where all of the elements are 0 or undef.
146bool ISD::isBuildVectorAllZeros(const SDNode *N) {
Chris Lattner547a16f2006-04-15 23:38:00 +0000147 // Look through a bit convert.
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000148 if (N->getOpcode() == ISD::BITCAST)
Gabor Greifba36cb52008-08-28 21:40:38 +0000149 N = N->getOperand(0).getNode();
Scott Michelfdc40a02009-02-17 22:15:04 +0000150
Evan Cheng4a147842006-03-26 09:50:58 +0000151 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
Scott Michelfdc40a02009-02-17 22:15:04 +0000152
Evan Chenga8df1662006-03-27 06:58:47 +0000153 unsigned i = 0, e = N->getNumOperands();
Scott Michelfdc40a02009-02-17 22:15:04 +0000154
Evan Chenga8df1662006-03-27 06:58:47 +0000155 // Skip over all of the undef values.
156 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
157 ++i;
Scott Michelfdc40a02009-02-17 22:15:04 +0000158
Evan Chenga8df1662006-03-27 06:58:47 +0000159 // Do not accept an all-undef vector.
160 if (i == e) return false;
Scott Michelfdc40a02009-02-17 22:15:04 +0000161
Dan Gohman68f32cb2009-06-04 16:49:15 +0000162 // Do not accept build_vectors that aren't all constants or which have non-0
Evan Chenga8df1662006-03-27 06:58:47 +0000163 // elements.
Dan Gohman475871a2008-07-27 21:46:04 +0000164 SDValue Zero = N->getOperand(i);
Jakub Staszak554c6762012-09-30 21:24:57 +0000165 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
166 if (!CN->isNullValue())
Evan Chenga8df1662006-03-27 06:58:47 +0000167 return false;
Jakub Staszak554c6762012-09-30 21:24:57 +0000168 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
169 if (!CFPN->getValueAPF().isPosZero())
Evan Chenga8df1662006-03-27 06:58:47 +0000170 return false;
171 } else
172 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +0000173
Dan Gohman68f32cb2009-06-04 16:49:15 +0000174 // Okay, we have at least one 0 value, check to see if the rest match or are
Evan Chenga8df1662006-03-27 06:58:47 +0000175 // undefs.
176 for (++i; i != e; ++i)
177 if (N->getOperand(i) != Zero &&
178 N->getOperand(i).getOpcode() != ISD::UNDEF)
179 return false;
180 return true;
Evan Cheng4a147842006-03-26 09:50:58 +0000181}
182
Evan Chengefec7512008-02-18 23:04:32 +0000183/// isScalarToVector - Return true if the specified node is a
184/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
185/// element is not an undef.
186bool ISD::isScalarToVector(const SDNode *N) {
187 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
188 return true;
189
190 if (N->getOpcode() != ISD::BUILD_VECTOR)
191 return false;
192 if (N->getOperand(0).getOpcode() == ISD::UNDEF)
193 return false;
194 unsigned NumElems = N->getNumOperands();
Mon P Wangcab98e32010-11-19 19:08:12 +0000195 if (NumElems == 1)
196 return false;
Evan Chengefec7512008-02-18 23:04:32 +0000197 for (unsigned i = 1; i < NumElems; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +0000198 SDValue V = N->getOperand(i);
Evan Chengefec7512008-02-18 23:04:32 +0000199 if (V.getOpcode() != ISD::UNDEF)
200 return false;
201 }
202 return true;
203}
204
Nadav Rotemb87bdac2012-07-15 08:38:23 +0000205/// allOperandsUndef - Return true if the node has at least one operand
206/// and all operands of the specified node are ISD::UNDEF.
207bool ISD::allOperandsUndef(const SDNode *N) {
208 // Return false if the node has no operands.
209 // This is "logically inconsistent" with the definition of "all" but
210 // is probably the desired behavior.
211 if (N->getNumOperands() == 0)
212 return false;
213
214 for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
215 if (N->getOperand(i).getOpcode() != ISD::UNDEF)
216 return false;
217
218 return true;
219}
220
Chris Lattnerc3aae252005-01-07 07:46:32 +0000221/// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
222/// when given the operation for (X op Y).
223ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
224 // To perform this operation, we just need to swap the L and G bits of the
225 // operation.
226 unsigned OldL = (Operation >> 2) & 1;
227 unsigned OldG = (Operation >> 1) & 1;
228 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
229 (OldL << 1) | // New G bit
Bill Wendlinga1dc6022008-10-19 20:51:12 +0000230 (OldG << 2)); // New L bit.
Chris Lattnerc3aae252005-01-07 07:46:32 +0000231}
232
233/// getSetCCInverse - Return the operation corresponding to !(X op Y), where
234/// 'op' is a valid SetCC operation.
235ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
236 unsigned Operation = Op;
237 if (isInteger)
238 Operation ^= 7; // Flip L, G, E bits, but not U.
239 else
240 Operation ^= 15; // Flip all of the condition bits.
Bill Wendlinga1dc6022008-10-19 20:51:12 +0000241
Chris Lattnerc3aae252005-01-07 07:46:32 +0000242 if (Operation > ISD::SETTRUE2)
Bill Wendlinga1dc6022008-10-19 20:51:12 +0000243 Operation &= ~8; // Don't let N and U bits get set.
244
Chris Lattnerc3aae252005-01-07 07:46:32 +0000245 return ISD::CondCode(Operation);
246}
247
248
249/// isSignedOp - For an integer comparison, return 1 if the comparison is a
250/// signed operation and 2 if the result is an unsigned comparison. Return zero
251/// if the operation does not depend on the sign of the input (setne and seteq).
252static int isSignedOp(ISD::CondCode Opcode) {
253 switch (Opcode) {
Torok Edwinc23197a2009-07-14 16:55:14 +0000254 default: llvm_unreachable("Illegal integer setcc operation!");
Chris Lattnerc3aae252005-01-07 07:46:32 +0000255 case ISD::SETEQ:
256 case ISD::SETNE: return 0;
257 case ISD::SETLT:
258 case ISD::SETLE:
259 case ISD::SETGT:
260 case ISD::SETGE: return 1;
261 case ISD::SETULT:
262 case ISD::SETULE:
263 case ISD::SETUGT:
264 case ISD::SETUGE: return 2;
265 }
266}
267
268/// getSetCCOrOperation - Return the result of a logical OR between different
269/// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function
270/// returns SETCC_INVALID if it is not possible to represent the resultant
271/// comparison.
272ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
273 bool isInteger) {
274 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
275 // Cannot fold a signed integer setcc with an unsigned integer setcc.
276 return ISD::SETCC_INVALID;
Misha Brukmanedf128a2005-04-21 22:36:52 +0000277
Chris Lattnerc3aae252005-01-07 07:46:32 +0000278 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
Misha Brukmanedf128a2005-04-21 22:36:52 +0000279
Chris Lattnerc3aae252005-01-07 07:46:32 +0000280 // If the N and U bits get set then the resultant comparison DOES suddenly
281 // care about orderedness, and is true when ordered.
282 if (Op > ISD::SETTRUE2)
Chris Lattnere41102b2006-05-12 17:03:46 +0000283 Op &= ~16; // Clear the U bit if the N bit is set.
Scott Michelfdc40a02009-02-17 22:15:04 +0000284
Chris Lattnere41102b2006-05-12 17:03:46 +0000285 // Canonicalize illegal integer setcc's.
286 if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
287 Op = ISD::SETNE;
Scott Michelfdc40a02009-02-17 22:15:04 +0000288
Chris Lattnerc3aae252005-01-07 07:46:32 +0000289 return ISD::CondCode(Op);
290}
291
292/// getSetCCAndOperation - Return the result of a logical AND between different
293/// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This
294/// function returns zero if it is not possible to represent the resultant
295/// comparison.
296ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
297 bool isInteger) {
298 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
299 // Cannot fold a signed setcc with an unsigned setcc.
Misha Brukmanedf128a2005-04-21 22:36:52 +0000300 return ISD::SETCC_INVALID;
Chris Lattnerc3aae252005-01-07 07:46:32 +0000301
302 // Combine all of the condition bits.
Chris Lattnera83385f2006-04-27 05:01:07 +0000303 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
Scott Michelfdc40a02009-02-17 22:15:04 +0000304
Chris Lattnera83385f2006-04-27 05:01:07 +0000305 // Canonicalize illegal integer setcc's.
306 if (isInteger) {
307 switch (Result) {
308 default: break;
Chris Lattner883a52d2006-06-28 18:29:47 +0000309 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
Dan Gohmand64a78c2008-05-14 18:17:09 +0000310 case ISD::SETOEQ: // SETEQ & SETU[LG]E
Chris Lattner883a52d2006-06-28 18:29:47 +0000311 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
312 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
313 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
Chris Lattnera83385f2006-04-27 05:01:07 +0000314 }
315 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000316
Chris Lattnera83385f2006-04-27 05:01:07 +0000317 return Result;
Chris Lattnerc3aae252005-01-07 07:46:32 +0000318}
319
Jim Laskey58b968b2005-08-17 20:08:02 +0000320//===----------------------------------------------------------------------===//
Jim Laskey583bd472006-10-27 23:46:08 +0000321// SDNode Profile Support
322//===----------------------------------------------------------------------===//
323
Jim Laskeydef69b92006-10-27 23:52:51 +0000324/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
325///
Jim Laskey583bd472006-10-27 23:46:08 +0000326static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
327 ID.AddInteger(OpC);
328}
329
330/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
331/// solely with their pointer.
Dan Gohman844731a2008-05-13 00:00:25 +0000332static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
Scott Michelfdc40a02009-02-17 22:15:04 +0000333 ID.AddPointer(VTList.VTs);
Jim Laskey583bd472006-10-27 23:46:08 +0000334}
335
Jim Laskeydef69b92006-10-27 23:52:51 +0000336/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
337///
Jim Laskey583bd472006-10-27 23:46:08 +0000338static void AddNodeIDOperands(FoldingSetNodeID &ID,
Dan Gohman475871a2008-07-27 21:46:04 +0000339 const SDValue *Ops, unsigned NumOps) {
Chris Lattner63e3f142007-02-04 07:28:00 +0000340 for (; NumOps; --NumOps, ++Ops) {
Gabor Greifba36cb52008-08-28 21:40:38 +0000341 ID.AddPointer(Ops->getNode());
Gabor Greif99a6cb92008-08-26 22:36:50 +0000342 ID.AddInteger(Ops->getResNo());
Chris Lattner63e3f142007-02-04 07:28:00 +0000343 }
Jim Laskey583bd472006-10-27 23:46:08 +0000344}
345
Dan Gohman6d9cdd52008-07-07 18:26:29 +0000346/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
347///
348static void AddNodeIDOperands(FoldingSetNodeID &ID,
349 const SDUse *Ops, unsigned NumOps) {
350 for (; NumOps; --NumOps, ++Ops) {
Dan Gohmane7852d02009-01-26 04:35:06 +0000351 ID.AddPointer(Ops->getNode());
352 ID.AddInteger(Ops->getResNo());
Dan Gohman6d9cdd52008-07-07 18:26:29 +0000353 }
354}
355
Jim Laskey583bd472006-10-27 23:46:08 +0000356static void AddNodeIDNode(FoldingSetNodeID &ID,
Scott Michelfdc40a02009-02-17 22:15:04 +0000357 unsigned short OpC, SDVTList VTList,
Dan Gohman475871a2008-07-27 21:46:04 +0000358 const SDValue *OpList, unsigned N) {
Jim Laskey583bd472006-10-27 23:46:08 +0000359 AddNodeIDOpcode(ID, OpC);
360 AddNodeIDValueTypes(ID, VTList);
361 AddNodeIDOperands(ID, OpList, N);
362}
363
Duncan Sands0dc40452008-10-27 15:30:53 +0000364/// AddNodeIDCustom - If this is an SDNode with special info, add this info to
365/// the NodeID data.
366static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000367 switch (N->getOpcode()) {
Chris Lattner2a4ed822009-06-25 21:21:14 +0000368 case ISD::TargetExternalSymbol:
369 case ISD::ExternalSymbol:
Torok Edwinc23197a2009-07-14 16:55:14 +0000370 llvm_unreachable("Should only be used on nodes with operands");
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000371 default: break; // Normal nodes don't need extra info.
372 case ISD::TargetConstant:
373 case ISD::Constant:
Dan Gohman4fbd7962008-09-12 18:08:03 +0000374 ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue());
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000375 break;
376 case ISD::TargetConstantFP:
Dale Johanneseneaf08942007-08-31 04:03:46 +0000377 case ISD::ConstantFP: {
Dan Gohman4fbd7962008-09-12 18:08:03 +0000378 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000379 break;
Dale Johanneseneaf08942007-08-31 04:03:46 +0000380 }
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000381 case ISD::TargetGlobalAddress:
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +0000382 case ISD::GlobalAddress:
383 case ISD::TargetGlobalTLSAddress:
384 case ISD::GlobalTLSAddress: {
Dan Gohmanb8d2f552008-08-20 15:58:01 +0000385 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000386 ID.AddPointer(GA->getGlobal());
387 ID.AddInteger(GA->getOffset());
Chris Lattner2a4ed822009-06-25 21:21:14 +0000388 ID.AddInteger(GA->getTargetFlags());
Pete Cooper32ecfb42012-07-30 20:23:19 +0000389 ID.AddInteger(GA->getAddressSpace());
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000390 break;
391 }
392 case ISD::BasicBlock:
393 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
394 break;
395 case ISD::Register:
396 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
397 break;
Jakob Stoklund Olesen9cf37e82012-01-18 23:52:12 +0000398 case ISD::RegisterMask:
399 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
400 break;
Dan Gohman69de1932008-02-06 22:27:42 +0000401 case ISD::SRCVALUE:
402 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
403 break;
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000404 case ISD::FrameIndex:
405 case ISD::TargetFrameIndex:
406 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
407 break;
408 case ISD::JumpTable:
409 case ISD::TargetJumpTable:
410 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
Chris Lattnerf5a55462009-06-25 21:35:31 +0000411 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000412 break;
413 case ISD::ConstantPool:
414 case ISD::TargetConstantPool: {
Dan Gohmanb8d2f552008-08-20 15:58:01 +0000415 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000416 ID.AddInteger(CP->getAlignment());
417 ID.AddInteger(CP->getOffset());
418 if (CP->isMachineConstantPoolEntry())
Jim Grosbach5405d582011-09-27 20:59:33 +0000419 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000420 else
421 ID.AddPointer(CP->getConstVal());
Chris Lattnerf5a55462009-06-25 21:35:31 +0000422 ID.AddInteger(CP->getTargetFlags());
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000423 break;
424 }
Jakob Stoklund Olesen74500bd2012-08-07 22:37:05 +0000425 case ISD::TargetIndex: {
426 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
427 ID.AddInteger(TI->getIndex());
428 ID.AddInteger(TI->getOffset());
429 ID.AddInteger(TI->getTargetFlags());
430 break;
431 }
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000432 case ISD::LOAD: {
Dan Gohmanb8d2f552008-08-20 15:58:01 +0000433 const LoadSDNode *LD = cast<LoadSDNode>(N);
Duncan Sands3b3adbb2008-06-06 12:49:32 +0000434 ID.AddInteger(LD->getMemoryVT().getRawBits());
Dan Gohmana7ce7412009-02-03 00:08:45 +0000435 ID.AddInteger(LD->getRawSubclassData());
Pete Cooper32ecfb42012-07-30 20:23:19 +0000436 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000437 break;
438 }
439 case ISD::STORE: {
Dan Gohmanb8d2f552008-08-20 15:58:01 +0000440 const StoreSDNode *ST = cast<StoreSDNode>(N);
Duncan Sands3b3adbb2008-06-06 12:49:32 +0000441 ID.AddInteger(ST->getMemoryVT().getRawBits());
Dan Gohmana7ce7412009-02-03 00:08:45 +0000442 ID.AddInteger(ST->getRawSubclassData());
Pete Cooper32ecfb42012-07-30 20:23:19 +0000443 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
Dale Johannesen2041a0e2007-03-30 21:38:07 +0000444 break;
445 }
Dan Gohman0b1d4a72008-12-23 21:37:04 +0000446 case ISD::ATOMIC_CMP_SWAP:
447 case ISD::ATOMIC_SWAP:
448 case ISD::ATOMIC_LOAD_ADD:
449 case ISD::ATOMIC_LOAD_SUB:
450 case ISD::ATOMIC_LOAD_AND:
451 case ISD::ATOMIC_LOAD_OR:
452 case ISD::ATOMIC_LOAD_XOR:
453 case ISD::ATOMIC_LOAD_NAND:
454 case ISD::ATOMIC_LOAD_MIN:
455 case ISD::ATOMIC_LOAD_MAX:
456 case ISD::ATOMIC_LOAD_UMIN:
Eli Friedman327236c2011-08-24 20:50:09 +0000457 case ISD::ATOMIC_LOAD_UMAX:
458 case ISD::ATOMIC_LOAD:
459 case ISD::ATOMIC_STORE: {
Dan Gohmanb8d2f552008-08-20 15:58:01 +0000460 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
Dan Gohmana7ce7412009-02-03 00:08:45 +0000461 ID.AddInteger(AT->getMemoryVT().getRawBits());
462 ID.AddInteger(AT->getRawSubclassData());
Pete Cooper32ecfb42012-07-30 20:23:19 +0000463 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
464 break;
465 }
466 case ISD::PREFETCH: {
467 const MemSDNode *PF = cast<MemSDNode>(N);
468 ID.AddInteger(PF->getPointerInfo().getAddrSpace());
Mon P Wang28873102008-06-25 08:15:39 +0000469 break;
Jim Laskey583bd472006-10-27 23:46:08 +0000470 }
Nate Begeman9008ca62009-04-27 18:41:29 +0000471 case ISD::VECTOR_SHUFFLE: {
472 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
Eric Christopher4d7c18c2009-08-22 00:40:45 +0000473 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
Nate Begeman9008ca62009-04-27 18:41:29 +0000474 i != e; ++i)
475 ID.AddInteger(SVN->getMaskElt(i));
476 break;
477 }
Dan Gohman8c2b5252009-10-30 01:27:03 +0000478 case ISD::TargetBlockAddress:
479 case ISD::BlockAddress: {
Michael Liao6c7ccaa2012-09-12 21:43:09 +0000480 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
481 ID.AddPointer(BA->getBlockAddress());
482 ID.AddInteger(BA->getOffset());
483 ID.AddInteger(BA->getTargetFlags());
Dan Gohman8c2b5252009-10-30 01:27:03 +0000484 break;
485 }
Mon P Wang28873102008-06-25 08:15:39 +0000486 } // end switch (N->getOpcode())
Pete Cooper32ecfb42012-07-30 20:23:19 +0000487
488 // Target specific memory nodes could also have address spaces to check.
489 if (N->isTargetMemoryOpcode())
490 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
Jim Laskey583bd472006-10-27 23:46:08 +0000491}
492
Duncan Sands0dc40452008-10-27 15:30:53 +0000493/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
494/// data.
495static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
496 AddNodeIDOpcode(ID, N->getOpcode());
497 // Add the return value info.
498 AddNodeIDValueTypes(ID, N->getVTList());
499 // Add the operand info.
500 AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands());
501
502 // Handle SDNode leafs with special info.
503 AddNodeIDCustom(ID, N);
504}
505
Dan Gohmanb8d2f552008-08-20 15:58:01 +0000506/// encodeMemSDNodeFlags - Generic routine for computing a value for use in
David Greene1157f792010-02-17 20:21:42 +0000507/// the CSE map that carries volatility, temporalness, indexing mode, and
Dan Gohmana7ce7412009-02-03 00:08:45 +0000508/// extension/truncation information.
Dan Gohmanb8d2f552008-08-20 15:58:01 +0000509///
Bill Wendlinga1dc6022008-10-19 20:51:12 +0000510static inline unsigned
David Greene1157f792010-02-17 20:21:42 +0000511encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
Pete Cooperd752e0f2011-11-08 18:42:53 +0000512 bool isNonTemporal, bool isInvariant) {
Dan Gohmana7ce7412009-02-03 00:08:45 +0000513 assert((ConvType & 3) == ConvType &&
514 "ConvType may not require more than 2 bits!");
515 assert((AM & 7) == AM &&
516 "AM may not require more than 3 bits!");
517 return ConvType |
518 (AM << 2) |
David Greene1157f792010-02-17 20:21:42 +0000519 (isVolatile << 5) |
Pete Cooperd752e0f2011-11-08 18:42:53 +0000520 (isNonTemporal << 6) |
521 (isInvariant << 7);
Dan Gohmanb8d2f552008-08-20 15:58:01 +0000522}
523
Jim Laskey583bd472006-10-27 23:46:08 +0000524//===----------------------------------------------------------------------===//
Jim Laskey58b968b2005-08-17 20:08:02 +0000525// SelectionDAG Class
526//===----------------------------------------------------------------------===//
Chris Lattnerb48da392005-01-23 04:39:44 +0000527
Duncan Sands0dc40452008-10-27 15:30:53 +0000528/// doNotCSE - Return true if CSE should not be performed for this node.
529static bool doNotCSE(SDNode *N) {
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +0000530 if (N->getValueType(0) == MVT::Glue)
Duncan Sands0dc40452008-10-27 15:30:53 +0000531 return true; // Never CSE anything that produces a flag.
532
533 switch (N->getOpcode()) {
534 default: break;
535 case ISD::HANDLENODE:
Duncan Sands0dc40452008-10-27 15:30:53 +0000536 case ISD::EH_LABEL:
Duncan Sands0dc40452008-10-27 15:30:53 +0000537 return true; // Never CSE these nodes.
538 }
539
540 // Check that remaining values produced are not flags.
541 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +0000542 if (N->getValueType(i) == MVT::Glue)
Duncan Sands0dc40452008-10-27 15:30:53 +0000543 return true; // Never CSE anything that produces a flag.
544
545 return false;
546}
547
Chris Lattner0e12e6e2005-01-07 21:09:16 +0000548/// RemoveDeadNodes - This method deletes all unreachable nodes in the
Chris Lattner190a4182006-08-04 17:45:20 +0000549/// SelectionDAG.
550void SelectionDAG::RemoveDeadNodes() {
Chris Lattner0e12e6e2005-01-07 21:09:16 +0000551 // Create a dummy node (which is not added to allnodes), that adds a reference
552 // to the root node, preventing it from being deleted.
Chris Lattner95038592005-10-05 06:35:28 +0000553 HandleSDNode Dummy(getRoot());
Chris Lattner0e12e6e2005-01-07 21:09:16 +0000554
Chris Lattner190a4182006-08-04 17:45:20 +0000555 SmallVector<SDNode*, 128> DeadNodes;
Scott Michelfdc40a02009-02-17 22:15:04 +0000556
Chris Lattner190a4182006-08-04 17:45:20 +0000557 // Add all obviously-dead nodes to the DeadNodes worklist.
Chris Lattnerde202b32005-11-09 23:47:37 +0000558 for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
Chris Lattner190a4182006-08-04 17:45:20 +0000559 if (I->use_empty())
560 DeadNodes.push_back(I);
561
Dan Gohman0fe9c6e2008-07-07 20:57:48 +0000562 RemoveDeadNodes(DeadNodes);
Scott Michelfdc40a02009-02-17 22:15:04 +0000563
Dan Gohman0fe9c6e2008-07-07 20:57:48 +0000564 // If the root changed (e.g. it was a dead load, update the root).
565 setRoot(Dummy.getValue());
566}
567
568/// RemoveDeadNodes - This method deletes the unreachable nodes in the
569/// given list, and any nodes that become unreachable as a result.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000570void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
Dan Gohman0fe9c6e2008-07-07 20:57:48 +0000571
Chris Lattner190a4182006-08-04 17:45:20 +0000572 // Process the worklist, deleting the nodes and adding their uses to the
573 // worklist.
574 while (!DeadNodes.empty()) {
Dan Gohmane7852d02009-01-26 04:35:06 +0000575 SDNode *N = DeadNodes.pop_back_val();
Scott Michelfdc40a02009-02-17 22:15:04 +0000576
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000577 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
578 DUL->NodeDeleted(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +0000579
Chris Lattner190a4182006-08-04 17:45:20 +0000580 // Take the node out of the appropriate CSE map.
581 RemoveNodeFromCSEMaps(N);
582
583 // Next, brutally remove the operand list. This is safe to do, as there are
584 // no cycles in the graph.
Dan Gohmane7852d02009-01-26 04:35:06 +0000585 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
586 SDUse &Use = *I++;
587 SDNode *Operand = Use.getNode();
588 Use.set(SDValue());
589
Chris Lattner190a4182006-08-04 17:45:20 +0000590 // Now that we removed this operand, see if there are no uses of it left.
591 if (Operand->use_empty())
592 DeadNodes.push_back(Operand);
Chris Lattnerf469cb62005-11-08 18:52:27 +0000593 }
Bill Wendlinga1dc6022008-10-19 20:51:12 +0000594
Dan Gohmanc5336122009-01-19 22:39:36 +0000595 DeallocateNode(N);
Chris Lattnerf469cb62005-11-08 18:52:27 +0000596 }
Chris Lattner0e12e6e2005-01-07 21:09:16 +0000597}
598
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000599void SelectionDAG::RemoveDeadNode(SDNode *N){
Dan Gohmane8be6c62008-07-17 19:10:17 +0000600 SmallVector<SDNode*, 16> DeadNodes(1, N);
Eli Friedman2efa35f2011-11-08 01:25:24 +0000601
602 // Create a dummy node that adds a reference to the root node, preventing
603 // it from being deleted. (This matters if the root is an operand of the
604 // dead node.)
605 HandleSDNode Dummy(getRoot());
606
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000607 RemoveDeadNodes(DeadNodes);
Evan Cheng130a6472006-10-12 20:34:05 +0000608}
609
Chris Lattnerc26aefa2005-08-29 21:59:31 +0000610void SelectionDAG::DeleteNode(SDNode *N) {
Chris Lattnerc26aefa2005-08-29 21:59:31 +0000611 // First take this out of the appropriate CSE map.
612 RemoveNodeFromCSEMaps(N);
613
Scott Michelfdc40a02009-02-17 22:15:04 +0000614 // Finally, remove uses due to operands of this node, remove from the
Chris Lattner1e111c72005-09-07 05:37:01 +0000615 // AllNodes list, and delete the node.
616 DeleteNodeNotInCSEMaps(N);
617}
618
619void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
Dan Gohmane77f89d2009-01-25 16:20:37 +0000620 assert(N != AllNodes.begin() && "Cannot delete the entry node!");
621 assert(N->use_empty() && "Cannot delete a node that is not dead!");
Dan Gohmanc5336122009-01-19 22:39:36 +0000622
Dan Gohmanf06c8352008-09-30 18:30:35 +0000623 // Drop all of the operands and decrement used node's use counts.
Dan Gohmane7852d02009-01-26 04:35:06 +0000624 N->DropOperands();
Bill Wendlinga1dc6022008-10-19 20:51:12 +0000625
Dan Gohmanc5336122009-01-19 22:39:36 +0000626 DeallocateNode(N);
627}
628
629void SelectionDAG::DeallocateNode(SDNode *N) {
630 if (N->OperandsNeedDelete)
Chris Lattner63e3f142007-02-04 07:28:00 +0000631 delete[] N->OperandList;
Scott Michelfdc40a02009-02-17 22:15:04 +0000632
Dan Gohmanc5336122009-01-19 22:39:36 +0000633 // Set the opcode to DELETED_NODE to help catch bugs when node
634 // memory is reallocated.
635 N->NodeType = ISD::DELETED_NODE;
636
Dan Gohman11467282008-08-26 01:44:34 +0000637 NodeAllocator.Deallocate(AllNodes.remove(N));
Daniel Dunbar819309e2009-12-16 20:10:05 +0000638
639 // Remove the ordering of this node.
Bill Wendling187361b2010-01-23 10:26:57 +0000640 Ordering->remove(N);
Dale Johannesenbfdf7f32010-03-10 22:13:47 +0000641
Evan Chengbfcb3052010-03-25 01:38:16 +0000642 // If any of the SDDbgValue nodes refer to this SDNode, invalidate them.
Benjamin Kramer22a54c12011-06-18 13:13:44 +0000643 ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N);
Evan Chengbfcb3052010-03-25 01:38:16 +0000644 for (unsigned i = 0, e = DbgVals.size(); i != e; ++i)
645 DbgVals[i]->setIsInvalidated();
Chris Lattnerc26aefa2005-08-29 21:59:31 +0000646}
647
Chris Lattner149c58c2005-08-16 18:17:10 +0000648/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
649/// correspond to it. This is useful when we're about to delete or repurpose
650/// the node. We don't want future request for structurally identical nodes
651/// to return N anymore.
Dan Gohman095cc292008-09-13 01:54:27 +0000652bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
Chris Lattner6621e3b2005-09-02 19:15:44 +0000653 bool Erased = false;
Chris Lattner0e12e6e2005-01-07 21:09:16 +0000654 switch (N->getOpcode()) {
Dan Gohman095cc292008-09-13 01:54:27 +0000655 case ISD::HANDLENODE: return false; // noop.
Chris Lattner7cf7e3f2005-08-09 20:20:18 +0000656 case ISD::CONDCODE:
657 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
658 "Cond code doesn't exist!");
Chris Lattner6621e3b2005-09-02 19:15:44 +0000659 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0;
Chris Lattner7cf7e3f2005-08-09 20:20:18 +0000660 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0;
661 break;
Bill Wendling056292f2008-09-16 21:48:12 +0000662 case ISD::ExternalSymbol:
663 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
Chris Lattner0e12e6e2005-01-07 21:09:16 +0000664 break;
Chris Lattner1af22312009-06-25 18:45:50 +0000665 case ISD::TargetExternalSymbol: {
666 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
667 Erased = TargetExternalSymbols.erase(
668 std::pair<std::string,unsigned char>(ESN->getSymbol(),
669 ESN->getTargetFlags()));
Andrew Lenharth2a2de662005-10-23 03:40:17 +0000670 break;
Chris Lattner1af22312009-06-25 18:45:50 +0000671 }
Duncan Sandsf411b832007-10-17 13:49:58 +0000672 case ISD::VALUETYPE: {
Owen Andersone50ed302009-08-10 22:56:29 +0000673 EVT VT = cast<VTSDNode>(N)->getVT();
Duncan Sands83ec4b62008-06-06 12:08:01 +0000674 if (VT.isExtended()) {
Duncan Sandsf411b832007-10-17 13:49:58 +0000675 Erased = ExtendedValueTypeNodes.erase(VT);
676 } else {
Owen Anderson825b72b2009-08-11 20:47:22 +0000677 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != 0;
678 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = 0;
Duncan Sandsf411b832007-10-17 13:49:58 +0000679 }
Chris Lattner15e4b012005-07-10 00:07:11 +0000680 break;
Duncan Sandsf411b832007-10-17 13:49:58 +0000681 }
Chris Lattner0e12e6e2005-01-07 21:09:16 +0000682 default:
Chris Lattner4a283e92006-08-11 18:38:11 +0000683 // Remove it from the CSE Map.
Duncan Sandsa30b7d22010-12-12 13:22:50 +0000684 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
685 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
Chris Lattner4a283e92006-08-11 18:38:11 +0000686 Erased = CSEMap.RemoveNode(N);
Chris Lattner0e12e6e2005-01-07 21:09:16 +0000687 break;
688 }
Chris Lattner6621e3b2005-09-02 19:15:44 +0000689#ifndef NDEBUG
Scott Michelfdc40a02009-02-17 22:15:04 +0000690 // Verify that the node was actually in one of the CSE maps, unless it has a
Chris Lattner6621e3b2005-09-02 19:15:44 +0000691 // flag result (which cannot be CSE'd) or is one of the special cases that are
692 // not subject to CSE.
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +0000693 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
Duncan Sands0dc40452008-10-27 15:30:53 +0000694 !N->isMachineOpcode() && !doNotCSE(N)) {
Dan Gohmanb5bec2b2007-06-19 14:13:56 +0000695 N->dump(this);
David Greene55d146e2010-01-05 01:24:36 +0000696 dbgs() << "\n";
Torok Edwinc23197a2009-07-14 16:55:14 +0000697 llvm_unreachable("Node is not in map!");
Chris Lattner6621e3b2005-09-02 19:15:44 +0000698 }
699#endif
Dan Gohman095cc292008-09-13 01:54:27 +0000700 return Erased;
Chris Lattner0e12e6e2005-01-07 21:09:16 +0000701}
702
Dan Gohman39946102009-01-25 16:29:12 +0000703/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
704/// maps and modified in place. Add it back to the CSE maps, unless an identical
705/// node already exists, in which case transfer all its users to the existing
706/// node. This transfer can potentially trigger recursive merging.
Chris Lattner8b8749f2005-08-17 19:00:20 +0000707///
Dan Gohman39946102009-01-25 16:29:12 +0000708void
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000709SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
Dan Gohman39946102009-01-25 16:29:12 +0000710 // For node types that aren't CSE'd, just act as if no identical node
711 // already exists.
712 if (!doNotCSE(N)) {
713 SDNode *Existing = CSEMap.GetOrInsertNode(N);
714 if (Existing != N) {
715 // If there was already an existing matching node, use ReplaceAllUsesWith
716 // to replace the dead one with the existing one. This can cause
717 // recursive merging of other unrelated nodes down the line.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000718 ReplaceAllUsesWith(N, Existing);
Evan Cheng71e86852008-07-08 20:06:39 +0000719
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000720 // N is now dead. Inform the listeners and delete it.
721 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
722 DUL->NodeDeleted(N, Existing);
Dan Gohman39946102009-01-25 16:29:12 +0000723 DeleteNodeNotInCSEMaps(N);
724 return;
725 }
726 }
Evan Cheng71e86852008-07-08 20:06:39 +0000727
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000728 // If the node doesn't already exist, we updated it. Inform listeners.
729 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
730 DUL->NodeUpdated(N);
Chris Lattner8b8749f2005-08-17 19:00:20 +0000731}
732
Chris Lattnerdf6eb302006-01-28 09:32:45 +0000733/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
Scott Michelfdc40a02009-02-17 22:15:04 +0000734/// were replaced with those specified. If this node is never memoized,
Chris Lattnerdf6eb302006-01-28 09:32:45 +0000735/// return null, otherwise return a pointer to the slot it would take. If a
736/// node already exists with these operands, the slot will be non-null.
Dan Gohman475871a2008-07-27 21:46:04 +0000737SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
Chris Lattnera5682852006-08-07 23:03:03 +0000738 void *&InsertPos) {
Duncan Sands0dc40452008-10-27 15:30:53 +0000739 if (doNotCSE(N))
740 return 0;
Evan Cheng71e86852008-07-08 20:06:39 +0000741
Dan Gohman475871a2008-07-27 21:46:04 +0000742 SDValue Ops[] = { Op };
Jim Laskey583bd472006-10-27 23:46:08 +0000743 FoldingSetNodeID ID;
Chris Lattner63e3f142007-02-04 07:28:00 +0000744 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1);
Duncan Sands0dc40452008-10-27 15:30:53 +0000745 AddNodeIDCustom(ID, N);
Daniel Dunbar819309e2009-12-16 20:10:05 +0000746 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
Daniel Dunbar819309e2009-12-16 20:10:05 +0000747 return Node;
Chris Lattnerdf6eb302006-01-28 09:32:45 +0000748}
749
750/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
Scott Michelfdc40a02009-02-17 22:15:04 +0000751/// were replaced with those specified. If this node is never memoized,
Chris Lattnerdf6eb302006-01-28 09:32:45 +0000752/// return null, otherwise return a pointer to the slot it would take. If a
753/// node already exists with these operands, the slot will be non-null.
Scott Michelfdc40a02009-02-17 22:15:04 +0000754SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
Dan Gohman475871a2008-07-27 21:46:04 +0000755 SDValue Op1, SDValue Op2,
Chris Lattnera5682852006-08-07 23:03:03 +0000756 void *&InsertPos) {
Duncan Sands0dc40452008-10-27 15:30:53 +0000757 if (doNotCSE(N))
758 return 0;
759
Dan Gohman475871a2008-07-27 21:46:04 +0000760 SDValue Ops[] = { Op1, Op2 };
Jim Laskey583bd472006-10-27 23:46:08 +0000761 FoldingSetNodeID ID;
Chris Lattner63e3f142007-02-04 07:28:00 +0000762 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2);
Duncan Sands0dc40452008-10-27 15:30:53 +0000763 AddNodeIDCustom(ID, N);
Daniel Dunbar819309e2009-12-16 20:10:05 +0000764 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
Daniel Dunbar819309e2009-12-16 20:10:05 +0000765 return Node;
Chris Lattnera5682852006-08-07 23:03:03 +0000766}
767
768
769/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
Scott Michelfdc40a02009-02-17 22:15:04 +0000770/// were replaced with those specified. If this node is never memoized,
Chris Lattnera5682852006-08-07 23:03:03 +0000771/// return null, otherwise return a pointer to the slot it would take. If a
772/// node already exists with these operands, the slot will be non-null.
Scott Michelfdc40a02009-02-17 22:15:04 +0000773SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
Dan Gohman475871a2008-07-27 21:46:04 +0000774 const SDValue *Ops,unsigned NumOps,
Chris Lattnera5682852006-08-07 23:03:03 +0000775 void *&InsertPos) {
Duncan Sands0dc40452008-10-27 15:30:53 +0000776 if (doNotCSE(N))
777 return 0;
Evan Cheng71e86852008-07-08 20:06:39 +0000778
Jim Laskey583bd472006-10-27 23:46:08 +0000779 FoldingSetNodeID ID;
Dan Gohman575e2f42007-06-04 15:49:41 +0000780 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps);
Duncan Sands0dc40452008-10-27 15:30:53 +0000781 AddNodeIDCustom(ID, N);
Daniel Dunbar819309e2009-12-16 20:10:05 +0000782 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
Daniel Dunbar819309e2009-12-16 20:10:05 +0000783 return Node;
Chris Lattnerdf6eb302006-01-28 09:32:45 +0000784}
Chris Lattner8b8749f2005-08-17 19:00:20 +0000785
Benjamin Kramer3ca13632010-11-20 15:53:24 +0000786#ifndef NDEBUG
Duncan Sands59d2dad2010-11-20 11:25:00 +0000787/// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid.
788static void VerifyNodeCommon(SDNode *N) {
Duncan Sandsd038e042008-07-21 10:20:31 +0000789 switch (N->getOpcode()) {
790 default:
791 break;
Duncan Sandsd22ec5f2008-10-29 14:22:20 +0000792 case ISD::BUILD_PAIR: {
Owen Andersone50ed302009-08-10 22:56:29 +0000793 EVT VT = N->getValueType(0);
Duncan Sandsd22ec5f2008-10-29 14:22:20 +0000794 assert(N->getNumValues() == 1 && "Too many results!");
795 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
796 "Wrong return type!");
797 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
798 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
799 "Mismatched operand types!");
800 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
801 "Wrong operand type!");
802 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
803 "Wrong return type size");
804 break;
805 }
Duncan Sandsd038e042008-07-21 10:20:31 +0000806 case ISD::BUILD_VECTOR: {
Duncan Sandsd22ec5f2008-10-29 14:22:20 +0000807 assert(N->getNumValues() == 1 && "Too many results!");
808 assert(N->getValueType(0).isVector() && "Wrong return type!");
Duncan Sandsd038e042008-07-21 10:20:31 +0000809 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
Duncan Sandsd22ec5f2008-10-29 14:22:20 +0000810 "Wrong number of operands!");
Owen Andersone50ed302009-08-10 22:56:29 +0000811 EVT EltVT = N->getValueType(0).getVectorElementType();
Eli Friedman9db817f2011-09-09 21:04:06 +0000812 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
Rafael Espindola15684b22009-04-24 12:40:33 +0000813 assert((I->getValueType() == EltVT ||
Nate Begeman9008ca62009-04-27 18:41:29 +0000814 (EltVT.isInteger() && I->getValueType().isInteger() &&
815 EltVT.bitsLE(I->getValueType()))) &&
816 "Wrong operand type!");
Eli Friedman9db817f2011-09-09 21:04:06 +0000817 assert(I->getValueType() == N->getOperand(0).getValueType() &&
818 "Operands must all have the same type");
819 }
Duncan Sandsd038e042008-07-21 10:20:31 +0000820 break;
821 }
822 }
823}
824
Duncan Sands59d2dad2010-11-20 11:25:00 +0000825/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
826static void VerifySDNode(SDNode *N) {
827 // The SDNode allocators cannot be used to allocate nodes with fields that are
828 // not present in an SDNode!
829 assert(!isa<MemSDNode>(N) && "Bad MemSDNode!");
830 assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!");
831 assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!");
832 assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!");
833 assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!");
834 assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!");
835 assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!");
836 assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!");
837 assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!");
838 assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!");
839 assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!");
840 assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!");
841 assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!");
842 assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!");
843 assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!");
844 assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!");
845 assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!");
846 assert(!isa<VTSDNode>(N) && "Bad VTSDNode!");
847 assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!");
848
849 VerifyNodeCommon(N);
850}
851
852/// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is
853/// invalid.
854static void VerifyMachineNode(SDNode *N) {
855 // The MachineNode allocators cannot be used to allocate nodes with fields
856 // that are not present in a MachineNode!
857 // Currently there are no such nodes.
858
859 VerifyNodeCommon(N);
860}
Benjamin Kramer3ca13632010-11-20 15:53:24 +0000861#endif // NDEBUG
Duncan Sands59d2dad2010-11-20 11:25:00 +0000862
Owen Andersone50ed302009-08-10 22:56:29 +0000863/// getEVTAlignment - Compute the default alignment value for the
Dan Gohman9c6e70e2008-07-08 23:46:32 +0000864/// given type.
865///
Owen Andersone50ed302009-08-10 22:56:29 +0000866unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
Chris Lattnerdb125cf2011-07-18 04:54:35 +0000867 Type *Ty = VT == MVT::iPTR ?
Owen Anderson1d0be152009-08-13 21:58:54 +0000868 PointerType::get(Type::getInt8Ty(*getContext()), 0) :
Owen Anderson23b9b192009-08-12 00:36:31 +0000869 VT.getTypeForEVT(*getContext());
Dan Gohman9c6e70e2008-07-08 23:46:32 +0000870
Micah Villmow3574eca2012-10-08 16:38:25 +0000871 return TLI.getDataLayout()->getABITypeAlignment(Ty);
Dan Gohman9c6e70e2008-07-08 23:46:32 +0000872}
Chris Lattner0e12e6e2005-01-07 21:09:16 +0000873
Dale Johannesen92570c42009-02-07 02:15:05 +0000874// EntryNode could meaningfully have debug info if we can find it...
Devang Patel0508d042011-12-15 18:21:18 +0000875SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
Dan Gohmanff7a5622010-05-11 17:31:57 +0000876 : TM(tm), TLI(*tm.getTargetLowering()), TSI(*tm.getSelectionDAGInfo()),
Andrew Trickac6d9be2013-05-25 02:42:55 +0000877 TTI(0), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
Chandler Carruthe4b4edd2013-01-05 12:32:17 +0000878 getVTList(MVT::Other)),
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000879 Root(getEntryNode()), Ordering(0), UpdateListeners(0) {
Dan Gohmanf350b272008-08-23 02:25:05 +0000880 AllNodes.push_back(&EntryNode);
Bill Wendling187361b2010-01-23 10:26:57 +0000881 Ordering = new SDNodeOrdering();
Dale Johannesenbfdf7f32010-03-10 22:13:47 +0000882 DbgInfo = new SDDbgInfo();
Dan Gohman6f179662008-08-23 00:50:30 +0000883}
884
Chandler Carruthe4b4edd2013-01-05 12:32:17 +0000885void SelectionDAG::init(MachineFunction &mf, const TargetTransformInfo *tti) {
Dan Gohman7c3234c2008-08-27 23:52:12 +0000886 MF = &mf;
Chandler Carruthe4b4edd2013-01-05 12:32:17 +0000887 TTI = tti;
Eric Christopher4d7c18c2009-08-22 00:40:45 +0000888 Context = &mf.getFunction()->getContext();
Dan Gohman7c3234c2008-08-27 23:52:12 +0000889}
890
Chris Lattner78ec3112003-08-11 14:57:33 +0000891SelectionDAG::~SelectionDAG() {
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000892 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
Dan Gohmanf350b272008-08-23 02:25:05 +0000893 allnodes_clear();
Daniel Dunbar819309e2009-12-16 20:10:05 +0000894 delete Ordering;
Dale Johannesenbfdf7f32010-03-10 22:13:47 +0000895 delete DbgInfo;
Dan Gohmanf350b272008-08-23 02:25:05 +0000896}
897
898void SelectionDAG::allnodes_clear() {
Dan Gohman11467282008-08-26 01:44:34 +0000899 assert(&*AllNodes.begin() == &EntryNode);
900 AllNodes.remove(AllNodes.begin());
Dan Gohmanc5336122009-01-19 22:39:36 +0000901 while (!AllNodes.empty())
902 DeallocateNode(AllNodes.begin());
Chris Lattner78ec3112003-08-11 14:57:33 +0000903}
904
Dan Gohman7c3234c2008-08-27 23:52:12 +0000905void SelectionDAG::clear() {
Dan Gohmanf350b272008-08-23 02:25:05 +0000906 allnodes_clear();
907 OperandAllocator.Reset();
908 CSEMap.clear();
909
910 ExtendedValueTypeNodes.clear();
Bill Wendling056292f2008-09-16 21:48:12 +0000911 ExternalSymbols.clear();
912 TargetExternalSymbols.clear();
Dan Gohmanf350b272008-08-23 02:25:05 +0000913 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
914 static_cast<CondCodeSDNode*>(0));
915 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
916 static_cast<SDNode*>(0));
917
Dan Gohmane7852d02009-01-26 04:35:06 +0000918 EntryNode.UseList = 0;
Dan Gohmanf350b272008-08-23 02:25:05 +0000919 AllNodes.push_back(&EntryNode);
920 Root = getEntryNode();
Dan Gohman27445f02010-06-18 15:40:58 +0000921 Ordering->clear();
Evan Cheng31441b72010-03-29 20:48:30 +0000922 DbgInfo->clear();
Dan Gohmanf350b272008-08-23 02:25:05 +0000923}
924
Andrew Trickac6d9be2013-05-25 02:42:55 +0000925SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
Nadav Rotema3c42f32011-09-27 11:16:47 +0000926 return VT.bitsGT(Op.getValueType()) ?
927 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
928 getNode(ISD::TRUNCATE, DL, VT, Op);
929}
930
Andrew Trickac6d9be2013-05-25 02:42:55 +0000931SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
Duncan Sands3a66a682009-10-13 21:04:12 +0000932 return VT.bitsGT(Op.getValueType()) ?
933 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
934 getNode(ISD::TRUNCATE, DL, VT, Op);
935}
936
Andrew Trickac6d9be2013-05-25 02:42:55 +0000937SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
Duncan Sands3a66a682009-10-13 21:04:12 +0000938 return VT.bitsGT(Op.getValueType()) ?
939 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
940 getNode(ISD::TRUNCATE, DL, VT, Op);
941}
942
Andrew Trickac6d9be2013-05-25 02:42:55 +0000943SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
Dan Gohman87862e72009-12-11 21:31:27 +0000944 assert(!VT.isVector() &&
945 "getZeroExtendInReg should use the vector element type instead of "
946 "the vector type!");
Bill Wendling6ce610f2009-01-30 22:23:15 +0000947 if (Op.getValueType() == VT) return Op;
Dan Gohman87862e72009-12-11 21:31:27 +0000948 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
949 APInt Imm = APInt::getLowBitsSet(BitWidth,
Bill Wendling6ce610f2009-01-30 22:23:15 +0000950 VT.getSizeInBits());
951 return getNode(ISD::AND, DL, Op.getValueType(), Op,
952 getConstant(Imm, Op.getValueType()));
953}
954
Bob Wilson4c245462009-01-22 17:39:32 +0000955/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
956///
Andrew Trickac6d9be2013-05-25 02:42:55 +0000957SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
Chris Lattner5cf0b6e2010-02-24 22:44:06 +0000958 EVT EltVT = VT.getScalarType();
Dan Gohmanbd209ab2009-04-20 22:51:43 +0000959 SDValue NegOne =
960 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
Bill Wendling41b9d272009-01-30 22:11:22 +0000961 return getNode(ISD::XOR, DL, VT, Val, NegOne);
962}
963
Owen Andersone50ed302009-08-10 22:56:29 +0000964SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) {
Chris Lattner5cf0b6e2010-02-24 22:44:06 +0000965 EVT EltVT = VT.getScalarType();
Dan Gohmance9bc122009-01-27 20:39:34 +0000966 assert((EltVT.getSizeInBits() >= 64 ||
967 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
968 "getConstant with a uint64_t value that doesn't fit in the type!");
Duncan Sands83ec4b62008-06-06 12:08:01 +0000969 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT);
Dan Gohman6394b092008-02-08 22:59:30 +0000970}
971
Owen Andersone50ed302009-08-10 22:56:29 +0000972SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) {
Owen Andersoneed707b2009-07-24 23:12:02 +0000973 return getConstant(*ConstantInt::get(*Context, Val), VT, isT);
Dan Gohman4fbd7962008-09-12 18:08:03 +0000974}
975
Owen Andersone50ed302009-08-10 22:56:29 +0000976SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) {
Duncan Sands83ec4b62008-06-06 12:08:01 +0000977 assert(VT.isInteger() && "Cannot create FP integer constant!");
Dan Gohman89081322007-12-12 22:21:26 +0000978
Chris Lattner5cf0b6e2010-02-24 22:44:06 +0000979 EVT EltVT = VT.getScalarType();
Duncan Sands28b77e92011-09-06 19:07:46 +0000980 const ConstantInt *Elt = &Val;
Chris Lattner61b09412006-08-11 21:01:22 +0000981
Duncan Sands28b77e92011-09-06 19:07:46 +0000982 // In some cases the vector type is legal but the element type is illegal and
983 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
984 // inserted value (the type does not need to match the vector element type).
985 // Any extra bits introduced will be truncated away.
986 if (VT.isVector() && TLI.getTypeAction(*getContext(), EltVT) ==
987 TargetLowering::TypePromoteInteger) {
988 EltVT = TLI.getTypeToTransformTo(*getContext(), EltVT);
989 APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
990 Elt = ConstantInt::get(*getContext(), NewVal);
991 }
992
993 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
994 "APInt size does not match type size!");
Chris Lattner61b09412006-08-11 21:01:22 +0000995 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
Jim Laskey583bd472006-10-27 23:46:08 +0000996 FoldingSetNodeID ID;
Dan Gohman6d9cdd52008-07-07 18:26:29 +0000997 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0);
Duncan Sands28b77e92011-09-06 19:07:46 +0000998 ID.AddPointer(Elt);
Chris Lattner61b09412006-08-11 21:01:22 +0000999 void *IP = 0;
Dan Gohman89081322007-12-12 22:21:26 +00001000 SDNode *N = NULL;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001001 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
Duncan Sands83ec4b62008-06-06 12:08:01 +00001002 if (!VT.isVector())
Dan Gohman475871a2008-07-27 21:46:04 +00001003 return SDValue(N, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001004
Dan Gohman89081322007-12-12 22:21:26 +00001005 if (!N) {
Duncan Sands28b77e92011-09-06 19:07:46 +00001006 N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT);
Dan Gohman89081322007-12-12 22:21:26 +00001007 CSEMap.InsertNode(N, IP);
1008 AllNodes.push_back(N);
1009 }
1010
Dan Gohman475871a2008-07-27 21:46:04 +00001011 SDValue Result(N, 0);
Duncan Sands83ec4b62008-06-06 12:08:01 +00001012 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001013 SmallVector<SDValue, 8> Ops;
Duncan Sands83ec4b62008-06-06 12:08:01 +00001014 Ops.assign(VT.getVectorNumElements(), Result);
Andrew Trickac6d9be2013-05-25 02:42:55 +00001015 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size());
Dan Gohman89081322007-12-12 22:21:26 +00001016 }
1017 return Result;
Chris Lattner78ec3112003-08-11 14:57:33 +00001018}
1019
Dan Gohman475871a2008-07-27 21:46:04 +00001020SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
Chris Lattner0bd48932008-01-17 07:00:52 +00001021 return getConstant(Val, TLI.getPointerTy(), isTarget);
1022}
1023
1024
Owen Andersone50ed302009-08-10 22:56:29 +00001025SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
Owen Anderson6f83c9c2009-07-27 20:59:43 +00001026 return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
Dan Gohman4fbd7962008-09-12 18:08:03 +00001027}
1028
Owen Andersone50ed302009-08-10 22:56:29 +00001029SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
Duncan Sands83ec4b62008-06-06 12:08:01 +00001030 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
Scott Michelfdc40a02009-02-17 22:15:04 +00001031
Chris Lattner5cf0b6e2010-02-24 22:44:06 +00001032 EVT EltVT = VT.getScalarType();
Chris Lattnerc3aae252005-01-07 07:46:32 +00001033
Chris Lattnerd8658612005-02-17 20:17:32 +00001034 // Do the map lookup using the actual bit pattern for the floating point
1035 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1036 // we don't have issues with SNANs.
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001037 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
Jim Laskey583bd472006-10-27 23:46:08 +00001038 FoldingSetNodeID ID;
Dan Gohman6d9cdd52008-07-07 18:26:29 +00001039 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0);
Dan Gohman4fbd7962008-09-12 18:08:03 +00001040 ID.AddPointer(&V);
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001041 void *IP = 0;
Evan Chengc908dcd2007-06-29 21:36:04 +00001042 SDNode *N = NULL;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001043 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
Duncan Sands83ec4b62008-06-06 12:08:01 +00001044 if (!VT.isVector())
Dan Gohman475871a2008-07-27 21:46:04 +00001045 return SDValue(N, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001046
Evan Chengc908dcd2007-06-29 21:36:04 +00001047 if (!N) {
Dan Gohman95531882010-03-18 18:49:47 +00001048 N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
Evan Chengc908dcd2007-06-29 21:36:04 +00001049 CSEMap.InsertNode(N, IP);
1050 AllNodes.push_back(N);
1051 }
1052
Dan Gohman475871a2008-07-27 21:46:04 +00001053 SDValue Result(N, 0);
Duncan Sands83ec4b62008-06-06 12:08:01 +00001054 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001055 SmallVector<SDValue, 8> Ops;
Duncan Sands83ec4b62008-06-06 12:08:01 +00001056 Ops.assign(VT.getVectorNumElements(), Result);
Andrew Trickac6d9be2013-05-25 02:42:55 +00001057 // FIXME SDLoc info might be appropriate here
1058 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size());
Dan Gohman7f321562007-06-25 16:23:39 +00001059 }
1060 return Result;
Chris Lattnerc3aae252005-01-07 07:46:32 +00001061}
1062
Owen Andersone50ed302009-08-10 22:56:29 +00001063SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
Chris Lattner5cf0b6e2010-02-24 22:44:06 +00001064 EVT EltVT = VT.getScalarType();
Owen Anderson825b72b2009-08-11 20:47:22 +00001065 if (EltVT==MVT::f32)
Dale Johannesenf04afdb2007-08-30 00:23:21 +00001066 return getConstantFP(APFloat((float)Val), VT, isTarget);
Dale Johannesen0a406ae2010-05-07 21:35:53 +00001067 else if (EltVT==MVT::f64)
Dale Johannesenf04afdb2007-08-30 00:23:21 +00001068 return getConstantFP(APFloat(Val), VT, isTarget);
Hal Finkel6eb7a422012-12-30 19:03:32 +00001069 else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
1070 EltVT==MVT::f16) {
Dale Johannesen0a406ae2010-05-07 21:35:53 +00001071 bool ignored;
1072 APFloat apf = APFloat(Val);
Tim Northover0a29cb02013-01-22 09:46:31 +00001073 apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
Dale Johannesen0a406ae2010-05-07 21:35:53 +00001074 &ignored);
1075 return getConstantFP(apf, VT, isTarget);
Craig Topper5e25ee82012-02-05 08:31:47 +00001076 } else
1077 llvm_unreachable("Unsupported type in getConstantFP");
Dale Johannesenf04afdb2007-08-30 00:23:21 +00001078}
1079
Andrew Trickac6d9be2013-05-25 02:42:55 +00001080SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
Owen Andersone50ed302009-08-10 22:56:29 +00001081 EVT VT, int64_t Offset,
Chris Lattner2a4ed822009-06-25 21:21:14 +00001082 bool isTargetGA,
1083 unsigned char TargetFlags) {
1084 assert((TargetFlags == 0 || isTargetGA) &&
1085 "Cannot set target flags on target-independent globals");
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001086
Dan Gohman6520e202008-10-18 02:06:02 +00001087 // Truncate (with sign-extension) the offset value to the pointer size.
Richard Smith1144af32012-08-24 23:29:28 +00001088 unsigned BitWidth = TLI.getPointerTy().getSizeInBits();
Dan Gohman6520e202008-10-18 02:06:02 +00001089 if (BitWidth < 64)
Richard Smith1144af32012-08-24 23:29:28 +00001090 Offset = SignExtend64(Offset, BitWidth);
Dan Gohman6520e202008-10-18 02:06:02 +00001091
Anton Korobeynikov4d86e2a2008-03-11 22:38:53 +00001092 const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV);
1093 if (!GVar) {
Anton Korobeynikovc73ede02008-03-22 07:53:40 +00001094 // If GV is an alias then use the aliasee for determining thread-localness.
Anton Korobeynikov4d86e2a2008-03-11 22:38:53 +00001095 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
Anton Korobeynikov19e861a2008-09-09 20:05:04 +00001096 GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false));
Anton Korobeynikov4d86e2a2008-03-11 22:38:53 +00001097 }
1098
Chris Lattner2a4ed822009-06-25 21:21:14 +00001099 unsigned Opc;
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00001100 if (GVar && GVar->isThreadLocal())
1101 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1102 else
1103 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
Anton Korobeynikov4d86e2a2008-03-11 22:38:53 +00001104
Jim Laskey583bd472006-10-27 23:46:08 +00001105 FoldingSetNodeID ID;
Dan Gohman6d9cdd52008-07-07 18:26:29 +00001106 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
Chris Lattner61b09412006-08-11 21:01:22 +00001107 ID.AddPointer(GV);
1108 ID.AddInteger(Offset);
Chris Lattner2a4ed822009-06-25 21:21:14 +00001109 ID.AddInteger(TargetFlags);
Pete Cooper32ecfb42012-07-30 20:23:19 +00001110 ID.AddInteger(GV->getType()->getAddressSpace());
Chris Lattner61b09412006-08-11 21:01:22 +00001111 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001112 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman74692c02009-01-25 16:21:38 +00001113 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001114
Andrew Trickac6d9be2013-05-25 02:42:55 +00001115 SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
1116 DL.getDebugLoc(), GV, VT,
Dan Gohman95531882010-03-18 18:49:47 +00001117 Offset, TargetFlags);
Chris Lattner61b09412006-08-11 21:01:22 +00001118 CSEMap.InsertNode(N, IP);
Chris Lattnerc3aae252005-01-07 07:46:32 +00001119 AllNodes.push_back(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001120 return SDValue(N, 0);
Chris Lattnerc3aae252005-01-07 07:46:32 +00001121}
1122
Owen Andersone50ed302009-08-10 22:56:29 +00001123SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001124 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
Jim Laskey583bd472006-10-27 23:46:08 +00001125 FoldingSetNodeID ID;
Dan Gohman6d9cdd52008-07-07 18:26:29 +00001126 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001127 ID.AddInteger(FI);
1128 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001129 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00001130 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001131
Dan Gohman95531882010-03-18 18:49:47 +00001132 SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001133 CSEMap.InsertNode(N, IP);
Chris Lattnerafb2dd42005-08-25 00:43:01 +00001134 AllNodes.push_back(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001135 return SDValue(N, 0);
Chris Lattnerafb2dd42005-08-25 00:43:01 +00001136}
1137
Owen Andersone50ed302009-08-10 22:56:29 +00001138SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
Chris Lattnerf5a55462009-06-25 21:35:31 +00001139 unsigned char TargetFlags) {
1140 assert((TargetFlags == 0 || isTarget) &&
1141 "Cannot set target flags on target-independent jump tables");
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001142 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
Jim Laskey583bd472006-10-27 23:46:08 +00001143 FoldingSetNodeID ID;
Dan Gohman6d9cdd52008-07-07 18:26:29 +00001144 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001145 ID.AddInteger(JTI);
Chris Lattnerf5a55462009-06-25 21:35:31 +00001146 ID.AddInteger(TargetFlags);
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001147 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001148 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00001149 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001150
Dan Gohman95531882010-03-18 18:49:47 +00001151 SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
1152 TargetFlags);
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001153 CSEMap.InsertNode(N, IP);
Nate Begeman37efe672006-04-22 18:53:45 +00001154 AllNodes.push_back(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001155 return SDValue(N, 0);
Nate Begeman37efe672006-04-22 18:53:45 +00001156}
1157
Dan Gohman46510a72010-04-15 01:51:59 +00001158SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
Dan Gohman475871a2008-07-27 21:46:04 +00001159 unsigned Alignment, int Offset,
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001160 bool isTarget,
Chris Lattnerf5a55462009-06-25 21:35:31 +00001161 unsigned char TargetFlags) {
1162 assert((TargetFlags == 0 || isTarget) &&
1163 "Cannot set target flags on target-independent globals");
Dan Gohman50284d82008-09-16 22:05:41 +00001164 if (Alignment == 0)
Micah Villmow3574eca2012-10-08 16:38:25 +00001165 Alignment = TLI.getDataLayout()->getPrefTypeAlignment(C->getType());
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001166 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
Jim Laskey583bd472006-10-27 23:46:08 +00001167 FoldingSetNodeID ID;
Dan Gohman6d9cdd52008-07-07 18:26:29 +00001168 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001169 ID.AddInteger(Alignment);
1170 ID.AddInteger(Offset);
Chris Lattner130fc132006-08-14 20:12:44 +00001171 ID.AddPointer(C);
Chris Lattnerf5a55462009-06-25 21:35:31 +00001172 ID.AddInteger(TargetFlags);
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001173 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001174 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00001175 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001176
Dan Gohman95531882010-03-18 18:49:47 +00001177 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
1178 Alignment, TargetFlags);
Chris Lattnerc9f8f412006-08-11 21:55:30 +00001179 CSEMap.InsertNode(N, IP);
Chris Lattner4025a9c2005-08-25 05:03:06 +00001180 AllNodes.push_back(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001181 return SDValue(N, 0);
Chris Lattner4025a9c2005-08-25 05:03:06 +00001182}
1183
Chris Lattnerc3aae252005-01-07 07:46:32 +00001184
Owen Andersone50ed302009-08-10 22:56:29 +00001185SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
Dan Gohman475871a2008-07-27 21:46:04 +00001186 unsigned Alignment, int Offset,
Chris Lattnerf5a55462009-06-25 21:35:31 +00001187 bool isTarget,
1188 unsigned char TargetFlags) {
1189 assert((TargetFlags == 0 || isTarget) &&
1190 "Cannot set target flags on target-independent globals");
Dan Gohman50284d82008-09-16 22:05:41 +00001191 if (Alignment == 0)
Micah Villmow3574eca2012-10-08 16:38:25 +00001192 Alignment = TLI.getDataLayout()->getPrefTypeAlignment(C->getType());
Evan Chengd6594ae2006-09-12 21:00:35 +00001193 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
Jim Laskey583bd472006-10-27 23:46:08 +00001194 FoldingSetNodeID ID;
Dan Gohman6d9cdd52008-07-07 18:26:29 +00001195 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
Evan Chengd6594ae2006-09-12 21:00:35 +00001196 ID.AddInteger(Alignment);
1197 ID.AddInteger(Offset);
Jim Grosbach5405d582011-09-27 20:59:33 +00001198 C->addSelectionDAGCSEId(ID);
Chris Lattnerf5a55462009-06-25 21:35:31 +00001199 ID.AddInteger(TargetFlags);
Evan Chengd6594ae2006-09-12 21:00:35 +00001200 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001201 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00001202 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001203
Dan Gohman95531882010-03-18 18:49:47 +00001204 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
1205 Alignment, TargetFlags);
Evan Chengd6594ae2006-09-12 21:00:35 +00001206 CSEMap.InsertNode(N, IP);
1207 AllNodes.push_back(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001208 return SDValue(N, 0);
Evan Chengd6594ae2006-09-12 21:00:35 +00001209}
1210
Jakob Stoklund Olesen74500bd2012-08-07 22:37:05 +00001211SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1212 unsigned char TargetFlags) {
1213 FoldingSetNodeID ID;
1214 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), 0, 0);
1215 ID.AddInteger(Index);
1216 ID.AddInteger(Offset);
1217 ID.AddInteger(TargetFlags);
1218 void *IP = 0;
1219 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1220 return SDValue(E, 0);
1221
1222 SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
1223 TargetFlags);
1224 CSEMap.InsertNode(N, IP);
1225 AllNodes.push_back(N);
1226 return SDValue(N, 0);
1227}
1228
Dan Gohman475871a2008-07-27 21:46:04 +00001229SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
Jim Laskey583bd472006-10-27 23:46:08 +00001230 FoldingSetNodeID ID;
Owen Anderson825b72b2009-08-11 20:47:22 +00001231 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0);
Chris Lattner61b09412006-08-11 21:01:22 +00001232 ID.AddPointer(MBB);
1233 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001234 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00001235 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001236
Dan Gohman95531882010-03-18 18:49:47 +00001237 SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
Chris Lattner61b09412006-08-11 21:01:22 +00001238 CSEMap.InsertNode(N, IP);
Chris Lattnerc3aae252005-01-07 07:46:32 +00001239 AllNodes.push_back(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001240 return SDValue(N, 0);
Chris Lattnerc3aae252005-01-07 07:46:32 +00001241}
1242
Owen Andersone50ed302009-08-10 22:56:29 +00001243SDValue SelectionDAG::getValueType(EVT VT) {
Owen Anderson825b72b2009-08-11 20:47:22 +00001244 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1245 ValueTypeNodes.size())
1246 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
Chris Lattner15e4b012005-07-10 00:07:11 +00001247
Duncan Sands83ec4b62008-06-06 12:08:01 +00001248 SDNode *&N = VT.isExtended() ?
Owen Anderson825b72b2009-08-11 20:47:22 +00001249 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
Duncan Sandsf411b832007-10-17 13:49:58 +00001250
Dan Gohman475871a2008-07-27 21:46:04 +00001251 if (N) return SDValue(N, 0);
Dan Gohman95531882010-03-18 18:49:47 +00001252 N = new (NodeAllocator) VTSDNode(VT);
Duncan Sandsf411b832007-10-17 13:49:58 +00001253 AllNodes.push_back(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001254 return SDValue(N, 0);
Chris Lattner15e4b012005-07-10 00:07:11 +00001255}
1256
Owen Andersone50ed302009-08-10 22:56:29 +00001257SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
Bill Wendling056292f2008-09-16 21:48:12 +00001258 SDNode *&N = ExternalSymbols[Sym];
Dan Gohman475871a2008-07-27 21:46:04 +00001259 if (N) return SDValue(N, 0);
Dan Gohman95531882010-03-18 18:49:47 +00001260 N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
Andrew Lenharth2a2de662005-10-23 03:40:17 +00001261 AllNodes.push_back(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001262 return SDValue(N, 0);
Andrew Lenharth2a2de662005-10-23 03:40:17 +00001263}
1264
Owen Andersone50ed302009-08-10 22:56:29 +00001265SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
Chris Lattner1af22312009-06-25 18:45:50 +00001266 unsigned char TargetFlags) {
1267 SDNode *&N =
1268 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
1269 TargetFlags)];
Dan Gohman475871a2008-07-27 21:46:04 +00001270 if (N) return SDValue(N, 0);
Dan Gohman95531882010-03-18 18:49:47 +00001271 N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
Chris Lattnerc3aae252005-01-07 07:46:32 +00001272 AllNodes.push_back(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001273 return SDValue(N, 0);
Chris Lattnerc3aae252005-01-07 07:46:32 +00001274}
1275
Dan Gohman475871a2008-07-27 21:46:04 +00001276SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
Chris Lattner7cf7e3f2005-08-09 20:20:18 +00001277 if ((unsigned)Cond >= CondCodeNodes.size())
1278 CondCodeNodes.resize(Cond+1);
Duncan Sands83ec4b62008-06-06 12:08:01 +00001279
Chris Lattner079a27a2005-08-09 20:40:02 +00001280 if (CondCodeNodes[Cond] == 0) {
Dan Gohman95531882010-03-18 18:49:47 +00001281 CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
Dan Gohman0e5f1302008-07-07 23:02:41 +00001282 CondCodeNodes[Cond] = N;
1283 AllNodes.push_back(N);
Chris Lattner079a27a2005-08-09 20:40:02 +00001284 }
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001285
Dan Gohman475871a2008-07-27 21:46:04 +00001286 return SDValue(CondCodeNodes[Cond], 0);
Chris Lattner7cf7e3f2005-08-09 20:20:18 +00001287}
1288
Nate Begeman5a5ca152009-04-29 05:20:52 +00001289// commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
1290// the shuffle mask M that point at N1 to point at N2, and indices that point
1291// N2 to point at N1.
Nate Begeman9008ca62009-04-27 18:41:29 +00001292static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
1293 std::swap(N1, N2);
1294 int NElts = M.size();
1295 for (int i = 0; i != NElts; ++i) {
1296 if (M[i] >= NElts)
1297 M[i] -= NElts;
1298 else if (M[i] >= 0)
1299 M[i] += NElts;
1300 }
1301}
1302
Andrew Trickac6d9be2013-05-25 02:42:55 +00001303SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
Nate Begeman9008ca62009-04-27 18:41:29 +00001304 SDValue N2, const int *Mask) {
1305 assert(N1.getValueType() == N2.getValueType() && "Invalid VECTOR_SHUFFLE");
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001306 assert(VT.isVector() && N1.getValueType().isVector() &&
Nate Begeman9008ca62009-04-27 18:41:29 +00001307 "Vector Shuffle VTs must be a vectors");
1308 assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType()
1309 && "Vector Shuffle VTs must have same element type");
1310
1311 // Canonicalize shuffle undef, undef -> undef
1312 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
Dan Gohman2e4284d2009-07-09 00:46:33 +00001313 return getUNDEF(VT);
Nate Begeman9008ca62009-04-27 18:41:29 +00001314
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001315 // Validate that all indices in Mask are within the range of the elements
Nate Begeman9008ca62009-04-27 18:41:29 +00001316 // input to the shuffle.
Nate Begeman5a5ca152009-04-29 05:20:52 +00001317 unsigned NElts = VT.getVectorNumElements();
Nate Begeman9008ca62009-04-27 18:41:29 +00001318 SmallVector<int, 8> MaskVec;
Nate Begeman5a5ca152009-04-29 05:20:52 +00001319 for (unsigned i = 0; i != NElts; ++i) {
1320 assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
Nate Begeman9008ca62009-04-27 18:41:29 +00001321 MaskVec.push_back(Mask[i]);
1322 }
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001323
Nate Begeman9008ca62009-04-27 18:41:29 +00001324 // Canonicalize shuffle v, v -> v, undef
1325 if (N1 == N2) {
1326 N2 = getUNDEF(VT);
Nate Begeman5a5ca152009-04-29 05:20:52 +00001327 for (unsigned i = 0; i != NElts; ++i)
1328 if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
Nate Begeman9008ca62009-04-27 18:41:29 +00001329 }
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001330
Nate Begeman9008ca62009-04-27 18:41:29 +00001331 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1332 if (N1.getOpcode() == ISD::UNDEF)
1333 commuteShuffle(N1, N2, MaskVec);
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001334
Nate Begeman9008ca62009-04-27 18:41:29 +00001335 // Canonicalize all index into lhs, -> shuffle lhs, undef
1336 // Canonicalize all index into rhs, -> shuffle rhs, undef
1337 bool AllLHS = true, AllRHS = true;
1338 bool N2Undef = N2.getOpcode() == ISD::UNDEF;
Nate Begeman5a5ca152009-04-29 05:20:52 +00001339 for (unsigned i = 0; i != NElts; ++i) {
1340 if (MaskVec[i] >= (int)NElts) {
Nate Begeman9008ca62009-04-27 18:41:29 +00001341 if (N2Undef)
1342 MaskVec[i] = -1;
1343 else
1344 AllLHS = false;
1345 } else if (MaskVec[i] >= 0) {
1346 AllRHS = false;
1347 }
1348 }
1349 if (AllLHS && AllRHS)
1350 return getUNDEF(VT);
Nate Begeman5a5ca152009-04-29 05:20:52 +00001351 if (AllLHS && !N2Undef)
Nate Begeman9008ca62009-04-27 18:41:29 +00001352 N2 = getUNDEF(VT);
1353 if (AllRHS) {
1354 N1 = getUNDEF(VT);
1355 commuteShuffle(N1, N2, MaskVec);
1356 }
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001357
Nate Begeman9008ca62009-04-27 18:41:29 +00001358 // If Identity shuffle, or all shuffle in to undef, return that node.
1359 bool AllUndef = true;
1360 bool Identity = true;
Nate Begeman5a5ca152009-04-29 05:20:52 +00001361 for (unsigned i = 0; i != NElts; ++i) {
1362 if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
Nate Begeman9008ca62009-04-27 18:41:29 +00001363 if (MaskVec[i] >= 0) AllUndef = false;
1364 }
Dan Gohman2e4284d2009-07-09 00:46:33 +00001365 if (Identity && NElts == N1.getValueType().getVectorNumElements())
Nate Begeman9008ca62009-04-27 18:41:29 +00001366 return N1;
1367 if (AllUndef)
1368 return getUNDEF(VT);
1369
1370 FoldingSetNodeID ID;
1371 SDValue Ops[2] = { N1, N2 };
1372 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2);
Nate Begeman5a5ca152009-04-29 05:20:52 +00001373 for (unsigned i = 0; i != NElts; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00001374 ID.AddInteger(MaskVec[i]);
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001375
Nate Begeman9008ca62009-04-27 18:41:29 +00001376 void* IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001377 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Nate Begeman9008ca62009-04-27 18:41:29 +00001378 return SDValue(E, 0);
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001379
Nate Begeman9008ca62009-04-27 18:41:29 +00001380 // Allocate the mask array for the node out of the BumpPtrAllocator, since
1381 // SDNode doesn't have access to it. This memory will be "leaked" when
1382 // the node is deallocated, but recovered when the NodeAllocator is released.
1383 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1384 memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001385
Dan Gohman95531882010-03-18 18:49:47 +00001386 ShuffleVectorSDNode *N =
Andrew Trickac6d9be2013-05-25 02:42:55 +00001387 new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(), dl.getDebugLoc(), N1, N2, MaskAlloc);
Nate Begeman9008ca62009-04-27 18:41:29 +00001388 CSEMap.InsertNode(N, IP);
1389 AllNodes.push_back(N);
1390 return SDValue(N, 0);
1391}
1392
Andrew Trickac6d9be2013-05-25 02:42:55 +00001393SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
Dale Johannesena04b7572009-02-03 23:04:43 +00001394 SDValue Val, SDValue DTy,
1395 SDValue STy, SDValue Rnd, SDValue Sat,
1396 ISD::CvtCode Code) {
Mon P Wangb0e341b2009-02-05 04:47:42 +00001397 // If the src and dest types are the same and the conversion is between
1398 // integer types of the same sign or two floats, no conversion is necessary.
1399 if (DTy == STy &&
1400 (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
Dale Johannesena04b7572009-02-03 23:04:43 +00001401 return Val;
1402
1403 FoldingSetNodeID ID;
Mon P Wangbdea3482009-11-07 04:46:25 +00001404 SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
1405 AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5);
Dale Johannesena04b7572009-02-03 23:04:43 +00001406 void* IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001407 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dale Johannesena04b7572009-02-03 23:04:43 +00001408 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001409
Andrew Trickac6d9be2013-05-25 02:42:55 +00001410 CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(), dl.getDebugLoc(), Ops, 5,
Dan Gohman95531882010-03-18 18:49:47 +00001411 Code);
Dale Johannesena04b7572009-02-03 23:04:43 +00001412 CSEMap.InsertNode(N, IP);
1413 AllNodes.push_back(N);
1414 return SDValue(N, 0);
1415}
1416
Owen Andersone50ed302009-08-10 22:56:29 +00001417SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
Jim Laskey583bd472006-10-27 23:46:08 +00001418 FoldingSetNodeID ID;
Dan Gohman6d9cdd52008-07-07 18:26:29 +00001419 AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0);
Chris Lattner61b09412006-08-11 21:01:22 +00001420 ID.AddInteger(RegNo);
1421 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001422 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00001423 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001424
Dan Gohman95531882010-03-18 18:49:47 +00001425 SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
Chris Lattner61b09412006-08-11 21:01:22 +00001426 CSEMap.InsertNode(N, IP);
1427 AllNodes.push_back(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001428 return SDValue(N, 0);
Chris Lattner61b09412006-08-11 21:01:22 +00001429}
1430
Jakob Stoklund Olesen9cf37e82012-01-18 23:52:12 +00001431SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1432 FoldingSetNodeID ID;
1433 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), 0, 0);
1434 ID.AddPointer(RegMask);
1435 void *IP = 0;
1436 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1437 return SDValue(E, 0);
1438
1439 SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
1440 CSEMap.InsertNode(N, IP);
1441 AllNodes.push_back(N);
1442 return SDValue(N, 0);
1443}
1444
Andrew Trickac6d9be2013-05-25 02:42:55 +00001445SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
Dale Johannesene8c17332009-01-29 00:47:48 +00001446 FoldingSetNodeID ID;
1447 SDValue Ops[] = { Root };
Chris Lattner7561d482010-03-14 02:33:54 +00001448 AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1);
1449 ID.AddPointer(Label);
Dale Johannesene8c17332009-01-29 00:47:48 +00001450 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001451 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dale Johannesene8c17332009-01-29 00:47:48 +00001452 return SDValue(E, 0);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001453
Andrew Trickac6d9be2013-05-25 02:42:55 +00001454 SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(), dl.getDebugLoc(), Root, Label);
Dale Johannesene8c17332009-01-29 00:47:48 +00001455 CSEMap.InsertNode(N, IP);
1456 AllNodes.push_back(N);
1457 return SDValue(N, 0);
1458}
1459
Chris Lattner7561d482010-03-14 02:33:54 +00001460
Dan Gohman46510a72010-04-15 01:51:59 +00001461SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
Michael Liao6c7ccaa2012-09-12 21:43:09 +00001462 int64_t Offset,
Dan Gohman29cbade2009-11-20 23:18:13 +00001463 bool isTarget,
1464 unsigned char TargetFlags) {
Dan Gohman8c2b5252009-10-30 01:27:03 +00001465 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
1466
1467 FoldingSetNodeID ID;
Dan Gohman29cbade2009-11-20 23:18:13 +00001468 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
Dan Gohman8c2b5252009-10-30 01:27:03 +00001469 ID.AddPointer(BA);
Michael Liao6c7ccaa2012-09-12 21:43:09 +00001470 ID.AddInteger(Offset);
Dan Gohman29cbade2009-11-20 23:18:13 +00001471 ID.AddInteger(TargetFlags);
Dan Gohman8c2b5252009-10-30 01:27:03 +00001472 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001473 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman8c2b5252009-10-30 01:27:03 +00001474 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001475
Michael Liao6c7ccaa2012-09-12 21:43:09 +00001476 SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
1477 TargetFlags);
Dan Gohman8c2b5252009-10-30 01:27:03 +00001478 CSEMap.InsertNode(N, IP);
1479 AllNodes.push_back(N);
1480 return SDValue(N, 0);
1481}
1482
Dan Gohman475871a2008-07-27 21:46:04 +00001483SDValue SelectionDAG::getSrcValue(const Value *V) {
Duncan Sands1df98592010-02-16 11:11:14 +00001484 assert((!V || V->getType()->isPointerTy()) &&
Chris Lattner61b09412006-08-11 21:01:22 +00001485 "SrcValue is not a pointer?");
1486
Jim Laskey583bd472006-10-27 23:46:08 +00001487 FoldingSetNodeID ID;
Owen Anderson825b72b2009-08-11 20:47:22 +00001488 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0);
Chris Lattner61b09412006-08-11 21:01:22 +00001489 ID.AddPointer(V);
Dan Gohman69de1932008-02-06 22:27:42 +00001490
Chris Lattner61b09412006-08-11 21:01:22 +00001491 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00001492 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00001493 return SDValue(E, 0);
Dan Gohman69de1932008-02-06 22:27:42 +00001494
Dan Gohman95531882010-03-18 18:49:47 +00001495 SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
Dan Gohman69de1932008-02-06 22:27:42 +00001496 CSEMap.InsertNode(N, IP);
1497 AllNodes.push_back(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001498 return SDValue(N, 0);
Dan Gohman69de1932008-02-06 22:27:42 +00001499}
1500
Chris Lattnerdecc2672010-04-07 05:20:54 +00001501/// getMDNode - Return an MDNodeSDNode which holds an MDNode.
1502SDValue SelectionDAG::getMDNode(const MDNode *MD) {
1503 FoldingSetNodeID ID;
1504 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0);
1505 ID.AddPointer(MD);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001506
Chris Lattnerdecc2672010-04-07 05:20:54 +00001507 void *IP = 0;
1508 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1509 return SDValue(E, 0);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001510
Chris Lattnerdecc2672010-04-07 05:20:54 +00001511 SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
1512 CSEMap.InsertNode(N, IP);
1513 AllNodes.push_back(N);
1514 return SDValue(N, 0);
1515}
1516
1517
Duncan Sands92abc622009-01-31 15:50:11 +00001518/// getShiftAmountOperand - Return the specified value casted to
1519/// the target's desired shift amount type.
Owen Anderson6154f6c2011-03-07 18:29:47 +00001520SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
Owen Andersone50ed302009-08-10 22:56:29 +00001521 EVT OpTy = Op.getValueType();
Michael Liaoa6b20ce2013-03-01 18:40:30 +00001522 EVT ShTy = TLI.getShiftAmountTy(LHSTy);
Duncan Sands92abc622009-01-31 15:50:11 +00001523 if (OpTy == ShTy || OpTy.isVector()) return Op;
1524
1525 ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
Andrew Trickac6d9be2013-05-25 02:42:55 +00001526 return getNode(Opcode, SDLoc(Op), ShTy, Op);
Duncan Sands92abc622009-01-31 15:50:11 +00001527}
1528
Chris Lattner37ce9df2007-10-15 17:47:20 +00001529/// CreateStackTemporary - Create a stack temporary, suitable for holding the
1530/// specified value type.
Owen Andersone50ed302009-08-10 22:56:29 +00001531SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
Chris Lattner37ce9df2007-10-15 17:47:20 +00001532 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
Dan Gohman4e918b22009-09-23 21:07:02 +00001533 unsigned ByteSize = VT.getStoreSize();
Chris Lattnerdb125cf2011-07-18 04:54:35 +00001534 Type *Ty = VT.getTypeForEVT(*getContext());
Mon P Wang364d73d2008-07-05 20:40:31 +00001535 unsigned StackAlign =
Micah Villmow3574eca2012-10-08 16:38:25 +00001536 std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
Scott Michelfdc40a02009-02-17 22:15:04 +00001537
David Greene3f2bf852009-11-12 20:49:22 +00001538 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
Chris Lattner37ce9df2007-10-15 17:47:20 +00001539 return getFrameIndex(FrameIdx, TLI.getPointerTy());
1540}
1541
Duncan Sands47d9dcc2008-12-09 21:33:20 +00001542/// CreateStackTemporary - Create a stack temporary suitable for holding
1543/// either of the specified value types.
Owen Andersone50ed302009-08-10 22:56:29 +00001544SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
Duncan Sands47d9dcc2008-12-09 21:33:20 +00001545 unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
1546 VT2.getStoreSizeInBits())/8;
Chris Lattnerdb125cf2011-07-18 04:54:35 +00001547 Type *Ty1 = VT1.getTypeForEVT(*getContext());
1548 Type *Ty2 = VT2.getTypeForEVT(*getContext());
Micah Villmow3574eca2012-10-08 16:38:25 +00001549 const DataLayout *TD = TLI.getDataLayout();
Duncan Sands47d9dcc2008-12-09 21:33:20 +00001550 unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
1551 TD->getPrefTypeAlignment(Ty2));
1552
1553 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
David Greene3f2bf852009-11-12 20:49:22 +00001554 int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
Duncan Sands47d9dcc2008-12-09 21:33:20 +00001555 return getFrameIndex(FrameIdx, TLI.getPointerTy());
1556}
1557
Owen Andersone50ed302009-08-10 22:56:29 +00001558SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
Andrew Trickac6d9be2013-05-25 02:42:55 +00001559 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
Chris Lattnerc3aae252005-01-07 07:46:32 +00001560 // These setcc operations always fold.
1561 switch (Cond) {
1562 default: break;
1563 case ISD::SETFALSE:
Chris Lattnerf30b73b2005-01-18 02:52:03 +00001564 case ISD::SETFALSE2: return getConstant(0, VT);
Chris Lattnerc3aae252005-01-07 07:46:32 +00001565 case ISD::SETTRUE:
Chris Lattnerf30b73b2005-01-18 02:52:03 +00001566 case ISD::SETTRUE2: return getConstant(1, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00001567
Chris Lattnera83385f2006-04-27 05:01:07 +00001568 case ISD::SETOEQ:
1569 case ISD::SETOGT:
1570 case ISD::SETOGE:
1571 case ISD::SETOLT:
1572 case ISD::SETOLE:
1573 case ISD::SETONE:
1574 case ISD::SETO:
1575 case ISD::SETUO:
1576 case ISD::SETUEQ:
1577 case ISD::SETUNE:
Duncan Sands83ec4b62008-06-06 12:08:01 +00001578 assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
Chris Lattnera83385f2006-04-27 05:01:07 +00001579 break;
Chris Lattnerc3aae252005-01-07 07:46:32 +00001580 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001581
Gabor Greifba36cb52008-08-28 21:40:38 +00001582 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
Dan Gohman6c231502008-02-29 01:47:35 +00001583 const APInt &C2 = N2C->getAPIntValue();
Gabor Greifba36cb52008-08-28 21:40:38 +00001584 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
Dan Gohman6c231502008-02-29 01:47:35 +00001585 const APInt &C1 = N1C->getAPIntValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00001586
Chris Lattnerc3aae252005-01-07 07:46:32 +00001587 switch (Cond) {
Torok Edwinc23197a2009-07-14 16:55:14 +00001588 default: llvm_unreachable("Unknown integer setcc!");
Chris Lattnerf30b73b2005-01-18 02:52:03 +00001589 case ISD::SETEQ: return getConstant(C1 == C2, VT);
1590 case ISD::SETNE: return getConstant(C1 != C2, VT);
Dan Gohman6c231502008-02-29 01:47:35 +00001591 case ISD::SETULT: return getConstant(C1.ult(C2), VT);
1592 case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
1593 case ISD::SETULE: return getConstant(C1.ule(C2), VT);
1594 case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
1595 case ISD::SETLT: return getConstant(C1.slt(C2), VT);
1596 case ISD::SETGT: return getConstant(C1.sgt(C2), VT);
1597 case ISD::SETLE: return getConstant(C1.sle(C2), VT);
1598 case ISD::SETGE: return getConstant(C1.sge(C2), VT);
Chris Lattnerc3aae252005-01-07 07:46:32 +00001599 }
Chris Lattnerc3aae252005-01-07 07:46:32 +00001600 }
Chris Lattner67255a12005-04-07 18:14:58 +00001601 }
Gabor Greifba36cb52008-08-28 21:40:38 +00001602 if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
1603 if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
Dale Johanneseneaf08942007-08-31 04:03:46 +00001604 APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
Chris Lattnerc3aae252005-01-07 07:46:32 +00001605 switch (Cond) {
Dale Johanneseneaf08942007-08-31 04:03:46 +00001606 default: break;
Scott Michelfdc40a02009-02-17 22:15:04 +00001607 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
Dale Johannesene8d72302009-02-06 23:05:02 +00001608 return getUNDEF(VT);
Dale Johannesenee847682007-08-31 17:03:33 +00001609 // fall through
1610 case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00001611 case ISD::SETNE: if (R==APFloat::cmpUnordered)
Dale Johannesene8d72302009-02-06 23:05:02 +00001612 return getUNDEF(VT);
Dale Johannesenee847682007-08-31 17:03:33 +00001613 // fall through
1614 case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
Dale Johanneseneaf08942007-08-31 04:03:46 +00001615 R==APFloat::cmpLessThan, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00001616 case ISD::SETLT: if (R==APFloat::cmpUnordered)
Dale Johannesene8d72302009-02-06 23:05:02 +00001617 return getUNDEF(VT);
Dale Johannesenee847682007-08-31 17:03:33 +00001618 // fall through
1619 case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00001620 case ISD::SETGT: if (R==APFloat::cmpUnordered)
Dale Johannesene8d72302009-02-06 23:05:02 +00001621 return getUNDEF(VT);
Dale Johannesenee847682007-08-31 17:03:33 +00001622 // fall through
1623 case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00001624 case ISD::SETLE: if (R==APFloat::cmpUnordered)
Dale Johannesene8d72302009-02-06 23:05:02 +00001625 return getUNDEF(VT);
Dale Johannesenee847682007-08-31 17:03:33 +00001626 // fall through
1627 case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
Dale Johanneseneaf08942007-08-31 04:03:46 +00001628 R==APFloat::cmpEqual, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00001629 case ISD::SETGE: if (R==APFloat::cmpUnordered)
Dale Johannesene8d72302009-02-06 23:05:02 +00001630 return getUNDEF(VT);
Dale Johannesenee847682007-08-31 17:03:33 +00001631 // fall through
1632 case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
Dale Johanneseneaf08942007-08-31 04:03:46 +00001633 R==APFloat::cmpEqual, VT);
1634 case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT);
1635 case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT);
1636 case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
1637 R==APFloat::cmpEqual, VT);
1638 case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
1639 case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
1640 R==APFloat::cmpLessThan, VT);
1641 case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
1642 R==APFloat::cmpUnordered, VT);
1643 case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
1644 case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
Chris Lattnerc3aae252005-01-07 07:46:32 +00001645 }
1646 } else {
1647 // Ensure that the constant occurs on the RHS.
Dale Johannesenff97d4f2009-02-03 00:47:48 +00001648 return getSetCC(dl, VT, N2, N1, ISD::getSetCCSwappedOperands(Cond));
Chris Lattnerc3aae252005-01-07 07:46:32 +00001649 }
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00001650 }
1651
Chris Lattner7cf7e3f2005-08-09 20:20:18 +00001652 // Could not fold it.
Dan Gohman475871a2008-07-27 21:46:04 +00001653 return SDValue();
Chris Lattnerc3aae252005-01-07 07:46:32 +00001654}
1655
Dan Gohman2e68b6f2008-02-25 21:11:39 +00001656/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
1657/// use this predicate to simplify operations downstream.
Dan Gohman475871a2008-07-27 21:46:04 +00001658bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
Chris Lattnera4f73182009-07-07 23:28:46 +00001659 // This predicate is not safe for vector operations.
1660 if (Op.getValueType().isVector())
1661 return false;
Eric Christopher4d7c18c2009-08-22 00:40:45 +00001662
Dan Gohman87862e72009-12-11 21:31:27 +00001663 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
Dan Gohman2e68b6f2008-02-25 21:11:39 +00001664 return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
1665}
1666
Dan Gohmanea859be2007-06-22 14:59:07 +00001667/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
1668/// this predicate to simplify operations downstream. Mask is known to be zero
1669/// for bits that V cannot have.
Scott Michelfdc40a02009-02-17 22:15:04 +00001670bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
Dan Gohmanea859be2007-06-22 14:59:07 +00001671 unsigned Depth) const {
Dan Gohman2e68b6f2008-02-25 21:11:39 +00001672 APInt KnownZero, KnownOne;
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001673 ComputeMaskedBits(Op, KnownZero, KnownOne, Depth);
Scott Michelfdc40a02009-02-17 22:15:04 +00001674 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
Dan Gohmanea859be2007-06-22 14:59:07 +00001675 return (KnownZero & Mask) == Mask;
1676}
1677
1678/// ComputeMaskedBits - Determine which of the bits specified in Mask are
1679/// known to be either zero or one and return them in the KnownZero/KnownOne
1680/// bitsets. This code only analyzes bits in Mask, in order to short-circuit
1681/// processing.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001682void SelectionDAG::ComputeMaskedBits(SDValue Op, APInt &KnownZero,
1683 APInt &KnownOne, unsigned Depth) const {
1684 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
Dan Gohmand9fe41c2008-02-13 23:13:32 +00001685
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001686 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001687 if (Depth == 6)
Dan Gohmanea859be2007-06-22 14:59:07 +00001688 return; // Limit search depth.
Scott Michelfdc40a02009-02-17 22:15:04 +00001689
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001690 APInt KnownZero2, KnownOne2;
Dan Gohmanea859be2007-06-22 14:59:07 +00001691
1692 switch (Op.getOpcode()) {
1693 case ISD::Constant:
1694 // We know all of the bits for a constant!
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001695 KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
1696 KnownZero = ~KnownOne;
Dan Gohmanea859be2007-06-22 14:59:07 +00001697 return;
1698 case ISD::AND:
1699 // If either the LHS or the RHS are Zero, the result is zero.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001700 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1701 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001702 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1703 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
Dan Gohmanea859be2007-06-22 14:59:07 +00001704
1705 // Output known-1 bits are only known if set in both the LHS & RHS.
1706 KnownOne &= KnownOne2;
1707 // Output known-0 are known to be clear if zero in either the LHS | RHS.
1708 KnownZero |= KnownZero2;
1709 return;
1710 case ISD::OR:
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001711 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1712 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001713 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1714 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
1715
Dan Gohmanea859be2007-06-22 14:59:07 +00001716 // Output known-0 bits are only known if clear in both the LHS & RHS.
1717 KnownZero &= KnownZero2;
1718 // Output known-1 are known to be set if set in either the LHS | RHS.
1719 KnownOne |= KnownOne2;
1720 return;
1721 case ISD::XOR: {
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001722 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1723 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001724 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1725 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
1726
Dan Gohmanea859be2007-06-22 14:59:07 +00001727 // Output known-0 bits are known if clear or set in both the LHS & RHS.
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001728 APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
Dan Gohmanea859be2007-06-22 14:59:07 +00001729 // Output known-1 are known to be set if set in only one of the LHS, RHS.
1730 KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
1731 KnownZero = KnownZeroOut;
1732 return;
1733 }
Dan Gohman23e8b712008-04-28 17:02:21 +00001734 case ISD::MUL: {
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001735 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1736 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
Dan Gohman23e8b712008-04-28 17:02:21 +00001737 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1738 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
1739
1740 // If low bits are zero in either operand, output low known-0 bits.
1741 // Also compute a conserative estimate for high known-0 bits.
1742 // More trickiness is possible, but this is sufficient for the
1743 // interesting case of alignment computation.
Jay Foad7a874dd2010-12-01 08:53:58 +00001744 KnownOne.clearAllBits();
Dan Gohman23e8b712008-04-28 17:02:21 +00001745 unsigned TrailZ = KnownZero.countTrailingOnes() +
1746 KnownZero2.countTrailingOnes();
1747 unsigned LeadZ = std::max(KnownZero.countLeadingOnes() +
Dan Gohman42ac9292008-05-07 00:35:55 +00001748 KnownZero2.countLeadingOnes(),
1749 BitWidth) - BitWidth;
Dan Gohman23e8b712008-04-28 17:02:21 +00001750
1751 TrailZ = std::min(TrailZ, BitWidth);
1752 LeadZ = std::min(LeadZ, BitWidth);
1753 KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
1754 APInt::getHighBitsSet(BitWidth, LeadZ);
Dan Gohman23e8b712008-04-28 17:02:21 +00001755 return;
1756 }
1757 case ISD::UDIV: {
1758 // For the purposes of computing leading zeros we can conservatively
1759 // treat a udiv as a logical right shift by the power of 2 known to
Dan Gohman1d9cd502008-05-02 21:30:02 +00001760 // be less than the denominator.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001761 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
Dan Gohman23e8b712008-04-28 17:02:21 +00001762 unsigned LeadZ = KnownZero2.countLeadingOnes();
1763
Jay Foad7a874dd2010-12-01 08:53:58 +00001764 KnownOne2.clearAllBits();
1765 KnownZero2.clearAllBits();
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001766 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
Dan Gohman1d9cd502008-05-02 21:30:02 +00001767 unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
1768 if (RHSUnknownLeadingOnes != BitWidth)
1769 LeadZ = std::min(BitWidth,
1770 LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
Dan Gohman23e8b712008-04-28 17:02:21 +00001771
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001772 KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
Dan Gohman23e8b712008-04-28 17:02:21 +00001773 return;
1774 }
Dan Gohmanea859be2007-06-22 14:59:07 +00001775 case ISD::SELECT:
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001776 ComputeMaskedBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
1777 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001778 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1779 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
1780
Dan Gohmanea859be2007-06-22 14:59:07 +00001781 // Only known if known in both the LHS and RHS.
1782 KnownOne &= KnownOne2;
1783 KnownZero &= KnownZero2;
1784 return;
1785 case ISD::SELECT_CC:
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001786 ComputeMaskedBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
1787 ComputeMaskedBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001788 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1789 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
1790
Dan Gohmanea859be2007-06-22 14:59:07 +00001791 // Only known if known in both the LHS and RHS.
1792 KnownOne &= KnownOne2;
1793 KnownZero &= KnownZero2;
1794 return;
Bill Wendling253174b2008-11-22 07:24:01 +00001795 case ISD::SADDO:
1796 case ISD::UADDO:
Bill Wendling74c37652008-12-09 22:08:41 +00001797 case ISD::SSUBO:
1798 case ISD::USUBO:
1799 case ISD::SMULO:
1800 case ISD::UMULO:
Bill Wendling253174b2008-11-22 07:24:01 +00001801 if (Op.getResNo() != 1)
1802 return;
Duncan Sands03228082008-11-23 15:47:28 +00001803 // The boolean result conforms to getBooleanContents. Fall through.
Dan Gohmanea859be2007-06-22 14:59:07 +00001804 case ISD::SETCC:
1805 // If we know the result of a setcc has the top bits zero, use this info.
Duncan Sands28b77e92011-09-06 19:07:46 +00001806 if (TLI.getBooleanContents(Op.getValueType().isVector()) ==
1807 TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1)
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001808 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
Dan Gohmanea859be2007-06-22 14:59:07 +00001809 return;
1810 case ISD::SHL:
Sylvestre Ledru94c22712012-09-27 10:14:43 +00001811 // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0
Dan Gohmanea859be2007-06-22 14:59:07 +00001812 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00001813 unsigned ShAmt = SA->getZExtValue();
Dan Gohmand4cf9922008-02-26 18:50:50 +00001814
1815 // If the shift count is an invalid immediate, don't do anything.
1816 if (ShAmt >= BitWidth)
1817 return;
1818
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001819 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001820 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
Dan Gohmand4cf9922008-02-26 18:50:50 +00001821 KnownZero <<= ShAmt;
1822 KnownOne <<= ShAmt;
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001823 // low bits known zero.
Dan Gohmand4cf9922008-02-26 18:50:50 +00001824 KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
Dan Gohmanea859be2007-06-22 14:59:07 +00001825 }
1826 return;
1827 case ISD::SRL:
Sylvestre Ledru94c22712012-09-27 10:14:43 +00001828 // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0
Dan Gohmanea859be2007-06-22 14:59:07 +00001829 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00001830 unsigned ShAmt = SA->getZExtValue();
Dan Gohmanea859be2007-06-22 14:59:07 +00001831
Dan Gohmand4cf9922008-02-26 18:50:50 +00001832 // If the shift count is an invalid immediate, don't do anything.
1833 if (ShAmt >= BitWidth)
1834 return;
1835
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001836 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001837 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001838 KnownZero = KnownZero.lshr(ShAmt);
1839 KnownOne = KnownOne.lshr(ShAmt);
Dan Gohmanea859be2007-06-22 14:59:07 +00001840
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001841 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
Dan Gohmanea859be2007-06-22 14:59:07 +00001842 KnownZero |= HighBits; // High bits known zero.
1843 }
1844 return;
1845 case ISD::SRA:
1846 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00001847 unsigned ShAmt = SA->getZExtValue();
Dan Gohmanea859be2007-06-22 14:59:07 +00001848
Dan Gohmand4cf9922008-02-26 18:50:50 +00001849 // If the shift count is an invalid immediate, don't do anything.
1850 if (ShAmt >= BitWidth)
1851 return;
1852
Dan Gohmanea859be2007-06-22 14:59:07 +00001853 // If any of the demanded bits are produced by the sign extension, we also
1854 // demand the input sign bit.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001855 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
Scott Michelfdc40a02009-02-17 22:15:04 +00001856
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001857 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001858 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001859 KnownZero = KnownZero.lshr(ShAmt);
1860 KnownOne = KnownOne.lshr(ShAmt);
Scott Michelfdc40a02009-02-17 22:15:04 +00001861
Dan Gohmanea859be2007-06-22 14:59:07 +00001862 // Handle the sign bits.
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001863 APInt SignBit = APInt::getSignBit(BitWidth);
1864 SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask.
Scott Michelfdc40a02009-02-17 22:15:04 +00001865
Dan Gohmanca93a432008-02-20 16:30:17 +00001866 if (KnownZero.intersects(SignBit)) {
Dan Gohmanea859be2007-06-22 14:59:07 +00001867 KnownZero |= HighBits; // New bits are known zero.
Dan Gohmanca93a432008-02-20 16:30:17 +00001868 } else if (KnownOne.intersects(SignBit)) {
Dan Gohmanea859be2007-06-22 14:59:07 +00001869 KnownOne |= HighBits; // New bits are known one.
1870 }
1871 }
1872 return;
1873 case ISD::SIGN_EXTEND_INREG: {
Owen Andersone50ed302009-08-10 22:56:29 +00001874 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
Dan Gohmand1996362010-01-09 02:13:55 +00001875 unsigned EBits = EVT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00001876
1877 // Sign extension. Compute the demanded bits in the result that are not
Dan Gohmanea859be2007-06-22 14:59:07 +00001878 // present in the input.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001879 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
Dan Gohmanea859be2007-06-22 14:59:07 +00001880
Dan Gohman977a76f2008-02-13 22:28:48 +00001881 APInt InSignBit = APInt::getSignBit(EBits);
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001882 APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
Scott Michelfdc40a02009-02-17 22:15:04 +00001883
Dan Gohmanea859be2007-06-22 14:59:07 +00001884 // If the sign extended bits are demanded, we know that the sign
1885 // bit is demanded.
Jay Foad40f8f622010-12-07 08:25:19 +00001886 InSignBit = InSignBit.zext(BitWidth);
Dan Gohman977a76f2008-02-13 22:28:48 +00001887 if (NewBits.getBoolValue())
Dan Gohmanea859be2007-06-22 14:59:07 +00001888 InputDemandedBits |= InSignBit;
Scott Michelfdc40a02009-02-17 22:15:04 +00001889
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001890 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
1891 KnownOne &= InputDemandedBits;
1892 KnownZero &= InputDemandedBits;
Scott Michelfdc40a02009-02-17 22:15:04 +00001893 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1894
Dan Gohmanea859be2007-06-22 14:59:07 +00001895 // If the sign bit of the input is known set or clear, then we know the
1896 // top bits of the result.
Dan Gohmanca93a432008-02-20 16:30:17 +00001897 if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear
Dan Gohmanea859be2007-06-22 14:59:07 +00001898 KnownZero |= NewBits;
1899 KnownOne &= ~NewBits;
Dan Gohmanca93a432008-02-20 16:30:17 +00001900 } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set
Dan Gohmanea859be2007-06-22 14:59:07 +00001901 KnownOne |= NewBits;
1902 KnownZero &= ~NewBits;
1903 } else { // Input sign bit unknown
1904 KnownZero &= ~NewBits;
1905 KnownOne &= ~NewBits;
1906 }
1907 return;
1908 }
1909 case ISD::CTTZ:
Chandler Carruth63974b22011-12-13 01:56:10 +00001910 case ISD::CTTZ_ZERO_UNDEF:
Dan Gohmanea859be2007-06-22 14:59:07 +00001911 case ISD::CTLZ:
Chandler Carruth63974b22011-12-13 01:56:10 +00001912 case ISD::CTLZ_ZERO_UNDEF:
Dan Gohmanea859be2007-06-22 14:59:07 +00001913 case ISD::CTPOP: {
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001914 unsigned LowBits = Log2_32(BitWidth)+1;
1915 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
Jay Foad7a874dd2010-12-01 08:53:58 +00001916 KnownOne.clearAllBits();
Dan Gohmanea859be2007-06-22 14:59:07 +00001917 return;
1918 }
1919 case ISD::LOAD: {
Rafael Espindola95d594c2012-03-31 18:14:00 +00001920 LoadSDNode *LD = cast<LoadSDNode>(Op);
Nadav Rotem77451752013-03-20 22:53:44 +00001921 // If this is a ZEXTLoad and we are looking at the loaded value.
1922 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
Owen Andersone50ed302009-08-10 22:56:29 +00001923 EVT VT = LD->getMemoryVT();
Dan Gohmand1996362010-01-09 02:13:55 +00001924 unsigned MemBits = VT.getScalarType().getSizeInBits();
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001925 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
Rafael Espindola95d594c2012-03-31 18:14:00 +00001926 } else if (const MDNode *Ranges = LD->getRanges()) {
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001927 computeMaskedBitsLoad(*Ranges, KnownZero);
Dan Gohmanea859be2007-06-22 14:59:07 +00001928 }
1929 return;
1930 }
1931 case ISD::ZERO_EXTEND: {
Owen Andersone50ed302009-08-10 22:56:29 +00001932 EVT InVT = Op.getOperand(0).getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00001933 unsigned InBits = InVT.getScalarType().getSizeInBits();
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001934 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
Jay Foad40f8f622010-12-07 08:25:19 +00001935 KnownZero = KnownZero.trunc(InBits);
1936 KnownOne = KnownOne.trunc(InBits);
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001937 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
Jay Foad40f8f622010-12-07 08:25:19 +00001938 KnownZero = KnownZero.zext(BitWidth);
1939 KnownOne = KnownOne.zext(BitWidth);
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001940 KnownZero |= NewBits;
Dan Gohmanea859be2007-06-22 14:59:07 +00001941 return;
1942 }
1943 case ISD::SIGN_EXTEND: {
Owen Andersone50ed302009-08-10 22:56:29 +00001944 EVT InVT = Op.getOperand(0).getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00001945 unsigned InBits = InVT.getScalarType().getSizeInBits();
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001946 APInt InSignBit = APInt::getSignBit(InBits);
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001947 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001948
Jay Foad40f8f622010-12-07 08:25:19 +00001949 KnownZero = KnownZero.trunc(InBits);
1950 KnownOne = KnownOne.trunc(InBits);
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001951 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
Dan Gohman977a76f2008-02-13 22:28:48 +00001952
1953 // Note if the sign bit is known to be zero or one.
1954 bool SignBitKnownZero = KnownZero.isNegative();
1955 bool SignBitKnownOne = KnownOne.isNegative();
1956 assert(!(SignBitKnownZero && SignBitKnownOne) &&
1957 "Sign bit can't be known to be both zero and one!");
1958
Jay Foad40f8f622010-12-07 08:25:19 +00001959 KnownZero = KnownZero.zext(BitWidth);
1960 KnownOne = KnownOne.zext(BitWidth);
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00001961
Dan Gohman977a76f2008-02-13 22:28:48 +00001962 // If the sign bit is known zero or one, the top bits match.
1963 if (SignBitKnownZero)
Dan Gohmanea859be2007-06-22 14:59:07 +00001964 KnownZero |= NewBits;
Dan Gohman977a76f2008-02-13 22:28:48 +00001965 else if (SignBitKnownOne)
Dan Gohmanea859be2007-06-22 14:59:07 +00001966 KnownOne |= NewBits;
Dan Gohmanea859be2007-06-22 14:59:07 +00001967 return;
1968 }
1969 case ISD::ANY_EXTEND: {
Evan Cheng2766a472012-12-06 19:13:27 +00001970 EVT InVT = Op.getOperand(0).getValueType();
1971 unsigned InBits = InVT.getScalarType().getSizeInBits();
1972 KnownZero = KnownZero.trunc(InBits);
1973 KnownOne = KnownOne.trunc(InBits);
1974 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
1975 KnownZero = KnownZero.zext(BitWidth);
1976 KnownOne = KnownOne.zext(BitWidth);
Dan Gohmanea859be2007-06-22 14:59:07 +00001977 return;
1978 }
1979 case ISD::TRUNCATE: {
Owen Andersone50ed302009-08-10 22:56:29 +00001980 EVT InVT = Op.getOperand(0).getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00001981 unsigned InBits = InVT.getScalarType().getSizeInBits();
Jay Foad40f8f622010-12-07 08:25:19 +00001982 KnownZero = KnownZero.zext(InBits);
1983 KnownOne = KnownOne.zext(InBits);
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001984 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001985 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
Jay Foad40f8f622010-12-07 08:25:19 +00001986 KnownZero = KnownZero.trunc(BitWidth);
1987 KnownOne = KnownOne.trunc(BitWidth);
Dan Gohmanea859be2007-06-22 14:59:07 +00001988 break;
1989 }
1990 case ISD::AssertZext: {
Owen Andersone50ed302009-08-10 22:56:29 +00001991 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
Duncan Sands83ec4b62008-06-06 12:08:01 +00001992 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001993 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
1994 KnownZero |= (~InMask);
Nadav Rotem7ee0e5a2012-07-16 18:34:53 +00001995 KnownOne &= (~KnownZero);
Dan Gohmanea859be2007-06-22 14:59:07 +00001996 return;
1997 }
Chris Lattnerd268a492007-12-22 21:26:52 +00001998 case ISD::FGETSIGN:
1999 // All bits are zero except the low bit.
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00002000 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
Chris Lattnerd268a492007-12-22 21:26:52 +00002001 return;
Scott Michelfdc40a02009-02-17 22:15:04 +00002002
Dan Gohman23e8b712008-04-28 17:02:21 +00002003 case ISD::SUB: {
2004 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
2005 // We know that the top bits of C-X are clear if X contains less bits
2006 // than C (i.e. no wrap-around can happen). For example, 20-X is
2007 // positive if we can prove that X is >= 0 and < 16.
2008 if (CLHS->getAPIntValue().isNonNegative()) {
2009 unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
2010 // NLZ can't be BitWidth with no sign bit
2011 APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002012 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
Dan Gohman23e8b712008-04-28 17:02:21 +00002013
2014 // If all of the MaskV bits are known to be zero, then we know the
2015 // output top bits are zero, because we now know that the output is
2016 // from [0-C].
2017 if ((KnownZero2 & MaskV) == MaskV) {
2018 unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
2019 // Top bits known zero.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002020 KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
Dan Gohman23e8b712008-04-28 17:02:21 +00002021 }
2022 }
2023 }
2024 }
2025 // fall through
Chris Lattnerda605882010-12-19 20:38:28 +00002026 case ISD::ADD:
2027 case ISD::ADDE: {
Dan Gohmanea859be2007-06-22 14:59:07 +00002028 // Output known-0 bits are known if clear or set in both the low clear bits
2029 // common to both LHS & RHS. For example, 8+(X<<3) is known to have the
2030 // low 3 bits clear.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002031 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00002032 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
Dan Gohman23e8b712008-04-28 17:02:21 +00002033 unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
2034
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002035 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00002036 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
Dan Gohman23e8b712008-04-28 17:02:21 +00002037 KnownZeroOut = std::min(KnownZeroOut,
2038 KnownZero2.countTrailingOnes());
2039
Chris Lattnerda605882010-12-19 20:38:28 +00002040 if (Op.getOpcode() == ISD::ADD) {
2041 KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
2042 return;
2043 }
2044
2045 // With ADDE, a carry bit may be added in, so we can only use this
2046 // information if we know (at least) that the low two bits are clear. We
2047 // then return to the caller that the low bit is unknown but that other bits
2048 // are known zero.
2049 if (KnownZeroOut >= 2) // ADDE
2050 KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
Dan Gohmanea859be2007-06-22 14:59:07 +00002051 return;
2052 }
Dan Gohman23e8b712008-04-28 17:02:21 +00002053 case ISD::SREM:
2054 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
Duncan Sands5c2873a2010-01-29 09:45:26 +00002055 const APInt &RA = Rem->getAPIntValue().abs();
2056 if (RA.isPowerOf2()) {
2057 APInt LowBits = RA - 1;
Dan Gohman23e8b712008-04-28 17:02:21 +00002058 APInt Mask2 = LowBits | APInt::getSignBit(BitWidth);
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002059 ComputeMaskedBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
Dan Gohmanea859be2007-06-22 14:59:07 +00002060
Duncan Sands5c2873a2010-01-29 09:45:26 +00002061 // The low bits of the first operand are unchanged by the srem.
2062 KnownZero = KnownZero2 & LowBits;
2063 KnownOne = KnownOne2 & LowBits;
Dan Gohmanea859be2007-06-22 14:59:07 +00002064
Duncan Sands5c2873a2010-01-29 09:45:26 +00002065 // If the first operand is non-negative or has all low bits zero, then
2066 // the upper bits are all zero.
2067 if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
2068 KnownZero |= ~LowBits;
2069
2070 // If the first operand is negative and not all low bits are zero, then
2071 // the upper bits are all one.
2072 if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
2073 KnownOne |= ~LowBits;
Dan Gohman23e8b712008-04-28 17:02:21 +00002074 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
Dan Gohmanea859be2007-06-22 14:59:07 +00002075 }
2076 }
2077 return;
Dan Gohman23e8b712008-04-28 17:02:21 +00002078 case ISD::UREM: {
2079 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
Dan Gohman9b44c1f2008-07-03 00:52:03 +00002080 const APInt &RA = Rem->getAPIntValue();
Dan Gohman23e1df82008-05-06 00:51:48 +00002081 if (RA.isPowerOf2()) {
2082 APInt LowBits = (RA - 1);
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002083 KnownZero |= ~LowBits;
2084 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne,Depth+1);
Dan Gohman23e8b712008-04-28 17:02:21 +00002085 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
2086 break;
2087 }
2088 }
2089
2090 // Since the result is less than or equal to either operand, any leading
2091 // zero bits in either operand must also exist in the result.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002092 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2093 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
Dan Gohman23e8b712008-04-28 17:02:21 +00002094
2095 uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
2096 KnownZero2.countLeadingOnes());
Jay Foad7a874dd2010-12-01 08:53:58 +00002097 KnownOne.clearAllBits();
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002098 KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
Dan Gohman23e8b712008-04-28 17:02:21 +00002099 return;
Dan Gohmanea859be2007-06-22 14:59:07 +00002100 }
Chris Lattner0a9481f2011-02-13 22:25:43 +00002101 case ISD::FrameIndex:
2102 case ISD::TargetFrameIndex:
2103 if (unsigned Align = InferPtrAlignment(Op)) {
2104 // The low bits are known zero if the pointer is aligned.
2105 KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
2106 return;
2107 }
2108 break;
Owen Anderson95771af2011-02-25 21:41:48 +00002109
Dan Gohmanea859be2007-06-22 14:59:07 +00002110 default:
Evan Chengb5a55d92011-05-24 01:48:22 +00002111 if (Op.getOpcode() < ISD::BUILTIN_OP_END)
2112 break;
2113 // Fallthrough
Dan Gohmanea859be2007-06-22 14:59:07 +00002114 case ISD::INTRINSIC_WO_CHAIN:
2115 case ISD::INTRINSIC_W_CHAIN:
2116 case ISD::INTRINSIC_VOID:
Evan Chengb5a55d92011-05-24 01:48:22 +00002117 // Allow the target to implement this method for its nodes.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002118 TLI.computeMaskedBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
Dan Gohmanea859be2007-06-22 14:59:07 +00002119 return;
2120 }
2121}
2122
2123/// ComputeNumSignBits - Return the number of times the sign bit of the
2124/// register is replicated into the other bits. We know that at least 1 bit
2125/// is always equal to the sign bit (itself), but other cases can give us
2126/// information. For example, immediately after an "SRA X, 2", we know that
2127/// the top 3 bits are all equal to each other, so we return 3.
Dan Gohman475871a2008-07-27 21:46:04 +00002128unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
Owen Andersone50ed302009-08-10 22:56:29 +00002129 EVT VT = Op.getValueType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00002130 assert(VT.isInteger() && "Invalid VT!");
Dan Gohman87862e72009-12-11 21:31:27 +00002131 unsigned VTBits = VT.getScalarType().getSizeInBits();
Dan Gohmanea859be2007-06-22 14:59:07 +00002132 unsigned Tmp, Tmp2;
Dan Gohmana332f172008-05-23 02:28:01 +00002133 unsigned FirstAnswer = 1;
Scott Michelfdc40a02009-02-17 22:15:04 +00002134
Dan Gohmanea859be2007-06-22 14:59:07 +00002135 if (Depth == 6)
2136 return 1; // Limit search depth.
2137
2138 switch (Op.getOpcode()) {
2139 default: break;
2140 case ISD::AssertSext:
Duncan Sands83ec4b62008-06-06 12:08:01 +00002141 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
Dan Gohmanea859be2007-06-22 14:59:07 +00002142 return VTBits-Tmp+1;
2143 case ISD::AssertZext:
Duncan Sands83ec4b62008-06-06 12:08:01 +00002144 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
Dan Gohmanea859be2007-06-22 14:59:07 +00002145 return VTBits-Tmp;
Scott Michelfdc40a02009-02-17 22:15:04 +00002146
Dan Gohmanea859be2007-06-22 14:59:07 +00002147 case ISD::Constant: {
Dan Gohmanbb271ff2008-03-03 23:35:36 +00002148 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
Cameron Zwarich9b6af8d2011-02-24 10:00:20 +00002149 return Val.getNumSignBits();
Dan Gohmanea859be2007-06-22 14:59:07 +00002150 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002151
Dan Gohmanea859be2007-06-22 14:59:07 +00002152 case ISD::SIGN_EXTEND:
Dan Gohman87862e72009-12-11 21:31:27 +00002153 Tmp = VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
Dan Gohmanea859be2007-06-22 14:59:07 +00002154 return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
Scott Michelfdc40a02009-02-17 22:15:04 +00002155
Dan Gohmanea859be2007-06-22 14:59:07 +00002156 case ISD::SIGN_EXTEND_INREG:
2157 // Max of the input and what this extends.
Dan Gohmand1996362010-01-09 02:13:55 +00002158 Tmp =
2159 cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
Dan Gohmanea859be2007-06-22 14:59:07 +00002160 Tmp = VTBits-Tmp+1;
Scott Michelfdc40a02009-02-17 22:15:04 +00002161
Dan Gohmanea859be2007-06-22 14:59:07 +00002162 Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2163 return std::max(Tmp, Tmp2);
2164
2165 case ISD::SRA:
2166 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2167 // SRA X, C -> adds C sign bits.
2168 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00002169 Tmp += C->getZExtValue();
Dan Gohmanea859be2007-06-22 14:59:07 +00002170 if (Tmp > VTBits) Tmp = VTBits;
2171 }
2172 return Tmp;
2173 case ISD::SHL:
2174 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2175 // shl destroys sign bits.
2176 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00002177 if (C->getZExtValue() >= VTBits || // Bad shift.
2178 C->getZExtValue() >= Tmp) break; // Shifted all sign bits out.
2179 return Tmp - C->getZExtValue();
Dan Gohmanea859be2007-06-22 14:59:07 +00002180 }
2181 break;
2182 case ISD::AND:
2183 case ISD::OR:
2184 case ISD::XOR: // NOT is handled here.
Dan Gohmana332f172008-05-23 02:28:01 +00002185 // Logical binary ops preserve the number of sign bits at the worst.
Dan Gohmanea859be2007-06-22 14:59:07 +00002186 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
Dan Gohmana332f172008-05-23 02:28:01 +00002187 if (Tmp != 1) {
2188 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2189 FirstAnswer = std::min(Tmp, Tmp2);
2190 // We computed what we know about the sign bits as our first
2191 // answer. Now proceed to the generic code that uses
2192 // ComputeMaskedBits, and pick whichever answer is better.
2193 }
2194 break;
Dan Gohmanea859be2007-06-22 14:59:07 +00002195
2196 case ISD::SELECT:
Dan Gohmanc84941b2008-05-20 20:59:51 +00002197 Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
Dan Gohmanea859be2007-06-22 14:59:07 +00002198 if (Tmp == 1) return 1; // Early out.
Dan Gohmanc84941b2008-05-20 20:59:51 +00002199 Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
Dan Gohmanea859be2007-06-22 14:59:07 +00002200 return std::min(Tmp, Tmp2);
Bill Wendling253174b2008-11-22 07:24:01 +00002201
2202 case ISD::SADDO:
2203 case ISD::UADDO:
Bill Wendling74c37652008-12-09 22:08:41 +00002204 case ISD::SSUBO:
2205 case ISD::USUBO:
2206 case ISD::SMULO:
2207 case ISD::UMULO:
Bill Wendling253174b2008-11-22 07:24:01 +00002208 if (Op.getResNo() != 1)
2209 break;
Duncan Sands03228082008-11-23 15:47:28 +00002210 // The boolean result conforms to getBooleanContents. Fall through.
Dan Gohmanea859be2007-06-22 14:59:07 +00002211 case ISD::SETCC:
2212 // If setcc returns 0/-1, all bits are sign bits.
Duncan Sands28b77e92011-09-06 19:07:46 +00002213 if (TLI.getBooleanContents(Op.getValueType().isVector()) ==
Duncan Sands03228082008-11-23 15:47:28 +00002214 TargetLowering::ZeroOrNegativeOneBooleanContent)
Dan Gohmanea859be2007-06-22 14:59:07 +00002215 return VTBits;
2216 break;
2217 case ISD::ROTL:
2218 case ISD::ROTR:
2219 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00002220 unsigned RotAmt = C->getZExtValue() & (VTBits-1);
Scott Michelfdc40a02009-02-17 22:15:04 +00002221
Dan Gohmanea859be2007-06-22 14:59:07 +00002222 // Handle rotate right by N like a rotate left by 32-N.
2223 if (Op.getOpcode() == ISD::ROTR)
2224 RotAmt = (VTBits-RotAmt) & (VTBits-1);
2225
2226 // If we aren't rotating out all of the known-in sign bits, return the
2227 // number that are left. This handles rotl(sext(x), 1) for example.
2228 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2229 if (Tmp > RotAmt+1) return Tmp-RotAmt;
2230 }
2231 break;
2232 case ISD::ADD:
2233 // Add can have at most one carry bit. Thus we know that the output
2234 // is, at worst, one more bit than the inputs.
2235 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2236 if (Tmp == 1) return 1; // Early out.
Scott Michelfdc40a02009-02-17 22:15:04 +00002237
Dan Gohmanea859be2007-06-22 14:59:07 +00002238 // Special case decrementing a value (ADD X, -1):
Dan Gohman0001e562009-02-24 02:00:40 +00002239 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
Dan Gohmanea859be2007-06-22 14:59:07 +00002240 if (CRHS->isAllOnesValue()) {
Dan Gohmanb3564aa2008-02-27 01:23:58 +00002241 APInt KnownZero, KnownOne;
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002242 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00002243
Dan Gohmanea859be2007-06-22 14:59:07 +00002244 // If the input is known to be 0 or 1, the output is 0/-1, which is all
2245 // sign bits set.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002246 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
Dan Gohmanea859be2007-06-22 14:59:07 +00002247 return VTBits;
Scott Michelfdc40a02009-02-17 22:15:04 +00002248
Dan Gohmanea859be2007-06-22 14:59:07 +00002249 // If we are subtracting one from a positive number, there is no carry
2250 // out of the result.
Dan Gohmanb3564aa2008-02-27 01:23:58 +00002251 if (KnownZero.isNegative())
Dan Gohmanea859be2007-06-22 14:59:07 +00002252 return Tmp;
2253 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002254
Dan Gohmanea859be2007-06-22 14:59:07 +00002255 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2256 if (Tmp2 == 1) return 1;
David Blaikie4d6ccb52012-01-20 21:51:11 +00002257 return std::min(Tmp, Tmp2)-1;
Scott Michelfdc40a02009-02-17 22:15:04 +00002258
Dan Gohmanea859be2007-06-22 14:59:07 +00002259 case ISD::SUB:
2260 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2261 if (Tmp2 == 1) return 1;
Scott Michelfdc40a02009-02-17 22:15:04 +00002262
Dan Gohmanea859be2007-06-22 14:59:07 +00002263 // Handle NEG.
2264 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
Dan Gohman002e5d02008-03-13 22:13:53 +00002265 if (CLHS->isNullValue()) {
Dan Gohmanb3564aa2008-02-27 01:23:58 +00002266 APInt KnownZero, KnownOne;
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002267 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
Dan Gohmanea859be2007-06-22 14:59:07 +00002268 // If the input is known to be 0 or 1, the output is 0/-1, which is all
2269 // sign bits set.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002270 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
Dan Gohmanea859be2007-06-22 14:59:07 +00002271 return VTBits;
Scott Michelfdc40a02009-02-17 22:15:04 +00002272
Dan Gohmanea859be2007-06-22 14:59:07 +00002273 // If the input is known to be positive (the sign bit is known clear),
2274 // the output of the NEG has the same number of sign bits as the input.
Dan Gohmanb3564aa2008-02-27 01:23:58 +00002275 if (KnownZero.isNegative())
Dan Gohmanea859be2007-06-22 14:59:07 +00002276 return Tmp2;
Scott Michelfdc40a02009-02-17 22:15:04 +00002277
Dan Gohmanea859be2007-06-22 14:59:07 +00002278 // Otherwise, we treat this like a SUB.
2279 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002280
Dan Gohmanea859be2007-06-22 14:59:07 +00002281 // Sub can have at most one carry bit. Thus we know that the output
2282 // is, at worst, one more bit than the inputs.
2283 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2284 if (Tmp == 1) return 1; // Early out.
David Blaikie4d6ccb52012-01-20 21:51:11 +00002285 return std::min(Tmp, Tmp2)-1;
Dan Gohmanea859be2007-06-22 14:59:07 +00002286 case ISD::TRUNCATE:
2287 // FIXME: it's tricky to do anything useful for this, but it is an important
2288 // case for targets like X86.
2289 break;
2290 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002291
Nadav Rotem77451752013-03-20 22:53:44 +00002292 // If we are looking at the loaded value of the SDNode.
2293 if (Op.getResNo() == 0) {
2294 // Handle LOADX separately here. EXTLOAD case will fallthrough.
2295 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
2296 unsigned ExtType = LD->getExtensionType();
2297 switch (ExtType) {
2298 default: break;
2299 case ISD::SEXTLOAD: // '17' bits known
2300 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
2301 return VTBits-Tmp+1;
2302 case ISD::ZEXTLOAD: // '16' bits known
2303 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
2304 return VTBits-Tmp;
2305 }
Dan Gohmanea859be2007-06-22 14:59:07 +00002306 }
2307 }
2308
2309 // Allow the target to implement this method for its nodes.
2310 if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
Scott Michelfdc40a02009-02-17 22:15:04 +00002311 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
Dan Gohmanea859be2007-06-22 14:59:07 +00002312 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2313 Op.getOpcode() == ISD::INTRINSIC_VOID) {
2314 unsigned NumBits = TLI.ComputeNumSignBitsForTargetNode(Op, Depth);
Dan Gohmana332f172008-05-23 02:28:01 +00002315 if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
Dan Gohmanea859be2007-06-22 14:59:07 +00002316 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002317
Dan Gohmanea859be2007-06-22 14:59:07 +00002318 // Finally, if we can prove that the top bits of the result are 0's or 1's,
2319 // use this information.
Dan Gohmanb3564aa2008-02-27 01:23:58 +00002320 APInt KnownZero, KnownOne;
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002321 ComputeMaskedBits(Op, KnownZero, KnownOne, Depth);
Scott Michelfdc40a02009-02-17 22:15:04 +00002322
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00002323 APInt Mask;
Dan Gohmanb3564aa2008-02-27 01:23:58 +00002324 if (KnownZero.isNegative()) { // sign bit is 0
Dan Gohmanea859be2007-06-22 14:59:07 +00002325 Mask = KnownZero;
Dan Gohmanb3564aa2008-02-27 01:23:58 +00002326 } else if (KnownOne.isNegative()) { // sign bit is 1;
Dan Gohmanea859be2007-06-22 14:59:07 +00002327 Mask = KnownOne;
2328 } else {
2329 // Nothing known.
Dan Gohmana332f172008-05-23 02:28:01 +00002330 return FirstAnswer;
Dan Gohmanea859be2007-06-22 14:59:07 +00002331 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002332
Dan Gohmanea859be2007-06-22 14:59:07 +00002333 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine
2334 // the number of identical bits in the top of the input value.
Dan Gohmanb3564aa2008-02-27 01:23:58 +00002335 Mask = ~Mask;
2336 Mask <<= Mask.getBitWidth()-VTBits;
Dan Gohmanea859be2007-06-22 14:59:07 +00002337 // Return # leading zeros. We use 'min' here in case Val was zero before
2338 // shifting. We don't want to return '64' as for an i32 "0".
Dan Gohmana332f172008-05-23 02:28:01 +00002339 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
Dan Gohmanea859be2007-06-22 14:59:07 +00002340}
2341
Chris Lattner0a9481f2011-02-13 22:25:43 +00002342/// isBaseWithConstantOffset - Return true if the specified operand is an
2343/// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
2344/// ISD::OR with a ConstantSDNode that is guaranteed to have the same
2345/// semantics as an ADD. This handles the equivalence:
Sylvestre Ledru94c22712012-09-27 10:14:43 +00002346/// X|Cst == X+Cst iff X&Cst = 0.
Chris Lattner0a9481f2011-02-13 22:25:43 +00002347bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
2348 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
2349 !isa<ConstantSDNode>(Op.getOperand(1)))
2350 return false;
Owen Anderson95771af2011-02-25 21:41:48 +00002351
2352 if (Op.getOpcode() == ISD::OR &&
Chris Lattner0a9481f2011-02-13 22:25:43 +00002353 !MaskedValueIsZero(Op.getOperand(0),
2354 cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
2355 return false;
Owen Anderson95771af2011-02-25 21:41:48 +00002356
Chris Lattner0a9481f2011-02-13 22:25:43 +00002357 return true;
2358}
2359
2360
Dan Gohman8d44b282009-09-03 20:34:31 +00002361bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
2362 // If we're told that NaNs won't happen, assume they won't.
Nick Lewycky8a8d4792011-12-02 22:16:29 +00002363 if (getTarget().Options.NoNaNsFPMath)
Dan Gohman8d44b282009-09-03 20:34:31 +00002364 return true;
2365
2366 // If the value is a constant, we can obviously see if it is a NaN or not.
2367 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
2368 return !C->getValueAPF().isNaN();
2369
2370 // TODO: Recognize more cases here.
2371
2372 return false;
2373}
Chris Lattner51dabfb2006-10-14 00:41:01 +00002374
Dan Gohmane8326932010-02-24 06:52:40 +00002375bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
2376 // If the value is a constant, we can obviously see if it is a zero or not.
2377 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
2378 return !C->isZero();
2379
2380 // TODO: Recognize more cases here.
Evan Chengb5a55d92011-05-24 01:48:22 +00002381 switch (Op.getOpcode()) {
2382 default: break;
2383 case ISD::OR:
2384 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
2385 return !C->isNullValue();
2386 break;
2387 }
Dan Gohmane8326932010-02-24 06:52:40 +00002388
2389 return false;
2390}
2391
2392bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
2393 // Check the obvious case.
2394 if (A == B) return true;
2395
2396 // For for negative and positive zero.
2397 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
2398 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
2399 if (CA->isZero() && CB->isZero()) return true;
2400
2401 // Otherwise they may not be equal.
2402 return false;
2403}
2404
Chris Lattnerc3aae252005-01-07 07:46:32 +00002405/// getNode - Gets or creates the specified node.
Chris Lattner78ec3112003-08-11 14:57:33 +00002406///
Andrew Trickac6d9be2013-05-25 02:42:55 +00002407SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
Jim Laskey583bd472006-10-27 23:46:08 +00002408 FoldingSetNodeID ID;
Dan Gohman6d9cdd52008-07-07 18:26:29 +00002409 AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0);
Chris Lattner4a283e92006-08-11 18:38:11 +00002410 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00002411 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00002412 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00002413
Andrew Trickac6d9be2013-05-25 02:42:55 +00002414 SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), getVTList(VT));
Chris Lattner4a283e92006-08-11 18:38:11 +00002415 CSEMap.InsertNode(N, IP);
Scott Michelfdc40a02009-02-17 22:15:04 +00002416
Chris Lattner4a283e92006-08-11 18:38:11 +00002417 AllNodes.push_back(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00002418#ifndef NDEBUG
Duncan Sands59d2dad2010-11-20 11:25:00 +00002419 VerifySDNode(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00002420#endif
Dan Gohman475871a2008-07-27 21:46:04 +00002421 return SDValue(N, 0);
Chris Lattnerc3aae252005-01-07 07:46:32 +00002422}
2423
Andrew Trickac6d9be2013-05-25 02:42:55 +00002424SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
Owen Andersone50ed302009-08-10 22:56:29 +00002425 EVT VT, SDValue Operand) {
Chris Lattner94683772005-12-23 05:30:37 +00002426 // Constant fold unary operations with an integer constant operand.
Gabor Greifba36cb52008-08-28 21:40:38 +00002427 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
Dan Gohman6c231502008-02-29 01:47:35 +00002428 const APInt &Val = C->getAPIntValue();
Chris Lattnerc3aae252005-01-07 07:46:32 +00002429 switch (Opcode) {
2430 default: break;
Evan Chengeb49c4e2008-03-06 17:42:34 +00002431 case ISD::SIGN_EXTEND:
Jay Foad40f8f622010-12-07 08:25:19 +00002432 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT);
Chris Lattner4ed11b42005-09-02 00:17:32 +00002433 case ISD::ANY_EXTEND:
Dan Gohman6c231502008-02-29 01:47:35 +00002434 case ISD::ZERO_EXTEND:
Evan Chengeb49c4e2008-03-06 17:42:34 +00002435 case ISD::TRUNCATE:
Jay Foad40f8f622010-12-07 08:25:19 +00002436 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT);
Dale Johannesen73328d12007-09-19 23:55:34 +00002437 case ISD::UINT_TO_FP:
2438 case ISD::SINT_TO_FP: {
Tim Northover0a29cb02013-01-22 09:46:31 +00002439 APFloat apf(EVTToAPFloatSemantics(VT),
2440 APInt::getNullValue(VT.getSizeInBits()));
Scott Michelfdc40a02009-02-17 22:15:04 +00002441 (void)apf.convertFromAPInt(Val,
Dan Gohman6c231502008-02-29 01:47:35 +00002442 Opcode==ISD::SINT_TO_FP,
2443 APFloat::rmNearestTiesToEven);
Dale Johannesen73328d12007-09-19 23:55:34 +00002444 return getConstantFP(apf, VT);
2445 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002446 case ISD::BITCAST:
Owen Anderson825b72b2009-08-11 20:47:22 +00002447 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
Tim Northover0a29cb02013-01-22 09:46:31 +00002448 return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
Owen Anderson825b72b2009-08-11 20:47:22 +00002449 else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
Tim Northover0a29cb02013-01-22 09:46:31 +00002450 return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
Chris Lattner94683772005-12-23 05:30:37 +00002451 break;
Nate Begeman1b5db7a2006-01-16 08:07:10 +00002452 case ISD::BSWAP:
Dan Gohman6c231502008-02-29 01:47:35 +00002453 return getConstant(Val.byteSwap(), VT);
Nate Begeman1b5db7a2006-01-16 08:07:10 +00002454 case ISD::CTPOP:
Dan Gohman6c231502008-02-29 01:47:35 +00002455 return getConstant(Val.countPopulation(), VT);
Nate Begeman1b5db7a2006-01-16 08:07:10 +00002456 case ISD::CTLZ:
Chandler Carruth63974b22011-12-13 01:56:10 +00002457 case ISD::CTLZ_ZERO_UNDEF:
Dan Gohman6c231502008-02-29 01:47:35 +00002458 return getConstant(Val.countLeadingZeros(), VT);
Nate Begeman1b5db7a2006-01-16 08:07:10 +00002459 case ISD::CTTZ:
Chandler Carruth63974b22011-12-13 01:56:10 +00002460 case ISD::CTTZ_ZERO_UNDEF:
Dan Gohman6c231502008-02-29 01:47:35 +00002461 return getConstant(Val.countTrailingZeros(), VT);
Chris Lattnerc3aae252005-01-07 07:46:32 +00002462 }
2463 }
2464
Dale Johannesenc4dd3c32007-08-31 23:34:27 +00002465 // Constant fold unary operations with a floating point constant operand.
Gabor Greifba36cb52008-08-28 21:40:38 +00002466 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
Dale Johannesenc4dd3c32007-08-31 23:34:27 +00002467 APFloat V = C->getValueAPF(); // make copy
Ulrich Weigande669c932012-10-29 18:35:49 +00002468 switch (Opcode) {
2469 case ISD::FNEG:
2470 V.changeSign();
2471 return getConstantFP(V, VT);
2472 case ISD::FABS:
2473 V.clearSign();
2474 return getConstantFP(V, VT);
2475 case ISD::FCEIL: {
2476 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
2477 if (fs == APFloat::opOK || fs == APFloat::opInexact)
Dale Johannesendb44bf82007-10-16 23:38:29 +00002478 return getConstantFP(V, VT);
Ulrich Weigande669c932012-10-29 18:35:49 +00002479 break;
2480 }
2481 case ISD::FTRUNC: {
2482 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
2483 if (fs == APFloat::opOK || fs == APFloat::opInexact)
Dale Johannesendb44bf82007-10-16 23:38:29 +00002484 return getConstantFP(V, VT);
Ulrich Weigande669c932012-10-29 18:35:49 +00002485 break;
2486 }
2487 case ISD::FFLOOR: {
2488 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
2489 if (fs == APFloat::opOK || fs == APFloat::opInexact)
Dale Johannesendb44bf82007-10-16 23:38:29 +00002490 return getConstantFP(V, VT);
Ulrich Weigande669c932012-10-29 18:35:49 +00002491 break;
2492 }
2493 case ISD::FP_EXTEND: {
2494 bool ignored;
2495 // This can return overflow, underflow, or inexact; we don't care.
2496 // FIXME need to be more flexible about rounding mode.
Tim Northover0a29cb02013-01-22 09:46:31 +00002497 (void)V.convert(EVTToAPFloatSemantics(VT),
Ulrich Weigande669c932012-10-29 18:35:49 +00002498 APFloat::rmNearestTiesToEven, &ignored);
2499 return getConstantFP(V, VT);
2500 }
2501 case ISD::FP_TO_SINT:
2502 case ISD::FP_TO_UINT: {
2503 integerPart x[2];
2504 bool ignored;
2505 assert(integerPartWidth >= 64);
2506 // FIXME need to be more flexible about rounding mode.
2507 APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
2508 Opcode==ISD::FP_TO_SINT,
2509 APFloat::rmTowardZero, &ignored);
2510 if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual
Dale Johannesenc4dd3c32007-08-31 23:34:27 +00002511 break;
Ulrich Weigande669c932012-10-29 18:35:49 +00002512 APInt api(VT.getSizeInBits(), x);
2513 return getConstant(api, VT);
2514 }
2515 case ISD::BITCAST:
2516 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
2517 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
2518 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
2519 return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
2520 break;
Chris Lattnerc3aae252005-01-07 07:46:32 +00002521 }
Dale Johannesenc4dd3c32007-08-31 23:34:27 +00002522 }
Chris Lattnerc3aae252005-01-07 07:46:32 +00002523
Gabor Greifba36cb52008-08-28 21:40:38 +00002524 unsigned OpOpcode = Operand.getNode()->getOpcode();
Chris Lattnerc3aae252005-01-07 07:46:32 +00002525 switch (Opcode) {
Chris Lattnera93ec3e2005-01-21 18:01:22 +00002526 case ISD::TokenFactor:
Duncan Sandsaaffa052008-12-01 11:41:29 +00002527 case ISD::MERGE_VALUES:
Dan Gohman7f8613e2008-08-14 20:04:46 +00002528 case ISD::CONCAT_VECTORS:
Duncan Sandsaaffa052008-12-01 11:41:29 +00002529 return Operand; // Factor, merge or concat of one node? No need.
Torok Edwinc23197a2009-07-14 16:55:14 +00002530 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
Chris Lattnerff33cc42007-04-09 05:23:13 +00002531 case ISD::FP_EXTEND:
Duncan Sands83ec4b62008-06-06 12:08:01 +00002532 assert(VT.isFloatingPoint() &&
2533 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
Chris Lattner7e2e0332008-01-16 17:59:31 +00002534 if (Operand.getValueType() == VT) return Operand; // noop conversion.
Dan Gohman2e141d72009-12-14 23:40:38 +00002535 assert((!VT.isVector() ||
2536 VT.getVectorNumElements() ==
2537 Operand.getValueType().getVectorNumElements()) &&
2538 "Vector element count mismatch!");
Chris Lattner5d03f212008-03-11 06:21:08 +00002539 if (Operand.getOpcode() == ISD::UNDEF)
Dale Johannesene8d72302009-02-06 23:05:02 +00002540 return getUNDEF(VT);
Chris Lattnerff33cc42007-04-09 05:23:13 +00002541 break;
Chris Lattner5d03f212008-03-11 06:21:08 +00002542 case ISD::SIGN_EXTEND:
Duncan Sands83ec4b62008-06-06 12:08:01 +00002543 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
Chris Lattnerff33cc42007-04-09 05:23:13 +00002544 "Invalid SIGN_EXTEND!");
Chris Lattnerc3aae252005-01-07 07:46:32 +00002545 if (Operand.getValueType() == VT) return Operand; // noop extension
Dan Gohman2e141d72009-12-14 23:40:38 +00002546 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
2547 "Invalid sext node, dst < src!");
2548 assert((!VT.isVector() ||
2549 VT.getVectorNumElements() ==
2550 Operand.getValueType().getVectorNumElements()) &&
2551 "Vector element count mismatch!");
Nadav Rotem0c8607b2013-01-20 08:35:56 +00002552 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
2553 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
2554 else if (OpOpcode == ISD::UNDEF)
Evan Chengbf34a5e2011-03-15 02:22:10 +00002555 // sext(undef) = 0, because the top bits will all be the same.
2556 return getConstant(0, VT);
Chris Lattnerc3aae252005-01-07 07:46:32 +00002557 break;
2558 case ISD::ZERO_EXTEND:
Duncan Sands83ec4b62008-06-06 12:08:01 +00002559 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
Chris Lattnerff33cc42007-04-09 05:23:13 +00002560 "Invalid ZERO_EXTEND!");
Chris Lattnerc3aae252005-01-07 07:46:32 +00002561 if (Operand.getValueType() == VT) return Operand; // noop extension
Dan Gohman2e141d72009-12-14 23:40:38 +00002562 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
2563 "Invalid zext node, dst < src!");
2564 assert((!VT.isVector() ||
2565 VT.getVectorNumElements() ==
2566 Operand.getValueType().getVectorNumElements()) &&
2567 "Vector element count mismatch!");
Chris Lattner5a6bace2005-04-07 19:43:53 +00002568 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
Scott Michelfdc40a02009-02-17 22:15:04 +00002569 return getNode(ISD::ZERO_EXTEND, DL, VT,
Dale Johannesendbfd8db2009-02-03 01:55:44 +00002570 Operand.getNode()->getOperand(0));
Evan Chengbf34a5e2011-03-15 02:22:10 +00002571 else if (OpOpcode == ISD::UNDEF)
2572 // zext(undef) = 0, because the top bits will be zero.
2573 return getConstant(0, VT);
Chris Lattnerc3aae252005-01-07 07:46:32 +00002574 break;
Chris Lattner4ed11b42005-09-02 00:17:32 +00002575 case ISD::ANY_EXTEND:
Duncan Sands83ec4b62008-06-06 12:08:01 +00002576 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
Chris Lattnerff33cc42007-04-09 05:23:13 +00002577 "Invalid ANY_EXTEND!");
Chris Lattner4ed11b42005-09-02 00:17:32 +00002578 if (Operand.getValueType() == VT) return Operand; // noop extension
Dan Gohman2e141d72009-12-14 23:40:38 +00002579 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
2580 "Invalid anyext node, dst < src!");
2581 assert((!VT.isVector() ||
2582 VT.getVectorNumElements() ==
2583 Operand.getValueType().getVectorNumElements()) &&
2584 "Vector element count mismatch!");
Dan Gohman4e39e9d2010-06-24 14:30:44 +00002585
Dan Gohman9e86a732010-06-18 00:08:30 +00002586 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
2587 OpOpcode == ISD::ANY_EXTEND)
Chris Lattner4ed11b42005-09-02 00:17:32 +00002588 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
Dale Johannesendbfd8db2009-02-03 01:55:44 +00002589 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
Evan Cheng5ae1da92011-03-14 18:15:55 +00002590 else if (OpOpcode == ISD::UNDEF)
2591 return getUNDEF(VT);
Dan Gohman4e39e9d2010-06-24 14:30:44 +00002592
2593 // (ext (trunx x)) -> x
2594 if (OpOpcode == ISD::TRUNCATE) {
2595 SDValue OpOp = Operand.getNode()->getOperand(0);
2596 if (OpOp.getValueType() == VT)
2597 return OpOp;
2598 }
Chris Lattner4ed11b42005-09-02 00:17:32 +00002599 break;
Chris Lattnerc3aae252005-01-07 07:46:32 +00002600 case ISD::TRUNCATE:
Duncan Sands83ec4b62008-06-06 12:08:01 +00002601 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
Chris Lattnerff33cc42007-04-09 05:23:13 +00002602 "Invalid TRUNCATE!");
Chris Lattnerc3aae252005-01-07 07:46:32 +00002603 if (Operand.getValueType() == VT) return Operand; // noop truncate
Dan Gohman2e141d72009-12-14 23:40:38 +00002604 assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
2605 "Invalid truncate node, src < dst!");
2606 assert((!VT.isVector() ||
2607 VT.getVectorNumElements() ==
2608 Operand.getValueType().getVectorNumElements()) &&
2609 "Vector element count mismatch!");
Chris Lattnerc3aae252005-01-07 07:46:32 +00002610 if (OpOpcode == ISD::TRUNCATE)
Dale Johannesendbfd8db2009-02-03 01:55:44 +00002611 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
Craig Topper799ea5c2012-01-15 01:05:11 +00002612 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
2613 OpOpcode == ISD::ANY_EXTEND) {
Chris Lattnerfd8c39b2005-01-07 21:56:24 +00002614 // If the source is smaller than the dest, we still need an extend.
Dan Gohman2e141d72009-12-14 23:40:38 +00002615 if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
2616 .bitsLT(VT.getScalarType()))
Dale Johannesendbfd8db2009-02-03 01:55:44 +00002617 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
Craig Topper799ea5c2012-01-15 01:05:11 +00002618 if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
Dale Johannesendbfd8db2009-02-03 01:55:44 +00002619 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
Craig Topper799ea5c2012-01-15 01:05:11 +00002620 return Operand.getNode()->getOperand(0);
Chris Lattnerfd8c39b2005-01-07 21:56:24 +00002621 }
Craig Topper799ea5c2012-01-15 01:05:11 +00002622 if (OpOpcode == ISD::UNDEF)
2623 return getUNDEF(VT);
Chris Lattnerc3aae252005-01-07 07:46:32 +00002624 break;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002625 case ISD::BITCAST:
Chris Lattner35481892005-12-23 00:16:34 +00002626 // Basic sanity checking.
Duncan Sands83ec4b62008-06-06 12:08:01 +00002627 assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002628 && "Cannot BITCAST between types of different sizes!");
Chris Lattner35481892005-12-23 00:16:34 +00002629 if (VT == Operand.getValueType()) return Operand; // noop conversion.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002630 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
2631 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
Chris Lattner08da55e2006-04-04 01:02:22 +00002632 if (OpOpcode == ISD::UNDEF)
Dale Johannesene8d72302009-02-06 23:05:02 +00002633 return getUNDEF(VT);
Chris Lattner35481892005-12-23 00:16:34 +00002634 break;
Chris Lattnerce872152006-03-19 06:31:19 +00002635 case ISD::SCALAR_TO_VECTOR:
Duncan Sands83ec4b62008-06-06 12:08:01 +00002636 assert(VT.isVector() && !Operand.getValueType().isVector() &&
Duncan Sandsb10b5ac2009-04-18 20:16:54 +00002637 (VT.getVectorElementType() == Operand.getValueType() ||
2638 (VT.getVectorElementType().isInteger() &&
2639 Operand.getValueType().isInteger() &&
2640 VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
Chris Lattnerce872152006-03-19 06:31:19 +00002641 "Illegal SCALAR_TO_VECTOR node!");
Chris Lattnerf3ba4342008-03-08 23:43:36 +00002642 if (OpOpcode == ISD::UNDEF)
Dale Johannesene8d72302009-02-06 23:05:02 +00002643 return getUNDEF(VT);
Chris Lattnerf3ba4342008-03-08 23:43:36 +00002644 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
2645 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
2646 isa<ConstantSDNode>(Operand.getOperand(1)) &&
2647 Operand.getConstantOperandVal(1) == 0 &&
2648 Operand.getOperand(0).getValueType() == VT)
2649 return Operand.getOperand(0);
Chris Lattnerce872152006-03-19 06:31:19 +00002650 break;
Chris Lattner485df9b2005-04-09 03:02:46 +00002651 case ISD::FNEG:
Mon P Wanga7b6cff2009-01-31 06:07:45 +00002652 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
Nick Lewycky8a8d4792011-12-02 22:16:29 +00002653 if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
Dale Johannesendbfd8db2009-02-03 01:55:44 +00002654 return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
Gabor Greifba36cb52008-08-28 21:40:38 +00002655 Operand.getNode()->getOperand(0));
Chris Lattner485df9b2005-04-09 03:02:46 +00002656 if (OpOpcode == ISD::FNEG) // --X -> X
Gabor Greifba36cb52008-08-28 21:40:38 +00002657 return Operand.getNode()->getOperand(0);
Chris Lattner485df9b2005-04-09 03:02:46 +00002658 break;
2659 case ISD::FABS:
2660 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
Dale Johannesendbfd8db2009-02-03 01:55:44 +00002661 return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
Chris Lattner485df9b2005-04-09 03:02:46 +00002662 break;
Chris Lattnerc3aae252005-01-07 07:46:32 +00002663 }
2664
Chris Lattner43247a12005-08-25 19:12:10 +00002665 SDNode *N;
Chris Lattner0b3e5252006-08-15 19:11:05 +00002666 SDVTList VTs = getVTList(VT);
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00002667 if (VT != MVT::Glue) { // Don't CSE flag producing nodes
Jim Laskey583bd472006-10-27 23:46:08 +00002668 FoldingSetNodeID ID;
Dan Gohman475871a2008-07-27 21:46:04 +00002669 SDValue Ops[1] = { Operand };
Chris Lattner63e3f142007-02-04 07:28:00 +00002670 AddNodeIDNode(ID, Opcode, VTs, Ops, 1);
Chris Lattnera5682852006-08-07 23:03:03 +00002671 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00002672 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00002673 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00002674
Andrew Trickac6d9be2013-05-25 02:42:55 +00002675 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Operand);
Chris Lattnera5682852006-08-07 23:03:03 +00002676 CSEMap.InsertNode(N, IP);
Chris Lattner43247a12005-08-25 19:12:10 +00002677 } else {
Andrew Trickac6d9be2013-05-25 02:42:55 +00002678 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Operand);
Chris Lattner43247a12005-08-25 19:12:10 +00002679 }
Duncan Sandsd038e042008-07-21 10:20:31 +00002680
Chris Lattnerc3aae252005-01-07 07:46:32 +00002681 AllNodes.push_back(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00002682#ifndef NDEBUG
Duncan Sands59d2dad2010-11-20 11:25:00 +00002683 VerifySDNode(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00002684#endif
Dan Gohman475871a2008-07-27 21:46:04 +00002685 return SDValue(N, 0);
Chris Lattner78ec3112003-08-11 14:57:33 +00002686}
2687
Benjamin Kramer49693102013-02-04 15:19:18 +00002688SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
2689 SDNode *Cst1, SDNode *Cst2) {
2690 SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
2691 SmallVector<SDValue, 4> Outputs;
2692 EVT SVT = VT.getScalarType();
Bill Wendling688d1c42008-09-24 10:16:24 +00002693
Benjamin Kramer49693102013-02-04 15:19:18 +00002694 ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
2695 ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
2696 if (Scalar1 && Scalar2) {
2697 // Scalar instruction.
2698 Inputs.push_back(std::make_pair(Scalar1, Scalar2));
2699 } else {
2700 // For vectors extract each constant element into Inputs so we can constant
2701 // fold them individually.
2702 BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
2703 BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
2704 if (!BV1 || !BV2)
2705 return SDValue();
2706
2707 assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
2708
2709 for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
2710 ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
2711 ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
2712 if (!V1 || !V2) // Not a constant, bail.
2713 return SDValue();
2714
2715 // Avoid BUILD_VECTOR nodes that perform implicit truncation.
2716 // FIXME: This is valid and could be handled by truncating the APInts.
2717 if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
2718 return SDValue();
2719
2720 Inputs.push_back(std::make_pair(V1, V2));
2721 }
Bill Wendling688d1c42008-09-24 10:16:24 +00002722 }
2723
Benjamin Kramer49693102013-02-04 15:19:18 +00002724 // We have a number of constant values, constant fold them element by element.
2725 for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
2726 const APInt &C1 = Inputs[I].first->getAPIntValue();
2727 const APInt &C2 = Inputs[I].second->getAPIntValue();
2728
2729 switch (Opcode) {
2730 case ISD::ADD:
2731 Outputs.push_back(getConstant(C1 + C2, SVT));
2732 break;
2733 case ISD::SUB:
2734 Outputs.push_back(getConstant(C1 - C2, SVT));
2735 break;
2736 case ISD::MUL:
2737 Outputs.push_back(getConstant(C1 * C2, SVT));
2738 break;
2739 case ISD::UDIV:
2740 if (!C2.getBoolValue())
2741 return SDValue();
2742 Outputs.push_back(getConstant(C1.udiv(C2), SVT));
2743 break;
2744 case ISD::UREM:
2745 if (!C2.getBoolValue())
2746 return SDValue();
2747 Outputs.push_back(getConstant(C1.urem(C2), SVT));
2748 break;
2749 case ISD::SDIV:
2750 if (!C2.getBoolValue())
2751 return SDValue();
2752 Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
2753 break;
2754 case ISD::SREM:
2755 if (!C2.getBoolValue())
2756 return SDValue();
2757 Outputs.push_back(getConstant(C1.srem(C2), SVT));
2758 break;
2759 case ISD::AND:
2760 Outputs.push_back(getConstant(C1 & C2, SVT));
2761 break;
2762 case ISD::OR:
2763 Outputs.push_back(getConstant(C1 | C2, SVT));
2764 break;
2765 case ISD::XOR:
2766 Outputs.push_back(getConstant(C1 ^ C2, SVT));
2767 break;
2768 case ISD::SHL:
2769 Outputs.push_back(getConstant(C1 << C2, SVT));
2770 break;
2771 case ISD::SRL:
2772 Outputs.push_back(getConstant(C1.lshr(C2), SVT));
2773 break;
2774 case ISD::SRA:
2775 Outputs.push_back(getConstant(C1.ashr(C2), SVT));
2776 break;
2777 case ISD::ROTL:
2778 Outputs.push_back(getConstant(C1.rotl(C2), SVT));
2779 break;
2780 case ISD::ROTR:
2781 Outputs.push_back(getConstant(C1.rotr(C2), SVT));
2782 break;
2783 default:
2784 return SDValue();
2785 }
2786 }
2787
2788 // Handle the scalar case first.
Silviu Baranga02066832013-04-25 09:32:33 +00002789 if (Scalar1 && Scalar2)
Benjamin Kramer49693102013-02-04 15:19:18 +00002790 return Outputs.back();
2791
2792 // Otherwise build a big vector out of the scalar elements we generated.
Andrew Trickac6d9be2013-05-25 02:42:55 +00002793 return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs.data(),
Benjamin Kramer49693102013-02-04 15:19:18 +00002794 Outputs.size());
Bill Wendling688d1c42008-09-24 10:16:24 +00002795}
2796
Andrew Trickac6d9be2013-05-25 02:42:55 +00002797SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
Benjamin Kramer49693102013-02-04 15:19:18 +00002798 SDValue N2) {
Gabor Greifba36cb52008-08-28 21:40:38 +00002799 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
2800 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
Chris Lattner76365122005-01-16 02:23:22 +00002801 switch (Opcode) {
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002802 default: break;
Chris Lattner39908e02005-01-19 18:01:40 +00002803 case ISD::TokenFactor:
Owen Anderson825b72b2009-08-11 20:47:22 +00002804 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
2805 N2.getValueType() == MVT::Other && "Invalid token factor!");
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002806 // Fold trivial token factors.
2807 if (N1.getOpcode() == ISD::EntryToken) return N2;
2808 if (N2.getOpcode() == ISD::EntryToken) return N1;
Dan Gohman38ac0622008-10-01 15:11:19 +00002809 if (N1 == N2) return N1;
Chris Lattner39908e02005-01-19 18:01:40 +00002810 break;
Dan Gohman7f8613e2008-08-14 20:04:46 +00002811 case ISD::CONCAT_VECTORS:
Nadav Rotemb87bdac2012-07-15 08:38:23 +00002812 // Concat of UNDEFs is UNDEF.
2813 if (N1.getOpcode() == ISD::UNDEF &&
2814 N2.getOpcode() == ISD::UNDEF)
2815 return getUNDEF(VT);
2816
Dan Gohman7f8613e2008-08-14 20:04:46 +00002817 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
2818 // one big BUILD_VECTOR.
2819 if (N1.getOpcode() == ISD::BUILD_VECTOR &&
2820 N2.getOpcode() == ISD::BUILD_VECTOR) {
Gabor Greif481c4c02010-07-22 17:18:03 +00002821 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
2822 N1.getNode()->op_end());
Dan Gohman403a8cd2010-06-21 19:47:52 +00002823 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
Evan Chenga87008d2009-02-25 22:49:59 +00002824 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size());
Dan Gohman7f8613e2008-08-14 20:04:46 +00002825 }
2826 break;
Chris Lattner76365122005-01-16 02:23:22 +00002827 case ISD::AND:
Dale Johannesen78995512010-05-15 18:38:02 +00002828 assert(VT.isInteger() && "This operator does not apply to FP types!");
2829 assert(N1.getValueType() == N2.getValueType() &&
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002830 N1.getValueType() == VT && "Binary operator types must match!");
2831 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
2832 // worth handling here.
Dan Gohman002e5d02008-03-13 22:13:53 +00002833 if (N2C && N2C->isNullValue())
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002834 return N2;
Chris Lattner9967c152008-01-26 01:05:42 +00002835 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X
2836 return N1;
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002837 break;
Chris Lattner76365122005-01-16 02:23:22 +00002838 case ISD::OR:
2839 case ISD::XOR:
Dan Gohman33b625b2008-06-02 22:27:05 +00002840 case ISD::ADD:
2841 case ISD::SUB:
Dale Johannesen78995512010-05-15 18:38:02 +00002842 assert(VT.isInteger() && "This operator does not apply to FP types!");
2843 assert(N1.getValueType() == N2.getValueType() &&
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002844 N1.getValueType() == VT && "Binary operator types must match!");
Dan Gohman33b625b2008-06-02 22:27:05 +00002845 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
2846 // it's worth handling here.
Dan Gohman002e5d02008-03-13 22:13:53 +00002847 if (N2C && N2C->isNullValue())
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002848 return N1;
2849 break;
Chris Lattner76365122005-01-16 02:23:22 +00002850 case ISD::UDIV:
2851 case ISD::UREM:
Chris Lattnere5eb6f82005-05-15 05:39:08 +00002852 case ISD::MULHU:
2853 case ISD::MULHS:
Chris Lattner76365122005-01-16 02:23:22 +00002854 case ISD::MUL:
2855 case ISD::SDIV:
2856 case ISD::SREM:
Dan Gohman760f86f2009-01-22 21:58:43 +00002857 assert(VT.isInteger() && "This operator does not apply to FP types!");
Dale Johannesen78995512010-05-15 18:38:02 +00002858 assert(N1.getValueType() == N2.getValueType() &&
2859 N1.getValueType() == VT && "Binary operator types must match!");
2860 break;
Chris Lattner01b3d732005-09-28 22:28:18 +00002861 case ISD::FADD:
2862 case ISD::FSUB:
2863 case ISD::FMUL:
2864 case ISD::FDIV:
2865 case ISD::FREM:
Nick Lewycky8a8d4792011-12-02 22:16:29 +00002866 if (getTarget().Options.UnsafeFPMath) {
Dan Gohmana90c8e62009-01-23 19:10:37 +00002867 if (Opcode == ISD::FADD) {
2868 // 0+x --> x
2869 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
2870 if (CFP->getValueAPF().isZero())
2871 return N2;
2872 // x+0 --> x
2873 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
2874 if (CFP->getValueAPF().isZero())
2875 return N1;
2876 } else if (Opcode == ISD::FSUB) {
2877 // x-0 --> x
2878 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
2879 if (CFP->getValueAPF().isZero())
2880 return N1;
Michael Ilseman10def392012-09-10 17:00:37 +00002881 } else if (Opcode == ISD::FMUL) {
2882 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
2883 SDValue V = N2;
2884
2885 // If the first operand isn't the constant, try the second
2886 if (!CFP) {
2887 CFP = dyn_cast<ConstantFPSDNode>(N2);
2888 V = N1;
2889 }
2890
2891 if (CFP) {
2892 // 0*x --> 0
2893 if (CFP->isZero())
2894 return SDValue(CFP,0);
2895 // 1*x --> x
2896 if (CFP->isExactlyValue(1.0))
2897 return V;
2898 }
Dan Gohmana90c8e62009-01-23 19:10:37 +00002899 }
Dan Gohman760f86f2009-01-22 21:58:43 +00002900 }
Dale Johannesen78995512010-05-15 18:38:02 +00002901 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
Chris Lattner76365122005-01-16 02:23:22 +00002902 assert(N1.getValueType() == N2.getValueType() &&
2903 N1.getValueType() == VT && "Binary operator types must match!");
2904 break;
Chris Lattnera09f8482006-03-05 05:09:38 +00002905 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
2906 assert(N1.getValueType() == VT &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00002907 N1.getValueType().isFloatingPoint() &&
2908 N2.getValueType().isFloatingPoint() &&
Chris Lattnera09f8482006-03-05 05:09:38 +00002909 "Invalid FCOPYSIGN!");
2910 break;
Chris Lattner76365122005-01-16 02:23:22 +00002911 case ISD::SHL:
2912 case ISD::SRA:
2913 case ISD::SRL:
Nate Begeman35ef9132006-01-11 21:21:00 +00002914 case ISD::ROTL:
2915 case ISD::ROTR:
Chris Lattner76365122005-01-16 02:23:22 +00002916 assert(VT == N1.getValueType() &&
2917 "Shift operators return type must be the same as their first arg");
Duncan Sands83ec4b62008-06-06 12:08:01 +00002918 assert(VT.isInteger() && N2.getValueType().isInteger() &&
Chris Lattner349db172008-07-02 17:01:57 +00002919 "Shifts only work on integers");
Michael Liaoa6b20ce2013-03-01 18:40:30 +00002920 assert((!VT.isVector() || VT == N2.getValueType()) &&
2921 "Vector shift amounts must be in the same as their first arg");
Chris Lattnere0751182011-02-13 19:09:16 +00002922 // Verify that the shift amount VT is bit enough to hold valid shift
2923 // amounts. This catches things like trying to shift an i1024 value by an
2924 // i8, which is easy to fall into in generic code that uses
2925 // TLI.getShiftAmount().
2926 assert(N2.getValueType().getSizeInBits() >=
Owen Anderson95771af2011-02-25 21:41:48 +00002927 Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
Chris Lattnere0751182011-02-13 19:09:16 +00002928 "Invalid use of small shift amount with oversized value!");
Chris Lattner349db172008-07-02 17:01:57 +00002929
2930 // Always fold shifts of i1 values so the code generator doesn't need to
2931 // handle them. Since we know the size of the shift has to be less than the
2932 // size of the value, the shift/rotate count is guaranteed to be zero.
Owen Anderson825b72b2009-08-11 20:47:22 +00002933 if (VT == MVT::i1)
Chris Lattner349db172008-07-02 17:01:57 +00002934 return N1;
Evan Chengd40d03e2010-01-06 19:38:29 +00002935 if (N2C && N2C->isNullValue())
2936 return N1;
Chris Lattner76365122005-01-16 02:23:22 +00002937 break;
Chris Lattner15e4b012005-07-10 00:07:11 +00002938 case ISD::FP_ROUND_INREG: {
Owen Andersone50ed302009-08-10 22:56:29 +00002939 EVT EVT = cast<VTSDNode>(N2)->getVT();
Chris Lattner15e4b012005-07-10 00:07:11 +00002940 assert(VT == N1.getValueType() && "Not an inreg round!");
Duncan Sands83ec4b62008-06-06 12:08:01 +00002941 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
Chris Lattner15e4b012005-07-10 00:07:11 +00002942 "Cannot FP_ROUND_INREG integer types");
Dan Gohmand1996362010-01-09 02:13:55 +00002943 assert(EVT.isVector() == VT.isVector() &&
Sylvestre Ledru94c22712012-09-27 10:14:43 +00002944 "FP_ROUND_INREG type should be vector iff the operand "
Dan Gohmand1996362010-01-09 02:13:55 +00002945 "type is vector!");
2946 assert((!EVT.isVector() ||
2947 EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
2948 "Vector element counts must match in FP_ROUND_INREG");
Duncan Sands8e4eb092008-06-08 20:54:56 +00002949 assert(EVT.bitsLE(VT) && "Not rounding down!");
Duncan Sands17001ce2011-10-18 12:44:00 +00002950 (void)EVT;
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002951 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding.
Chris Lattner15e4b012005-07-10 00:07:11 +00002952 break;
2953 }
Chris Lattner0bd48932008-01-17 07:00:52 +00002954 case ISD::FP_ROUND:
Duncan Sands83ec4b62008-06-06 12:08:01 +00002955 assert(VT.isFloatingPoint() &&
2956 N1.getValueType().isFloatingPoint() &&
Duncan Sands8e4eb092008-06-08 20:54:56 +00002957 VT.bitsLE(N1.getValueType()) &&
Chris Lattner0bd48932008-01-17 07:00:52 +00002958 isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002959 if (N1.getValueType() == VT) return N1; // noop conversion.
Chris Lattner0bd48932008-01-17 07:00:52 +00002960 break;
Nate Begeman56eb8682005-08-30 02:44:00 +00002961 case ISD::AssertSext:
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002962 case ISD::AssertZext: {
Owen Andersone50ed302009-08-10 22:56:29 +00002963 EVT EVT = cast<VTSDNode>(N2)->getVT();
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002964 assert(VT == N1.getValueType() && "Not an inreg extend!");
Duncan Sands83ec4b62008-06-06 12:08:01 +00002965 assert(VT.isInteger() && EVT.isInteger() &&
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002966 "Cannot *_EXTEND_INREG FP types");
Dan Gohman87862e72009-12-11 21:31:27 +00002967 assert(!EVT.isVector() &&
2968 "AssertSExt/AssertZExt type should be the vector element type "
2969 "rather than the vector type!");
Duncan Sands8e4eb092008-06-08 20:54:56 +00002970 assert(EVT.bitsLE(VT) && "Not extending!");
Duncan Sandsd885dbd2008-02-10 10:08:52 +00002971 if (VT == EVT) return N1; // noop assertion.
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002972 break;
2973 }
Chris Lattner15e4b012005-07-10 00:07:11 +00002974 case ISD::SIGN_EXTEND_INREG: {
Owen Andersone50ed302009-08-10 22:56:29 +00002975 EVT EVT = cast<VTSDNode>(N2)->getVT();
Chris Lattner15e4b012005-07-10 00:07:11 +00002976 assert(VT == N1.getValueType() && "Not an inreg extend!");
Duncan Sands83ec4b62008-06-06 12:08:01 +00002977 assert(VT.isInteger() && EVT.isInteger() &&
Chris Lattner15e4b012005-07-10 00:07:11 +00002978 "Cannot *_EXTEND_INREG FP types");
Dan Gohmand1996362010-01-09 02:13:55 +00002979 assert(EVT.isVector() == VT.isVector() &&
Sylvestre Ledru94c22712012-09-27 10:14:43 +00002980 "SIGN_EXTEND_INREG type should be vector iff the operand "
Dan Gohmand1996362010-01-09 02:13:55 +00002981 "type is vector!");
2982 assert((!EVT.isVector() ||
2983 EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
2984 "Vector element counts must match in SIGN_EXTEND_INREG");
2985 assert(EVT.bitsLE(VT) && "Not extending!");
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002986 if (EVT == VT) return N1; // Not actually extending
Chris Lattner15e4b012005-07-10 00:07:11 +00002987
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002988 if (N1C) {
Dan Gohman6c231502008-02-29 01:47:35 +00002989 APInt Val = N1C->getAPIntValue();
Dan Gohmand1996362010-01-09 02:13:55 +00002990 unsigned FromBits = EVT.getScalarType().getSizeInBits();
Dan Gohman6c231502008-02-29 01:47:35 +00002991 Val <<= Val.getBitWidth()-FromBits;
Evan Cheng433f6f62008-03-06 08:20:51 +00002992 Val = Val.ashr(Val.getBitWidth()-FromBits);
Chris Lattnerb9ebacd2006-05-06 23:05:41 +00002993 return getConstant(Val, VT);
2994 }
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00002995 break;
2996 }
2997 case ISD::EXTRACT_VECTOR_ELT:
Chris Lattnerf3ba4342008-03-08 23:43:36 +00002998 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
2999 if (N1.getOpcode() == ISD::UNDEF)
Dale Johannesene8d72302009-02-06 23:05:02 +00003000 return getUNDEF(VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00003001
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003002 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
3003 // expanding copies of large vectors from registers.
Dan Gohman6ab64222008-08-13 21:51:37 +00003004 if (N2C &&
3005 N1.getOpcode() == ISD::CONCAT_VECTORS &&
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003006 N1.getNumOperands() > 0) {
3007 unsigned Factor =
Duncan Sands83ec4b62008-06-06 12:08:01 +00003008 N1.getOperand(0).getValueType().getVectorNumElements();
Dale Johannesendbfd8db2009-02-03 01:55:44 +00003009 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003010 N1.getOperand(N2C->getZExtValue() / Factor),
3011 getConstant(N2C->getZExtValue() % Factor,
3012 N2.getValueType()));
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003013 }
3014
3015 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
3016 // expanding large vector constants.
Bob Wilsonb1303d02009-04-13 22:05:19 +00003017 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
3018 SDValue Elt = N1.getOperand(N2C->getZExtValue());
James Molloy8cd08bf2012-09-10 14:01:21 +00003019
3020 if (VT != Elt.getValueType())
Bob Wilsonb1303d02009-04-13 22:05:19 +00003021 // If the vector element type is not legal, the BUILD_VECTOR operands
James Molloy8cd08bf2012-09-10 14:01:21 +00003022 // are promoted and implicitly truncated, and the result implicitly
3023 // extended. Make that explicit here.
3024 Elt = getAnyExtOrTrunc(Elt, DL, VT);
Michael Ilseman06b96902012-09-10 16:56:31 +00003025
Bob Wilsonb1303d02009-04-13 22:05:19 +00003026 return Elt;
3027 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003028
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003029 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
3030 // operations are lowered to scalars.
Dan Gohman6ab64222008-08-13 21:51:37 +00003031 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
Mon P Wang87c46d82010-02-01 22:15:09 +00003032 // If the indices are the same, return the inserted element else
3033 // if the indices are known different, extract the element from
Dan Gohman197e88f2009-01-29 16:10:46 +00003034 // the original vector.
Bill Wendlingd71bb562010-04-30 22:19:17 +00003035 SDValue N1Op2 = N1.getOperand(2);
3036 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
3037
3038 if (N1Op2C && N2C) {
3039 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
3040 if (VT == N1.getOperand(1).getValueType())
3041 return N1.getOperand(1);
3042 else
3043 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
3044 }
3045
Dale Johannesendbfd8db2009-02-03 01:55:44 +00003046 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
Bill Wendlingd71bb562010-04-30 22:19:17 +00003047 }
Dan Gohman6ab64222008-08-13 21:51:37 +00003048 }
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003049 break;
3050 case ISD::EXTRACT_ELEMENT:
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003051 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
Duncan Sands041cde22008-06-25 20:24:48 +00003052 assert(!N1.getValueType().isVector() && !VT.isVector() &&
3053 (N1.getValueType().isInteger() == VT.isInteger()) &&
Eli Friedman6cdc1f42011-08-02 18:38:35 +00003054 N1.getValueType() != VT &&
Duncan Sands041cde22008-06-25 20:24:48 +00003055 "Wrong types for EXTRACT_ELEMENT!");
Duncan Sands25eb0432008-03-12 20:30:08 +00003056
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003057 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
3058 // 64-bit integers into 32-bit parts. Instead of building the extract of
Scott Michelfdc40a02009-02-17 22:15:04 +00003059 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003060 if (N1.getOpcode() == ISD::BUILD_PAIR)
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003061 return N1.getOperand(N2C->getZExtValue());
Duncan Sands25eb0432008-03-12 20:30:08 +00003062
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003063 // EXTRACT_ELEMENT of a constant int is also very common.
3064 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00003065 unsigned ElementSize = VT.getSizeInBits();
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003066 unsigned Shift = ElementSize * N2C->getZExtValue();
Dan Gohman4c931fc2008-03-24 16:38:05 +00003067 APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
3068 return getConstant(ShiftedVal.trunc(ElementSize), VT);
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003069 }
3070 break;
David Greene91585092011-01-26 15:38:49 +00003071 case ISD::EXTRACT_SUBVECTOR: {
3072 SDValue Index = N2;
3073 if (VT.isSimple() && N1.getValueType().isSimple()) {
3074 assert(VT.isVector() && N1.getValueType().isVector() &&
3075 "Extract subvector VTs must be a vectors!");
3076 assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() &&
3077 "Extract subvector VTs must have the same element type!");
3078 assert(VT.getSimpleVT() <= N1.getValueType().getSimpleVT() &&
3079 "Extract subvector must be from larger vector to smaller vector!");
3080
Matt Beaumont-Gay9a14a362011-02-01 22:12:50 +00003081 if (isa<ConstantSDNode>(Index.getNode())) {
3082 assert((VT.getVectorNumElements() +
3083 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
David Greene91585092011-01-26 15:38:49 +00003084 <= N1.getValueType().getVectorNumElements())
3085 && "Extract subvector overflow!");
3086 }
3087
3088 // Trivial extraction.
3089 if (VT.getSimpleVT() == N1.getValueType().getSimpleVT())
3090 return N1;
3091 }
Duncan Sandsf83b1f62008-02-20 17:38:09 +00003092 break;
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003093 }
David Greene91585092011-01-26 15:38:49 +00003094 }
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003095
Benjamin Kramer49693102013-02-04 15:19:18 +00003096 // Perform trivial constant folding.
3097 SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode());
3098 if (SV.getNode()) return SV;
3099
3100 // Canonicalize constant to RHS if commutative.
3101 if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
3102 std::swap(N1C, N2C);
3103 std::swap(N1, N2);
Chris Lattnerc3aae252005-01-07 07:46:32 +00003104 }
3105
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003106 // Constant fold FP operations.
Gabor Greifba36cb52008-08-28 21:40:38 +00003107 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
3108 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
Chris Lattner15e4b012005-07-10 00:07:11 +00003109 if (N1CFP) {
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003110 if (!N2CFP && isCommutativeBinOp(Opcode)) {
Benjamin Kramer49693102013-02-04 15:19:18 +00003111 // Canonicalize constant to RHS if commutative.
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003112 std::swap(N1CFP, N2CFP);
3113 std::swap(N1, N2);
Ulrich Weigande669c932012-10-29 18:35:49 +00003114 } else if (N2CFP) {
Dale Johannesenc4dd3c32007-08-31 23:34:27 +00003115 APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
3116 APFloat::opStatus s;
Chris Lattnerc3aae252005-01-07 07:46:32 +00003117 switch (Opcode) {
Scott Michelfdc40a02009-02-17 22:15:04 +00003118 case ISD::FADD:
Dale Johannesenc4dd3c32007-08-31 23:34:27 +00003119 s = V1.add(V2, APFloat::rmNearestTiesToEven);
Chris Lattner8e1f7ac2008-01-22 19:09:33 +00003120 if (s != APFloat::opInvalidOp)
Dale Johannesenc4dd3c32007-08-31 23:34:27 +00003121 return getConstantFP(V1, VT);
3122 break;
Scott Michelfdc40a02009-02-17 22:15:04 +00003123 case ISD::FSUB:
Dale Johannesenc4dd3c32007-08-31 23:34:27 +00003124 s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
3125 if (s!=APFloat::opInvalidOp)
3126 return getConstantFP(V1, VT);
3127 break;
3128 case ISD::FMUL:
3129 s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
3130 if (s!=APFloat::opInvalidOp)
3131 return getConstantFP(V1, VT);
3132 break;
Chris Lattner01b3d732005-09-28 22:28:18 +00003133 case ISD::FDIV:
Dale Johannesenc4dd3c32007-08-31 23:34:27 +00003134 s = V1.divide(V2, APFloat::rmNearestTiesToEven);
3135 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero)
3136 return getConstantFP(V1, VT);
Chris Lattnerc3aae252005-01-07 07:46:32 +00003137 break;
Chris Lattner01b3d732005-09-28 22:28:18 +00003138 case ISD::FREM :
Dale Johannesenc4dd3c32007-08-31 23:34:27 +00003139 s = V1.mod(V2, APFloat::rmNearestTiesToEven);
3140 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero)
3141 return getConstantFP(V1, VT);
Chris Lattnerc3aae252005-01-07 07:46:32 +00003142 break;
Dale Johannesenc4dd3c32007-08-31 23:34:27 +00003143 case ISD::FCOPYSIGN:
3144 V1.copySign(V2);
3145 return getConstantFP(V1, VT);
Chris Lattnerc3aae252005-01-07 07:46:32 +00003146 default: break;
3147 }
Chris Lattnerc3aae252005-01-07 07:46:32 +00003148 }
Owen Anderson06886aa2012-04-10 22:46:53 +00003149
3150 if (Opcode == ISD::FP_ROUND) {
3151 APFloat V = N1CFP->getValueAPF(); // make copy
3152 bool ignored;
3153 // This can return overflow, underflow, or inexact; we don't care.
3154 // FIXME need to be more flexible about rounding mode.
Tim Northover0a29cb02013-01-22 09:46:31 +00003155 (void)V.convert(EVTToAPFloatSemantics(VT),
Owen Anderson06886aa2012-04-10 22:46:53 +00003156 APFloat::rmNearestTiesToEven, &ignored);
3157 return getConstantFP(V, VT);
3158 }
Chris Lattner15e4b012005-07-10 00:07:11 +00003159 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003160
Chris Lattner62b57722006-04-20 05:39:12 +00003161 // Canonicalize an UNDEF to the RHS, even over a constant.
3162 if (N1.getOpcode() == ISD::UNDEF) {
3163 if (isCommutativeBinOp(Opcode)) {
3164 std::swap(N1, N2);
3165 } else {
3166 switch (Opcode) {
3167 case ISD::FP_ROUND_INREG:
3168 case ISD::SIGN_EXTEND_INREG:
3169 case ISD::SUB:
3170 case ISD::FSUB:
3171 case ISD::FDIV:
3172 case ISD::FREM:
Chris Lattner2cfd6742006-05-08 17:29:49 +00003173 case ISD::SRA:
Chris Lattner62b57722006-04-20 05:39:12 +00003174 return N1; // fold op(undef, arg2) -> undef
3175 case ISD::UDIV:
3176 case ISD::SDIV:
3177 case ISD::UREM:
3178 case ISD::SREM:
Chris Lattner2cfd6742006-05-08 17:29:49 +00003179 case ISD::SRL:
3180 case ISD::SHL:
Duncan Sands83ec4b62008-06-06 12:08:01 +00003181 if (!VT.isVector())
Chris Lattner964dd862007-04-25 00:00:45 +00003182 return getConstant(0, VT); // fold op(undef, arg2) -> 0
3183 // For vectors, we can't easily build an all zero vector, just return
3184 // the LHS.
3185 return N2;
Chris Lattner62b57722006-04-20 05:39:12 +00003186 }
3187 }
3188 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003189
3190 // Fold a bunch of operators when the RHS is undef.
Chris Lattner62b57722006-04-20 05:39:12 +00003191 if (N2.getOpcode() == ISD::UNDEF) {
3192 switch (Opcode) {
Evan Cheng26471c42008-03-25 20:08:07 +00003193 case ISD::XOR:
3194 if (N1.getOpcode() == ISD::UNDEF)
3195 // Handle undef ^ undef -> 0 special case. This is a common
3196 // idiom (misuse).
3197 return getConstant(0, VT);
3198 // fallthrough
Chris Lattner62b57722006-04-20 05:39:12 +00003199 case ISD::ADD:
Chris Lattner175415e2007-03-04 20:01:46 +00003200 case ISD::ADDC:
3201 case ISD::ADDE:
Chris Lattner62b57722006-04-20 05:39:12 +00003202 case ISD::SUB:
Chris Lattner62b57722006-04-20 05:39:12 +00003203 case ISD::UDIV:
3204 case ISD::SDIV:
3205 case ISD::UREM:
3206 case ISD::SREM:
Chris Lattner62b57722006-04-20 05:39:12 +00003207 return N2; // fold op(arg1, undef) -> undef
Dan Gohman77b81fe2009-06-04 17:12:12 +00003208 case ISD::FADD:
3209 case ISD::FSUB:
3210 case ISD::FMUL:
3211 case ISD::FDIV:
3212 case ISD::FREM:
Nick Lewycky8a8d4792011-12-02 22:16:29 +00003213 if (getTarget().Options.UnsafeFPMath)
Dan Gohman77b81fe2009-06-04 17:12:12 +00003214 return N2;
3215 break;
Scott Michelfdc40a02009-02-17 22:15:04 +00003216 case ISD::MUL:
Chris Lattner62b57722006-04-20 05:39:12 +00003217 case ISD::AND:
Chris Lattner2cfd6742006-05-08 17:29:49 +00003218 case ISD::SRL:
3219 case ISD::SHL:
Duncan Sands83ec4b62008-06-06 12:08:01 +00003220 if (!VT.isVector())
Chris Lattner964dd862007-04-25 00:00:45 +00003221 return getConstant(0, VT); // fold op(arg1, undef) -> 0
3222 // For vectors, we can't easily build an all zero vector, just return
3223 // the LHS.
3224 return N1;
Chris Lattner62b57722006-04-20 05:39:12 +00003225 case ISD::OR:
Duncan Sands83ec4b62008-06-06 12:08:01 +00003226 if (!VT.isVector())
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003227 return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
Chris Lattner964dd862007-04-25 00:00:45 +00003228 // For vectors, we can't easily build an all one vector, just return
3229 // the LHS.
3230 return N1;
Chris Lattner2cfd6742006-05-08 17:29:49 +00003231 case ISD::SRA:
3232 return N1;
Chris Lattner62b57722006-04-20 05:39:12 +00003233 }
3234 }
Chris Lattner15e4b012005-07-10 00:07:11 +00003235
Chris Lattner27e9b412005-05-11 18:57:39 +00003236 // Memoize this node if possible.
3237 SDNode *N;
Chris Lattner0b3e5252006-08-15 19:11:05 +00003238 SDVTList VTs = getVTList(VT);
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00003239 if (VT != MVT::Glue) {
Dan Gohman475871a2008-07-27 21:46:04 +00003240 SDValue Ops[] = { N1, N2 };
Jim Laskey583bd472006-10-27 23:46:08 +00003241 FoldingSetNodeID ID;
Chris Lattner63e3f142007-02-04 07:28:00 +00003242 AddNodeIDNode(ID, Opcode, VTs, Ops, 2);
Chris Lattnera5682852006-08-07 23:03:03 +00003243 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00003244 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00003245 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00003246
Andrew Trickac6d9be2013-05-25 02:42:55 +00003247 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
Chris Lattnera5682852006-08-07 23:03:03 +00003248 CSEMap.InsertNode(N, IP);
Chris Lattner27e9b412005-05-11 18:57:39 +00003249 } else {
Andrew Trickac6d9be2013-05-25 02:42:55 +00003250 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
Chris Lattner27e9b412005-05-11 18:57:39 +00003251 }
3252
Chris Lattnerc3aae252005-01-07 07:46:32 +00003253 AllNodes.push_back(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00003254#ifndef NDEBUG
Duncan Sands59d2dad2010-11-20 11:25:00 +00003255 VerifySDNode(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00003256#endif
Dan Gohman475871a2008-07-27 21:46:04 +00003257 return SDValue(N, 0);
Chris Lattnerc3aae252005-01-07 07:46:32 +00003258}
3259
Andrew Trickac6d9be2013-05-25 02:42:55 +00003260SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
Bill Wendling7ade28c2009-01-28 22:17:52 +00003261 SDValue N1, SDValue N2, SDValue N3) {
Chris Lattnerc3aae252005-01-07 07:46:32 +00003262 // Perform various simplifications.
Gabor Greifba36cb52008-08-28 21:40:38 +00003263 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
Chris Lattnerc3aae252005-01-07 07:46:32 +00003264 switch (Opcode) {
Owen Anderson58dcd202013-05-09 22:27:13 +00003265 case ISD::FMA: {
3266 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
3267 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
3268 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
3269 if (N1CFP && N2CFP && N3CFP) {
3270 APFloat V1 = N1CFP->getValueAPF();
3271 const APFloat &V2 = N2CFP->getValueAPF();
3272 const APFloat &V3 = N3CFP->getValueAPF();
3273 APFloat::opStatus s =
3274 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
3275 if (s != APFloat::opInvalidOp)
3276 return getConstantFP(V1, VT);
3277 }
3278 break;
3279 }
Dan Gohman7f8613e2008-08-14 20:04:46 +00003280 case ISD::CONCAT_VECTORS:
3281 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
3282 // one big BUILD_VECTOR.
3283 if (N1.getOpcode() == ISD::BUILD_VECTOR &&
3284 N2.getOpcode() == ISD::BUILD_VECTOR &&
3285 N3.getOpcode() == ISD::BUILD_VECTOR) {
Gabor Greif481c4c02010-07-22 17:18:03 +00003286 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
3287 N1.getNode()->op_end());
Dan Gohman403a8cd2010-06-21 19:47:52 +00003288 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
3289 Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
Evan Chenga87008d2009-02-25 22:49:59 +00003290 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size());
Dan Gohman7f8613e2008-08-14 20:04:46 +00003291 }
3292 break;
Chris Lattner7cf7e3f2005-08-09 20:20:18 +00003293 case ISD::SETCC: {
Chris Lattner51dabfb2006-10-14 00:41:01 +00003294 // Use FoldSetCC to simplify SETCC's.
Dale Johannesenff97d4f2009-02-03 00:47:48 +00003295 SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
Gabor Greifba36cb52008-08-28 21:40:38 +00003296 if (Simp.getNode()) return Simp;
Chris Lattner7cf7e3f2005-08-09 20:20:18 +00003297 break;
3298 }
Chris Lattnerc3aae252005-01-07 07:46:32 +00003299 case ISD::SELECT:
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00003300 if (N1C) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003301 if (N1C->getZExtValue())
David Blaikie4d6ccb52012-01-20 21:51:11 +00003302 return N2; // select true, X, Y -> X
3303 return N3; // select false, X, Y -> Y
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00003304 }
Chris Lattnerc3aae252005-01-07 07:46:32 +00003305
3306 if (N2 == N3) return N2; // select C, X, X -> X
Chris Lattnerc3aae252005-01-07 07:46:32 +00003307 break;
Chris Lattnerfb194b92006-03-19 23:56:04 +00003308 case ISD::VECTOR_SHUFFLE:
Torok Edwinc23197a2009-07-14 16:55:14 +00003309 llvm_unreachable("should use getVectorShuffle constructor!");
David Greenecfe33c42011-01-26 19:13:22 +00003310 case ISD::INSERT_SUBVECTOR: {
3311 SDValue Index = N3;
3312 if (VT.isSimple() && N1.getValueType().isSimple()
3313 && N2.getValueType().isSimple()) {
3314 assert(VT.isVector() && N1.getValueType().isVector() &&
3315 N2.getValueType().isVector() &&
3316 "Insert subvector VTs must be a vectors");
3317 assert(VT == N1.getValueType() &&
3318 "Dest and insert subvector source types must match!");
3319 assert(N2.getValueType().getSimpleVT() <= N1.getValueType().getSimpleVT() &&
3320 "Insert subvector must be from smaller vector to larger vector!");
Matt Beaumont-Gay9a14a362011-02-01 22:12:50 +00003321 if (isa<ConstantSDNode>(Index.getNode())) {
3322 assert((N2.getValueType().getVectorNumElements() +
3323 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
David Greenecfe33c42011-01-26 19:13:22 +00003324 <= VT.getVectorNumElements())
3325 && "Insert subvector overflow!");
3326 }
3327
3328 // Trivial insertion.
3329 if (VT.getSimpleVT() == N2.getValueType().getSimpleVT())
3330 return N2;
3331 }
3332 break;
3333 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003334 case ISD::BITCAST:
Dan Gohman7f321562007-06-25 16:23:39 +00003335 // Fold bit_convert nodes from a type to themselves.
3336 if (N1.getValueType() == VT)
3337 return N1;
Chris Lattner4829b1c2007-04-12 05:58:43 +00003338 break;
Chris Lattnerc3aae252005-01-07 07:46:32 +00003339 }
3340
Chris Lattner43247a12005-08-25 19:12:10 +00003341 // Memoize node if it doesn't produce a flag.
3342 SDNode *N;
Chris Lattner0b3e5252006-08-15 19:11:05 +00003343 SDVTList VTs = getVTList(VT);
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00003344 if (VT != MVT::Glue) {
Dan Gohman475871a2008-07-27 21:46:04 +00003345 SDValue Ops[] = { N1, N2, N3 };
Jim Laskey583bd472006-10-27 23:46:08 +00003346 FoldingSetNodeID ID;
Chris Lattner63e3f142007-02-04 07:28:00 +00003347 AddNodeIDNode(ID, Opcode, VTs, Ops, 3);
Chris Lattnera5682852006-08-07 23:03:03 +00003348 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00003349 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00003350 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00003351
Andrew Trickac6d9be2013-05-25 02:42:55 +00003352 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, N3);
Chris Lattnera5682852006-08-07 23:03:03 +00003353 CSEMap.InsertNode(N, IP);
Chris Lattner43247a12005-08-25 19:12:10 +00003354 } else {
Andrew Trickac6d9be2013-05-25 02:42:55 +00003355 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, N3);
Chris Lattner43247a12005-08-25 19:12:10 +00003356 }
Daniel Dunbar819309e2009-12-16 20:10:05 +00003357
Chris Lattnerc3aae252005-01-07 07:46:32 +00003358 AllNodes.push_back(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00003359#ifndef NDEBUG
Duncan Sands59d2dad2010-11-20 11:25:00 +00003360 VerifySDNode(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00003361#endif
Dan Gohman475871a2008-07-27 21:46:04 +00003362 return SDValue(N, 0);
Chris Lattnerc3aae252005-01-07 07:46:32 +00003363}
3364
Andrew Trickac6d9be2013-05-25 02:42:55 +00003365SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
Bill Wendling7ade28c2009-01-28 22:17:52 +00003366 SDValue N1, SDValue N2, SDValue N3,
3367 SDValue N4) {
Dan Gohman475871a2008-07-27 21:46:04 +00003368 SDValue Ops[] = { N1, N2, N3, N4 };
Bill Wendling7ade28c2009-01-28 22:17:52 +00003369 return getNode(Opcode, DL, VT, Ops, 4);
Andrew Lenharth2d86ea22005-04-27 20:10:01 +00003370}
3371
Andrew Trickac6d9be2013-05-25 02:42:55 +00003372SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
Bill Wendling7ade28c2009-01-28 22:17:52 +00003373 SDValue N1, SDValue N2, SDValue N3,
3374 SDValue N4, SDValue N5) {
Dan Gohman475871a2008-07-27 21:46:04 +00003375 SDValue Ops[] = { N1, N2, N3, N4, N5 };
Bill Wendling7ade28c2009-01-28 22:17:52 +00003376 return getNode(Opcode, DL, VT, Ops, 5);
Chris Lattner9fadb4c2005-07-10 00:29:18 +00003377}
3378
Dan Gohman98ca4f22009-08-05 01:29:28 +00003379/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
3380/// the incoming stack arguments to be loaded from the stack.
3381SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
3382 SmallVector<SDValue, 8> ArgChains;
3383
3384 // Include the original chain at the beginning of the list. When this is
3385 // used by target LowerCall hooks, this helps legalize find the
3386 // CALLSEQ_BEGIN node.
3387 ArgChains.push_back(Chain);
3388
3389 // Add a chain value for each stack argument.
3390 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
3391 UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
3392 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
3393 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
3394 if (FI->getIndex() < 0)
3395 ArgChains.push_back(SDValue(L, 1));
3396
3397 // Build a tokenfactor for all the chains.
Andrew Trickac6d9be2013-05-25 02:42:55 +00003398 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other,
Dan Gohman98ca4f22009-08-05 01:29:28 +00003399 &ArgChains[0], ArgChains.size());
3400}
3401
Dan Gohman707e0182008-04-12 04:36:06 +00003402/// getMemsetValue - Vectorized representation of the memset value
3403/// operand.
Owen Andersone50ed302009-08-10 22:56:29 +00003404static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
Andrew Trickac6d9be2013-05-25 02:42:55 +00003405 SDLoc dl) {
Evan Chengf28f8bc2010-04-02 19:36:14 +00003406 assert(Value.getOpcode() != ISD::UNDEF);
3407
Chris Lattner5cf0b6e2010-02-24 22:44:06 +00003408 unsigned NumBits = VT.getScalarType().getSizeInBits();
Dan Gohman707e0182008-04-12 04:36:06 +00003409 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
Benjamin Kramerad4da0f2013-02-20 13:00:06 +00003410 assert(C->getAPIntValue().getBitWidth() == 8);
3411 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
Duncan Sands83ec4b62008-06-06 12:08:01 +00003412 if (VT.isInteger())
Evan Chengf0df0312008-05-15 08:39:06 +00003413 return DAG.getConstant(Val, VT);
Tim Northover0a29cb02013-01-22 09:46:31 +00003414 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
Dan Gohman707e0182008-04-12 04:36:06 +00003415 }
Evan Chengf0df0312008-05-15 08:39:06 +00003416
Dale Johannesen0f502f62009-02-03 22:26:09 +00003417 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
Benjamin Kramer8c06aa12011-01-02 19:44:58 +00003418 if (NumBits > 8) {
3419 // Use a multiplication with 0x010101... to extend the input to the
3420 // required length.
Benjamin Kramerad4da0f2013-02-20 13:00:06 +00003421 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
Benjamin Kramer8c06aa12011-01-02 19:44:58 +00003422 Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
Evan Chengf0df0312008-05-15 08:39:06 +00003423 }
3424
3425 return Value;
Rafael Espindola5c0d6ed2007-10-19 10:41:11 +00003426}
3427
Dan Gohman707e0182008-04-12 04:36:06 +00003428/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
3429/// used when a memcpy is turned into a memset when the source is a constant
3430/// string ptr.
Andrew Trickac6d9be2013-05-25 02:42:55 +00003431static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
Chris Lattner18c7f802012-02-05 02:29:43 +00003432 const TargetLowering &TLI, StringRef Str) {
Evan Cheng0ff39b32008-06-30 07:31:25 +00003433 // Handle vector with all elements zero.
3434 if (Str.empty()) {
3435 if (VT.isInteger())
3436 return DAG.getConstant(0, VT);
Duncan Sandscdfad362010-11-03 12:17:33 +00003437 else if (VT == MVT::f32 || VT == MVT::f64)
Evan Cheng255f20f2010-04-01 06:04:33 +00003438 return DAG.getConstantFP(0.0, VT);
3439 else if (VT.isVector()) {
3440 unsigned NumElts = VT.getVectorNumElements();
3441 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003442 return DAG.getNode(ISD::BITCAST, dl, VT,
Evan Cheng255f20f2010-04-01 06:04:33 +00003443 DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
3444 EltVT, NumElts)));
3445 } else
3446 llvm_unreachable("Expected type!");
Evan Cheng0ff39b32008-06-30 07:31:25 +00003447 }
3448
Duncan Sands83ec4b62008-06-06 12:08:01 +00003449 assert(!VT.isVector() && "Can't handle vector type here!");
Evan Cheng4ff23d02013-01-10 22:13:27 +00003450 unsigned NumVTBits = VT.getSizeInBits();
3451 unsigned NumVTBytes = NumVTBits / 8;
Chris Lattner18c7f802012-02-05 02:29:43 +00003452 unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
3453
Evan Cheng4ff23d02013-01-10 22:13:27 +00003454 APInt Val(NumVTBits, 0);
Chris Lattner18c7f802012-02-05 02:29:43 +00003455 if (TLI.isLittleEndian()) {
3456 for (unsigned i = 0; i != NumBytes; ++i)
3457 Val |= (uint64_t)(unsigned char)Str[i] << i*8;
3458 } else {
3459 for (unsigned i = 0; i != NumBytes; ++i)
3460 Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
Dan Gohman707e0182008-04-12 04:36:06 +00003461 }
Chris Lattner18c7f802012-02-05 02:29:43 +00003462
Evan Chenge07f85e2012-12-11 23:26:14 +00003463 // If the "cost" of materializing the integer immediate is 1 or free, then
3464 // it is cost effective to turn the load into the immediate.
Chandler Carruthe4b4edd2013-01-05 12:32:17 +00003465 const TargetTransformInfo *TTI = DAG.getTargetTransformInfo();
3466 if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) < 2)
Evan Cheng376642e2012-12-10 23:21:26 +00003467 return DAG.getConstant(Val, VT);
3468 return SDValue(0, 0);
Rafael Espindola5c0d6ed2007-10-19 10:41:11 +00003469}
3470
Scott Michelfdc40a02009-02-17 22:15:04 +00003471/// getMemBasePlusOffset - Returns base and offset node for the
Evan Chengf0df0312008-05-15 08:39:06 +00003472///
Dan Gohman475871a2008-07-27 21:46:04 +00003473static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset,
Dan Gohman707e0182008-04-12 04:36:06 +00003474 SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00003475 EVT VT = Base.getValueType();
Andrew Trickac6d9be2013-05-25 02:42:55 +00003476 return DAG.getNode(ISD::ADD, SDLoc(Base),
Dale Johannesendbfd8db2009-02-03 01:55:44 +00003477 VT, Base, DAG.getConstant(Offset, VT));
Dan Gohman707e0182008-04-12 04:36:06 +00003478}
3479
Evan Chengf0df0312008-05-15 08:39:06 +00003480/// isMemSrcFromString - Returns true if memcpy source is a string constant.
3481///
Chris Lattner18c7f802012-02-05 02:29:43 +00003482static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
Evan Chengf0df0312008-05-15 08:39:06 +00003483 unsigned SrcDelta = 0;
3484 GlobalAddressSDNode *G = NULL;
3485 if (Src.getOpcode() == ISD::GlobalAddress)
3486 G = cast<GlobalAddressSDNode>(Src);
3487 else if (Src.getOpcode() == ISD::ADD &&
3488 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
3489 Src.getOperand(1).getOpcode() == ISD::Constant) {
3490 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003491 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
Evan Chengf0df0312008-05-15 08:39:06 +00003492 }
3493 if (!G)
3494 return false;
Dan Gohman707e0182008-04-12 04:36:06 +00003495
Chris Lattner18c7f802012-02-05 02:29:43 +00003496 return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
Evan Chengf0df0312008-05-15 08:39:06 +00003497}
Dan Gohman707e0182008-04-12 04:36:06 +00003498
Evan Cheng255f20f2010-04-01 06:04:33 +00003499/// FindOptimalMemOpLowering - Determines the optimial series memory ops
3500/// to replace the memset / memcpy. Return true if the number of memory ops
3501/// is below the threshold. It returns the types of the sequence of
3502/// memory ops to perform memset / memcpy by reference.
3503static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
Evan Cheng255f20f2010-04-01 06:04:33 +00003504 unsigned Limit, uint64_t Size,
3505 unsigned DstAlign, unsigned SrcAlign,
Evan Cheng946a3a92012-12-12 02:34:41 +00003506 bool IsMemset,
3507 bool ZeroMemset,
Evan Chengc3b0c342010-04-08 07:37:57 +00003508 bool MemcpyStrSrc,
Evan Cheng376642e2012-12-10 23:21:26 +00003509 bool AllowOverlap,
Evan Cheng255f20f2010-04-01 06:04:33 +00003510 SelectionDAG &DAG,
3511 const TargetLowering &TLI) {
3512 assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
3513 "Expecting memcpy / memset source to meet alignment requirement!");
Eric Christopher882e1e12011-07-06 22:41:18 +00003514 // If 'SrcAlign' is zero, that means the memory operation does not need to
3515 // load the value, i.e. memset or memcpy from constant string. Otherwise,
3516 // it's the inferred alignment of the source. 'DstAlign', on the other hand,
3517 // is the specified alignment of the memory operation. If it is zero, that
3518 // means it's possible to change the alignment of the destination.
3519 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
3520 // not need to be loaded.
Evan Chengf28f8bc2010-04-02 19:36:14 +00003521 EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
Evan Cheng946a3a92012-12-12 02:34:41 +00003522 IsMemset, ZeroMemset, MemcpyStrSrc,
Dan Gohman4bcf0a92010-04-16 20:22:43 +00003523 DAG.getMachineFunction());
Evan Chengf0df0312008-05-15 08:39:06 +00003524
Chris Lattneracee6472010-03-07 07:45:08 +00003525 if (VT == MVT::Other) {
Chandler Carruth426c2bf2012-11-01 09:14:31 +00003526 if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment() ||
Evan Cheng255f20f2010-04-01 06:04:33 +00003527 TLI.allowsUnalignedMemoryAccesses(VT)) {
Evan Cheng18141ee2010-04-05 23:33:29 +00003528 VT = TLI.getPointerTy();
Evan Chengf0df0312008-05-15 08:39:06 +00003529 } else {
Evan Cheng255f20f2010-04-01 06:04:33 +00003530 switch (DstAlign & 7) {
Owen Anderson825b72b2009-08-11 20:47:22 +00003531 case 0: VT = MVT::i64; break;
3532 case 4: VT = MVT::i32; break;
3533 case 2: VT = MVT::i16; break;
3534 default: VT = MVT::i8; break;
Evan Chengf0df0312008-05-15 08:39:06 +00003535 }
3536 }
3537
Owen Anderson766b5ef2009-08-11 21:59:30 +00003538 MVT LVT = MVT::i64;
Evan Chengf0df0312008-05-15 08:39:06 +00003539 while (!TLI.isTypeLegal(LVT))
Owen Anderson766b5ef2009-08-11 21:59:30 +00003540 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
Duncan Sands83ec4b62008-06-06 12:08:01 +00003541 assert(LVT.isInteger());
Evan Chengf0df0312008-05-15 08:39:06 +00003542
Duncan Sands8e4eb092008-06-08 20:54:56 +00003543 if (VT.bitsGT(LVT))
Evan Chengf0df0312008-05-15 08:39:06 +00003544 VT = LVT;
3545 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003546
Dan Gohman707e0182008-04-12 04:36:06 +00003547 unsigned NumMemOps = 0;
3548 while (Size != 0) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00003549 unsigned VTSize = VT.getSizeInBits() / 8;
Dan Gohman707e0182008-04-12 04:36:06 +00003550 while (VTSize > Size) {
Evan Chengf0df0312008-05-15 08:39:06 +00003551 // For now, only use non-vector load / store's for the left-over pieces.
Evan Cheng61f4dfe2012-12-12 00:42:09 +00003552 EVT NewVT = VT;
Evan Cheng376642e2012-12-10 23:21:26 +00003553 unsigned NewVTSize;
Evan Cheng61f4dfe2012-12-12 00:42:09 +00003554
3555 bool Found = false;
Evan Cheng255f20f2010-04-01 06:04:33 +00003556 if (VT.isVector() || VT.isFloatingPoint()) {
Evan Cheng376642e2012-12-10 23:21:26 +00003557 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
Evan Cheng61f4dfe2012-12-12 00:42:09 +00003558 if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
Evan Cheng7d342672012-12-12 01:32:07 +00003559 TLI.isSafeMemOpType(NewVT.getSimpleVT()))
Evan Cheng61f4dfe2012-12-12 00:42:09 +00003560 Found = true;
3561 else if (NewVT == MVT::i64 &&
3562 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
Evan Cheng7d342672012-12-12 01:32:07 +00003563 TLI.isSafeMemOpType(MVT::f64)) {
Evan Cheng61f4dfe2012-12-12 00:42:09 +00003564 // i64 is usually not legal on 32-bit targets, but f64 may be.
3565 NewVT = MVT::f64;
3566 Found = true;
Evan Cheng376642e2012-12-10 23:21:26 +00003567 }
Evan Cheng376642e2012-12-10 23:21:26 +00003568 }
3569
Evan Cheng61f4dfe2012-12-12 00:42:09 +00003570 if (!Found) {
3571 do {
3572 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
3573 if (NewVT == MVT::i8)
3574 break;
Evan Cheng7d342672012-12-12 01:32:07 +00003575 } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
Evan Cheng61f4dfe2012-12-12 00:42:09 +00003576 }
3577 NewVTSize = NewVT.getSizeInBits() / 8;
3578
Evan Cheng376642e2012-12-10 23:21:26 +00003579 // If the new VT cannot cover all of the remaining bits, then consider
3580 // issuing a (or a pair of) unaligned and overlapping load / store.
3581 // FIXME: Only does this for 64-bit or more since we don't have proper
3582 // cost model for unaligned load / store.
3583 bool Fast;
Evan Chenga16e49d2012-12-12 20:43:23 +00003584 if (NumMemOps && AllowOverlap &&
3585 VTSize >= 8 && NewVTSize < Size &&
Evan Cheng376642e2012-12-10 23:21:26 +00003586 TLI.allowsUnalignedMemoryAccesses(VT, &Fast) && Fast)
3587 VTSize = Size;
3588 else {
3589 VT = NewVT;
3590 VTSize = NewVTSize;
Evan Chengf0df0312008-05-15 08:39:06 +00003591 }
Dan Gohman707e0182008-04-12 04:36:06 +00003592 }
Dan Gohman707e0182008-04-12 04:36:06 +00003593
Evan Chenga16e49d2012-12-12 20:43:23 +00003594 if (++NumMemOps > Limit)
3595 return false;
3596
Dan Gohman707e0182008-04-12 04:36:06 +00003597 MemOps.push_back(VT);
3598 Size -= VTSize;
3599 }
3600
3601 return true;
3602}
3603
Andrew Trickac6d9be2013-05-25 02:42:55 +00003604static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
Evan Cheng87bd1912010-03-30 18:08:53 +00003605 SDValue Chain, SDValue Dst,
3606 SDValue Src, uint64_t Size,
Mon P Wang20adc9d2010-04-04 03:10:48 +00003607 unsigned Align, bool isVol,
3608 bool AlwaysInline,
Chris Lattnere72f2022010-09-21 05:40:29 +00003609 MachinePointerInfo DstPtrInfo,
3610 MachinePointerInfo SrcPtrInfo) {
Evan Chengf28f8bc2010-04-02 19:36:14 +00003611 // Turn a memcpy of undef to nop.
3612 if (Src.getOpcode() == ISD::UNDEF)
3613 return Chain;
Dan Gohman707e0182008-04-12 04:36:06 +00003614
Dan Gohman21323f32008-05-29 19:42:22 +00003615 // Expand memcpy to a series of load and store ops if the size operand falls
3616 // below a certain threshold.
Duncan Sands69300a22010-11-05 15:20:29 +00003617 // TODO: In the AlwaysInline case, if the size is big then generate a loop
3618 // rather than maybe a humongous number of loads and stores.
Evan Chengf28f8bc2010-04-02 19:36:14 +00003619 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Owen Andersone50ed302009-08-10 22:56:29 +00003620 std::vector<EVT> MemOps;
Evan Cheng255f20f2010-04-01 06:04:33 +00003621 bool DstAlignCanChange = false;
Evan Cheng05219282011-01-06 06:52:41 +00003622 MachineFunction &MF = DAG.getMachineFunction();
3623 MachineFrameInfo *MFI = MF.getFrameInfo();
Bill Wendling67658342012-10-09 07:45:08 +00003624 bool OptSize =
Bill Wendling831737d2012-12-30 10:32:01 +00003625 MF.getFunction()->getAttributes().
3626 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
Evan Cheng255f20f2010-04-01 06:04:33 +00003627 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
3628 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
3629 DstAlignCanChange = true;
3630 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
3631 if (Align > SrcAlign)
3632 SrcAlign = Align;
Chris Lattner18c7f802012-02-05 02:29:43 +00003633 StringRef Str;
Evan Cheng255f20f2010-04-01 06:04:33 +00003634 bool CopyFromStr = isMemSrcFromString(Src, Str);
3635 bool isZeroStr = CopyFromStr && Str.empty();
Evan Cheng05219282011-01-06 06:52:41 +00003636 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003637
Evan Cheng94107ba2010-04-01 18:19:11 +00003638 if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
Evan Cheng255f20f2010-04-01 06:04:33 +00003639 (DstAlignCanChange ? 0 : Align),
Evan Chengc3b0c342010-04-08 07:37:57 +00003640 (isZeroStr ? 0 : SrcAlign),
Evan Cheng946a3a92012-12-12 02:34:41 +00003641 false, false, CopyFromStr, true, DAG, TLI))
Dan Gohman475871a2008-07-27 21:46:04 +00003642 return SDValue();
Dan Gohman707e0182008-04-12 04:36:06 +00003643
Evan Cheng255f20f2010-04-01 06:04:33 +00003644 if (DstAlignCanChange) {
Chris Lattnerdb125cf2011-07-18 04:54:35 +00003645 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
Micah Villmow3574eca2012-10-08 16:38:25 +00003646 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
Lang Hames2d95e432013-01-31 20:23:43 +00003647
3648 // Don't promote to an alignment that would require dynamic stack
3649 // realignment.
3650 const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo();
3651 if (!TRI->needsStackRealignment(MF))
3652 while (NewAlign > Align &&
3653 TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
3654 NewAlign /= 2;
3655
Evan Cheng255f20f2010-04-01 06:04:33 +00003656 if (NewAlign > Align) {
3657 // Give the stack frame object a larger alignment if needed.
3658 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
3659 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
3660 Align = NewAlign;
3661 }
3662 }
Dan Gohman707e0182008-04-12 04:36:06 +00003663
Dan Gohman475871a2008-07-27 21:46:04 +00003664 SmallVector<SDValue, 8> OutChains;
Evan Chengf0df0312008-05-15 08:39:06 +00003665 unsigned NumMemOps = MemOps.size();
Evan Cheng0ff39b32008-06-30 07:31:25 +00003666 uint64_t SrcOff = 0, DstOff = 0;
Chris Lattner7453f8a2009-09-20 17:32:21 +00003667 for (unsigned i = 0; i != NumMemOps; ++i) {
Owen Andersone50ed302009-08-10 22:56:29 +00003668 EVT VT = MemOps[i];
Duncan Sands83ec4b62008-06-06 12:08:01 +00003669 unsigned VTSize = VT.getSizeInBits() / 8;
Dan Gohman475871a2008-07-27 21:46:04 +00003670 SDValue Value, Store;
Dan Gohman707e0182008-04-12 04:36:06 +00003671
Evan Cheng376642e2012-12-10 23:21:26 +00003672 if (VTSize > Size) {
3673 // Issuing an unaligned load / store pair that overlaps with the previous
3674 // pair. Adjust the offset accordingly.
3675 assert(i == NumMemOps-1 && i != 0);
3676 SrcOff -= VTSize - Size;
3677 DstOff -= VTSize - Size;
3678 }
3679
Evan Cheng255f20f2010-04-01 06:04:33 +00003680 if (CopyFromStr &&
3681 (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
Evan Chengf0df0312008-05-15 08:39:06 +00003682 // It's unlikely a store of a vector immediate can be done in a single
3683 // instruction. It would require a load from a constantpool first.
Evan Cheng255f20f2010-04-01 06:04:33 +00003684 // We only handle zero vectors here.
Evan Cheng0ff39b32008-06-30 07:31:25 +00003685 // FIXME: Handle other cases where store of vector immediate is done in
3686 // a single instruction.
Chris Lattner18c7f802012-02-05 02:29:43 +00003687 Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
Evan Cheng376642e2012-12-10 23:21:26 +00003688 if (Value.getNode())
3689 Store = DAG.getStore(Chain, dl, Value,
3690 getMemBasePlusOffset(Dst, DstOff, DAG),
3691 DstPtrInfo.getWithOffset(DstOff), isVol,
3692 false, Align);
3693 }
3694
3695 if (!Store.getNode()) {
Dale Johannesen08bc98e2009-06-22 20:59:07 +00003696 // The type might not be legal for the target. This should only happen
3697 // if the type is smaller than a legal type, as on PPC, so the right
Dale Johannesen8539cfd2009-06-24 17:11:31 +00003698 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
3699 // to Load/Store if NVT==VT.
Dale Johannesen08bc98e2009-06-22 20:59:07 +00003700 // FIXME does the case above also need this?
Owen Anderson23b9b192009-08-12 00:36:31 +00003701 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
Dale Johannesen8539cfd2009-06-24 17:11:31 +00003702 assert(NVT.bitsGE(VT));
Stuart Hastingsa9011292011-02-16 16:23:55 +00003703 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
Dale Johannesen8539cfd2009-06-24 17:11:31 +00003704 getMemBasePlusOffset(Src, SrcOff, DAG),
Chris Lattnere72f2022010-09-21 05:40:29 +00003705 SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
Evan Cheng255f20f2010-04-01 06:04:33 +00003706 MinAlign(SrcAlign, SrcOff));
Dale Johannesen8539cfd2009-06-24 17:11:31 +00003707 Store = DAG.getTruncStore(Chain, dl, Value,
David Greene1e559442010-02-15 17:00:31 +00003708 getMemBasePlusOffset(Dst, DstOff, DAG),
Chris Lattnere72f2022010-09-21 05:40:29 +00003709 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
3710 false, Align);
Dan Gohman707e0182008-04-12 04:36:06 +00003711 }
3712 OutChains.push_back(Store);
3713 SrcOff += VTSize;
3714 DstOff += VTSize;
Evan Cheng376642e2012-12-10 23:21:26 +00003715 Size -= VTSize;
Dan Gohman707e0182008-04-12 04:36:06 +00003716 }
3717
Owen Anderson825b72b2009-08-11 20:47:22 +00003718 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
Dan Gohman707e0182008-04-12 04:36:06 +00003719 &OutChains[0], OutChains.size());
3720}
3721
Andrew Trickac6d9be2013-05-25 02:42:55 +00003722static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
Evan Cheng255f20f2010-04-01 06:04:33 +00003723 SDValue Chain, SDValue Dst,
3724 SDValue Src, uint64_t Size,
Mon P Wang20adc9d2010-04-04 03:10:48 +00003725 unsigned Align, bool isVol,
3726 bool AlwaysInline,
Chris Lattnere72f2022010-09-21 05:40:29 +00003727 MachinePointerInfo DstPtrInfo,
3728 MachinePointerInfo SrcPtrInfo) {
Evan Chengf28f8bc2010-04-02 19:36:14 +00003729 // Turn a memmove of undef to nop.
3730 if (Src.getOpcode() == ISD::UNDEF)
3731 return Chain;
Dan Gohman21323f32008-05-29 19:42:22 +00003732
3733 // Expand memmove to a series of load and store ops if the size operand falls
3734 // below a certain threshold.
Evan Chengf28f8bc2010-04-02 19:36:14 +00003735 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Owen Andersone50ed302009-08-10 22:56:29 +00003736 std::vector<EVT> MemOps;
Evan Cheng255f20f2010-04-01 06:04:33 +00003737 bool DstAlignCanChange = false;
Evan Cheng05219282011-01-06 06:52:41 +00003738 MachineFunction &MF = DAG.getMachineFunction();
3739 MachineFrameInfo *MFI = MF.getFrameInfo();
Bill Wendling831737d2012-12-30 10:32:01 +00003740 bool OptSize = MF.getFunction()->getAttributes().
3741 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
Evan Cheng255f20f2010-04-01 06:04:33 +00003742 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
3743 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
3744 DstAlignCanChange = true;
3745 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
3746 if (Align > SrcAlign)
3747 SrcAlign = Align;
Evan Cheng05219282011-01-06 06:52:41 +00003748 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
Evan Cheng255f20f2010-04-01 06:04:33 +00003749
Evan Cheng94107ba2010-04-01 18:19:11 +00003750 if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
Evan Cheng946a3a92012-12-12 02:34:41 +00003751 (DstAlignCanChange ? 0 : Align), SrcAlign,
3752 false, false, false, false, DAG, TLI))
Dan Gohman475871a2008-07-27 21:46:04 +00003753 return SDValue();
Dan Gohman21323f32008-05-29 19:42:22 +00003754
Evan Cheng255f20f2010-04-01 06:04:33 +00003755 if (DstAlignCanChange) {
Chris Lattnerdb125cf2011-07-18 04:54:35 +00003756 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
Micah Villmow3574eca2012-10-08 16:38:25 +00003757 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
Evan Cheng255f20f2010-04-01 06:04:33 +00003758 if (NewAlign > Align) {
3759 // Give the stack frame object a larger alignment if needed.
3760 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
3761 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
3762 Align = NewAlign;
3763 }
3764 }
Dan Gohman21323f32008-05-29 19:42:22 +00003765
Evan Cheng255f20f2010-04-01 06:04:33 +00003766 uint64_t SrcOff = 0, DstOff = 0;
Dan Gohman475871a2008-07-27 21:46:04 +00003767 SmallVector<SDValue, 8> LoadValues;
3768 SmallVector<SDValue, 8> LoadChains;
3769 SmallVector<SDValue, 8> OutChains;
Dan Gohman21323f32008-05-29 19:42:22 +00003770 unsigned NumMemOps = MemOps.size();
3771 for (unsigned i = 0; i < NumMemOps; i++) {
Owen Andersone50ed302009-08-10 22:56:29 +00003772 EVT VT = MemOps[i];
Duncan Sands83ec4b62008-06-06 12:08:01 +00003773 unsigned VTSize = VT.getSizeInBits() / 8;
Dan Gohman475871a2008-07-27 21:46:04 +00003774 SDValue Value, Store;
Dan Gohman21323f32008-05-29 19:42:22 +00003775
Dale Johannesen0f502f62009-02-03 22:26:09 +00003776 Value = DAG.getLoad(VT, dl, Chain,
Dan Gohman21323f32008-05-29 19:42:22 +00003777 getMemBasePlusOffset(Src, SrcOff, DAG),
Chris Lattnere72f2022010-09-21 05:40:29 +00003778 SrcPtrInfo.getWithOffset(SrcOff), isVol,
Pete Cooperd752e0f2011-11-08 18:42:53 +00003779 false, false, SrcAlign);
Dan Gohman21323f32008-05-29 19:42:22 +00003780 LoadValues.push_back(Value);
3781 LoadChains.push_back(Value.getValue(1));
3782 SrcOff += VTSize;
3783 }
Owen Anderson825b72b2009-08-11 20:47:22 +00003784 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
Dan Gohman21323f32008-05-29 19:42:22 +00003785 &LoadChains[0], LoadChains.size());
3786 OutChains.clear();
3787 for (unsigned i = 0; i < NumMemOps; i++) {
Owen Andersone50ed302009-08-10 22:56:29 +00003788 EVT VT = MemOps[i];
Duncan Sands83ec4b62008-06-06 12:08:01 +00003789 unsigned VTSize = VT.getSizeInBits() / 8;
Dan Gohman475871a2008-07-27 21:46:04 +00003790 SDValue Value, Store;
Dan Gohman21323f32008-05-29 19:42:22 +00003791
Dale Johannesen0f502f62009-02-03 22:26:09 +00003792 Store = DAG.getStore(Chain, dl, LoadValues[i],
Dan Gohman21323f32008-05-29 19:42:22 +00003793 getMemBasePlusOffset(Dst, DstOff, DAG),
Chris Lattnere72f2022010-09-21 05:40:29 +00003794 DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
Dan Gohman21323f32008-05-29 19:42:22 +00003795 OutChains.push_back(Store);
3796 DstOff += VTSize;
3797 }
3798
Owen Anderson825b72b2009-08-11 20:47:22 +00003799 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
Dan Gohman21323f32008-05-29 19:42:22 +00003800 &OutChains[0], OutChains.size());
3801}
3802
Andrew Trickac6d9be2013-05-25 02:42:55 +00003803static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
Evan Cheng255f20f2010-04-01 06:04:33 +00003804 SDValue Chain, SDValue Dst,
3805 SDValue Src, uint64_t Size,
Mon P Wang20adc9d2010-04-04 03:10:48 +00003806 unsigned Align, bool isVol,
Chris Lattnere72f2022010-09-21 05:40:29 +00003807 MachinePointerInfo DstPtrInfo) {
Evan Chengf28f8bc2010-04-02 19:36:14 +00003808 // Turn a memset of undef to nop.
3809 if (Src.getOpcode() == ISD::UNDEF)
3810 return Chain;
Dan Gohman707e0182008-04-12 04:36:06 +00003811
3812 // Expand memset to a series of load/store ops if the size operand
3813 // falls below a certain threshold.
Evan Chengf28f8bc2010-04-02 19:36:14 +00003814 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Owen Andersone50ed302009-08-10 22:56:29 +00003815 std::vector<EVT> MemOps;
Evan Cheng255f20f2010-04-01 06:04:33 +00003816 bool DstAlignCanChange = false;
Evan Cheng05219282011-01-06 06:52:41 +00003817 MachineFunction &MF = DAG.getMachineFunction();
3818 MachineFrameInfo *MFI = MF.getFrameInfo();
Bill Wendling831737d2012-12-30 10:32:01 +00003819 bool OptSize = MF.getFunction()->getAttributes().
3820 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
Evan Cheng255f20f2010-04-01 06:04:33 +00003821 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
3822 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
3823 DstAlignCanChange = true;
Lang Hames15701f82011-10-26 23:50:43 +00003824 bool IsZeroVal =
Evan Chengf28f8bc2010-04-02 19:36:14 +00003825 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
Evan Cheng05219282011-01-06 06:52:41 +00003826 if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
Evan Cheng255f20f2010-04-01 06:04:33 +00003827 Size, (DstAlignCanChange ? 0 : Align), 0,
Evan Cheng946a3a92012-12-12 02:34:41 +00003828 true, IsZeroVal, false, true, DAG, TLI))
Dan Gohman475871a2008-07-27 21:46:04 +00003829 return SDValue();
Dan Gohman707e0182008-04-12 04:36:06 +00003830
Evan Cheng255f20f2010-04-01 06:04:33 +00003831 if (DstAlignCanChange) {
Chris Lattnerdb125cf2011-07-18 04:54:35 +00003832 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
Micah Villmow3574eca2012-10-08 16:38:25 +00003833 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
Evan Cheng255f20f2010-04-01 06:04:33 +00003834 if (NewAlign > Align) {
3835 // Give the stack frame object a larger alignment if needed.
3836 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
3837 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
3838 Align = NewAlign;
3839 }
3840 }
3841
Dan Gohman475871a2008-07-27 21:46:04 +00003842 SmallVector<SDValue, 8> OutChains;
Dan Gohman1f13c682008-04-28 17:15:20 +00003843 uint64_t DstOff = 0;
Dan Gohman707e0182008-04-12 04:36:06 +00003844 unsigned NumMemOps = MemOps.size();
Benjamin Kramer80220362011-01-02 19:57:05 +00003845
3846 // Find the largest store and generate the bit pattern for it.
3847 EVT LargestVT = MemOps[0];
3848 for (unsigned i = 1; i < NumMemOps; i++)
3849 if (MemOps[i].bitsGT(LargestVT))
3850 LargestVT = MemOps[i];
3851 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
3852
Dan Gohman707e0182008-04-12 04:36:06 +00003853 for (unsigned i = 0; i < NumMemOps; i++) {
Owen Andersone50ed302009-08-10 22:56:29 +00003854 EVT VT = MemOps[i];
Evan Cheng376642e2012-12-10 23:21:26 +00003855 unsigned VTSize = VT.getSizeInBits() / 8;
3856 if (VTSize > Size) {
3857 // Issuing an unaligned load / store pair that overlaps with the previous
3858 // pair. Adjust the offset accordingly.
3859 assert(i == NumMemOps-1 && i != 0);
3860 DstOff -= VTSize - Size;
3861 }
Benjamin Kramer80220362011-01-02 19:57:05 +00003862
3863 // If this store is smaller than the largest store see whether we can get
3864 // the smaller value for free with a truncate.
3865 SDValue Value = MemSetValue;
3866 if (VT.bitsLT(LargestVT)) {
3867 if (!LargestVT.isVector() && !VT.isVector() &&
3868 TLI.isTruncateFree(LargestVT, VT))
3869 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
3870 else
3871 Value = getMemsetValue(Src, VT, DAG, dl);
3872 }
3873 assert(Value.getValueType() == VT && "Value with wrong type.");
Dale Johannesen0f502f62009-02-03 22:26:09 +00003874 SDValue Store = DAG.getStore(Chain, dl, Value,
Chris Lattner7fe5e182008-10-28 07:10:51 +00003875 getMemBasePlusOffset(Dst, DstOff, DAG),
Chris Lattnere72f2022010-09-21 05:40:29 +00003876 DstPtrInfo.getWithOffset(DstOff),
Dale Johannesenb4ac2852010-11-18 01:35:23 +00003877 isVol, false, Align);
Dan Gohman707e0182008-04-12 04:36:06 +00003878 OutChains.push_back(Store);
Benjamin Kramer80220362011-01-02 19:57:05 +00003879 DstOff += VT.getSizeInBits() / 8;
Evan Cheng376642e2012-12-10 23:21:26 +00003880 Size -= VTSize;
Dan Gohman707e0182008-04-12 04:36:06 +00003881 }
3882
Owen Anderson825b72b2009-08-11 20:47:22 +00003883 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
Dan Gohman707e0182008-04-12 04:36:06 +00003884 &OutChains[0], OutChains.size());
3885}
3886
Andrew Trickac6d9be2013-05-25 02:42:55 +00003887SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
Dan Gohman475871a2008-07-27 21:46:04 +00003888 SDValue Src, SDValue Size,
Mon P Wang20adc9d2010-04-04 03:10:48 +00003889 unsigned Align, bool isVol, bool AlwaysInline,
Chris Lattnere72f2022010-09-21 05:40:29 +00003890 MachinePointerInfo DstPtrInfo,
3891 MachinePointerInfo SrcPtrInfo) {
Chandler Carruth7e6ffac2013-02-25 14:29:38 +00003892 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
Dan Gohman707e0182008-04-12 04:36:06 +00003893
3894 // Check to see if we should lower the memcpy to loads and stores first.
3895 // For cases within the target-specified limits, this is the best choice.
3896 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
3897 if (ConstantSize) {
3898 // Memcpy with size zero? Just return the original chain.
3899 if (ConstantSize->isNullValue())
3900 return Chain;
3901
Evan Cheng255f20f2010-04-01 06:04:33 +00003902 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
3903 ConstantSize->getZExtValue(),Align,
Chris Lattnere72f2022010-09-21 05:40:29 +00003904 isVol, false, DstPtrInfo, SrcPtrInfo);
Gabor Greifba36cb52008-08-28 21:40:38 +00003905 if (Result.getNode())
Dan Gohman707e0182008-04-12 04:36:06 +00003906 return Result;
3907 }
3908
3909 // Then check to see if we should lower the memcpy with target-specific
3910 // code. If the target chooses to do this, this is the next best.
Dan Gohman475871a2008-07-27 21:46:04 +00003911 SDValue Result =
Dan Gohmanff7a5622010-05-11 17:31:57 +00003912 TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align,
Mon P Wang20adc9d2010-04-04 03:10:48 +00003913 isVol, AlwaysInline,
Chris Lattnere72f2022010-09-21 05:40:29 +00003914 DstPtrInfo, SrcPtrInfo);
Gabor Greifba36cb52008-08-28 21:40:38 +00003915 if (Result.getNode())
Dan Gohman707e0182008-04-12 04:36:06 +00003916 return Result;
3917
3918 // If we really need inline code and the target declined to provide it,
3919 // use a (potentially long) sequence of loads and stores.
3920 if (AlwaysInline) {
3921 assert(ConstantSize && "AlwaysInline requires a constant size!");
Dale Johannesen0f502f62009-02-03 22:26:09 +00003922 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
Mon P Wang20adc9d2010-04-04 03:10:48 +00003923 ConstantSize->getZExtValue(), Align, isVol,
Chris Lattnere72f2022010-09-21 05:40:29 +00003924 true, DstPtrInfo, SrcPtrInfo);
Dan Gohman707e0182008-04-12 04:36:06 +00003925 }
3926
Dan Gohman7b55d362010-04-05 20:24:08 +00003927 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
3928 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
3929 // respect volatile, so they may do things like read or write memory
3930 // beyond the given memory regions. But fixing this isn't easy, and most
3931 // people don't care.
3932
Dan Gohman707e0182008-04-12 04:36:06 +00003933 // Emit a library call.
3934 TargetLowering::ArgListTy Args;
3935 TargetLowering::ArgListEntry Entry;
Chandler Carruthece6c6b2012-11-01 08:07:29 +00003936 Entry.Ty = TLI.getDataLayout()->getIntPtrType(*getContext());
Dan Gohman707e0182008-04-12 04:36:06 +00003937 Entry.Node = Dst; Args.push_back(Entry);
3938 Entry.Node = Src; Args.push_back(Entry);
3939 Entry.Node = Size; Args.push_back(Entry);
Andrew Trickac6d9be2013-05-25 02:42:55 +00003940 // FIXME: pass in SDLoc
Justin Holewinskid2ea0e12012-05-25 16:35:28 +00003941 TargetLowering::
3942 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()),
Anton Korobeynikov72977a42009-08-14 20:10:52 +00003943 false, false, false, false, 0,
Evan Cheng4bfcd4a2012-02-28 18:51:51 +00003944 TLI.getLibcallCallingConv(RTLIB::MEMCPY),
3945 /*isTailCall=*/false,
3946 /*doesNotReturn=*/false, /*isReturnValueUsed=*/false,
Eric Christopher4d7c18c2009-08-22 00:40:45 +00003947 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMCPY),
Sanjiv Guptaa114baa2009-07-30 09:12:56 +00003948 TLI.getPointerTy()),
Bill Wendling46ada192010-03-02 01:55:18 +00003949 Args, *this, dl);
Justin Holewinskid2ea0e12012-05-25 16:35:28 +00003950 std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
3951
Dan Gohman707e0182008-04-12 04:36:06 +00003952 return CallResult.second;
3953}
3954
Andrew Trickac6d9be2013-05-25 02:42:55 +00003955SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
Dan Gohman475871a2008-07-27 21:46:04 +00003956 SDValue Src, SDValue Size,
Mon P Wang20adc9d2010-04-04 03:10:48 +00003957 unsigned Align, bool isVol,
Chris Lattnere72f2022010-09-21 05:40:29 +00003958 MachinePointerInfo DstPtrInfo,
3959 MachinePointerInfo SrcPtrInfo) {
Chandler Carruth7e6ffac2013-02-25 14:29:38 +00003960 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
Dan Gohman707e0182008-04-12 04:36:06 +00003961
Dan Gohman21323f32008-05-29 19:42:22 +00003962 // Check to see if we should lower the memmove to loads and stores first.
3963 // For cases within the target-specified limits, this is the best choice.
3964 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
3965 if (ConstantSize) {
3966 // Memmove with size zero? Just return the original chain.
3967 if (ConstantSize->isNullValue())
3968 return Chain;
3969
Dan Gohman475871a2008-07-27 21:46:04 +00003970 SDValue Result =
Dale Johannesen0f502f62009-02-03 22:26:09 +00003971 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
Mon P Wang20adc9d2010-04-04 03:10:48 +00003972 ConstantSize->getZExtValue(), Align, isVol,
Chris Lattnere72f2022010-09-21 05:40:29 +00003973 false, DstPtrInfo, SrcPtrInfo);
Gabor Greifba36cb52008-08-28 21:40:38 +00003974 if (Result.getNode())
Dan Gohman21323f32008-05-29 19:42:22 +00003975 return Result;
3976 }
Dan Gohman707e0182008-04-12 04:36:06 +00003977
3978 // Then check to see if we should lower the memmove with target-specific
3979 // code. If the target chooses to do this, this is the next best.
Dan Gohman475871a2008-07-27 21:46:04 +00003980 SDValue Result =
Dan Gohmanff7a5622010-05-11 17:31:57 +00003981 TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol,
Chris Lattnere72f2022010-09-21 05:40:29 +00003982 DstPtrInfo, SrcPtrInfo);
Gabor Greifba36cb52008-08-28 21:40:38 +00003983 if (Result.getNode())
Dan Gohman707e0182008-04-12 04:36:06 +00003984 return Result;
3985
Mon P Wang01f0e852010-04-06 08:27:51 +00003986 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
3987 // not be safe. See memcpy above for more details.
3988
Dan Gohman707e0182008-04-12 04:36:06 +00003989 // Emit a library call.
3990 TargetLowering::ArgListTy Args;
3991 TargetLowering::ArgListEntry Entry;
Chandler Carruthece6c6b2012-11-01 08:07:29 +00003992 Entry.Ty = TLI.getDataLayout()->getIntPtrType(*getContext());
Dan Gohman707e0182008-04-12 04:36:06 +00003993 Entry.Node = Dst; Args.push_back(Entry);
3994 Entry.Node = Src; Args.push_back(Entry);
3995 Entry.Node = Size; Args.push_back(Entry);
Andrew Trickac6d9be2013-05-25 02:42:55 +00003996 // FIXME: pass in SDLoc
Justin Holewinskid2ea0e12012-05-25 16:35:28 +00003997 TargetLowering::
3998 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()),
Anton Korobeynikov72977a42009-08-14 20:10:52 +00003999 false, false, false, false, 0,
Evan Cheng4bfcd4a2012-02-28 18:51:51 +00004000 TLI.getLibcallCallingConv(RTLIB::MEMMOVE),
4001 /*isTailCall=*/false,
4002 /*doesNotReturn=*/false, /*isReturnValueUsed=*/false,
Eric Christopher4d7c18c2009-08-22 00:40:45 +00004003 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMMOVE),
Sanjiv Guptaa114baa2009-07-30 09:12:56 +00004004 TLI.getPointerTy()),
Bill Wendling46ada192010-03-02 01:55:18 +00004005 Args, *this, dl);
Justin Holewinskid2ea0e12012-05-25 16:35:28 +00004006 std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
4007
Dan Gohman707e0182008-04-12 04:36:06 +00004008 return CallResult.second;
4009}
4010
Andrew Trickac6d9be2013-05-25 02:42:55 +00004011SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
Dan Gohman475871a2008-07-27 21:46:04 +00004012 SDValue Src, SDValue Size,
Mon P Wang20adc9d2010-04-04 03:10:48 +00004013 unsigned Align, bool isVol,
Chris Lattnere72f2022010-09-21 05:40:29 +00004014 MachinePointerInfo DstPtrInfo) {
Chandler Carruth7e6ffac2013-02-25 14:29:38 +00004015 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
Dan Gohman707e0182008-04-12 04:36:06 +00004016
4017 // Check to see if we should lower the memset to stores first.
4018 // For cases within the target-specified limits, this is the best choice.
4019 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4020 if (ConstantSize) {
4021 // Memset with size zero? Just return the original chain.
4022 if (ConstantSize->isNullValue())
4023 return Chain;
4024
Mon P Wang20adc9d2010-04-04 03:10:48 +00004025 SDValue Result =
4026 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
Chris Lattnere72f2022010-09-21 05:40:29 +00004027 Align, isVol, DstPtrInfo);
Mon P Wang20adc9d2010-04-04 03:10:48 +00004028
Gabor Greifba36cb52008-08-28 21:40:38 +00004029 if (Result.getNode())
Dan Gohman707e0182008-04-12 04:36:06 +00004030 return Result;
4031 }
4032
4033 // Then check to see if we should lower the memset with target-specific
4034 // code. If the target chooses to do this, this is the next best.
Dan Gohman475871a2008-07-27 21:46:04 +00004035 SDValue Result =
Dan Gohmanff7a5622010-05-11 17:31:57 +00004036 TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol,
Chris Lattnere72f2022010-09-21 05:40:29 +00004037 DstPtrInfo);
Gabor Greifba36cb52008-08-28 21:40:38 +00004038 if (Result.getNode())
Dan Gohman707e0182008-04-12 04:36:06 +00004039 return Result;
4040
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004041 // Emit a library call.
Chandler Carruthece6c6b2012-11-01 08:07:29 +00004042 Type *IntPtrTy = TLI.getDataLayout()->getIntPtrType(*getContext());
Dan Gohman707e0182008-04-12 04:36:06 +00004043 TargetLowering::ArgListTy Args;
4044 TargetLowering::ArgListEntry Entry;
4045 Entry.Node = Dst; Entry.Ty = IntPtrTy;
4046 Args.push_back(Entry);
4047 // Extend or truncate the argument to be an i32 value for the call.
Owen Anderson825b72b2009-08-11 20:47:22 +00004048 if (Src.getValueType().bitsGT(MVT::i32))
4049 Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src);
Dan Gohman707e0182008-04-12 04:36:06 +00004050 else
Owen Anderson825b72b2009-08-11 20:47:22 +00004051 Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src);
Owen Anderson1d0be152009-08-13 21:58:54 +00004052 Entry.Node = Src;
4053 Entry.Ty = Type::getInt32Ty(*getContext());
4054 Entry.isSExt = true;
Dan Gohman707e0182008-04-12 04:36:06 +00004055 Args.push_back(Entry);
Owen Anderson1d0be152009-08-13 21:58:54 +00004056 Entry.Node = Size;
4057 Entry.Ty = IntPtrTy;
4058 Entry.isSExt = false;
Dan Gohman707e0182008-04-12 04:36:06 +00004059 Args.push_back(Entry);
Andrew Trickac6d9be2013-05-25 02:42:55 +00004060 // FIXME: pass in SDLoc
Justin Holewinskid2ea0e12012-05-25 16:35:28 +00004061 TargetLowering::
4062 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()),
Anton Korobeynikov72977a42009-08-14 20:10:52 +00004063 false, false, false, false, 0,
Evan Cheng4bfcd4a2012-02-28 18:51:51 +00004064 TLI.getLibcallCallingConv(RTLIB::MEMSET),
4065 /*isTailCall=*/false,
4066 /*doesNotReturn*/false, /*isReturnValueUsed=*/false,
Eric Christopher4d7c18c2009-08-22 00:40:45 +00004067 getExternalSymbol(TLI.getLibcallName(RTLIB::MEMSET),
Sanjiv Guptaa114baa2009-07-30 09:12:56 +00004068 TLI.getPointerTy()),
Bill Wendling46ada192010-03-02 01:55:18 +00004069 Args, *this, dl);
Justin Holewinskid2ea0e12012-05-25 16:35:28 +00004070 std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
4071
Dan Gohman707e0182008-04-12 04:36:06 +00004072 return CallResult.second;
Rafael Espindola5c0d6ed2007-10-19 10:41:11 +00004073}
4074
Andrew Trickac6d9be2013-05-25 02:42:55 +00004075SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
Chris Lattner60bddc82010-09-21 04:53:42 +00004076 SDValue Chain, SDValue Ptr, SDValue Cmp,
4077 SDValue Swp, MachinePointerInfo PtrInfo,
Eli Friedman55ba8162011-07-29 03:05:32 +00004078 unsigned Alignment,
4079 AtomicOrdering Ordering,
Michael Ilseman06b96902012-09-10 16:56:31 +00004080 SynchronizationScope SynchScope) {
Dan Gohmanc76909a2009-09-25 20:36:54 +00004081 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4082 Alignment = getEVTAlignment(MemVT);
4083
Evan Chengff89dcb2009-10-18 18:16:27 +00004084 MachineFunction &MF = getMachineFunction();
Dan Gohmanc76909a2009-09-25 20:36:54 +00004085
Jakob Stoklund Olesen36d29bc2012-08-28 03:11:32 +00004086 // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE.
Dan Gohmanc76909a2009-09-25 20:36:54 +00004087 // For now, atomics are considered to be volatile always.
Eli Friedman981a0102011-09-07 02:23:42 +00004088 // FIXME: Volatile isn't really correct; we should keep track of atomic
4089 // orderings in the memoperand.
Jakob Stoklund Olesen36d29bc2012-08-28 03:11:32 +00004090 unsigned Flags = MachineMemOperand::MOVolatile;
4091 if (Opcode != ISD::ATOMIC_STORE)
4092 Flags |= MachineMemOperand::MOLoad;
4093 if (Opcode != ISD::ATOMIC_LOAD)
4094 Flags |= MachineMemOperand::MOStore;
Dan Gohmanc76909a2009-09-25 20:36:54 +00004095
4096 MachineMemOperand *MMO =
Chris Lattner60bddc82010-09-21 04:53:42 +00004097 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004098
Eli Friedman55ba8162011-07-29 03:05:32 +00004099 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO,
4100 Ordering, SynchScope);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004101}
4102
Andrew Trickac6d9be2013-05-25 02:42:55 +00004103SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
Dan Gohmanc76909a2009-09-25 20:36:54 +00004104 SDValue Chain,
4105 SDValue Ptr, SDValue Cmp,
Eli Friedman55ba8162011-07-29 03:05:32 +00004106 SDValue Swp, MachineMemOperand *MMO,
4107 AtomicOrdering Ordering,
4108 SynchronizationScope SynchScope) {
Dale Johannesene8c17332009-01-29 00:47:48 +00004109 assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
4110 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
4111
Owen Andersone50ed302009-08-10 22:56:29 +00004112 EVT VT = Cmp.getValueType();
Dale Johannesene8c17332009-01-29 00:47:48 +00004113
Owen Anderson825b72b2009-08-11 20:47:22 +00004114 SDVTList VTs = getVTList(VT, MVT::Other);
Dale Johannesene8c17332009-01-29 00:47:48 +00004115 FoldingSetNodeID ID;
Dan Gohmana7ce7412009-02-03 00:08:45 +00004116 ID.AddInteger(MemVT.getRawBits());
Dale Johannesene8c17332009-01-29 00:47:48 +00004117 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
4118 AddNodeIDNode(ID, Opcode, VTs, Ops, 4);
Pete Cooper32ecfb42012-07-30 20:23:19 +00004119 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
Dale Johannesene8c17332009-01-29 00:47:48 +00004120 void* IP = 0;
Dan Gohmanc76909a2009-09-25 20:36:54 +00004121 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4122 cast<AtomicSDNode>(E)->refineAlignment(MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004123 return SDValue(E, 0);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004124 }
Andrew Trickac6d9be2013-05-25 02:42:55 +00004125 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, MemVT, Chain,
Eli Friedman55ba8162011-07-29 03:05:32 +00004126 Ptr, Cmp, Swp, MMO, Ordering,
4127 SynchScope);
Dale Johannesene8c17332009-01-29 00:47:48 +00004128 CSEMap.InsertNode(N, IP);
4129 AllNodes.push_back(N);
4130 return SDValue(N, 0);
4131}
4132
Andrew Trickac6d9be2013-05-25 02:42:55 +00004133SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
Dale Johannesene8c17332009-01-29 00:47:48 +00004134 SDValue Chain,
Scott Michelfdc40a02009-02-17 22:15:04 +00004135 SDValue Ptr, SDValue Val,
Dale Johannesene8c17332009-01-29 00:47:48 +00004136 const Value* PtrVal,
Eli Friedman55ba8162011-07-29 03:05:32 +00004137 unsigned Alignment,
4138 AtomicOrdering Ordering,
4139 SynchronizationScope SynchScope) {
Dan Gohmanc76909a2009-09-25 20:36:54 +00004140 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4141 Alignment = getEVTAlignment(MemVT);
4142
Evan Chengff89dcb2009-10-18 18:16:27 +00004143 MachineFunction &MF = getMachineFunction();
Jakob Stoklund Olesen36d29bc2012-08-28 03:11:32 +00004144 // An atomic store does not load. An atomic load does not store.
Eli Friedman981a0102011-09-07 02:23:42 +00004145 // (An atomicrmw obviously both loads and stores.)
Jakob Stoklund Olesen36d29bc2012-08-28 03:11:32 +00004146 // For now, atomics are considered to be volatile always, and they are
4147 // chained as such.
Eli Friedman981a0102011-09-07 02:23:42 +00004148 // FIXME: Volatile isn't really correct; we should keep track of atomic
4149 // orderings in the memoperand.
Jakob Stoklund Olesen36d29bc2012-08-28 03:11:32 +00004150 unsigned Flags = MachineMemOperand::MOVolatile;
4151 if (Opcode != ISD::ATOMIC_STORE)
4152 Flags |= MachineMemOperand::MOLoad;
4153 if (Opcode != ISD::ATOMIC_LOAD)
4154 Flags |= MachineMemOperand::MOStore;
Dan Gohmanc76909a2009-09-25 20:36:54 +00004155
4156 MachineMemOperand *MMO =
Chris Lattner93a95ae2010-09-21 04:46:39 +00004157 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
Dan Gohmanc76909a2009-09-25 20:36:54 +00004158 MemVT.getStoreSize(), Alignment);
4159
Eli Friedman55ba8162011-07-29 03:05:32 +00004160 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
4161 Ordering, SynchScope);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004162}
4163
Andrew Trickac6d9be2013-05-25 02:42:55 +00004164SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
Dan Gohmanc76909a2009-09-25 20:36:54 +00004165 SDValue Chain,
4166 SDValue Ptr, SDValue Val,
Eli Friedman55ba8162011-07-29 03:05:32 +00004167 MachineMemOperand *MMO,
4168 AtomicOrdering Ordering,
4169 SynchronizationScope SynchScope) {
Dale Johannesene8c17332009-01-29 00:47:48 +00004170 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
4171 Opcode == ISD::ATOMIC_LOAD_SUB ||
4172 Opcode == ISD::ATOMIC_LOAD_AND ||
4173 Opcode == ISD::ATOMIC_LOAD_OR ||
4174 Opcode == ISD::ATOMIC_LOAD_XOR ||
4175 Opcode == ISD::ATOMIC_LOAD_NAND ||
Scott Michelfdc40a02009-02-17 22:15:04 +00004176 Opcode == ISD::ATOMIC_LOAD_MIN ||
Dale Johannesene8c17332009-01-29 00:47:48 +00004177 Opcode == ISD::ATOMIC_LOAD_MAX ||
Scott Michelfdc40a02009-02-17 22:15:04 +00004178 Opcode == ISD::ATOMIC_LOAD_UMIN ||
Dale Johannesene8c17332009-01-29 00:47:48 +00004179 Opcode == ISD::ATOMIC_LOAD_UMAX ||
Eli Friedman327236c2011-08-24 20:50:09 +00004180 Opcode == ISD::ATOMIC_SWAP ||
4181 Opcode == ISD::ATOMIC_STORE) &&
Dale Johannesene8c17332009-01-29 00:47:48 +00004182 "Invalid Atomic Op");
4183
Owen Andersone50ed302009-08-10 22:56:29 +00004184 EVT VT = Val.getValueType();
Dale Johannesene8c17332009-01-29 00:47:48 +00004185
Eli Friedman327236c2011-08-24 20:50:09 +00004186 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
4187 getVTList(VT, MVT::Other);
Dale Johannesene8c17332009-01-29 00:47:48 +00004188 FoldingSetNodeID ID;
Dan Gohmana7ce7412009-02-03 00:08:45 +00004189 ID.AddInteger(MemVT.getRawBits());
Dale Johannesene8c17332009-01-29 00:47:48 +00004190 SDValue Ops[] = {Chain, Ptr, Val};
4191 AddNodeIDNode(ID, Opcode, VTs, Ops, 3);
Pete Cooper32ecfb42012-07-30 20:23:19 +00004192 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
Dale Johannesene8c17332009-01-29 00:47:48 +00004193 void* IP = 0;
Dan Gohmanc76909a2009-09-25 20:36:54 +00004194 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4195 cast<AtomicSDNode>(E)->refineAlignment(MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004196 return SDValue(E, 0);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004197 }
Andrew Trickac6d9be2013-05-25 02:42:55 +00004198 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, MemVT, Chain,
Eli Friedman55ba8162011-07-29 03:05:32 +00004199 Ptr, Val, MMO,
4200 Ordering, SynchScope);
Dale Johannesene8c17332009-01-29 00:47:48 +00004201 CSEMap.InsertNode(N, IP);
4202 AllNodes.push_back(N);
4203 return SDValue(N, 0);
4204}
4205
Andrew Trickac6d9be2013-05-25 02:42:55 +00004206SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
Eli Friedman327236c2011-08-24 20:50:09 +00004207 EVT VT, SDValue Chain,
4208 SDValue Ptr,
4209 const Value* PtrVal,
4210 unsigned Alignment,
4211 AtomicOrdering Ordering,
4212 SynchronizationScope SynchScope) {
4213 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4214 Alignment = getEVTAlignment(MemVT);
4215
4216 MachineFunction &MF = getMachineFunction();
Jakob Stoklund Olesen36d29bc2012-08-28 03:11:32 +00004217 // An atomic store does not load. An atomic load does not store.
4218 // (An atomicrmw obviously both loads and stores.)
4219 // For now, atomics are considered to be volatile always, and they are
4220 // chained as such.
Eli Friedman981a0102011-09-07 02:23:42 +00004221 // FIXME: Volatile isn't really correct; we should keep track of atomic
4222 // orderings in the memoperand.
Jakob Stoklund Olesen36d29bc2012-08-28 03:11:32 +00004223 unsigned Flags = MachineMemOperand::MOVolatile;
4224 if (Opcode != ISD::ATOMIC_STORE)
4225 Flags |= MachineMemOperand::MOLoad;
4226 if (Opcode != ISD::ATOMIC_LOAD)
4227 Flags |= MachineMemOperand::MOStore;
Eli Friedman327236c2011-08-24 20:50:09 +00004228
4229 MachineMemOperand *MMO =
4230 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
4231 MemVT.getStoreSize(), Alignment);
4232
4233 return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO,
4234 Ordering, SynchScope);
4235}
4236
Andrew Trickac6d9be2013-05-25 02:42:55 +00004237SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
Eli Friedman327236c2011-08-24 20:50:09 +00004238 EVT VT, SDValue Chain,
4239 SDValue Ptr,
4240 MachineMemOperand *MMO,
4241 AtomicOrdering Ordering,
4242 SynchronizationScope SynchScope) {
4243 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
4244
4245 SDVTList VTs = getVTList(VT, MVT::Other);
4246 FoldingSetNodeID ID;
4247 ID.AddInteger(MemVT.getRawBits());
4248 SDValue Ops[] = {Chain, Ptr};
4249 AddNodeIDNode(ID, Opcode, VTs, Ops, 2);
Pete Cooper32ecfb42012-07-30 20:23:19 +00004250 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
Eli Friedman327236c2011-08-24 20:50:09 +00004251 void* IP = 0;
4252 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4253 cast<AtomicSDNode>(E)->refineAlignment(MMO);
4254 return SDValue(E, 0);
4255 }
Andrew Trickac6d9be2013-05-25 02:42:55 +00004256 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, MemVT, Chain,
Eli Friedman327236c2011-08-24 20:50:09 +00004257 Ptr, MMO, Ordering, SynchScope);
4258 CSEMap.InsertNode(N, IP);
4259 AllNodes.push_back(N);
4260 return SDValue(N, 0);
4261}
4262
Duncan Sands4bdcb612008-07-02 17:40:58 +00004263/// getMergeValues - Create a MERGE_VALUES node from the given operands.
Dale Johannesen54c94522009-02-02 20:47:48 +00004264SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps,
Andrew Trickac6d9be2013-05-25 02:42:55 +00004265 SDLoc dl) {
Dale Johannesen54c94522009-02-02 20:47:48 +00004266 if (NumOps == 1)
4267 return Ops[0];
4268
Owen Andersone50ed302009-08-10 22:56:29 +00004269 SmallVector<EVT, 4> VTs;
Dale Johannesen54c94522009-02-02 20:47:48 +00004270 VTs.reserve(NumOps);
4271 for (unsigned i = 0; i < NumOps; ++i)
4272 VTs.push_back(Ops[i].getValueType());
Scott Michelfdc40a02009-02-17 22:15:04 +00004273 return getNode(ISD::MERGE_VALUES, dl, getVTList(&VTs[0], NumOps),
Dale Johannesen54c94522009-02-02 20:47:48 +00004274 Ops, NumOps);
4275}
4276
Dan Gohman475871a2008-07-27 21:46:04 +00004277SDValue
Andrew Trickac6d9be2013-05-25 02:42:55 +00004278SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl,
Owen Andersone50ed302009-08-10 22:56:29 +00004279 const EVT *VTs, unsigned NumVTs,
Dale Johannesene8c17332009-01-29 00:47:48 +00004280 const SDValue *Ops, unsigned NumOps,
Chris Lattnere9ba5dd2010-09-21 04:57:15 +00004281 EVT MemVT, MachinePointerInfo PtrInfo,
Dale Johannesene8c17332009-01-29 00:47:48 +00004282 unsigned Align, bool Vol,
4283 bool ReadMem, bool WriteMem) {
4284 return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps,
Chris Lattnere9ba5dd2010-09-21 04:57:15 +00004285 MemVT, PtrInfo, Align, Vol,
Dale Johannesene8c17332009-01-29 00:47:48 +00004286 ReadMem, WriteMem);
4287}
4288
4289SDValue
Andrew Trickac6d9be2013-05-25 02:42:55 +00004290SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
Dale Johannesene8c17332009-01-29 00:47:48 +00004291 const SDValue *Ops, unsigned NumOps,
Chris Lattnere9ba5dd2010-09-21 04:57:15 +00004292 EVT MemVT, MachinePointerInfo PtrInfo,
Dale Johannesene8c17332009-01-29 00:47:48 +00004293 unsigned Align, bool Vol,
4294 bool ReadMem, bool WriteMem) {
Dan Gohmanc76909a2009-09-25 20:36:54 +00004295 if (Align == 0) // Ensure that codegen never sees alignment 0
4296 Align = getEVTAlignment(MemVT);
4297
4298 MachineFunction &MF = getMachineFunction();
4299 unsigned Flags = 0;
4300 if (WriteMem)
4301 Flags |= MachineMemOperand::MOStore;
4302 if (ReadMem)
4303 Flags |= MachineMemOperand::MOLoad;
4304 if (Vol)
4305 Flags |= MachineMemOperand::MOVolatile;
4306 MachineMemOperand *MMO =
Chris Lattnere9ba5dd2010-09-21 04:57:15 +00004307 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004308
4309 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO);
4310}
4311
4312SDValue
Andrew Trickac6d9be2013-05-25 02:42:55 +00004313SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
Dan Gohmanc76909a2009-09-25 20:36:54 +00004314 const SDValue *Ops, unsigned NumOps,
4315 EVT MemVT, MachineMemOperand *MMO) {
4316 assert((Opcode == ISD::INTRINSIC_VOID ||
4317 Opcode == ISD::INTRINSIC_W_CHAIN ||
Dale Johannesen1de4aa92010-10-26 23:11:10 +00004318 Opcode == ISD::PREFETCH ||
Nadav Rotemc05d3062012-09-06 09:17:37 +00004319 Opcode == ISD::LIFETIME_START ||
4320 Opcode == ISD::LIFETIME_END ||
Dan Gohmanc76909a2009-09-25 20:36:54 +00004321 (Opcode <= INT_MAX &&
4322 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
4323 "Opcode is not a memory-accessing opcode!");
4324
Dale Johannesene8c17332009-01-29 00:47:48 +00004325 // Memoize the node unless it returns a flag.
4326 MemIntrinsicSDNode *N;
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00004327 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
Dale Johannesene8c17332009-01-29 00:47:48 +00004328 FoldingSetNodeID ID;
4329 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
Pete Cooper32ecfb42012-07-30 20:23:19 +00004330 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
Dale Johannesene8c17332009-01-29 00:47:48 +00004331 void *IP = 0;
Dan Gohmanc76909a2009-09-25 20:36:54 +00004332 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4333 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004334 return SDValue(E, 0);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004335 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004336
Andrew Trickac6d9be2013-05-25 02:42:55 +00004337 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTList, Ops, NumOps,
Dan Gohman95531882010-03-18 18:49:47 +00004338 MemVT, MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004339 CSEMap.InsertNode(N, IP);
4340 } else {
Andrew Trickac6d9be2013-05-25 02:42:55 +00004341 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), VTList, Ops, NumOps,
Dan Gohman95531882010-03-18 18:49:47 +00004342 MemVT, MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004343 }
4344 AllNodes.push_back(N);
4345 return SDValue(N, 0);
4346}
4347
Chris Lattnerd0e139f2010-09-21 17:24:05 +00004348/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
4349/// MachinePointerInfo record from it. This is particularly useful because the
4350/// code generator has many cases where it doesn't bother passing in a
4351/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
4352static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
4353 // If this is FI+Offset, we can model it.
4354 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
4355 return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
4356
4357 // If this is (FI+Offset1)+Offset2, we can model it.
4358 if (Ptr.getOpcode() != ISD::ADD ||
4359 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
4360 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
4361 return MachinePointerInfo();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004362
Chris Lattnerd0e139f2010-09-21 17:24:05 +00004363 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
4364 return MachinePointerInfo::getFixedStack(FI, Offset+
4365 cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
4366}
4367
4368/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
4369/// MachinePointerInfo record from it. This is particularly useful because the
4370/// code generator has many cases where it doesn't bother passing in a
4371/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
4372static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
4373 // If the 'Offset' value isn't a constant, we can't handle this.
4374 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
4375 return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
4376 if (OffsetOp.getOpcode() == ISD::UNDEF)
4377 return InferPointerInfo(Ptr);
4378 return MachinePointerInfo();
4379}
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004380
Chris Lattnerd0e139f2010-09-21 17:24:05 +00004381
Chris Lattner5c5cb2a2010-09-21 05:10:45 +00004382SDValue
4383SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
Andrew Trickac6d9be2013-05-25 02:42:55 +00004384 EVT VT, SDLoc dl, SDValue Chain,
Chris Lattner5c5cb2a2010-09-21 05:10:45 +00004385 SDValue Ptr, SDValue Offset,
4386 MachinePointerInfo PtrInfo, EVT MemVT,
Pete Cooperd752e0f2011-11-08 18:42:53 +00004387 bool isVolatile, bool isNonTemporal, bool isInvariant,
Rafael Espindola95d594c2012-03-31 18:14:00 +00004388 unsigned Alignment, const MDNode *TBAAInfo,
4389 const MDNode *Ranges) {
Michael Ilseman06b96902012-09-10 16:56:31 +00004390 assert(Chain.getValueType() == MVT::Other &&
Nadav Rotemaeb86fa2011-07-14 10:37:54 +00004391 "Invalid chain type");
Chris Lattner5c5cb2a2010-09-21 05:10:45 +00004392 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4393 Alignment = getEVTAlignment(VT);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004394
Dan Gohmanc76909a2009-09-25 20:36:54 +00004395 unsigned Flags = MachineMemOperand::MOLoad;
4396 if (isVolatile)
4397 Flags |= MachineMemOperand::MOVolatile;
David Greene1e559442010-02-15 17:00:31 +00004398 if (isNonTemporal)
4399 Flags |= MachineMemOperand::MONonTemporal;
Pete Cooperd752e0f2011-11-08 18:42:53 +00004400 if (isInvariant)
4401 Flags |= MachineMemOperand::MOInvariant;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004402
Chris Lattnerd0e139f2010-09-21 17:24:05 +00004403 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
4404 // clients.
4405 if (PtrInfo.V == 0)
4406 PtrInfo = InferPointerInfo(Ptr, Offset);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004407
Chris Lattnerd0e139f2010-09-21 17:24:05 +00004408 MachineFunction &MF = getMachineFunction();
Dan Gohmanc76909a2009-09-25 20:36:54 +00004409 MachineMemOperand *MMO =
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00004410 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
Rafael Espindola95d594c2012-03-31 18:14:00 +00004411 TBAAInfo, Ranges);
Evan Chengbcc80172010-07-07 22:15:37 +00004412 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004413}
4414
4415SDValue
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004416SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
Andrew Trickac6d9be2013-05-25 02:42:55 +00004417 EVT VT, SDLoc dl, SDValue Chain,
Dan Gohmanc76909a2009-09-25 20:36:54 +00004418 SDValue Ptr, SDValue Offset, EVT MemVT,
4419 MachineMemOperand *MMO) {
Dan Gohman8a55ce42009-09-23 21:02:20 +00004420 if (VT == MemVT) {
Dale Johannesene8c17332009-01-29 00:47:48 +00004421 ExtType = ISD::NON_EXTLOAD;
4422 } else if (ExtType == ISD::NON_EXTLOAD) {
Dan Gohman8a55ce42009-09-23 21:02:20 +00004423 assert(VT == MemVT && "Non-extending load from different memory type!");
Dale Johannesene8c17332009-01-29 00:47:48 +00004424 } else {
4425 // Extending load.
Dan Gohman2e141d72009-12-14 23:40:38 +00004426 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
4427 "Should only be an extending load, not truncating!");
Dan Gohman8a55ce42009-09-23 21:02:20 +00004428 assert(VT.isInteger() == MemVT.isInteger() &&
Dale Johannesene8c17332009-01-29 00:47:48 +00004429 "Cannot convert from FP to Int or Int -> FP!");
Dan Gohman2e141d72009-12-14 23:40:38 +00004430 assert(VT.isVector() == MemVT.isVector() &&
4431 "Cannot use trunc store to convert to or from a vector!");
4432 assert((!VT.isVector() ||
4433 VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
4434 "Cannot use trunc store to change the number of vector elements!");
Dale Johannesene8c17332009-01-29 00:47:48 +00004435 }
4436
4437 bool Indexed = AM != ISD::UNINDEXED;
4438 assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
4439 "Unindexed load with an offset!");
4440
4441 SDVTList VTs = Indexed ?
Owen Anderson825b72b2009-08-11 20:47:22 +00004442 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
Dale Johannesene8c17332009-01-29 00:47:48 +00004443 SDValue Ops[] = { Chain, Ptr, Offset };
4444 FoldingSetNodeID ID;
4445 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
Dan Gohman8a55ce42009-09-23 21:02:20 +00004446 ID.AddInteger(MemVT.getRawBits());
David Greene1157f792010-02-17 20:21:42 +00004447 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
Michael Ilseman06b96902012-09-10 16:56:31 +00004448 MMO->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00004449 MMO->isInvariant()));
Pete Cooper32ecfb42012-07-30 20:23:19 +00004450 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
Dale Johannesene8c17332009-01-29 00:47:48 +00004451 void *IP = 0;
Dan Gohmanc76909a2009-09-25 20:36:54 +00004452 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4453 cast<LoadSDNode>(E)->refineAlignment(MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004454 return SDValue(E, 0);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004455 }
Andrew Trickac6d9be2013-05-25 02:42:55 +00004456 SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, AM, ExtType,
Dan Gohman95531882010-03-18 18:49:47 +00004457 MemVT, MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004458 CSEMap.InsertNode(N, IP);
4459 AllNodes.push_back(N);
4460 return SDValue(N, 0);
4461}
4462
Andrew Trickac6d9be2013-05-25 02:42:55 +00004463SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
Dale Johannesene8c17332009-01-29 00:47:48 +00004464 SDValue Chain, SDValue Ptr,
Chris Lattnere72f2022010-09-21 05:40:29 +00004465 MachinePointerInfo PtrInfo,
4466 bool isVolatile, bool isNonTemporal,
Michael Ilseman06b96902012-09-10 16:56:31 +00004467 bool isInvariant, unsigned Alignment,
Rafael Espindola95d594c2012-03-31 18:14:00 +00004468 const MDNode *TBAAInfo,
4469 const MDNode *Ranges) {
Chris Lattnere72f2022010-09-21 05:40:29 +00004470 SDValue Undef = getUNDEF(Ptr.getValueType());
4471 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
Rafael Espindola95d594c2012-03-31 18:14:00 +00004472 PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
4473 TBAAInfo, Ranges);
Chris Lattnere72f2022010-09-21 05:40:29 +00004474}
Dale Johannesene8c17332009-01-29 00:47:48 +00004475
Andrew Trickac6d9be2013-05-25 02:42:55 +00004476SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
Chris Lattnere72f2022010-09-21 05:40:29 +00004477 SDValue Chain, SDValue Ptr,
4478 MachinePointerInfo PtrInfo, EVT MemVT,
4479 bool isVolatile, bool isNonTemporal,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00004480 unsigned Alignment, const MDNode *TBAAInfo) {
Chris Lattnere72f2022010-09-21 05:40:29 +00004481 SDValue Undef = getUNDEF(Ptr.getValueType());
4482 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
Pete Cooperd752e0f2011-11-08 18:42:53 +00004483 PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00004484 TBAAInfo);
Chris Lattnere72f2022010-09-21 05:40:29 +00004485}
4486
4487
Dan Gohman475871a2008-07-27 21:46:04 +00004488SDValue
Andrew Trickac6d9be2013-05-25 02:42:55 +00004489SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
Dale Johannesene8c17332009-01-29 00:47:48 +00004490 SDValue Offset, ISD::MemIndexedMode AM) {
4491 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
4492 assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
4493 "Load is already a indexed load!");
Evan Chengbcc80172010-07-07 22:15:37 +00004494 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
Chris Lattnerd0e139f2010-09-21 17:24:05 +00004495 LD->getChain(), Base, Offset, LD->getPointerInfo(),
Michael Ilseman06b96902012-09-10 16:56:31 +00004496 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00004497 false, LD->getAlignment());
Dale Johannesene8c17332009-01-29 00:47:48 +00004498}
4499
Andrew Trickac6d9be2013-05-25 02:42:55 +00004500SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
Chris Lattner5c5cb2a2010-09-21 05:10:45 +00004501 SDValue Ptr, MachinePointerInfo PtrInfo,
David Greene1e559442010-02-15 17:00:31 +00004502 bool isVolatile, bool isNonTemporal,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00004503 unsigned Alignment, const MDNode *TBAAInfo) {
Michael Ilseman06b96902012-09-10 16:56:31 +00004504 assert(Chain.getValueType() == MVT::Other &&
Nadav Rotemaeb86fa2011-07-14 10:37:54 +00004505 "Invalid chain type");
Dale Johannesene8c17332009-01-29 00:47:48 +00004506 if (Alignment == 0) // Ensure that codegen never sees alignment 0
Dan Gohmanc76909a2009-09-25 20:36:54 +00004507 Alignment = getEVTAlignment(Val.getValueType());
Dale Johannesene8c17332009-01-29 00:47:48 +00004508
Dan Gohmanc76909a2009-09-25 20:36:54 +00004509 unsigned Flags = MachineMemOperand::MOStore;
4510 if (isVolatile)
4511 Flags |= MachineMemOperand::MOVolatile;
David Greene1e559442010-02-15 17:00:31 +00004512 if (isNonTemporal)
4513 Flags |= MachineMemOperand::MONonTemporal;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004514
Chris Lattnerd0e139f2010-09-21 17:24:05 +00004515 if (PtrInfo.V == 0)
4516 PtrInfo = InferPointerInfo(Ptr);
4517
4518 MachineFunction &MF = getMachineFunction();
Dan Gohmanc76909a2009-09-25 20:36:54 +00004519 MachineMemOperand *MMO =
Chris Lattner5c5cb2a2010-09-21 05:10:45 +00004520 MF.getMachineMemOperand(PtrInfo, Flags,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00004521 Val.getValueType().getStoreSize(), Alignment,
4522 TBAAInfo);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004523
4524 return getStore(Chain, dl, Val, Ptr, MMO);
4525}
4526
Andrew Trickac6d9be2013-05-25 02:42:55 +00004527SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
Dan Gohmanc76909a2009-09-25 20:36:54 +00004528 SDValue Ptr, MachineMemOperand *MMO) {
Michael Ilseman06b96902012-09-10 16:56:31 +00004529 assert(Chain.getValueType() == MVT::Other &&
Nadav Rotemaeb86fa2011-07-14 10:37:54 +00004530 "Invalid chain type");
Dan Gohmanc76909a2009-09-25 20:36:54 +00004531 EVT VT = Val.getValueType();
Owen Anderson825b72b2009-08-11 20:47:22 +00004532 SDVTList VTs = getVTList(MVT::Other);
Dale Johannesene8d72302009-02-06 23:05:02 +00004533 SDValue Undef = getUNDEF(Ptr.getValueType());
Dale Johannesene8c17332009-01-29 00:47:48 +00004534 SDValue Ops[] = { Chain, Val, Ptr, Undef };
4535 FoldingSetNodeID ID;
4536 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
Dale Johannesene8c17332009-01-29 00:47:48 +00004537 ID.AddInteger(VT.getRawBits());
David Greene1157f792010-02-17 20:21:42 +00004538 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00004539 MMO->isNonTemporal(), MMO->isInvariant()));
Pete Cooper32ecfb42012-07-30 20:23:19 +00004540 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
Dale Johannesene8c17332009-01-29 00:47:48 +00004541 void *IP = 0;
Dan Gohmanc76909a2009-09-25 20:36:54 +00004542 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4543 cast<StoreSDNode>(E)->refineAlignment(MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004544 return SDValue(E, 0);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004545 }
Andrew Trickac6d9be2013-05-25 02:42:55 +00004546 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, ISD::UNINDEXED,
Dan Gohman95531882010-03-18 18:49:47 +00004547 false, VT, MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004548 CSEMap.InsertNode(N, IP);
4549 AllNodes.push_back(N);
4550 return SDValue(N, 0);
4551}
4552
Andrew Trickac6d9be2013-05-25 02:42:55 +00004553SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
Chris Lattner5c5cb2a2010-09-21 05:10:45 +00004554 SDValue Ptr, MachinePointerInfo PtrInfo,
4555 EVT SVT,bool isVolatile, bool isNonTemporal,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00004556 unsigned Alignment,
4557 const MDNode *TBAAInfo) {
Michael Ilseman06b96902012-09-10 16:56:31 +00004558 assert(Chain.getValueType() == MVT::Other &&
Nadav Rotemaeb86fa2011-07-14 10:37:54 +00004559 "Invalid chain type");
Chris Lattner5c5cb2a2010-09-21 05:10:45 +00004560 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4561 Alignment = getEVTAlignment(SVT);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004562
Dan Gohmanc76909a2009-09-25 20:36:54 +00004563 unsigned Flags = MachineMemOperand::MOStore;
4564 if (isVolatile)
4565 Flags |= MachineMemOperand::MOVolatile;
David Greene1e559442010-02-15 17:00:31 +00004566 if (isNonTemporal)
4567 Flags |= MachineMemOperand::MONonTemporal;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004568
Chris Lattnerd0e139f2010-09-21 17:24:05 +00004569 if (PtrInfo.V == 0)
4570 PtrInfo = InferPointerInfo(Ptr);
4571
4572 MachineFunction &MF = getMachineFunction();
Dan Gohmanc76909a2009-09-25 20:36:54 +00004573 MachineMemOperand *MMO =
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00004574 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
4575 TBAAInfo);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004576
4577 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
4578}
4579
Andrew Trickac6d9be2013-05-25 02:42:55 +00004580SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
Dan Gohmanc76909a2009-09-25 20:36:54 +00004581 SDValue Ptr, EVT SVT,
4582 MachineMemOperand *MMO) {
Owen Andersone50ed302009-08-10 22:56:29 +00004583 EVT VT = Val.getValueType();
Dale Johannesene8c17332009-01-29 00:47:48 +00004584
Michael Ilseman06b96902012-09-10 16:56:31 +00004585 assert(Chain.getValueType() == MVT::Other &&
Nadav Rotemaeb86fa2011-07-14 10:37:54 +00004586 "Invalid chain type");
Dale Johannesene8c17332009-01-29 00:47:48 +00004587 if (VT == SVT)
Dan Gohmanc76909a2009-09-25 20:36:54 +00004588 return getStore(Chain, dl, Val, Ptr, MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004589
Dan Gohman2e141d72009-12-14 23:40:38 +00004590 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
4591 "Should only be a truncating store, not extending!");
Dale Johannesene8c17332009-01-29 00:47:48 +00004592 assert(VT.isInteger() == SVT.isInteger() &&
4593 "Can't do FP-INT conversion!");
Dan Gohman2e141d72009-12-14 23:40:38 +00004594 assert(VT.isVector() == SVT.isVector() &&
4595 "Cannot use trunc store to convert to or from a vector!");
4596 assert((!VT.isVector() ||
4597 VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
4598 "Cannot use trunc store to change the number of vector elements!");
Dale Johannesene8c17332009-01-29 00:47:48 +00004599
Owen Anderson825b72b2009-08-11 20:47:22 +00004600 SDVTList VTs = getVTList(MVT::Other);
Dale Johannesene8d72302009-02-06 23:05:02 +00004601 SDValue Undef = getUNDEF(Ptr.getValueType());
Dale Johannesene8c17332009-01-29 00:47:48 +00004602 SDValue Ops[] = { Chain, Val, Ptr, Undef };
4603 FoldingSetNodeID ID;
4604 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
Dale Johannesene8c17332009-01-29 00:47:48 +00004605 ID.AddInteger(SVT.getRawBits());
David Greene1157f792010-02-17 20:21:42 +00004606 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00004607 MMO->isNonTemporal(), MMO->isInvariant()));
Pete Cooper32ecfb42012-07-30 20:23:19 +00004608 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
Dale Johannesene8c17332009-01-29 00:47:48 +00004609 void *IP = 0;
Dan Gohmanc76909a2009-09-25 20:36:54 +00004610 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4611 cast<StoreSDNode>(E)->refineAlignment(MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004612 return SDValue(E, 0);
Dan Gohmanc76909a2009-09-25 20:36:54 +00004613 }
Andrew Trickac6d9be2013-05-25 02:42:55 +00004614 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, ISD::UNINDEXED,
Dan Gohman95531882010-03-18 18:49:47 +00004615 true, SVT, MMO);
Dale Johannesene8c17332009-01-29 00:47:48 +00004616 CSEMap.InsertNode(N, IP);
4617 AllNodes.push_back(N);
4618 return SDValue(N, 0);
4619}
4620
Dan Gohman475871a2008-07-27 21:46:04 +00004621SDValue
Andrew Trickac6d9be2013-05-25 02:42:55 +00004622SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
Dale Johannesene8c17332009-01-29 00:47:48 +00004623 SDValue Offset, ISD::MemIndexedMode AM) {
4624 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
4625 assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
4626 "Store is already a indexed store!");
Owen Anderson825b72b2009-08-11 20:47:22 +00004627 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
Dale Johannesene8c17332009-01-29 00:47:48 +00004628 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
4629 FoldingSetNodeID ID;
4630 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
Dale Johannesene8c17332009-01-29 00:47:48 +00004631 ID.AddInteger(ST->getMemoryVT().getRawBits());
Dan Gohmana7ce7412009-02-03 00:08:45 +00004632 ID.AddInteger(ST->getRawSubclassData());
Pete Cooper32ecfb42012-07-30 20:23:19 +00004633 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
Dale Johannesene8c17332009-01-29 00:47:48 +00004634 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00004635 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dale Johannesene8c17332009-01-29 00:47:48 +00004636 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00004637
Andrew Trickac6d9be2013-05-25 02:42:55 +00004638 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
Dan Gohman95531882010-03-18 18:49:47 +00004639 ST->isTruncatingStore(),
4640 ST->getMemoryVT(),
4641 ST->getMemOperand());
Dale Johannesene8c17332009-01-29 00:47:48 +00004642 CSEMap.InsertNode(N, IP);
4643 AllNodes.push_back(N);
4644 return SDValue(N, 0);
4645}
4646
Andrew Trickac6d9be2013-05-25 02:42:55 +00004647SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
Dale Johannesen0f502f62009-02-03 22:26:09 +00004648 SDValue Chain, SDValue Ptr,
Rafael Espindola72d13ff2010-06-26 18:22:20 +00004649 SDValue SV,
4650 unsigned Align) {
4651 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
4652 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4);
Dale Johannesen0f502f62009-02-03 22:26:09 +00004653}
4654
Andrew Trickac6d9be2013-05-25 02:42:55 +00004655SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
Bill Wendling7ade28c2009-01-28 22:17:52 +00004656 const SDUse *Ops, unsigned NumOps) {
Dan Gohman6d9cdd52008-07-07 18:26:29 +00004657 switch (NumOps) {
Bill Wendling7ade28c2009-01-28 22:17:52 +00004658 case 0: return getNode(Opcode, DL, VT);
4659 case 1: return getNode(Opcode, DL, VT, Ops[0]);
4660 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
4661 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
Dan Gohman6d9cdd52008-07-07 18:26:29 +00004662 default: break;
4663 }
4664
Dan Gohman475871a2008-07-27 21:46:04 +00004665 // Copy from an SDUse array into an SDValue array for use with
Dan Gohman6d9cdd52008-07-07 18:26:29 +00004666 // the regular getNode logic.
Dan Gohman475871a2008-07-27 21:46:04 +00004667 SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps);
Bill Wendling7ade28c2009-01-28 22:17:52 +00004668 return getNode(Opcode, DL, VT, &NewOps[0], NumOps);
Dan Gohman6d9cdd52008-07-07 18:26:29 +00004669}
4670
Andrew Trickac6d9be2013-05-25 02:42:55 +00004671SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
Bill Wendling7ade28c2009-01-28 22:17:52 +00004672 const SDValue *Ops, unsigned NumOps) {
Chris Lattnerf06f35e2006-08-08 01:09:31 +00004673 switch (NumOps) {
Bill Wendling7ade28c2009-01-28 22:17:52 +00004674 case 0: return getNode(Opcode, DL, VT);
4675 case 1: return getNode(Opcode, DL, VT, Ops[0]);
4676 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
4677 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
Chris Lattneref847df2005-04-09 03:27:28 +00004678 default: break;
Chris Lattnerc3aae252005-01-07 07:46:32 +00004679 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004680
Chris Lattneref847df2005-04-09 03:27:28 +00004681 switch (Opcode) {
4682 default: break;
Chris Lattner7b2880c2005-08-24 22:44:39 +00004683 case ISD::SELECT_CC: {
Chris Lattnerf06f35e2006-08-08 01:09:31 +00004684 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
Chris Lattner7b2880c2005-08-24 22:44:39 +00004685 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
4686 "LHS and RHS of condition must have same type!");
4687 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
4688 "True and False arms of SelectCC must have same type!");
4689 assert(Ops[2].getValueType() == VT &&
4690 "select_cc node must be of same type as true and false value!");
Chris Lattner7b2880c2005-08-24 22:44:39 +00004691 break;
4692 }
4693 case ISD::BR_CC: {
Chris Lattnerf06f35e2006-08-08 01:09:31 +00004694 assert(NumOps == 5 && "BR_CC takes 5 operands!");
Chris Lattner7b2880c2005-08-24 22:44:39 +00004695 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
4696 "LHS/RHS of comparison should match types!");
Chris Lattner7b2880c2005-08-24 22:44:39 +00004697 break;
4698 }
Chris Lattneref847df2005-04-09 03:27:28 +00004699 }
4700
Chris Lattner385328c2005-05-14 07:42:29 +00004701 // Memoize nodes.
Chris Lattner43247a12005-08-25 19:12:10 +00004702 SDNode *N;
Chris Lattner0b3e5252006-08-15 19:11:05 +00004703 SDVTList VTs = getVTList(VT);
Bill Wendling7ade28c2009-01-28 22:17:52 +00004704
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00004705 if (VT != MVT::Glue) {
Jim Laskey583bd472006-10-27 23:46:08 +00004706 FoldingSetNodeID ID;
4707 AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps);
Chris Lattnera5682852006-08-07 23:03:03 +00004708 void *IP = 0;
Bill Wendling7ade28c2009-01-28 22:17:52 +00004709
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00004710 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00004711 return SDValue(E, 0);
Bill Wendling7ade28c2009-01-28 22:17:52 +00004712
Andrew Trickac6d9be2013-05-25 02:42:55 +00004713 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops, NumOps);
Chris Lattnera5682852006-08-07 23:03:03 +00004714 CSEMap.InsertNode(N, IP);
Chris Lattner43247a12005-08-25 19:12:10 +00004715 } else {
Andrew Trickac6d9be2013-05-25 02:42:55 +00004716 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops, NumOps);
Chris Lattner43247a12005-08-25 19:12:10 +00004717 }
Bill Wendling7ade28c2009-01-28 22:17:52 +00004718
Chris Lattneref847df2005-04-09 03:27:28 +00004719 AllNodes.push_back(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00004720#ifndef NDEBUG
Duncan Sands59d2dad2010-11-20 11:25:00 +00004721 VerifySDNode(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00004722#endif
Dan Gohman475871a2008-07-27 21:46:04 +00004723 return SDValue(N, 0);
Chris Lattnerc3aae252005-01-07 07:46:32 +00004724}
4725
Andrew Trickac6d9be2013-05-25 02:42:55 +00004726SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
Benjamin Kramer3853f742013-03-07 20:33:29 +00004727 ArrayRef<EVT> ResultTys,
Bill Wendling7ade28c2009-01-28 22:17:52 +00004728 const SDValue *Ops, unsigned NumOps) {
Dan Gohmanfc166572009-04-09 23:54:40 +00004729 return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()),
Chris Lattner2fa6d3b2006-08-14 23:31:51 +00004730 Ops, NumOps);
4731}
4732
Andrew Trickac6d9be2013-05-25 02:42:55 +00004733SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
Owen Andersone50ed302009-08-10 22:56:29 +00004734 const EVT *VTs, unsigned NumVTs,
Bill Wendling7ade28c2009-01-28 22:17:52 +00004735 const SDValue *Ops, unsigned NumOps) {
Chris Lattner2fa6d3b2006-08-14 23:31:51 +00004736 if (NumVTs == 1)
Bill Wendling7ade28c2009-01-28 22:17:52 +00004737 return getNode(Opcode, DL, VTs[0], Ops, NumOps);
4738 return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps);
Scott Michelfdc40a02009-02-17 22:15:04 +00004739}
4740
Andrew Trickac6d9be2013-05-25 02:42:55 +00004741SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
Bill Wendling7ade28c2009-01-28 22:17:52 +00004742 const SDValue *Ops, unsigned NumOps) {
Chris Lattnerbe384162006-08-16 22:57:46 +00004743 if (VTList.NumVTs == 1)
Bill Wendling7ade28c2009-01-28 22:17:52 +00004744 return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps);
Chris Lattner89c34632005-05-14 06:20:26 +00004745
Daniel Dunbarcfb8a1b2009-07-19 01:38:38 +00004746#if 0
Chris Lattner5f056bf2005-07-10 01:55:33 +00004747 switch (Opcode) {
Chris Lattnere89083a2005-05-14 07:25:05 +00004748 // FIXME: figure out how to safely handle things like
4749 // int foo(int x) { return 1 << (x & 255); }
4750 // int bar() { return foo(256); }
Chris Lattnere89083a2005-05-14 07:25:05 +00004751 case ISD::SRA_PARTS:
4752 case ISD::SRL_PARTS:
4753 case ISD::SHL_PARTS:
4754 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
Owen Anderson825b72b2009-08-11 20:47:22 +00004755 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
Bill Wendling7ade28c2009-01-28 22:17:52 +00004756 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
Chris Lattnere89083a2005-05-14 07:25:05 +00004757 else if (N3.getOpcode() == ISD::AND)
4758 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
4759 // If the and is only masking out bits that cannot effect the shift,
4760 // eliminate the and.
Dan Gohmand1996362010-01-09 02:13:55 +00004761 unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
Chris Lattnere89083a2005-05-14 07:25:05 +00004762 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
Bill Wendling7ade28c2009-01-28 22:17:52 +00004763 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
Chris Lattnere89083a2005-05-14 07:25:05 +00004764 }
4765 break;
Chris Lattner5f056bf2005-07-10 01:55:33 +00004766 }
Daniel Dunbarcfb8a1b2009-07-19 01:38:38 +00004767#endif
Chris Lattner89c34632005-05-14 06:20:26 +00004768
Chris Lattner43247a12005-08-25 19:12:10 +00004769 // Memoize the node unless it returns a flag.
4770 SDNode *N;
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00004771 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
Jim Laskey583bd472006-10-27 23:46:08 +00004772 FoldingSetNodeID ID;
4773 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
Chris Lattnera5682852006-08-07 23:03:03 +00004774 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00004775 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Dan Gohman475871a2008-07-27 21:46:04 +00004776 return SDValue(E, 0);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00004777
Dan Gohman0e5f1302008-07-07 23:02:41 +00004778 if (NumOps == 1) {
Andrew Trickac6d9be2013-05-25 02:42:55 +00004779 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0]);
Dan Gohman0e5f1302008-07-07 23:02:41 +00004780 } else if (NumOps == 2) {
Andrew Trickac6d9be2013-05-25 02:42:55 +00004781 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0], Ops[1]);
Dan Gohman0e5f1302008-07-07 23:02:41 +00004782 } else if (NumOps == 3) {
Andrew Trickac6d9be2013-05-25 02:42:55 +00004783 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0], Ops[1],
Dan Gohman95531882010-03-18 18:49:47 +00004784 Ops[2]);
Dan Gohman0e5f1302008-07-07 23:02:41 +00004785 } else {
Andrew Trickac6d9be2013-05-25 02:42:55 +00004786 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops, NumOps);
Dan Gohman0e5f1302008-07-07 23:02:41 +00004787 }
Chris Lattnera5682852006-08-07 23:03:03 +00004788 CSEMap.InsertNode(N, IP);
Chris Lattner43247a12005-08-25 19:12:10 +00004789 } else {
Dan Gohman0e5f1302008-07-07 23:02:41 +00004790 if (NumOps == 1) {
Andrew Trickac6d9be2013-05-25 02:42:55 +00004791 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0]);
Dan Gohman0e5f1302008-07-07 23:02:41 +00004792 } else if (NumOps == 2) {
Andrew Trickac6d9be2013-05-25 02:42:55 +00004793 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0], Ops[1]);
Dan Gohman0e5f1302008-07-07 23:02:41 +00004794 } else if (NumOps == 3) {
Andrew Trickac6d9be2013-05-25 02:42:55 +00004795 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops[0], Ops[1],
Dan Gohman95531882010-03-18 18:49:47 +00004796 Ops[2]);
Dan Gohman0e5f1302008-07-07 23:02:41 +00004797 } else {
Andrew Trickac6d9be2013-05-25 02:42:55 +00004798 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, Ops, NumOps);
Dan Gohman0e5f1302008-07-07 23:02:41 +00004799 }
Chris Lattner43247a12005-08-25 19:12:10 +00004800 }
Chris Lattner5fa4fa42005-05-14 06:42:57 +00004801 AllNodes.push_back(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00004802#ifndef NDEBUG
Duncan Sands59d2dad2010-11-20 11:25:00 +00004803 VerifySDNode(N);
Duncan Sandsd038e042008-07-21 10:20:31 +00004804#endif
Dan Gohman475871a2008-07-27 21:46:04 +00004805 return SDValue(N, 0);
Chris Lattner89c34632005-05-14 06:20:26 +00004806}
4807
Andrew Trickac6d9be2013-05-25 02:42:55 +00004808SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
Bill Wendling7ade28c2009-01-28 22:17:52 +00004809 return getNode(Opcode, DL, VTList, 0, 0);
Dan Gohman08ce9762007-10-08 15:49:58 +00004810}
4811
Andrew Trickac6d9be2013-05-25 02:42:55 +00004812SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
Bill Wendling7ade28c2009-01-28 22:17:52 +00004813 SDValue N1) {
Dan Gohman475871a2008-07-27 21:46:04 +00004814 SDValue Ops[] = { N1 };
Bill Wendling7ade28c2009-01-28 22:17:52 +00004815 return getNode(Opcode, DL, VTList, Ops, 1);
Dan Gohman08ce9762007-10-08 15:49:58 +00004816}
4817
Andrew Trickac6d9be2013-05-25 02:42:55 +00004818SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
Bill Wendling7ade28c2009-01-28 22:17:52 +00004819 SDValue N1, SDValue N2) {
Dan Gohman475871a2008-07-27 21:46:04 +00004820 SDValue Ops[] = { N1, N2 };
Bill Wendling7ade28c2009-01-28 22:17:52 +00004821 return getNode(Opcode, DL, VTList, Ops, 2);
Dan Gohman08ce9762007-10-08 15:49:58 +00004822}
4823
Andrew Trickac6d9be2013-05-25 02:42:55 +00004824SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
Bill Wendling7ade28c2009-01-28 22:17:52 +00004825 SDValue N1, SDValue N2, SDValue N3) {
Dan Gohman475871a2008-07-27 21:46:04 +00004826 SDValue Ops[] = { N1, N2, N3 };
Bill Wendling7ade28c2009-01-28 22:17:52 +00004827 return getNode(Opcode, DL, VTList, Ops, 3);
Dan Gohman08ce9762007-10-08 15:49:58 +00004828}
4829
Andrew Trickac6d9be2013-05-25 02:42:55 +00004830SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
Bill Wendling7ade28c2009-01-28 22:17:52 +00004831 SDValue N1, SDValue N2, SDValue N3,
4832 SDValue N4) {
Dan Gohman475871a2008-07-27 21:46:04 +00004833 SDValue Ops[] = { N1, N2, N3, N4 };
Bill Wendling7ade28c2009-01-28 22:17:52 +00004834 return getNode(Opcode, DL, VTList, Ops, 4);
Dan Gohman08ce9762007-10-08 15:49:58 +00004835}
4836
Andrew Trickac6d9be2013-05-25 02:42:55 +00004837SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
Bill Wendling7ade28c2009-01-28 22:17:52 +00004838 SDValue N1, SDValue N2, SDValue N3,
4839 SDValue N4, SDValue N5) {
Dan Gohman475871a2008-07-27 21:46:04 +00004840 SDValue Ops[] = { N1, N2, N3, N4, N5 };
Bill Wendling7ade28c2009-01-28 22:17:52 +00004841 return getNode(Opcode, DL, VTList, Ops, 5);
Dan Gohman08ce9762007-10-08 15:49:58 +00004842}
4843
Owen Andersone50ed302009-08-10 22:56:29 +00004844SDVTList SelectionDAG::getVTList(EVT VT) {
Duncan Sandsaf47b112007-10-16 09:56:48 +00004845 return makeVTList(SDNode::getValueTypeList(VT), 1);
Chris Lattnera3255112005-11-08 23:30:28 +00004846}
4847
Owen Andersone50ed302009-08-10 22:56:29 +00004848SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
Dan Gohmane8be6c62008-07-17 19:10:17 +00004849 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(),
4850 E = VTList.rend(); I != E; ++I)
4851 if (I->NumVTs == 2 && I->VTs[0] == VT1 && I->VTs[1] == VT2)
4852 return *I;
4853
Owen Andersone50ed302009-08-10 22:56:29 +00004854 EVT *Array = Allocator.Allocate<EVT>(2);
Dan Gohmane8be6c62008-07-17 19:10:17 +00004855 Array[0] = VT1;
4856 Array[1] = VT2;
4857 SDVTList Result = makeVTList(Array, 2);
4858 VTList.push_back(Result);
4859 return Result;
Chris Lattnera3255112005-11-08 23:30:28 +00004860}
Dan Gohmane8be6c62008-07-17 19:10:17 +00004861
Owen Andersone50ed302009-08-10 22:56:29 +00004862SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
Dan Gohmane8be6c62008-07-17 19:10:17 +00004863 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(),
4864 E = VTList.rend(); I != E; ++I)
4865 if (I->NumVTs == 3 && I->VTs[0] == VT1 && I->VTs[1] == VT2 &&
4866 I->VTs[2] == VT3)
4867 return *I;
4868
Owen Andersone50ed302009-08-10 22:56:29 +00004869 EVT *Array = Allocator.Allocate<EVT>(3);
Dan Gohmane8be6c62008-07-17 19:10:17 +00004870 Array[0] = VT1;
4871 Array[1] = VT2;
4872 Array[2] = VT3;
4873 SDVTList Result = makeVTList(Array, 3);
4874 VTList.push_back(Result);
4875 return Result;
Chris Lattner70046e92006-08-15 17:46:01 +00004876}
4877
Owen Andersone50ed302009-08-10 22:56:29 +00004878SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
Bill Wendling13d6d442008-12-01 23:28:22 +00004879 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(),
4880 E = VTList.rend(); I != E; ++I)
4881 if (I->NumVTs == 4 && I->VTs[0] == VT1 && I->VTs[1] == VT2 &&
4882 I->VTs[2] == VT3 && I->VTs[3] == VT4)
4883 return *I;
4884
Anton Korobeynikov60283f92009-12-13 01:00:59 +00004885 EVT *Array = Allocator.Allocate<EVT>(4);
Bill Wendling13d6d442008-12-01 23:28:22 +00004886 Array[0] = VT1;
4887 Array[1] = VT2;
4888 Array[2] = VT3;
4889 Array[3] = VT4;
4890 SDVTList Result = makeVTList(Array, 4);
4891 VTList.push_back(Result);
4892 return Result;
4893}
4894
Owen Andersone50ed302009-08-10 22:56:29 +00004895SDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) {
Chris Lattner70046e92006-08-15 17:46:01 +00004896 switch (NumVTs) {
Torok Edwinc23197a2009-07-14 16:55:14 +00004897 case 0: llvm_unreachable("Cannot have nodes without results!");
Dan Gohman7f321562007-06-25 16:23:39 +00004898 case 1: return getVTList(VTs[0]);
Chris Lattner70046e92006-08-15 17:46:01 +00004899 case 2: return getVTList(VTs[0], VTs[1]);
4900 case 3: return getVTList(VTs[0], VTs[1], VTs[2]);
Anton Korobeynikovcc62c3c2009-12-19 02:04:00 +00004901 case 4: return getVTList(VTs[0], VTs[1], VTs[2], VTs[3]);
Chris Lattner70046e92006-08-15 17:46:01 +00004902 default: break;
4903 }
4904
Dan Gohmane8be6c62008-07-17 19:10:17 +00004905 for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(),
4906 E = VTList.rend(); I != E; ++I) {
4907 if (I->NumVTs != NumVTs || VTs[0] != I->VTs[0] || VTs[1] != I->VTs[1])
4908 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00004909
Benjamin Kramerb26e2912012-07-19 10:46:05 +00004910 if (std::equal(&VTs[2], &VTs[NumVTs], &I->VTs[2]))
Dan Gohmane8be6c62008-07-17 19:10:17 +00004911 return *I;
Chris Lattner70046e92006-08-15 17:46:01 +00004912 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004913
Owen Andersone50ed302009-08-10 22:56:29 +00004914 EVT *Array = Allocator.Allocate<EVT>(NumVTs);
Dan Gohmane8be6c62008-07-17 19:10:17 +00004915 std::copy(VTs, VTs+NumVTs, Array);
4916 SDVTList Result = makeVTList(Array, NumVTs);
4917 VTList.push_back(Result);
4918 return Result;
Chris Lattner2fa6d3b2006-08-14 23:31:51 +00004919}
4920
4921
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004922/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
4923/// specified operands. If the resultant node already exists in the DAG,
4924/// this does not modify the specified node, instead it returns the node that
4925/// already exists. If the resultant node does not exist in the DAG, the
4926/// input node is returned. As a degenerate case, if you specify the same
4927/// input operands as the node already has, the input node is returned.
Dan Gohman027657d2010-06-18 15:30:29 +00004928SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004929 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
Scott Michelfdc40a02009-02-17 22:15:04 +00004930
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004931 // Check to see if there is no change.
Dan Gohman027657d2010-06-18 15:30:29 +00004932 if (Op == N->getOperand(0)) return N;
Scott Michelfdc40a02009-02-17 22:15:04 +00004933
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004934 // See if the modified node already exists.
Chris Lattnera5682852006-08-07 23:03:03 +00004935 void *InsertPos = 0;
4936 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
Dan Gohman027657d2010-06-18 15:30:29 +00004937 return Existing;
Scott Michelfdc40a02009-02-17 22:15:04 +00004938
Dan Gohman79acd2b2008-07-21 22:38:59 +00004939 // Nope it doesn't. Remove the node from its current place in the maps.
Chris Lattnera5682852006-08-07 23:03:03 +00004940 if (InsertPos)
Dan Gohman095cc292008-09-13 01:54:27 +00004941 if (!RemoveNodeFromCSEMaps(N))
4942 InsertPos = 0;
Scott Michelfdc40a02009-02-17 22:15:04 +00004943
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004944 // Now we update the operands.
Dan Gohmane7852d02009-01-26 04:35:06 +00004945 N->OperandList[0].set(Op);
Scott Michelfdc40a02009-02-17 22:15:04 +00004946
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004947 // If this gets put into a CSE map, add it.
Chris Lattnera5682852006-08-07 23:03:03 +00004948 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
Dan Gohman027657d2010-06-18 15:30:29 +00004949 return N;
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004950}
4951
Dan Gohman027657d2010-06-18 15:30:29 +00004952SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004953 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
Scott Michelfdc40a02009-02-17 22:15:04 +00004954
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004955 // Check to see if there is no change.
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004956 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
Dan Gohman027657d2010-06-18 15:30:29 +00004957 return N; // No operands changed, just return the input node.
Scott Michelfdc40a02009-02-17 22:15:04 +00004958
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004959 // See if the modified node already exists.
Chris Lattnera5682852006-08-07 23:03:03 +00004960 void *InsertPos = 0;
4961 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
Dan Gohman027657d2010-06-18 15:30:29 +00004962 return Existing;
Scott Michelfdc40a02009-02-17 22:15:04 +00004963
Dan Gohman79acd2b2008-07-21 22:38:59 +00004964 // Nope it doesn't. Remove the node from its current place in the maps.
Chris Lattnera5682852006-08-07 23:03:03 +00004965 if (InsertPos)
Dan Gohman095cc292008-09-13 01:54:27 +00004966 if (!RemoveNodeFromCSEMaps(N))
4967 InsertPos = 0;
Scott Michelfdc40a02009-02-17 22:15:04 +00004968
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004969 // Now we update the operands.
Dan Gohmane7852d02009-01-26 04:35:06 +00004970 if (N->OperandList[0] != Op1)
4971 N->OperandList[0].set(Op1);
4972 if (N->OperandList[1] != Op2)
4973 N->OperandList[1].set(Op2);
Scott Michelfdc40a02009-02-17 22:15:04 +00004974
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004975 // If this gets put into a CSE map, add it.
Chris Lattnera5682852006-08-07 23:03:03 +00004976 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
Dan Gohman027657d2010-06-18 15:30:29 +00004977 return N;
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004978}
4979
Dan Gohman027657d2010-06-18 15:30:29 +00004980SDNode *SelectionDAG::
4981UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
Dan Gohman475871a2008-07-27 21:46:04 +00004982 SDValue Ops[] = { Op1, Op2, Op3 };
Chris Lattnerf06f35e2006-08-08 01:09:31 +00004983 return UpdateNodeOperands(N, Ops, 3);
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004984}
4985
Dan Gohman027657d2010-06-18 15:30:29 +00004986SDNode *SelectionDAG::
4987UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
Dan Gohman475871a2008-07-27 21:46:04 +00004988 SDValue Op3, SDValue Op4) {
4989 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
Chris Lattnerf06f35e2006-08-08 01:09:31 +00004990 return UpdateNodeOperands(N, Ops, 4);
Chris Lattnerdf6eb302006-01-28 09:32:45 +00004991}
4992
Dan Gohman027657d2010-06-18 15:30:29 +00004993SDNode *SelectionDAG::
4994UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
Dan Gohman475871a2008-07-27 21:46:04 +00004995 SDValue Op3, SDValue Op4, SDValue Op5) {
4996 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
Chris Lattnerf06f35e2006-08-08 01:09:31 +00004997 return UpdateNodeOperands(N, Ops, 5);
Chris Lattner809ec112006-01-28 10:09:25 +00004998}
4999
Dan Gohman027657d2010-06-18 15:30:29 +00005000SDNode *SelectionDAG::
5001UpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) {
Chris Lattnerf06f35e2006-08-08 01:09:31 +00005002 assert(N->getNumOperands() == NumOps &&
Chris Lattnerdf6eb302006-01-28 09:32:45 +00005003 "Update with wrong number of operands");
Scott Michelfdc40a02009-02-17 22:15:04 +00005004
Chris Lattnerdf6eb302006-01-28 09:32:45 +00005005 // Check to see if there is no change.
Chris Lattnerdf6eb302006-01-28 09:32:45 +00005006 bool AnyChange = false;
5007 for (unsigned i = 0; i != NumOps; ++i) {
5008 if (Ops[i] != N->getOperand(i)) {
5009 AnyChange = true;
5010 break;
5011 }
5012 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005013
Chris Lattnerdf6eb302006-01-28 09:32:45 +00005014 // No operands changed, just return the input node.
Dan Gohman027657d2010-06-18 15:30:29 +00005015 if (!AnyChange) return N;
Scott Michelfdc40a02009-02-17 22:15:04 +00005016
Chris Lattnerdf6eb302006-01-28 09:32:45 +00005017 // See if the modified node already exists.
Chris Lattnera5682852006-08-07 23:03:03 +00005018 void *InsertPos = 0;
Chris Lattnerf06f35e2006-08-08 01:09:31 +00005019 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos))
Dan Gohman027657d2010-06-18 15:30:29 +00005020 return Existing;
Scott Michelfdc40a02009-02-17 22:15:04 +00005021
Dan Gohman7ceda162008-05-02 00:05:03 +00005022 // Nope it doesn't. Remove the node from its current place in the maps.
Chris Lattnera5682852006-08-07 23:03:03 +00005023 if (InsertPos)
Dan Gohman095cc292008-09-13 01:54:27 +00005024 if (!RemoveNodeFromCSEMaps(N))
5025 InsertPos = 0;
Scott Michelfdc40a02009-02-17 22:15:04 +00005026
Chris Lattnerdf6eb302006-01-28 09:32:45 +00005027 // Now we update the operands.
Dan Gohmane7852d02009-01-26 04:35:06 +00005028 for (unsigned i = 0; i != NumOps; ++i)
5029 if (N->OperandList[i] != Ops[i])
5030 N->OperandList[i].set(Ops[i]);
Chris Lattnerdf6eb302006-01-28 09:32:45 +00005031
5032 // If this gets put into a CSE map, add it.
Chris Lattnera5682852006-08-07 23:03:03 +00005033 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
Dan Gohman027657d2010-06-18 15:30:29 +00005034 return N;
Chris Lattnerdf6eb302006-01-28 09:32:45 +00005035}
5036
Dan Gohman0fe9c6e2008-07-07 20:57:48 +00005037/// DropOperands - Release the operands and set this node to have
Dan Gohmane8be6c62008-07-17 19:10:17 +00005038/// zero operands.
Dan Gohman0fe9c6e2008-07-07 20:57:48 +00005039void SDNode::DropOperands() {
Dan Gohman0fe9c6e2008-07-07 20:57:48 +00005040 // Unlike the code in MorphNodeTo that does this, we don't need to
5041 // watch for dead nodes here.
Dan Gohmane7852d02009-01-26 04:35:06 +00005042 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
5043 SDUse &Use = *I++;
5044 Use.set(SDValue());
5045 }
Chris Lattnerd429bcd2007-02-04 02:49:29 +00005046}
Chris Lattner149c58c2005-08-16 18:17:10 +00005047
Dan Gohmane8be6c62008-07-17 19:10:17 +00005048/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
5049/// machine opcode.
Chris Lattnerc5e6c642005-12-01 18:00:57 +00005050///
Dan Gohmane8be6c62008-07-17 19:10:17 +00005051SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005052 EVT VT) {
Chris Lattner0b3e5252006-08-15 19:11:05 +00005053 SDVTList VTs = getVTList(VT);
Dan Gohmane8be6c62008-07-17 19:10:17 +00005054 return SelectNodeTo(N, MachineOpc, VTs, 0, 0);
Chris Lattner7651fa42005-08-24 23:00:29 +00005055}
Chris Lattner0fb094f2005-11-19 01:44:53 +00005056
Dan Gohmane8be6c62008-07-17 19:10:17 +00005057SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005058 EVT VT, SDValue Op1) {
Chris Lattner0b3e5252006-08-15 19:11:05 +00005059 SDVTList VTs = getVTList(VT);
Dan Gohman475871a2008-07-27 21:46:04 +00005060 SDValue Ops[] = { Op1 };
Dan Gohmane8be6c62008-07-17 19:10:17 +00005061 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1);
Chris Lattner149c58c2005-08-16 18:17:10 +00005062}
Chris Lattner0fb094f2005-11-19 01:44:53 +00005063
Dan Gohmane8be6c62008-07-17 19:10:17 +00005064SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005065 EVT VT, SDValue Op1,
Dan Gohman475871a2008-07-27 21:46:04 +00005066 SDValue Op2) {
Chris Lattner0b3e5252006-08-15 19:11:05 +00005067 SDVTList VTs = getVTList(VT);
Dan Gohman475871a2008-07-27 21:46:04 +00005068 SDValue Ops[] = { Op1, Op2 };
Dan Gohmane8be6c62008-07-17 19:10:17 +00005069 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2);
Chris Lattner149c58c2005-08-16 18:17:10 +00005070}
Chris Lattner0fb094f2005-11-19 01:44:53 +00005071
Dan Gohmane8be6c62008-07-17 19:10:17 +00005072SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005073 EVT VT, SDValue Op1,
Dan Gohman475871a2008-07-27 21:46:04 +00005074 SDValue Op2, SDValue Op3) {
Chris Lattner0b3e5252006-08-15 19:11:05 +00005075 SDVTList VTs = getVTList(VT);
Dan Gohman475871a2008-07-27 21:46:04 +00005076 SDValue Ops[] = { Op1, Op2, Op3 };
Dan Gohmane8be6c62008-07-17 19:10:17 +00005077 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3);
Chris Lattner149c58c2005-08-16 18:17:10 +00005078}
Chris Lattnerc975e1d2005-08-21 22:30:30 +00005079
Dan Gohmane8be6c62008-07-17 19:10:17 +00005080SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005081 EVT VT, const SDValue *Ops,
Evan Cheng694481e2006-08-27 08:08:54 +00005082 unsigned NumOps) {
Chris Lattner0b3e5252006-08-15 19:11:05 +00005083 SDVTList VTs = getVTList(VT);
Dan Gohmane8be6c62008-07-17 19:10:17 +00005084 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
Dan Gohmancd920d92008-07-02 23:23:19 +00005085}
5086
Dan Gohmane8be6c62008-07-17 19:10:17 +00005087SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005088 EVT VT1, EVT VT2, const SDValue *Ops,
Dan Gohmancd920d92008-07-02 23:23:19 +00005089 unsigned NumOps) {
5090 SDVTList VTs = getVTList(VT1, VT2);
Dan Gohmane8be6c62008-07-17 19:10:17 +00005091 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
Dan Gohmancd920d92008-07-02 23:23:19 +00005092}
5093
Dan Gohmane8be6c62008-07-17 19:10:17 +00005094SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005095 EVT VT1, EVT VT2) {
Dan Gohmancd920d92008-07-02 23:23:19 +00005096 SDVTList VTs = getVTList(VT1, VT2);
Dan Gohman475871a2008-07-27 21:46:04 +00005097 return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)0, 0);
Dan Gohmancd920d92008-07-02 23:23:19 +00005098}
5099
Dan Gohmane8be6c62008-07-17 19:10:17 +00005100SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005101 EVT VT1, EVT VT2, EVT VT3,
Dan Gohman475871a2008-07-27 21:46:04 +00005102 const SDValue *Ops, unsigned NumOps) {
Dan Gohmancd920d92008-07-02 23:23:19 +00005103 SDVTList VTs = getVTList(VT1, VT2, VT3);
Dan Gohmane8be6c62008-07-17 19:10:17 +00005104 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
Dan Gohmancd920d92008-07-02 23:23:19 +00005105}
5106
Bill Wendling13d6d442008-12-01 23:28:22 +00005107SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005108 EVT VT1, EVT VT2, EVT VT3, EVT VT4,
Bill Wendling13d6d442008-12-01 23:28:22 +00005109 const SDValue *Ops, unsigned NumOps) {
5110 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
5111 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
5112}
5113
Scott Michelfdc40a02009-02-17 22:15:04 +00005114SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005115 EVT VT1, EVT VT2,
Dan Gohman475871a2008-07-27 21:46:04 +00005116 SDValue Op1) {
Dan Gohmancd920d92008-07-02 23:23:19 +00005117 SDVTList VTs = getVTList(VT1, VT2);
Dan Gohman475871a2008-07-27 21:46:04 +00005118 SDValue Ops[] = { Op1 };
Dan Gohmane8be6c62008-07-17 19:10:17 +00005119 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1);
Andrew Lenharth7cf11b42006-01-23 21:51:14 +00005120}
Andrew Lenharth8c6f1ee2006-01-23 20:59:12 +00005121
Scott Michelfdc40a02009-02-17 22:15:04 +00005122SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005123 EVT VT1, EVT VT2,
Dan Gohman475871a2008-07-27 21:46:04 +00005124 SDValue Op1, SDValue Op2) {
Chris Lattner0b3e5252006-08-15 19:11:05 +00005125 SDVTList VTs = getVTList(VT1, VT2);
Dan Gohman475871a2008-07-27 21:46:04 +00005126 SDValue Ops[] = { Op1, Op2 };
Dan Gohmane8be6c62008-07-17 19:10:17 +00005127 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2);
Chris Lattner0fb094f2005-11-19 01:44:53 +00005128}
5129
Dan Gohmane8be6c62008-07-17 19:10:17 +00005130SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005131 EVT VT1, EVT VT2,
Scott Michelfdc40a02009-02-17 22:15:04 +00005132 SDValue Op1, SDValue Op2,
Dan Gohman475871a2008-07-27 21:46:04 +00005133 SDValue Op3) {
Chris Lattner0b3e5252006-08-15 19:11:05 +00005134 SDVTList VTs = getVTList(VT1, VT2);
Dan Gohman475871a2008-07-27 21:46:04 +00005135 SDValue Ops[] = { Op1, Op2, Op3 };
Dan Gohmane8be6c62008-07-17 19:10:17 +00005136 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3);
Dan Gohmancd920d92008-07-02 23:23:19 +00005137}
5138
Dan Gohmane8be6c62008-07-17 19:10:17 +00005139SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Owen Andersone50ed302009-08-10 22:56:29 +00005140 EVT VT1, EVT VT2, EVT VT3,
Scott Michelfdc40a02009-02-17 22:15:04 +00005141 SDValue Op1, SDValue Op2,
Bill Wendling13d6d442008-12-01 23:28:22 +00005142 SDValue Op3) {
5143 SDVTList VTs = getVTList(VT1, VT2, VT3);
5144 SDValue Ops[] = { Op1, Op2, Op3 };
5145 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3);
5146}
5147
5148SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
Dan Gohman475871a2008-07-27 21:46:04 +00005149 SDVTList VTs, const SDValue *Ops,
Dan Gohmancd920d92008-07-02 23:23:19 +00005150 unsigned NumOps) {
Chris Lattner5857e0a2010-02-23 23:01:35 +00005151 N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps);
5152 // Reset the NodeID to -1.
5153 N->setNodeId(-1);
5154 return N;
Dan Gohmane8be6c62008-07-17 19:10:17 +00005155}
5156
Andrew Trickac6d9be2013-05-25 02:42:55 +00005157/// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
Devang Patel0508d042011-12-15 18:21:18 +00005158/// the line number information on the merged node since it is not possible to
5159/// preserve the information that operation is associated with multiple lines.
5160/// This will make the debugger working better at -O0, were there is a higher
5161/// probability having other instructions associated with that line.
5162///
Andrew Trickac6d9be2013-05-25 02:42:55 +00005163/// For IROrder, we keep the smaller of the two
5164SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
Devang Patel0508d042011-12-15 18:21:18 +00005165 DebugLoc NLoc = N->getDebugLoc();
Andrew Trickac6d9be2013-05-25 02:42:55 +00005166 if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
5167 (OLoc.getDebugLoc() != NLoc)) {
Devang Patel0508d042011-12-15 18:21:18 +00005168 N->setDebugLoc(DebugLoc());
5169 }
Andrew Trickac6d9be2013-05-25 02:42:55 +00005170 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
5171 N->setIROrder(Order);
Devang Patel0508d042011-12-15 18:21:18 +00005172 return N;
5173}
5174
Chris Lattner21221e32010-02-28 21:36:14 +00005175/// MorphNodeTo - This *mutates* the specified node to have the specified
Dan Gohmane8be6c62008-07-17 19:10:17 +00005176/// return type, opcode, and operands.
5177///
5178/// Note that MorphNodeTo returns the resultant node. If there is already a
5179/// node of the specified opcode and operands, it returns that node instead of
Andrew Trickac6d9be2013-05-25 02:42:55 +00005180/// the current one. Note that the SDLoc need not be the same.
Dan Gohmane8be6c62008-07-17 19:10:17 +00005181///
5182/// Using MorphNodeTo is faster than creating a new node and swapping it in
5183/// with ReplaceAllUsesWith both because it often avoids allocating a new
Gabor Greifdc715632008-08-30 22:16:05 +00005184/// node, and because it doesn't require CSE recalculation for any of
Dan Gohmane8be6c62008-07-17 19:10:17 +00005185/// the node's users.
5186///
5187SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
Dan Gohman475871a2008-07-27 21:46:04 +00005188 SDVTList VTs, const SDValue *Ops,
Dan Gohmane8be6c62008-07-17 19:10:17 +00005189 unsigned NumOps) {
Dan Gohmancd920d92008-07-02 23:23:19 +00005190 // If an identical node already exists, use it.
Chris Lattnera5682852006-08-07 23:03:03 +00005191 void *IP = 0;
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00005192 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
Dan Gohmane8be6c62008-07-17 19:10:17 +00005193 FoldingSetNodeID ID;
5194 AddNodeIDNode(ID, Opc, VTs, Ops, NumOps);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00005195 if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
Andrew Trickac6d9be2013-05-25 02:42:55 +00005196 return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
Dan Gohmane8be6c62008-07-17 19:10:17 +00005197 }
Chris Lattnerc5e6c642005-12-01 18:00:57 +00005198
Dan Gohman095cc292008-09-13 01:54:27 +00005199 if (!RemoveNodeFromCSEMaps(N))
5200 IP = 0;
Chris Lattner67612a12007-02-04 02:32:44 +00005201
Dan Gohmane8be6c62008-07-17 19:10:17 +00005202 // Start the morphing.
5203 N->NodeType = Opc;
5204 N->ValueList = VTs.VTs;
5205 N->NumValues = VTs.NumVTs;
Scott Michelfdc40a02009-02-17 22:15:04 +00005206
Dan Gohmane8be6c62008-07-17 19:10:17 +00005207 // Clear the operands list, updating used nodes to remove this from their
5208 // use list. Keep track of any operands that become dead as a result.
5209 SmallPtrSet<SDNode*, 16> DeadNodeSet;
Dan Gohmane7852d02009-01-26 04:35:06 +00005210 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
5211 SDUse &Use = *I++;
5212 SDNode *Used = Use.getNode();
5213 Use.set(SDValue());
Dan Gohmane8be6c62008-07-17 19:10:17 +00005214 if (Used->use_empty())
5215 DeadNodeSet.insert(Used);
5216 }
5217
Dan Gohmanc76909a2009-09-25 20:36:54 +00005218 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
5219 // Initialize the memory references information.
5220 MN->setMemRefs(0, 0);
5221 // If NumOps is larger than the # of operands we can have in a
5222 // MachineSDNode, reallocate the operand list.
5223 if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
5224 if (MN->OperandsNeedDelete)
5225 delete[] MN->OperandList;
5226 if (NumOps > array_lengthof(MN->LocalOperands))
5227 // We're creating a final node that will live unmorphed for the
5228 // remainder of the current SelectionDAG iteration, so we can allocate
5229 // the operands directly out of a pool with no recycling metadata.
5230 MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
Dan Gohman95531882010-03-18 18:49:47 +00005231 Ops, NumOps);
Dan Gohmanc76909a2009-09-25 20:36:54 +00005232 else
5233 MN->InitOperands(MN->LocalOperands, Ops, NumOps);
5234 MN->OperandsNeedDelete = false;
5235 } else
5236 MN->InitOperands(MN->OperandList, Ops, NumOps);
5237 } else {
5238 // If NumOps is larger than the # of operands we currently have, reallocate
5239 // the operand list.
5240 if (NumOps > N->NumOperands) {
5241 if (N->OperandsNeedDelete)
5242 delete[] N->OperandList;
5243 N->InitOperands(new SDUse[NumOps], Ops, NumOps);
Dan Gohmane8be6c62008-07-17 19:10:17 +00005244 N->OperandsNeedDelete = true;
Dan Gohmanc76909a2009-09-25 20:36:54 +00005245 } else
Anton Korobeynikov443b2152009-10-22 00:15:17 +00005246 N->InitOperands(N->OperandList, Ops, NumOps);
Dan Gohmane8be6c62008-07-17 19:10:17 +00005247 }
5248
5249 // Delete any nodes that are still dead after adding the uses for the
5250 // new operands.
Chris Lattner7d892d62010-03-01 07:43:08 +00005251 if (!DeadNodeSet.empty()) {
5252 SmallVector<SDNode *, 16> DeadNodes;
5253 for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(),
5254 E = DeadNodeSet.end(); I != E; ++I)
5255 if ((*I)->use_empty())
5256 DeadNodes.push_back(*I);
5257 RemoveDeadNodes(DeadNodes);
5258 }
Dan Gohman0fe9c6e2008-07-07 20:57:48 +00005259
Dan Gohmane8be6c62008-07-17 19:10:17 +00005260 if (IP)
5261 CSEMap.InsertNode(N, IP); // Memoize the new node.
Evan Cheng95514ba2006-08-26 08:00:10 +00005262 return N;
Chris Lattner0fb094f2005-11-19 01:44:53 +00005263}
5264
Chris Lattner0fb094f2005-11-19 01:44:53 +00005265
Dan Gohman602b0c82009-09-25 18:54:59 +00005266/// getMachineNode - These are used for target selectors to create a new node
5267/// with specified return type(s), MachineInstr opcode, and operands.
Evan Cheng6ae46c42006-02-09 07:15:23 +00005268///
Dan Gohman602b0c82009-09-25 18:54:59 +00005269/// Note that getMachineNode returns the resultant node. If there is already a
Evan Cheng6ae46c42006-02-09 07:15:23 +00005270/// node of the specified opcode and operands, it returns that node instead of
5271/// the current one.
Dan Gohmanc81b7832009-10-10 01:29:16 +00005272MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005273SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
Dan Gohmanc76909a2009-09-25 20:36:54 +00005274 SDVTList VTs = getVTList(VT);
Dmitri Gribenko5c332db2013-05-05 00:40:33 +00005275 return getMachineNode(Opcode, dl, VTs, None);
Dale Johannesene8c17332009-01-29 00:47:48 +00005276}
Bill Wendling56ab1a22009-01-29 09:01:55 +00005277
Dan Gohmanc81b7832009-10-10 01:29:16 +00005278MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005279SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
Dan Gohmanc76909a2009-09-25 20:36:54 +00005280 SDVTList VTs = getVTList(VT);
5281 SDValue Ops[] = { Op1 };
Michael Liao2a8bea72013-04-19 22:22:57 +00005282 return getMachineNode(Opcode, dl, VTs, Ops);
Dale Johannesene8c17332009-01-29 00:47:48 +00005283}
Bill Wendling56ab1a22009-01-29 09:01:55 +00005284
Dan Gohmanc81b7832009-10-10 01:29:16 +00005285MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005286SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
Dan Gohmanc81b7832009-10-10 01:29:16 +00005287 SDValue Op1, SDValue Op2) {
Dan Gohmanc76909a2009-09-25 20:36:54 +00005288 SDVTList VTs = getVTList(VT);
5289 SDValue Ops[] = { Op1, Op2 };
Michael Liao2a8bea72013-04-19 22:22:57 +00005290 return getMachineNode(Opcode, dl, VTs, Ops);
Dale Johannesene8c17332009-01-29 00:47:48 +00005291}
Bill Wendling56ab1a22009-01-29 09:01:55 +00005292
Dan Gohmanc81b7832009-10-10 01:29:16 +00005293MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005294SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
Dan Gohmanc81b7832009-10-10 01:29:16 +00005295 SDValue Op1, SDValue Op2, SDValue Op3) {
Dan Gohmanc76909a2009-09-25 20:36:54 +00005296 SDVTList VTs = getVTList(VT);
5297 SDValue Ops[] = { Op1, Op2, Op3 };
Michael Liao2a8bea72013-04-19 22:22:57 +00005298 return getMachineNode(Opcode, dl, VTs, Ops);
Dale Johannesene8c17332009-01-29 00:47:48 +00005299}
Bill Wendling56ab1a22009-01-29 09:01:55 +00005300
Dan Gohmanc81b7832009-10-10 01:29:16 +00005301MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005302SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
Michael Liao2a8bea72013-04-19 22:22:57 +00005303 ArrayRef<SDValue> Ops) {
Dan Gohmanc76909a2009-09-25 20:36:54 +00005304 SDVTList VTs = getVTList(VT);
Michael Liao2a8bea72013-04-19 22:22:57 +00005305 return getMachineNode(Opcode, dl, VTs, Ops);
Dale Johannesene8c17332009-01-29 00:47:48 +00005306}
Bill Wendling56ab1a22009-01-29 09:01:55 +00005307
Dan Gohmanc81b7832009-10-10 01:29:16 +00005308MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005309SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
Dan Gohmanfc166572009-04-09 23:54:40 +00005310 SDVTList VTs = getVTList(VT1, VT2);
Dmitri Gribenko5c332db2013-05-05 00:40:33 +00005311 return getMachineNode(Opcode, dl, VTs, None);
Dale Johannesene8c17332009-01-29 00:47:48 +00005312}
Bill Wendling56ab1a22009-01-29 09:01:55 +00005313
Dan Gohmanc81b7832009-10-10 01:29:16 +00005314MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005315SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
Dan Gohmanc81b7832009-10-10 01:29:16 +00005316 EVT VT1, EVT VT2, SDValue Op1) {
Dan Gohmanfc166572009-04-09 23:54:40 +00005317 SDVTList VTs = getVTList(VT1, VT2);
Dan Gohmanc76909a2009-09-25 20:36:54 +00005318 SDValue Ops[] = { Op1 };
Michael Liao2a8bea72013-04-19 22:22:57 +00005319 return getMachineNode(Opcode, dl, VTs, Ops);
Dale Johannesene8c17332009-01-29 00:47:48 +00005320}
Bill Wendling56ab1a22009-01-29 09:01:55 +00005321
Dan Gohmanc81b7832009-10-10 01:29:16 +00005322MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005323SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
Dan Gohmanc81b7832009-10-10 01:29:16 +00005324 EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
Dan Gohmanfc166572009-04-09 23:54:40 +00005325 SDVTList VTs = getVTList(VT1, VT2);
Bill Wendling56ab1a22009-01-29 09:01:55 +00005326 SDValue Ops[] = { Op1, Op2 };
Michael Liao2a8bea72013-04-19 22:22:57 +00005327 return getMachineNode(Opcode, dl, VTs, Ops);
Bill Wendling56ab1a22009-01-29 09:01:55 +00005328}
5329
Dan Gohmanc81b7832009-10-10 01:29:16 +00005330MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005331SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
Dan Gohmanc81b7832009-10-10 01:29:16 +00005332 EVT VT1, EVT VT2, SDValue Op1,
5333 SDValue Op2, SDValue Op3) {
Dan Gohmanfc166572009-04-09 23:54:40 +00005334 SDVTList VTs = getVTList(VT1, VT2);
Bill Wendling56ab1a22009-01-29 09:01:55 +00005335 SDValue Ops[] = { Op1, Op2, Op3 };
Michael Liao2a8bea72013-04-19 22:22:57 +00005336 return getMachineNode(Opcode, dl, VTs, Ops);
Bill Wendling56ab1a22009-01-29 09:01:55 +00005337}
5338
Dan Gohmanc81b7832009-10-10 01:29:16 +00005339MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005340SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
Dan Gohmanc81b7832009-10-10 01:29:16 +00005341 EVT VT1, EVT VT2,
Michael Liao2a8bea72013-04-19 22:22:57 +00005342 ArrayRef<SDValue> Ops) {
Dan Gohmanfc166572009-04-09 23:54:40 +00005343 SDVTList VTs = getVTList(VT1, VT2);
Michael Liao2a8bea72013-04-19 22:22:57 +00005344 return getMachineNode(Opcode, dl, VTs, Ops);
Dale Johannesene8c17332009-01-29 00:47:48 +00005345}
Bill Wendling56ab1a22009-01-29 09:01:55 +00005346
Dan Gohmanc81b7832009-10-10 01:29:16 +00005347MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005348SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
Dan Gohmanc81b7832009-10-10 01:29:16 +00005349 EVT VT1, EVT VT2, EVT VT3,
5350 SDValue Op1, SDValue Op2) {
Dan Gohmanfc166572009-04-09 23:54:40 +00005351 SDVTList VTs = getVTList(VT1, VT2, VT3);
Dale Johannesene8c17332009-01-29 00:47:48 +00005352 SDValue Ops[] = { Op1, Op2 };
Michael Liao2a8bea72013-04-19 22:22:57 +00005353 return getMachineNode(Opcode, dl, VTs, Ops);
Dale Johannesene8c17332009-01-29 00:47:48 +00005354}
Bill Wendling56ab1a22009-01-29 09:01:55 +00005355
Dan Gohmanc81b7832009-10-10 01:29:16 +00005356MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005357SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
Dan Gohmanc81b7832009-10-10 01:29:16 +00005358 EVT VT1, EVT VT2, EVT VT3,
5359 SDValue Op1, SDValue Op2, SDValue Op3) {
Dan Gohmanfc166572009-04-09 23:54:40 +00005360 SDVTList VTs = getVTList(VT1, VT2, VT3);
Bill Wendling56ab1a22009-01-29 09:01:55 +00005361 SDValue Ops[] = { Op1, Op2, Op3 };
Michael Liao2a8bea72013-04-19 22:22:57 +00005362 return getMachineNode(Opcode, dl, VTs, Ops);
Evan Cheng6ae46c42006-02-09 07:15:23 +00005363}
Bill Wendling56ab1a22009-01-29 09:01:55 +00005364
Dan Gohmanc81b7832009-10-10 01:29:16 +00005365MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005366SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
Dan Gohmanc81b7832009-10-10 01:29:16 +00005367 EVT VT1, EVT VT2, EVT VT3,
Michael Liao2a8bea72013-04-19 22:22:57 +00005368 ArrayRef<SDValue> Ops) {
Dan Gohmanfc166572009-04-09 23:54:40 +00005369 SDVTList VTs = getVTList(VT1, VT2, VT3);
Michael Liao2a8bea72013-04-19 22:22:57 +00005370 return getMachineNode(Opcode, dl, VTs, Ops);
Dale Johannesene8c17332009-01-29 00:47:48 +00005371}
Bill Wendling56ab1a22009-01-29 09:01:55 +00005372
Dan Gohmanc81b7832009-10-10 01:29:16 +00005373MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005374SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
Dan Gohmanc81b7832009-10-10 01:29:16 +00005375 EVT VT2, EVT VT3, EVT VT4,
Michael Liao2a8bea72013-04-19 22:22:57 +00005376 ArrayRef<SDValue> Ops) {
Dan Gohmanfc166572009-04-09 23:54:40 +00005377 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
Michael Liao2a8bea72013-04-19 22:22:57 +00005378 return getMachineNode(Opcode, dl, VTs, Ops);
Bill Wendling56ab1a22009-01-29 09:01:55 +00005379}
5380
Dan Gohmanc81b7832009-10-10 01:29:16 +00005381MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005382SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
Benjamin Kramer3853f742013-03-07 20:33:29 +00005383 ArrayRef<EVT> ResultTys,
Michael Liao2a8bea72013-04-19 22:22:57 +00005384 ArrayRef<SDValue> Ops) {
Dan Gohmanc76909a2009-09-25 20:36:54 +00005385 SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size());
Michael Liao2a8bea72013-04-19 22:22:57 +00005386 return getMachineNode(Opcode, dl, VTs, Ops);
Dan Gohmanc76909a2009-09-25 20:36:54 +00005387}
5388
Dan Gohmanc81b7832009-10-10 01:29:16 +00005389MachineSDNode *
Andrew Trickac6d9be2013-05-25 02:42:55 +00005390SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
Michael Liao2a8bea72013-04-19 22:22:57 +00005391 ArrayRef<SDValue> OpsArray) {
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00005392 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
Dan Gohmanc76909a2009-09-25 20:36:54 +00005393 MachineSDNode *N;
Ted Kremenek584520e2011-01-23 17:05:06 +00005394 void *IP = 0;
Michael Liao2a8bea72013-04-19 22:22:57 +00005395 const SDValue *Ops = OpsArray.data();
5396 unsigned NumOps = OpsArray.size();
Dan Gohmanc76909a2009-09-25 20:36:54 +00005397
5398 if (DoCSE) {
5399 FoldingSetNodeID ID;
5400 AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps);
5401 IP = 0;
Devang Patel0508d042011-12-15 18:21:18 +00005402 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
Andrew Trickac6d9be2013-05-25 02:42:55 +00005403 return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
Devang Patel0508d042011-12-15 18:21:18 +00005404 }
Dan Gohmanc76909a2009-09-25 20:36:54 +00005405 }
5406
5407 // Allocate a new MachineSDNode.
Andrew Trickac6d9be2013-05-25 02:42:55 +00005408 N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
Dan Gohmanc76909a2009-09-25 20:36:54 +00005409
5410 // Initialize the operands list.
5411 if (NumOps > array_lengthof(N->LocalOperands))
5412 // We're creating a final node that will live unmorphed for the
5413 // remainder of the current SelectionDAG iteration, so we can allocate
5414 // the operands directly out of a pool with no recycling metadata.
5415 N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
5416 Ops, NumOps);
5417 else
5418 N->InitOperands(N->LocalOperands, Ops, NumOps);
5419 N->OperandsNeedDelete = false;
5420
5421 if (DoCSE)
5422 CSEMap.InsertNode(N, IP);
5423
5424 AllNodes.push_back(N);
5425#ifndef NDEBUG
Duncan Sands59d2dad2010-11-20 11:25:00 +00005426 VerifyMachineNode(N);
Dan Gohmanc76909a2009-09-25 20:36:54 +00005427#endif
5428 return N;
Dale Johannesene8c17332009-01-29 00:47:48 +00005429}
Evan Cheng6ae46c42006-02-09 07:15:23 +00005430
Dan Gohman6a402dc2009-08-19 18:16:17 +00005431/// getTargetExtractSubreg - A convenience function for creating
Chris Lattner518bb532010-02-09 19:54:29 +00005432/// TargetOpcode::EXTRACT_SUBREG nodes.
Dan Gohman6a402dc2009-08-19 18:16:17 +00005433SDValue
Andrew Trickac6d9be2013-05-25 02:42:55 +00005434SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
Dan Gohman6a402dc2009-08-19 18:16:17 +00005435 SDValue Operand) {
5436 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
Chris Lattner518bb532010-02-09 19:54:29 +00005437 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
Dan Gohman602b0c82009-09-25 18:54:59 +00005438 VT, Operand, SRIdxVal);
Dan Gohman6a402dc2009-08-19 18:16:17 +00005439 return SDValue(Subreg, 0);
5440}
5441
Bob Wilson5fcbf0d2009-10-08 18:49:46 +00005442/// getTargetInsertSubreg - A convenience function for creating
Chris Lattner518bb532010-02-09 19:54:29 +00005443/// TargetOpcode::INSERT_SUBREG nodes.
Bob Wilson5fcbf0d2009-10-08 18:49:46 +00005444SDValue
Andrew Trickac6d9be2013-05-25 02:42:55 +00005445SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
Bob Wilson5fcbf0d2009-10-08 18:49:46 +00005446 SDValue Operand, SDValue Subreg) {
5447 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
Chris Lattner518bb532010-02-09 19:54:29 +00005448 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
Bob Wilson5fcbf0d2009-10-08 18:49:46 +00005449 VT, Operand, Subreg, SRIdxVal);
5450 return SDValue(Result, 0);
5451}
5452
Evan Cheng08b11732008-03-22 01:55:50 +00005453/// getNodeIfExists - Get the specified node if it's already available, or
5454/// else return NULL.
5455SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
Dan Gohman475871a2008-07-27 21:46:04 +00005456 const SDValue *Ops, unsigned NumOps) {
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00005457 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
Evan Cheng08b11732008-03-22 01:55:50 +00005458 FoldingSetNodeID ID;
5459 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
5460 void *IP = 0;
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00005461 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
Evan Cheng08b11732008-03-22 01:55:50 +00005462 return E;
5463 }
5464 return NULL;
5465}
5466
Evan Cheng31441b72010-03-29 20:48:30 +00005467/// getDbgValue - Creates a SDDbgValue node.
5468///
5469SDDbgValue *
5470SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off,
5471 DebugLoc DL, unsigned O) {
5472 return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O);
5473}
5474
5475SDDbgValue *
Dan Gohman46510a72010-04-15 01:51:59 +00005476SelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off,
Evan Cheng31441b72010-03-29 20:48:30 +00005477 DebugLoc DL, unsigned O) {
5478 return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O);
5479}
5480
5481SDDbgValue *
5482SelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off,
5483 DebugLoc DL, unsigned O) {
5484 return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O);
5485}
5486
Dan Gohmana72d2a22010-03-03 21:33:37 +00005487namespace {
5488
Dan Gohmanba72b0c2010-03-04 19:11:28 +00005489/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
Dan Gohmana72d2a22010-03-03 21:33:37 +00005490/// pointed to by a use iterator is deleted, increment the use iterator
5491/// so that it doesn't dangle.
5492///
Dan Gohmana72d2a22010-03-03 21:33:37 +00005493class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
Dan Gohmana72d2a22010-03-03 21:33:37 +00005494 SDNode::use_iterator &UI;
5495 SDNode::use_iterator &UE;
5496
5497 virtual void NodeDeleted(SDNode *N, SDNode *E) {
5498 // Increment the iterator as needed.
5499 while (UI != UE && N == *UI)
5500 ++UI;
Dan Gohmana72d2a22010-03-03 21:33:37 +00005501 }
5502
5503public:
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005504 RAUWUpdateListener(SelectionDAG &d,
Dan Gohmana72d2a22010-03-03 21:33:37 +00005505 SDNode::use_iterator &ui,
5506 SDNode::use_iterator &ue)
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005507 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
Dan Gohmana72d2a22010-03-03 21:33:37 +00005508};
5509
5510}
5511
Evan Cheng99157a02006-08-07 22:13:29 +00005512/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
Chris Lattner8b8749f2005-08-17 19:00:20 +00005513/// This can cause recursive merging of nodes in the DAG.
5514///
Chris Lattner11d049c2008-02-03 03:35:22 +00005515/// This version assumes From has a single result value.
Chris Lattner8b52f212005-08-26 18:36:28 +00005516///
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005517void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
Gabor Greifba36cb52008-08-28 21:40:38 +00005518 SDNode *From = FromN.getNode();
Scott Michelfdc40a02009-02-17 22:15:04 +00005519 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
Chris Lattner8b52f212005-08-26 18:36:28 +00005520 "Cannot replace with this method!");
Gabor Greifba36cb52008-08-28 21:40:38 +00005521 assert(From != To.getNode() && "Cannot replace uses of with self");
Roman Levensteindc1adac2008-04-07 10:06:32 +00005522
Dan Gohman39946102009-01-25 16:29:12 +00005523 // Iterate over all the existing uses of From. New uses will be added
5524 // to the beginning of the use list, which we avoid visiting.
5525 // This specifically avoids visiting uses of From that arise while the
5526 // replacement is happening, because any such uses would be the result
5527 // of CSE: If an existing node looks like From after one of its operands
5528 // is replaced by To, we don't want to replace of all its users with To
5529 // too. See PR3018 for more info.
Dan Gohmandbe664a2009-01-19 21:44:21 +00005530 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005531 RAUWUpdateListener Listener(*this, UI, UE);
Dan Gohmandbe664a2009-01-19 21:44:21 +00005532 while (UI != UE) {
Dan Gohman39946102009-01-25 16:29:12 +00005533 SDNode *User = *UI;
Roman Levensteindc1adac2008-04-07 10:06:32 +00005534
Chris Lattner8b8749f2005-08-17 19:00:20 +00005535 // This node is about to morph, remove its old self from the CSE maps.
Dan Gohman39946102009-01-25 16:29:12 +00005536 RemoveNodeFromCSEMaps(User);
Chris Lattner8b52f212005-08-26 18:36:28 +00005537
Dan Gohman39946102009-01-25 16:29:12 +00005538 // A user can appear in a use list multiple times, and when this
5539 // happens the uses are usually next to each other in the list.
5540 // To help reduce the number of CSE recomputations, process all
5541 // the uses of this user that we can find this way.
5542 do {
Dan Gohmane7852d02009-01-26 04:35:06 +00005543 SDUse &Use = UI.getUse();
Dan Gohman39946102009-01-25 16:29:12 +00005544 ++UI;
Dan Gohmane7852d02009-01-26 04:35:06 +00005545 Use.set(To);
Dan Gohman39946102009-01-25 16:29:12 +00005546 } while (UI != UE && *UI == User);
5547
5548 // Now that we have modified User, add it back to the CSE maps. If it
5549 // already exists there, recursively merge the results together.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005550 AddModifiedNodeToCSEMaps(User);
Chris Lattner8b52f212005-08-26 18:36:28 +00005551 }
Dan Gohman65fd6562011-11-03 21:49:52 +00005552
5553 // If we just RAUW'd the root, take note.
5554 if (FromN == getRoot())
5555 setRoot(To);
Chris Lattner8b52f212005-08-26 18:36:28 +00005556}
5557
5558/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
5559/// This can cause recursive merging of nodes in the DAG.
5560///
Dan Gohmanc23e4962009-04-15 20:06:30 +00005561/// This version assumes that for each value of From, there is a
5562/// corresponding value in To in the same position with the same type.
Chris Lattner8b52f212005-08-26 18:36:28 +00005563///
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005564void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
Dan Gohmanc23e4962009-04-15 20:06:30 +00005565#ifndef NDEBUG
5566 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
5567 assert((!From->hasAnyUseOfValue(i) ||
5568 From->getValueType(i) == To->getValueType(i)) &&
5569 "Cannot use this version of ReplaceAllUsesWith!");
5570#endif
Dan Gohmane8be6c62008-07-17 19:10:17 +00005571
5572 // Handle the trivial case.
5573 if (From == To)
5574 return;
5575
Dan Gohmandbe664a2009-01-19 21:44:21 +00005576 // Iterate over just the existing users of From. See the comments in
5577 // the ReplaceAllUsesWith above.
5578 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005579 RAUWUpdateListener Listener(*this, UI, UE);
Dan Gohmandbe664a2009-01-19 21:44:21 +00005580 while (UI != UE) {
Dan Gohman39946102009-01-25 16:29:12 +00005581 SDNode *User = *UI;
Roman Levensteindc1adac2008-04-07 10:06:32 +00005582
Chris Lattner8b52f212005-08-26 18:36:28 +00005583 // This node is about to morph, remove its old self from the CSE maps.
Dan Gohman39946102009-01-25 16:29:12 +00005584 RemoveNodeFromCSEMaps(User);
Roman Levensteindc1adac2008-04-07 10:06:32 +00005585
Dan Gohman39946102009-01-25 16:29:12 +00005586 // A user can appear in a use list multiple times, and when this
5587 // happens the uses are usually next to each other in the list.
5588 // To help reduce the number of CSE recomputations, process all
5589 // the uses of this user that we can find this way.
5590 do {
Dan Gohmane7852d02009-01-26 04:35:06 +00005591 SDUse &Use = UI.getUse();
Dan Gohman39946102009-01-25 16:29:12 +00005592 ++UI;
Dan Gohmane7852d02009-01-26 04:35:06 +00005593 Use.setNode(To);
Dan Gohman39946102009-01-25 16:29:12 +00005594 } while (UI != UE && *UI == User);
5595
5596 // Now that we have modified User, add it back to the CSE maps. If it
5597 // already exists there, recursively merge the results together.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005598 AddModifiedNodeToCSEMaps(User);
Chris Lattner8b8749f2005-08-17 19:00:20 +00005599 }
Dan Gohman65fd6562011-11-03 21:49:52 +00005600
5601 // If we just RAUW'd the root, take note.
5602 if (From == getRoot().getNode())
5603 setRoot(SDValue(To, getRoot().getResNo()));
Chris Lattner8b8749f2005-08-17 19:00:20 +00005604}
5605
Chris Lattner8b52f212005-08-26 18:36:28 +00005606/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
5607/// This can cause recursive merging of nodes in the DAG.
5608///
5609/// This version can replace From with any result values. To must match the
5610/// number and types of values returned by From.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005611void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
Chris Lattner11d049c2008-02-03 03:35:22 +00005612 if (From->getNumValues() == 1) // Handle the simple case efficiently.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005613 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
Chris Lattner7b2880c2005-08-24 22:44:39 +00005614
Dan Gohmandbe664a2009-01-19 21:44:21 +00005615 // Iterate over just the existing users of From. See the comments in
5616 // the ReplaceAllUsesWith above.
5617 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005618 RAUWUpdateListener Listener(*this, UI, UE);
Dan Gohmandbe664a2009-01-19 21:44:21 +00005619 while (UI != UE) {
Dan Gohman39946102009-01-25 16:29:12 +00005620 SDNode *User = *UI;
Roman Levensteindc1adac2008-04-07 10:06:32 +00005621
Chris Lattner7b2880c2005-08-24 22:44:39 +00005622 // This node is about to morph, remove its old self from the CSE maps.
Dan Gohman39946102009-01-25 16:29:12 +00005623 RemoveNodeFromCSEMaps(User);
Roman Levensteindc1adac2008-04-07 10:06:32 +00005624
Dan Gohman39946102009-01-25 16:29:12 +00005625 // A user can appear in a use list multiple times, and when this
5626 // happens the uses are usually next to each other in the list.
5627 // To help reduce the number of CSE recomputations, process all
5628 // the uses of this user that we can find this way.
5629 do {
Dan Gohmane7852d02009-01-26 04:35:06 +00005630 SDUse &Use = UI.getUse();
5631 const SDValue &ToOp = To[Use.getResNo()];
Dan Gohman39946102009-01-25 16:29:12 +00005632 ++UI;
Dan Gohmane7852d02009-01-26 04:35:06 +00005633 Use.set(ToOp);
Dan Gohman39946102009-01-25 16:29:12 +00005634 } while (UI != UE && *UI == User);
5635
5636 // Now that we have modified User, add it back to the CSE maps. If it
5637 // already exists there, recursively merge the results together.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005638 AddModifiedNodeToCSEMaps(User);
Chris Lattner7b2880c2005-08-24 22:44:39 +00005639 }
Dan Gohman65fd6562011-11-03 21:49:52 +00005640
5641 // If we just RAUW'd the root, take note.
5642 if (From == getRoot().getNode())
5643 setRoot(SDValue(To[getRoot().getResNo()]));
Chris Lattner7b2880c2005-08-24 22:44:39 +00005644}
5645
Chris Lattner012f2412006-02-17 21:58:01 +00005646/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
Dan Gohmane7852d02009-01-26 04:35:06 +00005647/// uses of other values produced by From.getNode() alone. The Deleted
5648/// vector is handled the same way as for ReplaceAllUsesWith.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005649void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
Dan Gohmane8be6c62008-07-17 19:10:17 +00005650 // Handle the really simple, really trivial case efficiently.
5651 if (From == To) return;
5652
Chris Lattner012f2412006-02-17 21:58:01 +00005653 // Handle the simple, trivial, case efficiently.
Gabor Greifba36cb52008-08-28 21:40:38 +00005654 if (From.getNode()->getNumValues() == 1) {
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005655 ReplaceAllUsesWith(From, To);
Chris Lattner012f2412006-02-17 21:58:01 +00005656 return;
5657 }
Chris Lattnerf8dc0612008-02-03 06:49:24 +00005658
Dan Gohman39946102009-01-25 16:29:12 +00005659 // Iterate over just the existing users of From. See the comments in
5660 // the ReplaceAllUsesWith above.
5661 SDNode::use_iterator UI = From.getNode()->use_begin(),
5662 UE = From.getNode()->use_end();
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005663 RAUWUpdateListener Listener(*this, UI, UE);
Dan Gohman39946102009-01-25 16:29:12 +00005664 while (UI != UE) {
5665 SDNode *User = *UI;
5666 bool UserRemovedFromCSEMaps = false;
Chris Lattner012f2412006-02-17 21:58:01 +00005667
Dan Gohman39946102009-01-25 16:29:12 +00005668 // A user can appear in a use list multiple times, and when this
5669 // happens the uses are usually next to each other in the list.
5670 // To help reduce the number of CSE recomputations, process all
5671 // the uses of this user that we can find this way.
5672 do {
Dan Gohmane7852d02009-01-26 04:35:06 +00005673 SDUse &Use = UI.getUse();
Dan Gohman39946102009-01-25 16:29:12 +00005674
5675 // Skip uses of different values from the same node.
Dan Gohmane7852d02009-01-26 04:35:06 +00005676 if (Use.getResNo() != From.getResNo()) {
Dan Gohman39946102009-01-25 16:29:12 +00005677 ++UI;
5678 continue;
Chris Lattner012f2412006-02-17 21:58:01 +00005679 }
Dan Gohman39946102009-01-25 16:29:12 +00005680
5681 // If this node hasn't been modified yet, it's still in the CSE maps,
5682 // so remove its old self from the CSE maps.
5683 if (!UserRemovedFromCSEMaps) {
5684 RemoveNodeFromCSEMaps(User);
5685 UserRemovedFromCSEMaps = true;
5686 }
5687
5688 ++UI;
Dan Gohmane7852d02009-01-26 04:35:06 +00005689 Use.set(To);
Dan Gohman39946102009-01-25 16:29:12 +00005690 } while (UI != UE && *UI == User);
5691
5692 // We are iterating over all uses of the From node, so if a use
5693 // doesn't use the specific value, no changes are made.
5694 if (!UserRemovedFromCSEMaps)
5695 continue;
5696
Chris Lattner01d029b2007-10-15 06:10:22 +00005697 // Now that we have modified User, add it back to the CSE maps. If it
5698 // already exists there, recursively merge the results together.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005699 AddModifiedNodeToCSEMaps(User);
Dan Gohmane8be6c62008-07-17 19:10:17 +00005700 }
Dan Gohman65fd6562011-11-03 21:49:52 +00005701
5702 // If we just RAUW'd the root, take note.
5703 if (From == getRoot())
5704 setRoot(To);
Dan Gohmane8be6c62008-07-17 19:10:17 +00005705}
5706
Dan Gohman39946102009-01-25 16:29:12 +00005707namespace {
5708 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
5709 /// to record information about a use.
5710 struct UseMemo {
5711 SDNode *User;
5712 unsigned Index;
Dan Gohmane7852d02009-01-26 04:35:06 +00005713 SDUse *Use;
Dan Gohman39946102009-01-25 16:29:12 +00005714 };
Dan Gohmane7852d02009-01-26 04:35:06 +00005715
5716 /// operator< - Sort Memos by User.
5717 bool operator<(const UseMemo &L, const UseMemo &R) {
5718 return (intptr_t)L.User < (intptr_t)R.User;
5719 }
Dan Gohman39946102009-01-25 16:29:12 +00005720}
5721
Dan Gohmane8be6c62008-07-17 19:10:17 +00005722/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
Dan Gohmane7852d02009-01-26 04:35:06 +00005723/// uses of other values produced by From.getNode() alone. The same value
5724/// may appear in both the From and To list. The Deleted vector is
Dan Gohmane8be6c62008-07-17 19:10:17 +00005725/// handled the same way as for ReplaceAllUsesWith.
Dan Gohman475871a2008-07-27 21:46:04 +00005726void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
5727 const SDValue *To,
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005728 unsigned Num){
Dan Gohmane8be6c62008-07-17 19:10:17 +00005729 // Handle the simple, trivial case efficiently.
5730 if (Num == 1)
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005731 return ReplaceAllUsesOfValueWith(*From, *To);
Dan Gohmane8be6c62008-07-17 19:10:17 +00005732
Dan Gohman39946102009-01-25 16:29:12 +00005733 // Read up all the uses and make records of them. This helps
5734 // processing new uses that are introduced during the
5735 // replacement process.
5736 SmallVector<UseMemo, 4> Uses;
5737 for (unsigned i = 0; i != Num; ++i) {
5738 unsigned FromResNo = From[i].getResNo();
5739 SDNode *FromNode = From[i].getNode();
Scott Michelfdc40a02009-02-17 22:15:04 +00005740 for (SDNode::use_iterator UI = FromNode->use_begin(),
Dan Gohmane7852d02009-01-26 04:35:06 +00005741 E = FromNode->use_end(); UI != E; ++UI) {
5742 SDUse &Use = UI.getUse();
5743 if (Use.getResNo() == FromResNo) {
5744 UseMemo Memo = { *UI, i, &Use };
Dan Gohman39946102009-01-25 16:29:12 +00005745 Uses.push_back(Memo);
5746 }
Dan Gohmane7852d02009-01-26 04:35:06 +00005747 }
Dan Gohman39946102009-01-25 16:29:12 +00005748 }
Dan Gohmane8be6c62008-07-17 19:10:17 +00005749
Dan Gohmane7852d02009-01-26 04:35:06 +00005750 // Sort the uses, so that all the uses from a given User are together.
5751 std::sort(Uses.begin(), Uses.end());
5752
Dan Gohman39946102009-01-25 16:29:12 +00005753 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
5754 UseIndex != UseIndexEnd; ) {
Dan Gohmane8be6c62008-07-17 19:10:17 +00005755 // We know that this user uses some value of From. If it is the right
5756 // value, update it.
Dan Gohman39946102009-01-25 16:29:12 +00005757 SDNode *User = Uses[UseIndex].User;
5758
5759 // This node is about to morph, remove its old self from the CSE maps.
Dan Gohmane8be6c62008-07-17 19:10:17 +00005760 RemoveNodeFromCSEMaps(User);
Dan Gohman39946102009-01-25 16:29:12 +00005761
Dan Gohmane7852d02009-01-26 04:35:06 +00005762 // The Uses array is sorted, so all the uses for a given User
5763 // are next to each other in the list.
Dan Gohman39946102009-01-25 16:29:12 +00005764 // To help reduce the number of CSE recomputations, process all
5765 // the uses of this user that we can find this way.
5766 do {
5767 unsigned i = Uses[UseIndex].Index;
Dan Gohmane7852d02009-01-26 04:35:06 +00005768 SDUse &Use = *Uses[UseIndex].Use;
Dan Gohman39946102009-01-25 16:29:12 +00005769 ++UseIndex;
5770
Dan Gohmane7852d02009-01-26 04:35:06 +00005771 Use.set(To[i]);
Dan Gohman39946102009-01-25 16:29:12 +00005772 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
5773
Dan Gohmane8be6c62008-07-17 19:10:17 +00005774 // Now that we have modified User, add it back to the CSE maps. If it
5775 // already exists there, recursively merge the results together.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005776 AddModifiedNodeToCSEMaps(User);
Chris Lattner012f2412006-02-17 21:58:01 +00005777 }
5778}
5779
Evan Chenge6f35d82006-08-01 08:20:41 +00005780/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
Evan Chengc384d6c2006-08-02 22:00:34 +00005781/// based on their topological order. It returns the maximum id and a vector
5782/// of the SDNodes* in assigned order by reference.
Dan Gohmanf06c8352008-09-30 18:30:35 +00005783unsigned SelectionDAG::AssignTopologicalOrder() {
Evan Chengc384d6c2006-08-02 22:00:34 +00005784
Dan Gohmanf06c8352008-09-30 18:30:35 +00005785 unsigned DAGSize = 0;
Evan Chenge6f35d82006-08-01 08:20:41 +00005786
Dan Gohmanf06c8352008-09-30 18:30:35 +00005787 // SortedPos tracks the progress of the algorithm. Nodes before it are
5788 // sorted, nodes after it are unsorted. When the algorithm completes
5789 // it is at the end of the list.
5790 allnodes_iterator SortedPos = allnodes_begin();
5791
Dan Gohman3ebd0ee2008-11-21 19:10:41 +00005792 // Visit all the nodes. Move nodes with no operands to the front of
5793 // the list immediately. Annotate nodes that do have operands with their
Dan Gohmanf06c8352008-09-30 18:30:35 +00005794 // operand count. Before we do this, the Node Id fields of the nodes
5795 // may contain arbitrary values. After, the Node Id fields for nodes
5796 // before SortedPos will contain the topological sort index, and the
5797 // Node Id fields for nodes At SortedPos and after will contain the
5798 // count of outstanding operands.
5799 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
5800 SDNode *N = I++;
David Greenecf495bc2010-01-20 20:13:31 +00005801 checkForCycles(N);
Dan Gohmanf06c8352008-09-30 18:30:35 +00005802 unsigned Degree = N->getNumOperands();
5803 if (Degree == 0) {
5804 // A node with no uses, add it to the result array immediately.
5805 N->setNodeId(DAGSize++);
5806 allnodes_iterator Q = N;
5807 if (Q != SortedPos)
5808 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
David Greene221925e2010-01-20 00:59:23 +00005809 assert(SortedPos != AllNodes.end() && "Overran node list");
Dan Gohmanf06c8352008-09-30 18:30:35 +00005810 ++SortedPos;
5811 } else {
5812 // Temporarily use the Node Id as scratch space for the degree count.
5813 N->setNodeId(Degree);
Evan Chenge6f35d82006-08-01 08:20:41 +00005814 }
5815 }
5816
Chad Rosierf496dea2012-05-21 17:13:41 +00005817 // Visit all the nodes. As we iterate, move nodes into sorted order,
Dan Gohmanf06c8352008-09-30 18:30:35 +00005818 // such that by the time the end is reached all nodes will be sorted.
5819 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
5820 SDNode *N = I;
David Greenecf495bc2010-01-20 20:13:31 +00005821 checkForCycles(N);
David Greene221925e2010-01-20 00:59:23 +00005822 // N is in sorted position, so all its uses have one less operand
5823 // that needs to be sorted.
Dan Gohmanf06c8352008-09-30 18:30:35 +00005824 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
5825 UI != UE; ++UI) {
5826 SDNode *P = *UI;
5827 unsigned Degree = P->getNodeId();
David Greene221925e2010-01-20 00:59:23 +00005828 assert(Degree != 0 && "Invalid node degree");
Dan Gohmanf06c8352008-09-30 18:30:35 +00005829 --Degree;
5830 if (Degree == 0) {
5831 // All of P's operands are sorted, so P may sorted now.
5832 P->setNodeId(DAGSize++);
5833 if (P != SortedPos)
5834 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
David Greene221925e2010-01-20 00:59:23 +00005835 assert(SortedPos != AllNodes.end() && "Overran node list");
Dan Gohmanf06c8352008-09-30 18:30:35 +00005836 ++SortedPos;
5837 } else {
5838 // Update P's outstanding operand count.
5839 P->setNodeId(Degree);
5840 }
5841 }
David Greene221925e2010-01-20 00:59:23 +00005842 if (I == SortedPos) {
David Greene39143702010-02-09 23:03:05 +00005843#ifndef NDEBUG
5844 SDNode *S = ++I;
5845 dbgs() << "Overran sorted position:\n";
David Greene221925e2010-01-20 00:59:23 +00005846 S->dumprFull();
David Greene39143702010-02-09 23:03:05 +00005847#endif
5848 llvm_unreachable(0);
David Greene221925e2010-01-20 00:59:23 +00005849 }
Dan Gohmanf06c8352008-09-30 18:30:35 +00005850 }
5851
5852 assert(SortedPos == AllNodes.end() &&
5853 "Topological sort incomplete!");
5854 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
5855 "First node in topological sort is not the entry token!");
5856 assert(AllNodes.front().getNodeId() == 0 &&
5857 "First node in topological sort has non-zero id!");
5858 assert(AllNodes.front().getNumOperands() == 0 &&
5859 "First node in topological sort has operands!");
5860 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
5861 "Last node in topologic sort has unexpected id!");
5862 assert(AllNodes.back().use_empty() &&
5863 "Last node in topologic sort has users!");
Dan Gohman3ebd0ee2008-11-21 19:10:41 +00005864 assert(DAGSize == allnodes_size() && "Node count mismatch!");
Dan Gohmanf06c8352008-09-30 18:30:35 +00005865 return DAGSize;
Evan Chenge6f35d82006-08-01 08:20:41 +00005866}
5867
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00005868/// AssignOrdering - Assign an order to the SDNode.
Bill Wendling4533cac2010-01-28 21:51:40 +00005869void SelectionDAG::AssignOrdering(const SDNode *SD, unsigned Order) {
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00005870 assert(SD && "Trying to assign an order to a null node!");
Bill Wendling187361b2010-01-23 10:26:57 +00005871 Ordering->add(SD, Order);
Bill Wendlingb4e6a5d2009-12-18 23:32:53 +00005872}
Evan Chenge6f35d82006-08-01 08:20:41 +00005873
Bill Wendling0777e922009-12-21 21:59:52 +00005874/// GetOrdering - Get the order for the SDNode.
5875unsigned SelectionDAG::GetOrdering(const SDNode *SD) const {
5876 assert(SD && "Trying to get the order of a null node!");
Bill Wendling187361b2010-01-23 10:26:57 +00005877 return Ordering->getOrder(SD);
Bill Wendling0777e922009-12-21 21:59:52 +00005878}
5879
Evan Chengbfcb3052010-03-25 01:38:16 +00005880/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
5881/// value is produced by SD.
Dale Johannesenfdb42fa2010-04-26 20:06:49 +00005882void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
5883 DbgInfo->add(DB, SD, isParameter);
Evan Chengbfcb3052010-03-25 01:38:16 +00005884 if (SD)
5885 SD->setHasDebugValue(true);
Dale Johannesenbfdf7f32010-03-10 22:13:47 +00005886}
Evan Cheng091cba12006-07-27 06:39:06 +00005887
Devang Patela2e868d2011-01-25 23:27:42 +00005888/// TransferDbgValues - Transfer SDDbgValues.
5889void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
5890 if (From == To || !From.getNode()->getHasDebugValue())
5891 return;
5892 SDNode *FromNode = From.getNode();
5893 SDNode *ToNode = To.getNode();
Benjamin Kramer22a54c12011-06-18 13:13:44 +00005894 ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
Devang Patela778f5c2011-02-18 22:43:42 +00005895 SmallVector<SDDbgValue *, 2> ClonedDVs;
Benjamin Kramer22a54c12011-06-18 13:13:44 +00005896 for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
Devang Patela2e868d2011-01-25 23:27:42 +00005897 I != E; ++I) {
Devang Patela778f5c2011-02-18 22:43:42 +00005898 SDDbgValue *Dbg = *I;
5899 if (Dbg->getKind() == SDDbgValue::SDNODE) {
5900 SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(),
5901 Dbg->getOffset(), Dbg->getDebugLoc(),
5902 Dbg->getOrder());
5903 ClonedDVs.push_back(Clone);
Devang Patela2e868d2011-01-25 23:27:42 +00005904 }
5905 }
Devang Patela778f5c2011-02-18 22:43:42 +00005906 for (SmallVector<SDDbgValue *, 2>::iterator I = ClonedDVs.begin(),
5907 E = ClonedDVs.end(); I != E; ++I)
5908 AddDbgValue(*I, ToNode, false);
Devang Patela2e868d2011-01-25 23:27:42 +00005909}
5910
Jim Laskey58b968b2005-08-17 20:08:02 +00005911//===----------------------------------------------------------------------===//
5912// SDNode Class
5913//===----------------------------------------------------------------------===//
Chris Lattner149c58c2005-08-16 18:17:10 +00005914
Chris Lattner48b85922007-02-04 02:41:42 +00005915HandleSDNode::~HandleSDNode() {
Dan Gohman0fe9c6e2008-07-07 20:57:48 +00005916 DropOperands();
Chris Lattner48b85922007-02-04 02:41:42 +00005917}
5918
Andrew Trickac6d9be2013-05-25 02:42:55 +00005919GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
5920 DebugLoc DL, const GlobalValue *GA,
Owen Andersone50ed302009-08-10 22:56:29 +00005921 EVT VT, int64_t o, unsigned char TF)
Andrew Trickac6d9be2013-05-25 02:42:55 +00005922 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
Dan Gohman383b5f62010-04-17 15:32:28 +00005923 TheGlobal = GA;
Lauro Ramos Venancio2c5c1112007-04-21 20:56:26 +00005924}
Chris Lattner48b85922007-02-04 02:41:42 +00005925
Andrew Trickac6d9be2013-05-25 02:42:55 +00005926MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
5927 EVT memvt, MachineMemOperand *mmo)
5928 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
David Greene1157f792010-02-17 20:21:42 +00005929 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00005930 MMO->isNonTemporal(), MMO->isInvariant());
Dan Gohmanc76909a2009-09-25 20:36:54 +00005931 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
David Greene1157f792010-02-17 20:21:42 +00005932 assert(isNonTemporal() == MMO->isNonTemporal() &&
5933 "Non-temporal encoding error!");
Dan Gohmanc76909a2009-09-25 20:36:54 +00005934 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!");
Dale Johannesen3edb43e2009-01-28 21:18:29 +00005935}
5936
Andrew Trickac6d9be2013-05-25 02:42:55 +00005937MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005938 const SDValue *Ops, unsigned NumOps, EVT memvt,
Dan Gohmanc76909a2009-09-25 20:36:54 +00005939 MachineMemOperand *mmo)
Andrew Trickac6d9be2013-05-25 02:42:55 +00005940 : SDNode(Opc, Order, dl, VTs, Ops, NumOps),
Dan Gohmanc76909a2009-09-25 20:36:54 +00005941 MemoryVT(memvt), MMO(mmo) {
David Greene1157f792010-02-17 20:21:42 +00005942 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00005943 MMO->isNonTemporal(), MMO->isInvariant());
Dan Gohmanc76909a2009-09-25 20:36:54 +00005944 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
5945 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!");
Mon P Wang28873102008-06-25 08:15:39 +00005946}
5947
Jim Laskey583bd472006-10-27 23:46:08 +00005948/// Profile - Gather unique data for the node.
5949///
Dan Gohmanb8d2f552008-08-20 15:58:01 +00005950void SDNode::Profile(FoldingSetNodeID &ID) const {
Jim Laskey583bd472006-10-27 23:46:08 +00005951 AddNodeIDNode(ID, this);
5952}
5953
Owen Andersond8110fb2009-08-25 22:27:22 +00005954namespace {
5955 struct EVTArray {
5956 std::vector<EVT> VTs;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005957
Owen Andersond8110fb2009-08-25 22:27:22 +00005958 EVTArray() {
5959 VTs.reserve(MVT::LAST_VALUETYPE);
5960 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
5961 VTs.push_back(MVT((MVT::SimpleValueType)i));
5962 }
5963 };
5964}
5965
Owen Andersone50ed302009-08-10 22:56:29 +00005966static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs;
Owen Andersond8110fb2009-08-25 22:27:22 +00005967static ManagedStatic<EVTArray> SimpleVTArray;
Chris Lattner895a55e2009-08-22 04:07:34 +00005968static ManagedStatic<sys::SmartMutex<true> > VTMutex;
Owen Andersonb4459082009-06-25 17:09:00 +00005969
Chris Lattnera3255112005-11-08 23:30:28 +00005970/// getValueTypeList - Return a pointer to the specified value type.
5971///
Owen Andersone50ed302009-08-10 22:56:29 +00005972const EVT *SDNode::getValueTypeList(EVT VT) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00005973 if (VT.isExtended()) {
Owen Anderson02b10342009-08-22 06:32:36 +00005974 sys::SmartScopedLock<true> Lock(*VTMutex);
Owen Andersonb4459082009-06-25 17:09:00 +00005975 return &(*EVTs->insert(VT).first);
Duncan Sandsaf47b112007-10-16 09:56:48 +00005976 } else {
Duncan Sandscdfad362010-11-03 12:17:33 +00005977 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
Duncan Sandsad017dc2010-05-10 04:54:28 +00005978 "Value type out of range!");
Owen Andersond8110fb2009-08-25 22:27:22 +00005979 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
Duncan Sandsaf47b112007-10-16 09:56:48 +00005980 }
Chris Lattnera3255112005-11-08 23:30:28 +00005981}
Duncan Sandsaf47b112007-10-16 09:56:48 +00005982
Chris Lattner5c884562005-01-12 18:37:47 +00005983/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
5984/// indicated value. This method ignores uses of other values defined by this
5985/// operation.
Evan Cheng4ee62112006-02-05 06:29:23 +00005986bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
Chris Lattner5c884562005-01-12 18:37:47 +00005987 assert(Value < getNumValues() && "Bad value!");
5988
Roman Levensteindc1adac2008-04-07 10:06:32 +00005989 // TODO: Only iterate over uses of a given value of the node
5990 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
Dan Gohmane7852d02009-01-26 04:35:06 +00005991 if (UI.getUse().getResNo() == Value) {
Roman Levensteindc1adac2008-04-07 10:06:32 +00005992 if (NUses == 0)
5993 return false;
5994 --NUses;
5995 }
Chris Lattner5c884562005-01-12 18:37:47 +00005996 }
5997
5998 // Found exactly the right number of uses?
5999 return NUses == 0;
6000}
6001
6002
Evan Cheng33d55952007-08-02 05:29:38 +00006003/// hasAnyUseOfValue - Return true if there are any use of the indicated
6004/// value. This method ignores uses of other values defined by this operation.
6005bool SDNode::hasAnyUseOfValue(unsigned Value) const {
6006 assert(Value < getNumValues() && "Bad value!");
6007
Dan Gohman1373c1c2008-07-09 22:39:01 +00006008 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
Dan Gohmane7852d02009-01-26 04:35:06 +00006009 if (UI.getUse().getResNo() == Value)
Dan Gohman1373c1c2008-07-09 22:39:01 +00006010 return true;
Evan Cheng33d55952007-08-02 05:29:38 +00006011
6012 return false;
6013}
6014
6015
Dan Gohman2a629952008-07-27 18:06:42 +00006016/// isOnlyUserOf - Return true if this node is the only use of N.
Evan Chenge6e97e62006-11-03 07:31:32 +00006017///
Dan Gohman2a629952008-07-27 18:06:42 +00006018bool SDNode::isOnlyUserOf(SDNode *N) const {
Evan Cheng4ee62112006-02-05 06:29:23 +00006019 bool Seen = false;
6020 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
Dan Gohman89684502008-07-27 20:43:25 +00006021 SDNode *User = *I;
Evan Cheng4ee62112006-02-05 06:29:23 +00006022 if (User == this)
6023 Seen = true;
6024 else
6025 return false;
6026 }
6027
6028 return Seen;
6029}
6030
Evan Chenge6e97e62006-11-03 07:31:32 +00006031/// isOperand - Return true if this node is an operand of N.
6032///
Dan Gohman475871a2008-07-27 21:46:04 +00006033bool SDValue::isOperandOf(SDNode *N) const {
Evan Chengbfa284f2006-03-03 06:42:32 +00006034 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
6035 if (*this == N->getOperand(i))
6036 return true;
6037 return false;
6038}
6039
Evan Cheng917be682008-03-04 00:41:45 +00006040bool SDNode::isOperandOf(SDNode *N) const {
Evan Cheng80d8eaa2006-03-03 06:24:54 +00006041 for (unsigned i = 0, e = N->NumOperands; i != e; ++i)
Dan Gohmane7852d02009-01-26 04:35:06 +00006042 if (this == N->OperandList[i].getNode())
Evan Cheng80d8eaa2006-03-03 06:24:54 +00006043 return true;
6044 return false;
6045}
Evan Cheng4ee62112006-02-05 06:29:23 +00006046
Chris Lattner572dee72008-01-16 05:49:24 +00006047/// reachesChainWithoutSideEffects - Return true if this operand (which must
Scott Michelfdc40a02009-02-17 22:15:04 +00006048/// be a chain) reaches the specified operand without crossing any
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006049/// side-effecting instructions on any chain path. In practice, this looks
6050/// through token factors and non-volatile loads. In order to remain efficient,
Owen Anderson14ac1dd2010-09-18 04:45:14 +00006051/// this only looks a couple of nodes in, it does not do an exhaustive search.
Scott Michelfdc40a02009-02-17 22:15:04 +00006052bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
Chris Lattner572dee72008-01-16 05:49:24 +00006053 unsigned Depth) const {
6054 if (*this == Dest) return true;
Scott Michelfdc40a02009-02-17 22:15:04 +00006055
Chris Lattner572dee72008-01-16 05:49:24 +00006056 // Don't search too deeply, we just want to be able to see through
6057 // TokenFactor's etc.
6058 if (Depth == 0) return false;
Scott Michelfdc40a02009-02-17 22:15:04 +00006059
Chris Lattner572dee72008-01-16 05:49:24 +00006060 // If this is a token factor, all inputs to the TF happen in parallel. If any
Owen Anderson14ac1dd2010-09-18 04:45:14 +00006061 // of the operands of the TF does not reach dest, then we cannot do the xform.
Chris Lattner572dee72008-01-16 05:49:24 +00006062 if (getOpcode() == ISD::TokenFactor) {
6063 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
Owen Anderson14ac1dd2010-09-18 04:45:14 +00006064 if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1))
6065 return false;
6066 return true;
Chris Lattner572dee72008-01-16 05:49:24 +00006067 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006068
Chris Lattner572dee72008-01-16 05:49:24 +00006069 // Loads don't have side effects, look through them.
6070 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
6071 if (!Ld->isVolatile())
6072 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
6073 }
6074 return false;
6075}
6076
Lang Hames944520f2011-07-07 04:31:51 +00006077/// hasPredecessor - Return true if N is a predecessor of this node.
6078/// N is either an operand of this node, or can be reached by recursively
6079/// traversing up the operands.
6080/// NOTE: This is an expensive method. Use it carefully.
6081bool SDNode::hasPredecessor(const SDNode *N) const {
6082 SmallPtrSet<const SDNode *, 32> Visited;
6083 SmallVector<const SDNode *, 16> Worklist;
6084 return hasPredecessorHelper(N, Visited, Worklist);
6085}
Dan Gohman6688d612009-10-28 03:44:30 +00006086
Lang Hames944520f2011-07-07 04:31:51 +00006087bool SDNode::hasPredecessorHelper(const SDNode *N,
6088 SmallPtrSet<const SDNode *, 32> &Visited,
6089 SmallVector<const SDNode *, 16> &Worklist) const {
6090 if (Visited.empty()) {
6091 Worklist.push_back(this);
6092 } else {
6093 // Take a look in the visited set. If we've already encountered this node
6094 // we needn't search further.
6095 if (Visited.count(N))
6096 return true;
6097 }
6098
6099 // Haven't visited N yet. Continue the search.
6100 while (!Worklist.empty()) {
6101 const SDNode *M = Worklist.pop_back_val();
6102 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
6103 SDNode *Op = M->getOperand(i).getNode();
Dan Gohman6688d612009-10-28 03:44:30 +00006104 if (Visited.insert(Op))
6105 Worklist.push_back(Op);
Lang Hames944520f2011-07-07 04:31:51 +00006106 if (Op == N)
6107 return true;
Dan Gohman6688d612009-10-28 03:44:30 +00006108 }
Lang Hames944520f2011-07-07 04:31:51 +00006109 }
Dan Gohman6688d612009-10-28 03:44:30 +00006110
6111 return false;
Evan Chengc5fc57d2006-11-03 03:05:24 +00006112}
6113
Evan Chengc5484282006-10-04 00:56:09 +00006114uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
6115 assert(Num < NumOperands && "Invalid child # of SDNode!");
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00006116 return cast<ConstantSDNode>(OperandList[Num])->getZExtValue();
Evan Chengc5484282006-10-04 00:56:09 +00006117}
6118
Mon P Wangcd6e7252009-11-30 02:42:02 +00006119SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
6120 assert(N->getNumValues() == 1 &&
6121 "Can't unroll a vector with multiple results!");
6122
6123 EVT VT = N->getValueType(0);
6124 unsigned NE = VT.getVectorNumElements();
6125 EVT EltVT = VT.getVectorElementType();
Andrew Trickac6d9be2013-05-25 02:42:55 +00006126 SDLoc dl(N);
Mon P Wangcd6e7252009-11-30 02:42:02 +00006127
6128 SmallVector<SDValue, 8> Scalars;
6129 SmallVector<SDValue, 4> Operands(N->getNumOperands());
6130
6131 // If ResNE is 0, fully unroll the vector op.
6132 if (ResNE == 0)
6133 ResNE = NE;
6134 else if (NE > ResNE)
6135 NE = ResNE;
6136
6137 unsigned i;
6138 for (i= 0; i != NE; ++i) {
Bill Wendlingd71bb562010-04-30 22:19:17 +00006139 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
Mon P Wangcd6e7252009-11-30 02:42:02 +00006140 SDValue Operand = N->getOperand(j);
6141 EVT OperandVT = Operand.getValueType();
6142 if (OperandVT.isVector()) {
6143 // A vector operand; extract a single element.
6144 EVT OperandEltVT = OperandVT.getVectorElementType();
6145 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl,
6146 OperandEltVT,
6147 Operand,
Owen Andersonf7710af2011-05-02 22:25:45 +00006148 getConstant(i, TLI.getPointerTy()));
Mon P Wangcd6e7252009-11-30 02:42:02 +00006149 } else {
6150 // A scalar operand; just use it as is.
6151 Operands[j] = Operand;
6152 }
6153 }
6154
6155 switch (N->getOpcode()) {
6156 default:
6157 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
6158 &Operands[0], Operands.size()));
6159 break;
Nadav Rotemaec58612011-09-13 19:17:42 +00006160 case ISD::VSELECT:
6161 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT,
6162 &Operands[0], Operands.size()));
6163 break;
Mon P Wangcd6e7252009-11-30 02:42:02 +00006164 case ISD::SHL:
6165 case ISD::SRA:
6166 case ISD::SRL:
6167 case ISD::ROTL:
6168 case ISD::ROTR:
6169 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
Owen Anderson6154f6c2011-03-07 18:29:47 +00006170 getShiftAmountOperand(Operands[0].getValueType(),
6171 Operands[1])));
Mon P Wangcd6e7252009-11-30 02:42:02 +00006172 break;
Dan Gohmand1996362010-01-09 02:13:55 +00006173 case ISD::SIGN_EXTEND_INREG:
6174 case ISD::FP_ROUND_INREG: {
6175 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
6176 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
6177 Operands[0],
6178 getValueType(ExtVT)));
6179 }
Mon P Wangcd6e7252009-11-30 02:42:02 +00006180 }
6181 }
6182
6183 for (; i < ResNE; ++i)
6184 Scalars.push_back(getUNDEF(EltVT));
6185
6186 return getNode(ISD::BUILD_VECTOR, dl,
6187 EVT::getVectorVT(*getContext(), EltVT, ResNE),
6188 &Scalars[0], Scalars.size());
6189}
6190
Evan Cheng64fa4a92009-12-09 01:36:00 +00006191
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006192/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a
6193/// location that is 'Dist' units away from the location that the 'Base' load
Evan Cheng64fa4a92009-12-09 01:36:00 +00006194/// is loading from.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006195bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base,
Evan Cheng64fa4a92009-12-09 01:36:00 +00006196 unsigned Bytes, int Dist) const {
6197 if (LD->getChain() != Base->getChain())
6198 return false;
6199 EVT VT = LD->getValueType(0);
6200 if (VT.getSizeInBits() / 8 != Bytes)
6201 return false;
6202
6203 SDValue Loc = LD->getOperand(1);
6204 SDValue BaseLoc = Base->getOperand(1);
6205 if (Loc.getOpcode() == ISD::FrameIndex) {
6206 if (BaseLoc.getOpcode() != ISD::FrameIndex)
6207 return false;
6208 const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo();
6209 int FI = cast<FrameIndexSDNode>(Loc)->getIndex();
6210 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex();
6211 int FS = MFI->getObjectSize(FI);
6212 int BFS = MFI->getObjectSize(BFI);
6213 if (FS != BFS || FS != (int)Bytes) return false;
6214 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes);
6215 }
Chris Lattner0a9481f2011-02-13 22:25:43 +00006216
6217 // Handle X+C
6218 if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc &&
6219 cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes)
6220 return true;
Evan Cheng64fa4a92009-12-09 01:36:00 +00006221
Dan Gohman46510a72010-04-15 01:51:59 +00006222 const GlobalValue *GV1 = NULL;
6223 const GlobalValue *GV2 = NULL;
Evan Cheng64fa4a92009-12-09 01:36:00 +00006224 int64_t Offset1 = 0;
6225 int64_t Offset2 = 0;
6226 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1);
6227 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2);
6228 if (isGA1 && isGA2 && GV1 == GV2)
6229 return Offset1 == (Offset2 + Dist*Bytes);
6230 return false;
6231}
6232
6233
Evan Chengf2dc5c72009-12-09 01:04:59 +00006234/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
6235/// it cannot be inferred.
Evan Cheng7ced2e02009-12-09 01:10:37 +00006236unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
Evan Cheng7bd64782009-12-09 01:53:58 +00006237 // If this is a GlobalAddress + cst, return the alignment.
Dan Gohman46510a72010-04-15 01:51:59 +00006238 const GlobalValue *GV;
Evan Cheng7bd64782009-12-09 01:53:58 +00006239 int64_t GVOffset = 0;
Evan Cheng255f20f2010-04-01 06:04:33 +00006240 if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
Eli Friedmanc4c2a022011-11-28 22:48:22 +00006241 unsigned PtrWidth = TLI.getPointerTy().getSizeInBits();
Eli Friedmanc4c2a022011-11-28 22:48:22 +00006242 APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0);
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00006243 llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne,
Micah Villmow3574eca2012-10-08 16:38:25 +00006244 TLI.getDataLayout());
Eli Friedmanc4c2a022011-11-28 22:48:22 +00006245 unsigned AlignBits = KnownZero.countTrailingOnes();
6246 unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
6247 if (Align)
6248 return MinAlign(Align, GVOffset);
Evan Cheng255f20f2010-04-01 06:04:33 +00006249 }
Evan Cheng7bd64782009-12-09 01:53:58 +00006250
Evan Chengf2dc5c72009-12-09 01:04:59 +00006251 // If this is a direct reference to a stack slot, use information about the
6252 // stack slot's alignment.
6253 int FrameIdx = 1 << 31;
6254 int64_t FrameOffset = 0;
6255 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
6256 FrameIdx = FI->getIndex();
Chris Lattner0a9481f2011-02-13 22:25:43 +00006257 } else if (isBaseWithConstantOffset(Ptr) &&
Evan Chengf2dc5c72009-12-09 01:04:59 +00006258 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
Chris Lattner0a9481f2011-02-13 22:25:43 +00006259 // Handle FI+Cst
Evan Chengf2dc5c72009-12-09 01:04:59 +00006260 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
6261 FrameOffset = Ptr.getConstantOperandVal(1);
6262 }
6263
6264 if (FrameIdx != (1 << 31)) {
Evan Chengf2dc5c72009-12-09 01:04:59 +00006265 const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo();
Evan Chengde2ace12009-12-09 01:17:24 +00006266 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
6267 FrameOffset);
Evan Chengde2ace12009-12-09 01:17:24 +00006268 return FIInfoAlign;
Evan Chengf2dc5c72009-12-09 01:04:59 +00006269 }
6270
6271 return 0;
6272}
6273
Sanjiv Guptaa3518a12009-04-29 04:43:24 +00006274// getAddressSpace - Return the address space this GlobalAddress belongs to.
6275unsigned GlobalAddressSDNode::getAddressSpace() const {
6276 return getGlobal()->getType()->getAddressSpace();
6277}
6278
6279
Chris Lattnerdb125cf2011-07-18 04:54:35 +00006280Type *ConstantPoolSDNode::getType() const {
Evan Chengd6594ae2006-09-12 21:00:35 +00006281 if (isMachineConstantPoolEntry())
6282 return Val.MachineCPVal->getType();
6283 return Val.ConstVal->getType();
6284}
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006285
Bob Wilson24e338e2009-03-02 23:24:16 +00006286bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue,
6287 APInt &SplatUndef,
6288 unsigned &SplatBitSize,
6289 bool &HasAnyUndefs,
Dale Johannesen1e608812009-11-13 01:45:18 +00006290 unsigned MinSplatBits,
6291 bool isBigEndian) {
Owen Andersone50ed302009-08-10 22:56:29 +00006292 EVT VT = getValueType(0);
Bob Wilson24e338e2009-03-02 23:24:16 +00006293 assert(VT.isVector() && "Expected a vector type");
6294 unsigned sz = VT.getSizeInBits();
6295 if (MinSplatBits > sz)
6296 return false;
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006297
Bob Wilson24e338e2009-03-02 23:24:16 +00006298 SplatValue = APInt(sz, 0);
6299 SplatUndef = APInt(sz, 0);
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006300
Bob Wilson24e338e2009-03-02 23:24:16 +00006301 // Get the bits. Bits with undefined values (when the corresponding element
6302 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
6303 // in SplatValue. If any of the values are not constant, give up and return
6304 // false.
6305 unsigned int nOps = getNumOperands();
6306 assert(nOps > 0 && "isConstantSplat has 0-size build vector");
6307 unsigned EltBitSize = VT.getVectorElementType().getSizeInBits();
Dale Johannesen1e608812009-11-13 01:45:18 +00006308
6309 for (unsigned j = 0; j < nOps; ++j) {
6310 unsigned i = isBigEndian ? nOps-1-j : j;
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006311 SDValue OpVal = getOperand(i);
Dale Johannesen1e608812009-11-13 01:45:18 +00006312 unsigned BitPos = j * EltBitSize;
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006313
Bob Wilson24e338e2009-03-02 23:24:16 +00006314 if (OpVal.getOpcode() == ISD::UNDEF)
Dale Johannesen1e608812009-11-13 01:45:18 +00006315 SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize);
Bob Wilson24e338e2009-03-02 23:24:16 +00006316 else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal))
Jay Foad40f8f622010-12-07 08:25:19 +00006317 SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize).
Dan Gohman58c25872010-04-12 02:24:01 +00006318 zextOrTrunc(sz) << BitPos;
Bob Wilson24e338e2009-03-02 23:24:16 +00006319 else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal))
Bob Wilsond344b882009-03-04 17:47:01 +00006320 SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos;
Bob Wilson24e338e2009-03-02 23:24:16 +00006321 else
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006322 return false;
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006323 }
6324
Bob Wilson24e338e2009-03-02 23:24:16 +00006325 // The build_vector is all constants or undefs. Find the smallest element
6326 // size that splats the vector.
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006327
Bob Wilson24e338e2009-03-02 23:24:16 +00006328 HasAnyUndefs = (SplatUndef != 0);
6329 while (sz > 8) {
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006330
Bob Wilson24e338e2009-03-02 23:24:16 +00006331 unsigned HalfSize = sz / 2;
Jay Foad40f8f622010-12-07 08:25:19 +00006332 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
6333 APInt LowValue = SplatValue.trunc(HalfSize);
6334 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
6335 APInt LowUndef = SplatUndef.trunc(HalfSize);
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006336
Bob Wilson24e338e2009-03-02 23:24:16 +00006337 // If the two halves do not match (ignoring undef bits), stop here.
6338 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
6339 MinSplatBits > HalfSize)
6340 break;
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006341
Bob Wilson24e338e2009-03-02 23:24:16 +00006342 SplatValue = HighValue | LowValue;
6343 SplatUndef = HighUndef & LowUndef;
Eric Christopher4d7c18c2009-08-22 00:40:45 +00006344
Bob Wilson24e338e2009-03-02 23:24:16 +00006345 sz = HalfSize;
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006346 }
6347
Bob Wilson24e338e2009-03-02 23:24:16 +00006348 SplatBitSize = sz;
Bob Wilsona27ea9e2009-03-01 01:13:55 +00006349 return true;
6350}
Nate Begeman9008ca62009-04-27 18:41:29 +00006351
Owen Andersone50ed302009-08-10 22:56:29 +00006352bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
Nate Begeman5a5ca152009-04-29 05:20:52 +00006353 // Find the first non-undef value in the shuffle mask.
6354 unsigned i, e;
6355 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
6356 /* search */;
6357
Nate Begemana6415752009-04-29 18:13:31 +00006358 assert(i != e && "VECTOR_SHUFFLE node with all undef indices!");
Eric Christopher4d7c18c2009-08-22 00:40:45 +00006359
Nate Begeman5a5ca152009-04-29 05:20:52 +00006360 // Make sure all remaining elements are either undef or the same as the first
6361 // non-undef value.
6362 for (int Idx = Mask[i]; i != e; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00006363 if (Mask[i] >= 0 && Mask[i] != Idx)
6364 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00006365 return true;
6366}
David Greenecf495bc2010-01-20 20:13:31 +00006367
Chris Lattner46ca5ef2010-02-24 21:34:04 +00006368#ifdef XDEBUG
David Greenecf495bc2010-01-20 20:13:31 +00006369static void checkForCyclesHelper(const SDNode *N,
Chris Lattner46ca5ef2010-02-24 21:34:04 +00006370 SmallPtrSet<const SDNode*, 32> &Visited,
6371 SmallPtrSet<const SDNode*, 32> &Checked) {
6372 // If this node has already been checked, don't check it again.
6373 if (Checked.count(N))
David Greenee3d97c72010-02-23 17:37:50 +00006374 return;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006375
Chris Lattner46ca5ef2010-02-24 21:34:04 +00006376 // If a node has already been visited on this depth-first walk, reject it as
6377 // a cycle.
6378 if (!Visited.insert(N)) {
David Greenecf495bc2010-01-20 20:13:31 +00006379 dbgs() << "Offending node:\n";
6380 N->dumprFull();
Chris Lattner46ca5ef2010-02-24 21:34:04 +00006381 errs() << "Detected cycle in SelectionDAG\n";
6382 abort();
David Greenecf495bc2010-01-20 20:13:31 +00006383 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006384
Chris Lattner46ca5ef2010-02-24 21:34:04 +00006385 for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
6386 checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006387
Chris Lattner46ca5ef2010-02-24 21:34:04 +00006388 Checked.insert(N);
6389 Visited.erase(N);
David Greenecf495bc2010-01-20 20:13:31 +00006390}
Chris Lattner46ca5ef2010-02-24 21:34:04 +00006391#endif
David Greenecf495bc2010-01-20 20:13:31 +00006392
6393void llvm::checkForCycles(const llvm::SDNode *N) {
6394#ifdef XDEBUG
6395 assert(N && "Checking nonexistant SDNode");
Chris Lattner46ca5ef2010-02-24 21:34:04 +00006396 SmallPtrSet<const SDNode*, 32> visited;
6397 SmallPtrSet<const SDNode*, 32> checked;
David Greenee3d97c72010-02-23 17:37:50 +00006398 checkForCyclesHelper(N, visited, checked);
David Greenecf495bc2010-01-20 20:13:31 +00006399#endif
6400}
6401
6402void llvm::checkForCycles(const llvm::SelectionDAG *DAG) {
6403 checkForCycles(DAG->getRoot().getNode());
6404}