blob: 0203353ec4b6c72a4d47715be3d29992f552f770 [file] [log] [blame]
Chris Lattner7a125372005-11-16 22:59:19 +00001//===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===//
Chris Lattnerc961eea2005-11-16 01:54:32 +00002//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner4ee451d2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Chris Lattnerc961eea2005-11-16 01:54:32 +00007//
8//===----------------------------------------------------------------------===//
9//
10// This file defines a DAG pattern matching instruction selector for X86,
11// converting from a legalized dag to a X86 dag.
12//
13//===----------------------------------------------------------------------===//
14
Evan Cheng2ef88a02006-08-07 22:28:20 +000015#define DEBUG_TYPE "x86-isel"
Chris Lattnerc961eea2005-11-16 01:54:32 +000016#include "X86.h"
Evan Cheng8700e142006-01-11 06:09:51 +000017#include "X86InstrBuilder.h"
Evan Chengc4c62572006-03-13 23:20:37 +000018#include "X86ISelLowering.h"
Evan Cheng0475ab52008-01-05 00:41:47 +000019#include "X86MachineFunctionInfo.h"
Chris Lattner92cb0af2006-01-11 01:15:34 +000020#include "X86RegisterInfo.h"
Chris Lattnerc961eea2005-11-16 01:54:32 +000021#include "X86Subtarget.h"
Evan Chengc4c62572006-03-13 23:20:37 +000022#include "X86TargetMachine.h"
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +000023#include "llvm/GlobalValue.h"
Chris Lattner92cb0af2006-01-11 01:15:34 +000024#include "llvm/Instructions.h"
Chris Lattner420736d2006-03-25 06:47:10 +000025#include "llvm/Intrinsics.h"
Chris Lattner92cb0af2006-01-11 01:15:34 +000026#include "llvm/Support/CFG.h"
Reid Spencer7aa8a452007-01-12 23:22:14 +000027#include "llvm/Type.h"
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +000028#include "llvm/CodeGen/MachineConstantPool.h"
Chris Lattnerc961eea2005-11-16 01:54:32 +000029#include "llvm/CodeGen/MachineFunction.h"
Evan Chengaaca22c2006-01-10 20:26:56 +000030#include "llvm/CodeGen/MachineFrameInfo.h"
Chris Lattner92cb0af2006-01-11 01:15:34 +000031#include "llvm/CodeGen/MachineInstrBuilder.h"
Chris Lattner84bc5422007-12-31 04:13:23 +000032#include "llvm/CodeGen/MachineRegisterInfo.h"
Chris Lattnerc961eea2005-11-16 01:54:32 +000033#include "llvm/CodeGen/SelectionDAGISel.h"
34#include "llvm/Target/TargetMachine.h"
Evan Chengb7a75a52008-09-26 23:41:32 +000035#include "llvm/Target/TargetOptions.h"
Chris Lattnera4f0b3a2006-08-27 12:54:02 +000036#include "llvm/Support/Compiler.h"
Evan Cheng25ab6902006-09-08 06:48:29 +000037#include "llvm/Support/Debug.h"
38#include "llvm/Support/MathExtras.h"
Dale Johannesen50dd1d02008-08-11 23:46:25 +000039#include "llvm/Support/Streams.h"
Evan Chengcdda25d2008-04-25 08:22:20 +000040#include "llvm/ADT/SmallPtrSet.h"
Chris Lattnerc961eea2005-11-16 01:54:32 +000041#include "llvm/ADT/Statistic.h"
42using namespace llvm;
43
Evan Cheng4d952322009-03-31 01:13:53 +000044#include "llvm/Support/CommandLine.h"
45static cl::opt<bool> AvoidDupAddrCompute("x86-avoid-dup-address", cl::Hidden);
46
Chris Lattner95b2c7d2006-12-19 22:59:26 +000047STATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor");
48
Chris Lattnerc961eea2005-11-16 01:54:32 +000049//===----------------------------------------------------------------------===//
50// Pattern Matcher Implementation
51//===----------------------------------------------------------------------===//
52
53namespace {
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +000054 /// X86ISelAddressMode - This corresponds to X86AddressMode, but uses
Dan Gohman475871a2008-07-27 21:46:04 +000055 /// SDValue's instead of register numbers for the leaves of the matched
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +000056 /// tree.
57 struct X86ISelAddressMode {
58 enum {
59 RegBase,
Chris Lattnerd74ea2b2006-05-24 17:04:05 +000060 FrameIndexBase
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +000061 } BaseType;
62
63 struct { // This is really a union, discriminated by BaseType!
Dan Gohman475871a2008-07-27 21:46:04 +000064 SDValue Reg;
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +000065 int FrameIndex;
66 } Base;
67
Evan Chengbe3bf422008-02-07 08:53:49 +000068 bool isRIPRel; // RIP as base?
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +000069 unsigned Scale;
Dan Gohman475871a2008-07-27 21:46:04 +000070 SDValue IndexReg;
Dan Gohman27cae7b2008-11-11 15:52:29 +000071 int32_t Disp;
Rafael Espindola094fad32009-04-08 21:14:34 +000072 SDValue Segment;
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +000073 GlobalValue *GV;
Evan Cheng51a9ed92006-02-25 10:09:08 +000074 Constant *CP;
Evan Cheng25ab6902006-09-08 06:48:29 +000075 const char *ES;
76 int JT;
Evan Cheng51a9ed92006-02-25 10:09:08 +000077 unsigned Align; // CP alignment.
Chris Lattnerb8afeb92009-06-26 05:51:45 +000078 unsigned char SymbolFlags; // X86II::MO_*
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +000079
80 X86ISelAddressMode()
Evan Cheng25ab6902006-09-08 06:48:29 +000081 : BaseType(RegBase), isRIPRel(false), Scale(1), IndexReg(), Disp(0),
Chris Lattnerb8afeb92009-06-26 05:51:45 +000082 Segment(), GV(0), CP(0), ES(0), JT(-1), Align(0), SymbolFlags(0) {
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +000083 }
Dan Gohman2d0a1cc2009-02-07 00:43:41 +000084
85 bool hasSymbolicDisplacement() const {
86 return GV != 0 || CP != 0 || ES != 0 || JT != -1;
87 }
88
Dale Johannesen50dd1d02008-08-11 23:46:25 +000089 void dump() {
90 cerr << "X86ISelAddressMode " << this << "\n";
Gabor Greif93c53e52008-08-31 15:37:04 +000091 cerr << "Base.Reg ";
92 if (Base.Reg.getNode() != 0) Base.Reg.getNode()->dump();
93 else cerr << "nul";
Dale Johannesen50dd1d02008-08-11 23:46:25 +000094 cerr << " Base.FrameIndex " << Base.FrameIndex << "\n";
95 cerr << "isRIPRel " << isRIPRel << " Scale" << Scale << "\n";
Gabor Greif93c53e52008-08-31 15:37:04 +000096 cerr << "IndexReg ";
97 if (IndexReg.getNode() != 0) IndexReg.getNode()->dump();
98 else cerr << "nul";
Dale Johannesen50dd1d02008-08-11 23:46:25 +000099 cerr << " Disp " << Disp << "\n";
100 cerr << "GV "; if (GV) GV->dump();
101 else cerr << "nul";
102 cerr << " CP "; if (CP) CP->dump();
103 else cerr << "nul";
104 cerr << "\n";
105 cerr << "ES "; if (ES) cerr << ES; else cerr << "nul";
106 cerr << " JT" << JT << " Align" << Align << "\n";
107 }
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000108 };
109}
110
111namespace {
Chris Lattnerc961eea2005-11-16 01:54:32 +0000112 //===--------------------------------------------------------------------===//
113 /// ISel - X86 specific code to select X86 machine instructions for
114 /// SelectionDAG operations.
115 ///
Chris Lattner2c79de82006-06-28 23:27:49 +0000116 class VISIBILITY_HIDDEN X86DAGToDAGISel : public SelectionDAGISel {
Chris Lattnerc961eea2005-11-16 01:54:32 +0000117 /// X86Lowering - This object fully describes how to lower LLVM code to an
118 /// X86-specific SelectionDAG.
Dan Gohmanda8ac5f2008-10-03 16:55:19 +0000119 X86TargetLowering &X86Lowering;
Chris Lattnerc961eea2005-11-16 01:54:32 +0000120
121 /// Subtarget - Keep a pointer to the X86Subtarget around so that we can
122 /// make the right decision when generating code for different targets.
123 const X86Subtarget *Subtarget;
Evan Cheng7ccced62006-02-18 00:15:05 +0000124
Evan Chengdb8d56b2008-06-30 20:45:06 +0000125 /// CurBB - Current BB being isel'd.
126 ///
127 MachineBasicBlock *CurBB;
128
Evan Chengb7a75a52008-09-26 23:41:32 +0000129 /// OptForSize - If true, selector should try to optimize for code size
130 /// instead of performance.
131 bool OptForSize;
132
Chris Lattnerc961eea2005-11-16 01:54:32 +0000133 public:
Bill Wendling98a366d2009-04-29 23:29:43 +0000134 explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOpt::Level OptLevel)
Bill Wendlingbe8cc2a2009-04-29 00:15:41 +0000135 : SelectionDAGISel(tm, OptLevel),
Dan Gohmanc5534622009-06-03 20:20:00 +0000136 X86Lowering(*tm.getTargetLowering()),
137 Subtarget(&tm.getSubtarget<X86Subtarget>()),
Devang Patel4ae641f2008-10-01 23:18:38 +0000138 OptForSize(false) {}
Chris Lattnerc961eea2005-11-16 01:54:32 +0000139
140 virtual const char *getPassName() const {
141 return "X86 DAG->DAG Instruction Selection";
142 }
143
Evan Chengdb8d56b2008-06-30 20:45:06 +0000144 /// InstructionSelect - This callback is invoked by
Chris Lattnerc961eea2005-11-16 01:54:32 +0000145 /// SelectionDAGISel when it has created a SelectionDAG for us to codegen.
Dan Gohmanf350b272008-08-23 02:25:05 +0000146 virtual void InstructionSelect();
Evan Chengdb8d56b2008-06-30 20:45:06 +0000147
Anton Korobeynikov2fe12592007-09-25 21:52:30 +0000148 virtual void EmitFunctionEntryCode(Function &Fn, MachineFunction &MF);
149
Evan Cheng884c70c2008-11-27 00:49:46 +0000150 virtual
151 bool IsLegalAndProfitableToFold(SDNode *N, SDNode *U, SDNode *Root) const;
Evan Chenga8df1b42006-07-27 16:44:36 +0000152
Chris Lattnerc961eea2005-11-16 01:54:32 +0000153// Include the pieces autogenerated from the target description.
154#include "X86GenDAGISel.inc"
155
156 private:
Dan Gohman475871a2008-07-27 21:46:04 +0000157 SDNode *Select(SDValue N);
Dale Johannesen48c1bc22008-10-02 18:53:47 +0000158 SDNode *SelectAtomic64(SDNode *Node, unsigned Opc);
Chris Lattnerc961eea2005-11-16 01:54:32 +0000159
Rafael Espindola094fad32009-04-08 21:14:34 +0000160 bool MatchSegmentBaseAddress(SDValue N, X86ISelAddressMode &AM);
161 bool MatchLoad(SDValue N, X86ISelAddressMode &AM);
Rafael Espindola49a168d2009-04-12 21:55:03 +0000162 bool MatchWrapper(SDValue N, X86ISelAddressMode &AM);
Dan Gohman475871a2008-07-27 21:46:04 +0000163 bool MatchAddress(SDValue N, X86ISelAddressMode &AM,
Rafael Espindola523249f2009-03-31 16:16:57 +0000164 unsigned Depth = 0);
165 bool MatchAddressBase(SDValue N, X86ISelAddressMode &AM);
Dan Gohman475871a2008-07-27 21:46:04 +0000166 bool SelectAddr(SDValue Op, SDValue N, SDValue &Base,
Rafael Espindola094fad32009-04-08 21:14:34 +0000167 SDValue &Scale, SDValue &Index, SDValue &Disp,
168 SDValue &Segment);
Dan Gohman475871a2008-07-27 21:46:04 +0000169 bool SelectLEAAddr(SDValue Op, SDValue N, SDValue &Base,
170 SDValue &Scale, SDValue &Index, SDValue &Disp);
Chris Lattner5c0b16d2009-06-20 20:38:48 +0000171 bool SelectTLSADDRAddr(SDValue Op, SDValue N, SDValue &Base,
172 SDValue &Scale, SDValue &Index, SDValue &Disp);
Dan Gohman475871a2008-07-27 21:46:04 +0000173 bool SelectScalarSSELoad(SDValue Op, SDValue Pred,
174 SDValue N, SDValue &Base, SDValue &Scale,
175 SDValue &Index, SDValue &Disp,
Rafael Espindola094fad32009-04-08 21:14:34 +0000176 SDValue &Segment,
Dan Gohman475871a2008-07-27 21:46:04 +0000177 SDValue &InChain, SDValue &OutChain);
178 bool TryFoldLoad(SDValue P, SDValue N,
179 SDValue &Base, SDValue &Scale,
Rafael Espindola094fad32009-04-08 21:14:34 +0000180 SDValue &Index, SDValue &Disp,
181 SDValue &Segment);
Dan Gohmanf350b272008-08-23 02:25:05 +0000182 void PreprocessForRMW();
183 void PreprocessForFPConvert();
Evan Cheng2ef88a02006-08-07 22:28:20 +0000184
Chris Lattnerc0bad572006-06-08 18:03:49 +0000185 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
186 /// inline asm expressions.
Dan Gohman475871a2008-07-27 21:46:04 +0000187 virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
Chris Lattnerc0bad572006-06-08 18:03:49 +0000188 char ConstraintCode,
Dan Gohmanf350b272008-08-23 02:25:05 +0000189 std::vector<SDValue> &OutOps);
Chris Lattnerc0bad572006-06-08 18:03:49 +0000190
Anton Korobeynikov2fe12592007-09-25 21:52:30 +0000191 void EmitSpecialCodeForMain(MachineBasicBlock *BB, MachineFrameInfo *MFI);
192
Dan Gohman475871a2008-07-27 21:46:04 +0000193 inline void getAddressOperands(X86ISelAddressMode &AM, SDValue &Base,
194 SDValue &Scale, SDValue &Index,
Rafael Espindola094fad32009-04-08 21:14:34 +0000195 SDValue &Disp, SDValue &Segment) {
Evan Chenge5280532005-12-12 21:49:40 +0000196 Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase) ?
Evan Cheng25ab6902006-09-08 06:48:29 +0000197 CurDAG->getTargetFrameIndex(AM.Base.FrameIndex, TLI.getPointerTy()) :
198 AM.Base.Reg;
Evan Chengbdce7b42005-12-17 09:13:43 +0000199 Scale = getI8Imm(AM.Scale);
Evan Chenge5280532005-12-12 21:49:40 +0000200 Index = AM.IndexReg;
Evan Cheng25ab6902006-09-08 06:48:29 +0000201 // These are 32-bit even in 64-bit mode since RIP relative offset
202 // is 32-bit.
203 if (AM.GV)
Chris Lattnerb8afeb92009-06-26 05:51:45 +0000204 Disp = CurDAG->getTargetGlobalAddress(AM.GV, MVT::i32, AM.Disp,
205 AM.SymbolFlags);
Evan Cheng25ab6902006-09-08 06:48:29 +0000206 else if (AM.CP)
Gabor Greif93c53e52008-08-31 15:37:04 +0000207 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32,
Chris Lattnerb8afeb92009-06-26 05:51:45 +0000208 AM.Align, AM.Disp, AM.SymbolFlags);
Evan Cheng25ab6902006-09-08 06:48:29 +0000209 else if (AM.ES)
Chris Lattnerb8afeb92009-06-26 05:51:45 +0000210 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
Evan Cheng25ab6902006-09-08 06:48:29 +0000211 else if (AM.JT != -1)
Chris Lattnerb8afeb92009-06-26 05:51:45 +0000212 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
Evan Cheng25ab6902006-09-08 06:48:29 +0000213 else
Dan Gohman27cae7b2008-11-11 15:52:29 +0000214 Disp = CurDAG->getTargetConstant(AM.Disp, MVT::i32);
Rafael Espindola094fad32009-04-08 21:14:34 +0000215
216 if (AM.Segment.getNode())
217 Segment = AM.Segment;
218 else
219 Segment = CurDAG->getRegister(0, MVT::i32);
Evan Chenge5280532005-12-12 21:49:40 +0000220 }
221
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000222 /// getI8Imm - Return a target constant with the specified value, of type
223 /// i8.
Dan Gohman475871a2008-07-27 21:46:04 +0000224 inline SDValue getI8Imm(unsigned Imm) {
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000225 return CurDAG->getTargetConstant(Imm, MVT::i8);
226 }
227
Chris Lattnerc961eea2005-11-16 01:54:32 +0000228 /// getI16Imm - Return a target constant with the specified value, of type
229 /// i16.
Dan Gohman475871a2008-07-27 21:46:04 +0000230 inline SDValue getI16Imm(unsigned Imm) {
Chris Lattnerc961eea2005-11-16 01:54:32 +0000231 return CurDAG->getTargetConstant(Imm, MVT::i16);
232 }
233
234 /// getI32Imm - Return a target constant with the specified value, of type
235 /// i32.
Dan Gohman475871a2008-07-27 21:46:04 +0000236 inline SDValue getI32Imm(unsigned Imm) {
Chris Lattnerc961eea2005-11-16 01:54:32 +0000237 return CurDAG->getTargetConstant(Imm, MVT::i32);
238 }
Evan Chengf597dc72006-02-10 22:24:32 +0000239
Dan Gohman8b746962008-09-23 18:22:58 +0000240 /// getGlobalBaseReg - Return an SDNode that returns the value of
241 /// the global base register. Output instructions required to
242 /// initialize the global base register, if necessary.
243 ///
Evan Cheng9ade2182006-08-26 05:34:46 +0000244 SDNode *getGlobalBaseReg();
Evan Cheng7ccced62006-02-18 00:15:05 +0000245
Dan Gohmanc5534622009-06-03 20:20:00 +0000246 /// getTargetMachine - Return a reference to the TargetMachine, casted
247 /// to the target-specific type.
248 const X86TargetMachine &getTargetMachine() {
249 return static_cast<const X86TargetMachine &>(TM);
250 }
251
252 /// getInstrInfo - Return a reference to the TargetInstrInfo, casted
253 /// to the target-specific type.
254 const X86InstrInfo *getInstrInfo() {
255 return getTargetMachine().getInstrInfo();
256 }
257
Evan Cheng23addc02006-02-10 22:46:26 +0000258#ifndef NDEBUG
259 unsigned Indent;
260#endif
Chris Lattnerc961eea2005-11-16 01:54:32 +0000261 };
262}
263
Evan Chengf4b4c412006-08-08 00:31:00 +0000264
Evan Cheng884c70c2008-11-27 00:49:46 +0000265bool X86DAGToDAGISel::IsLegalAndProfitableToFold(SDNode *N, SDNode *U,
266 SDNode *Root) const {
Bill Wendling98a366d2009-04-29 23:29:43 +0000267 if (OptLevel == CodeGenOpt::None) return false;
Evan Cheng27e1fe92006-10-14 08:33:25 +0000268
Evan Cheng884c70c2008-11-27 00:49:46 +0000269 if (U == Root)
270 switch (U->getOpcode()) {
271 default: break;
272 case ISD::ADD:
273 case ISD::ADDC:
274 case ISD::ADDE:
275 case ISD::AND:
276 case ISD::OR:
277 case ISD::XOR: {
Rafael Espindoladbcfb302009-04-10 10:09:34 +0000278 SDValue Op1 = U->getOperand(1);
279
Evan Cheng884c70c2008-11-27 00:49:46 +0000280 // If the other operand is a 8-bit immediate we should fold the immediate
281 // instead. This reduces code size.
282 // e.g.
283 // movl 4(%esp), %eax
284 // addl $4, %eax
285 // vs.
286 // movl $4, %eax
287 // addl 4(%esp), %eax
288 // The former is 2 bytes shorter. In case where the increment is 1, then
289 // the saving can be 4 bytes (by using incl %eax).
Rafael Espindoladbcfb302009-04-10 10:09:34 +0000290 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Op1))
Dan Gohman9a49d312009-03-14 02:07:16 +0000291 if (Imm->getAPIntValue().isSignedIntN(8))
292 return false;
Rafael Espindoladbcfb302009-04-10 10:09:34 +0000293
294 // If the other operand is a TLS address, we should fold it instead.
295 // This produces
296 // movl %gs:0, %eax
297 // leal i@NTPOFF(%eax), %eax
298 // instead of
299 // movl $i@NTPOFF, %eax
300 // addl %gs:0, %eax
301 // if the block also has an access to a second TLS address this will save
302 // a load.
303 // FIXME: This is probably also true for non TLS addresses.
304 if (Op1.getOpcode() == X86ISD::Wrapper) {
305 SDValue Val = Op1.getOperand(0);
306 if (Val.getOpcode() == ISD::TargetGlobalTLSAddress)
307 return false;
308 }
Evan Cheng884c70c2008-11-27 00:49:46 +0000309 }
310 }
311
Anton Korobeynikovc1c6ef82009-05-08 18:51:58 +0000312 // Proceed to 'generic' cycle finder code
313 return SelectionDAGISel::IsLegalAndProfitableToFold(N, U, Root);
Evan Chenga8df1b42006-07-27 16:44:36 +0000314}
315
Evan Cheng70e674e2006-08-28 20:10:17 +0000316/// MoveBelowTokenFactor - Replace TokenFactor operand with load's chain operand
317/// and move load below the TokenFactor. Replace store's chain operand with
318/// load's chain result.
Dan Gohmanf350b272008-08-23 02:25:05 +0000319static void MoveBelowTokenFactor(SelectionDAG *CurDAG, SDValue Load,
Dan Gohman475871a2008-07-27 21:46:04 +0000320 SDValue Store, SDValue TF) {
Evan Chengab6c3bb2008-08-25 21:27:18 +0000321 SmallVector<SDValue, 4> Ops;
Gabor Greifba36cb52008-08-28 21:40:38 +0000322 for (unsigned i = 0, e = TF.getNode()->getNumOperands(); i != e; ++i)
323 if (Load.getNode() == TF.getOperand(i).getNode())
Evan Chengab6c3bb2008-08-25 21:27:18 +0000324 Ops.push_back(Load.getOperand(0));
Evan Cheng70e674e2006-08-28 20:10:17 +0000325 else
Evan Chengab6c3bb2008-08-25 21:27:18 +0000326 Ops.push_back(TF.getOperand(i));
Dan Gohmanf350b272008-08-23 02:25:05 +0000327 CurDAG->UpdateNodeOperands(TF, &Ops[0], Ops.size());
328 CurDAG->UpdateNodeOperands(Load, TF, Load.getOperand(1), Load.getOperand(2));
329 CurDAG->UpdateNodeOperands(Store, Load.getValue(1), Store.getOperand(1),
330 Store.getOperand(2), Store.getOperand(3));
Evan Cheng70e674e2006-08-28 20:10:17 +0000331}
332
Evan Chengcd0baf22008-05-23 21:23:16 +0000333/// isRMWLoad - Return true if N is a load that's part of RMW sub-DAG.
334///
Dan Gohman475871a2008-07-27 21:46:04 +0000335static bool isRMWLoad(SDValue N, SDValue Chain, SDValue Address,
336 SDValue &Load) {
Evan Chengcd0baf22008-05-23 21:23:16 +0000337 if (N.getOpcode() == ISD::BIT_CONVERT)
338 N = N.getOperand(0);
339
340 LoadSDNode *LD = dyn_cast<LoadSDNode>(N);
341 if (!LD || LD->isVolatile())
342 return false;
343 if (LD->getAddressingMode() != ISD::UNINDEXED)
344 return false;
345
346 ISD::LoadExtType ExtType = LD->getExtensionType();
347 if (ExtType != ISD::NON_EXTLOAD && ExtType != ISD::EXTLOAD)
348 return false;
349
350 if (N.hasOneUse() &&
351 N.getOperand(1) == Address &&
Gabor Greifba36cb52008-08-28 21:40:38 +0000352 N.getNode()->isOperandOf(Chain.getNode())) {
Evan Chengcd0baf22008-05-23 21:23:16 +0000353 Load = N;
354 return true;
355 }
356 return false;
357}
358
Evan Chengab6c3bb2008-08-25 21:27:18 +0000359/// MoveBelowCallSeqStart - Replace CALLSEQ_START operand with load's chain
360/// operand and move load below the call's chain operand.
361static void MoveBelowCallSeqStart(SelectionDAG *CurDAG, SDValue Load,
Evan Cheng5b2e5892009-01-26 18:43:34 +0000362 SDValue Call, SDValue CallSeqStart) {
Evan Chengab6c3bb2008-08-25 21:27:18 +0000363 SmallVector<SDValue, 8> Ops;
Evan Cheng5b2e5892009-01-26 18:43:34 +0000364 SDValue Chain = CallSeqStart.getOperand(0);
365 if (Chain.getNode() == Load.getNode())
366 Ops.push_back(Load.getOperand(0));
367 else {
368 assert(Chain.getOpcode() == ISD::TokenFactor &&
369 "Unexpected CallSeqStart chain operand");
370 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i)
371 if (Chain.getOperand(i).getNode() == Load.getNode())
372 Ops.push_back(Load.getOperand(0));
373 else
374 Ops.push_back(Chain.getOperand(i));
375 SDValue NewChain =
Dale Johannesened2eee62009-02-06 01:31:28 +0000376 CurDAG->getNode(ISD::TokenFactor, Load.getDebugLoc(),
377 MVT::Other, &Ops[0], Ops.size());
Evan Cheng5b2e5892009-01-26 18:43:34 +0000378 Ops.clear();
379 Ops.push_back(NewChain);
380 }
381 for (unsigned i = 1, e = CallSeqStart.getNumOperands(); i != e; ++i)
382 Ops.push_back(CallSeqStart.getOperand(i));
383 CurDAG->UpdateNodeOperands(CallSeqStart, &Ops[0], Ops.size());
Evan Chengab6c3bb2008-08-25 21:27:18 +0000384 CurDAG->UpdateNodeOperands(Load, Call.getOperand(0),
385 Load.getOperand(1), Load.getOperand(2));
386 Ops.clear();
Gabor Greifba36cb52008-08-28 21:40:38 +0000387 Ops.push_back(SDValue(Load.getNode(), 1));
388 for (unsigned i = 1, e = Call.getNode()->getNumOperands(); i != e; ++i)
Evan Chengab6c3bb2008-08-25 21:27:18 +0000389 Ops.push_back(Call.getOperand(i));
390 CurDAG->UpdateNodeOperands(Call, &Ops[0], Ops.size());
391}
392
393/// isCalleeLoad - Return true if call address is a load and it can be
394/// moved below CALLSEQ_START and the chains leading up to the call.
395/// Return the CALLSEQ_START by reference as a second output.
396static bool isCalleeLoad(SDValue Callee, SDValue &Chain) {
Gabor Greifba36cb52008-08-28 21:40:38 +0000397 if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse())
Evan Chengab6c3bb2008-08-25 21:27:18 +0000398 return false;
Gabor Greifba36cb52008-08-28 21:40:38 +0000399 LoadSDNode *LD = dyn_cast<LoadSDNode>(Callee.getNode());
Evan Chengab6c3bb2008-08-25 21:27:18 +0000400 if (!LD ||
401 LD->isVolatile() ||
402 LD->getAddressingMode() != ISD::UNINDEXED ||
403 LD->getExtensionType() != ISD::NON_EXTLOAD)
404 return false;
405
406 // Now let's find the callseq_start.
407 while (Chain.getOpcode() != ISD::CALLSEQ_START) {
408 if (!Chain.hasOneUse())
409 return false;
410 Chain = Chain.getOperand(0);
411 }
Evan Cheng5b2e5892009-01-26 18:43:34 +0000412
413 if (Chain.getOperand(0).getNode() == Callee.getNode())
414 return true;
415 if (Chain.getOperand(0).getOpcode() == ISD::TokenFactor &&
416 Callee.getValue(1).isOperandOf(Chain.getOperand(0).getNode()))
417 return true;
418 return false;
Evan Chengab6c3bb2008-08-25 21:27:18 +0000419}
420
421
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000422/// PreprocessForRMW - Preprocess the DAG to make instruction selection better.
Bill Wendlingbe8cc2a2009-04-29 00:15:41 +0000423/// This is only run if not in -O0 mode.
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000424/// This allows the instruction selector to pick more read-modify-write
425/// instructions. This is a common case:
Evan Cheng70e674e2006-08-28 20:10:17 +0000426///
427/// [Load chain]
428/// ^
429/// |
430/// [Load]
431/// ^ ^
432/// | |
433/// / \-
434/// / |
435/// [TokenFactor] [Op]
436/// ^ ^
437/// | |
438/// \ /
439/// \ /
440/// [Store]
441///
442/// The fact the store's chain operand != load's chain will prevent the
443/// (store (op (load))) instruction from being selected. We can transform it to:
444///
445/// [Load chain]
446/// ^
447/// |
448/// [TokenFactor]
449/// ^
450/// |
451/// [Load]
452/// ^ ^
453/// | |
454/// | \-
455/// | |
456/// | [Op]
457/// | ^
458/// | |
459/// \ /
460/// \ /
461/// [Store]
Dan Gohmanf350b272008-08-23 02:25:05 +0000462void X86DAGToDAGISel::PreprocessForRMW() {
463 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
464 E = CurDAG->allnodes_end(); I != E; ++I) {
Evan Chengab6c3bb2008-08-25 21:27:18 +0000465 if (I->getOpcode() == X86ISD::CALL) {
466 /// Also try moving call address load from outside callseq_start to just
467 /// before the call to allow it to be folded.
468 ///
469 /// [Load chain]
470 /// ^
471 /// |
472 /// [Load]
473 /// ^ ^
474 /// | |
475 /// / \--
476 /// / |
477 ///[CALLSEQ_START] |
478 /// ^ |
479 /// | |
480 /// [LOAD/C2Reg] |
481 /// | |
482 /// \ /
483 /// \ /
484 /// [CALL]
485 SDValue Chain = I->getOperand(0);
486 SDValue Load = I->getOperand(1);
487 if (!isCalleeLoad(Load, Chain))
488 continue;
489 MoveBelowCallSeqStart(CurDAG, Load, SDValue(I, 0), Chain);
490 ++NumLoadMoved;
491 continue;
492 }
493
Evan Cheng8b2794a2006-10-13 21:14:26 +0000494 if (!ISD::isNON_TRUNCStore(I))
Evan Cheng70e674e2006-08-28 20:10:17 +0000495 continue;
Dan Gohman475871a2008-07-27 21:46:04 +0000496 SDValue Chain = I->getOperand(0);
Evan Chengab6c3bb2008-08-25 21:27:18 +0000497
Gabor Greifba36cb52008-08-28 21:40:38 +0000498 if (Chain.getNode()->getOpcode() != ISD::TokenFactor)
Evan Cheng70e674e2006-08-28 20:10:17 +0000499 continue;
500
Dan Gohman475871a2008-07-27 21:46:04 +0000501 SDValue N1 = I->getOperand(1);
502 SDValue N2 = I->getOperand(2);
Duncan Sands83ec4b62008-06-06 12:08:01 +0000503 if ((N1.getValueType().isFloatingPoint() &&
504 !N1.getValueType().isVector()) ||
Evan Cheng780413d2006-08-29 18:37:37 +0000505 !N1.hasOneUse())
Evan Cheng70e674e2006-08-28 20:10:17 +0000506 continue;
507
508 bool RModW = false;
Dan Gohman475871a2008-07-27 21:46:04 +0000509 SDValue Load;
Gabor Greifba36cb52008-08-28 21:40:38 +0000510 unsigned Opcode = N1.getNode()->getOpcode();
Evan Cheng70e674e2006-08-28 20:10:17 +0000511 switch (Opcode) {
Evan Chengab6c3bb2008-08-25 21:27:18 +0000512 case ISD::ADD:
513 case ISD::MUL:
514 case ISD::AND:
515 case ISD::OR:
516 case ISD::XOR:
517 case ISD::ADDC:
518 case ISD::ADDE:
519 case ISD::VECTOR_SHUFFLE: {
520 SDValue N10 = N1.getOperand(0);
521 SDValue N11 = N1.getOperand(1);
522 RModW = isRMWLoad(N10, Chain, N2, Load);
523 if (!RModW)
524 RModW = isRMWLoad(N11, Chain, N2, Load);
525 break;
526 }
527 case ISD::SUB:
528 case ISD::SHL:
529 case ISD::SRA:
530 case ISD::SRL:
531 case ISD::ROTL:
532 case ISD::ROTR:
533 case ISD::SUBC:
534 case ISD::SUBE:
535 case X86ISD::SHLD:
536 case X86ISD::SHRD: {
537 SDValue N10 = N1.getOperand(0);
538 RModW = isRMWLoad(N10, Chain, N2, Load);
539 break;
540 }
Evan Cheng70e674e2006-08-28 20:10:17 +0000541 }
542
Evan Cheng82a35b32006-08-29 06:44:17 +0000543 if (RModW) {
Dan Gohmanf350b272008-08-23 02:25:05 +0000544 MoveBelowTokenFactor(CurDAG, Load, SDValue(I, 0), Chain);
Evan Cheng82a35b32006-08-29 06:44:17 +0000545 ++NumLoadMoved;
546 }
Evan Cheng70e674e2006-08-28 20:10:17 +0000547 }
548}
549
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000550
551/// PreprocessForFPConvert - Walk over the dag lowering fpround and fpextend
552/// nodes that target the FP stack to be store and load to the stack. This is a
553/// gross hack. We would like to simply mark these as being illegal, but when
554/// we do that, legalize produces these when it expands calls, then expands
555/// these in the same legalize pass. We would like dag combine to be able to
556/// hack on these between the call expansion and the node legalization. As such
557/// this pass basically does "really late" legalization of these inline with the
558/// X86 isel pass.
Dan Gohmanf350b272008-08-23 02:25:05 +0000559void X86DAGToDAGISel::PreprocessForFPConvert() {
560 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
561 E = CurDAG->allnodes_end(); I != E; ) {
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000562 SDNode *N = I++; // Preincrement iterator to avoid invalidation issues.
563 if (N->getOpcode() != ISD::FP_ROUND && N->getOpcode() != ISD::FP_EXTEND)
564 continue;
565
566 // If the source and destination are SSE registers, then this is a legal
567 // conversion that should not be lowered.
Duncan Sands83ec4b62008-06-06 12:08:01 +0000568 MVT SrcVT = N->getOperand(0).getValueType();
569 MVT DstVT = N->getValueType(0);
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000570 bool SrcIsSSE = X86Lowering.isScalarFPTypeInSSEReg(SrcVT);
571 bool DstIsSSE = X86Lowering.isScalarFPTypeInSSEReg(DstVT);
572 if (SrcIsSSE && DstIsSSE)
573 continue;
574
Chris Lattner6fa2f9c2008-03-09 07:05:32 +0000575 if (!SrcIsSSE && !DstIsSSE) {
576 // If this is an FPStack extension, it is a noop.
577 if (N->getOpcode() == ISD::FP_EXTEND)
578 continue;
579 // If this is a value-preserving FPStack truncation, it is a noop.
580 if (N->getConstantOperandVal(1))
581 continue;
582 }
583
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000584 // Here we could have an FP stack truncation or an FPStack <-> SSE convert.
585 // FPStack has extload and truncstore. SSE can fold direct loads into other
586 // operations. Based on this, decide what we want to do.
Duncan Sands83ec4b62008-06-06 12:08:01 +0000587 MVT MemVT;
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000588 if (N->getOpcode() == ISD::FP_ROUND)
589 MemVT = DstVT; // FP_ROUND must use DstVT, we can't do a 'trunc load'.
590 else
591 MemVT = SrcIsSSE ? SrcVT : DstVT;
592
Dan Gohmanf350b272008-08-23 02:25:05 +0000593 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
Dale Johannesend8392542009-02-03 21:48:12 +0000594 DebugLoc dl = N->getDebugLoc();
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000595
596 // FIXME: optimize the case where the src/dest is a load or store?
Dale Johannesend8392542009-02-03 21:48:12 +0000597 SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(), dl,
Dan Gohmanf350b272008-08-23 02:25:05 +0000598 N->getOperand(0),
599 MemTmp, NULL, 0, MemVT);
Dale Johannesend8392542009-02-03 21:48:12 +0000600 SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, MemTmp,
Dan Gohmanf350b272008-08-23 02:25:05 +0000601 NULL, 0, MemVT);
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000602
603 // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the
604 // extload we created. This will cause general havok on the dag because
605 // anything below the conversion could be folded into other existing nodes.
606 // To avoid invalidating 'I', back it up to the convert node.
607 --I;
Dan Gohmanf350b272008-08-23 02:25:05 +0000608 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000609
610 // Now that we did that, the node is dead. Increment the iterator to the
611 // next node to process, then delete N.
612 ++I;
Dan Gohmanf350b272008-08-23 02:25:05 +0000613 CurDAG->DeleteNode(N);
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000614 }
615}
616
Chris Lattnerc961eea2005-11-16 01:54:32 +0000617/// InstructionSelectBasicBlock - This callback is invoked by SelectionDAGISel
618/// when it has created a SelectionDAG for us to codegen.
Dan Gohmanf350b272008-08-23 02:25:05 +0000619void X86DAGToDAGISel::InstructionSelect() {
Evan Chengdb8d56b2008-06-30 20:45:06 +0000620 CurBB = BB; // BB can change as result of isel.
Devang Patele76225a2008-10-06 18:03:39 +0000621 const Function *F = CurDAG->getMachineFunction().getFunction();
622 OptForSize = F->hasFnAttr(Attribute::OptimizeForSize);
Chris Lattnerc961eea2005-11-16 01:54:32 +0000623
Evan Chengdb8d56b2008-06-30 20:45:06 +0000624 DEBUG(BB->dump());
Bill Wendling98a366d2009-04-29 23:29:43 +0000625 if (OptLevel != CodeGenOpt::None)
Dan Gohmanf350b272008-08-23 02:25:05 +0000626 PreprocessForRMW();
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000627
Bill Wendlingbe8cc2a2009-04-29 00:15:41 +0000628 // FIXME: This should only happen when not compiled with -O0.
Dan Gohmanf350b272008-08-23 02:25:05 +0000629 PreprocessForFPConvert();
Evan Cheng70e674e2006-08-28 20:10:17 +0000630
Chris Lattnerc961eea2005-11-16 01:54:32 +0000631 // Codegen the basic block.
Evan Chengf597dc72006-02-10 22:24:32 +0000632#ifndef NDEBUG
Bill Wendling6345d752006-11-17 07:52:03 +0000633 DOUT << "===== Instruction selection begins:\n";
Evan Cheng23addc02006-02-10 22:46:26 +0000634 Indent = 0;
Evan Chengf597dc72006-02-10 22:24:32 +0000635#endif
David Greene8ad4c002008-10-27 21:56:29 +0000636 SelectRoot(*CurDAG);
Evan Chengf597dc72006-02-10 22:24:32 +0000637#ifndef NDEBUG
Bill Wendling6345d752006-11-17 07:52:03 +0000638 DOUT << "===== Instruction selection ends:\n";
Evan Chengf597dc72006-02-10 22:24:32 +0000639#endif
Evan Cheng63ce5682006-07-28 00:10:59 +0000640
Dan Gohmanf350b272008-08-23 02:25:05 +0000641 CurDAG->RemoveDeadNodes();
Evan Chengdb8d56b2008-06-30 20:45:06 +0000642}
Chris Lattnerc961eea2005-11-16 01:54:32 +0000643
Anton Korobeynikov2fe12592007-09-25 21:52:30 +0000644/// EmitSpecialCodeForMain - Emit any code that needs to be executed only in
645/// the main function.
646void X86DAGToDAGISel::EmitSpecialCodeForMain(MachineBasicBlock *BB,
647 MachineFrameInfo *MFI) {
648 const TargetInstrInfo *TII = TM.getInstrInfo();
649 if (Subtarget->isTargetCygMing())
Dale Johannesen8d13f8f2009-02-13 02:33:27 +0000650 BuildMI(BB, DebugLoc::getUnknownLoc(),
651 TII->get(X86::CALLpcrel32)).addExternalSymbol("__main");
Anton Korobeynikov2fe12592007-09-25 21:52:30 +0000652}
653
654void X86DAGToDAGISel::EmitFunctionEntryCode(Function &Fn, MachineFunction &MF) {
655 // If this is main, emit special code for main.
656 MachineBasicBlock *BB = MF.begin();
657 if (Fn.hasExternalLinkage() && Fn.getName() == "main")
658 EmitSpecialCodeForMain(BB, MF.getFrameInfo());
659}
660
Rafael Espindola094fad32009-04-08 21:14:34 +0000661
662bool X86DAGToDAGISel::MatchSegmentBaseAddress(SDValue N,
663 X86ISelAddressMode &AM) {
664 assert(N.getOpcode() == X86ISD::SegmentBaseAddress);
665 SDValue Segment = N.getOperand(0);
666
667 if (AM.Segment.getNode() == 0) {
668 AM.Segment = Segment;
669 return false;
670 }
671
672 return true;
673}
674
675bool X86DAGToDAGISel::MatchLoad(SDValue N, X86ISelAddressMode &AM) {
676 // This optimization is valid because the GNU TLS model defines that
677 // gs:0 (or fs:0 on X86-64) contains its own address.
678 // For more information see http://people.redhat.com/drepper/tls.pdf
679
680 SDValue Address = N.getOperand(1);
681 if (Address.getOpcode() == X86ISD::SegmentBaseAddress &&
682 !MatchSegmentBaseAddress (Address, AM))
683 return false;
684
685 return true;
686}
687
Rafael Espindola49a168d2009-04-12 21:55:03 +0000688bool X86DAGToDAGISel::MatchWrapper(SDValue N, X86ISelAddressMode &AM) {
Dan Gohmanc5534622009-06-03 20:20:00 +0000689 bool SymbolicAddressesAreRIPRel =
690 getTargetMachine().symbolicAddressesAreRIPRel();
Rafael Espindola49a168d2009-04-12 21:55:03 +0000691 bool is64Bit = Subtarget->is64Bit();
692 DOUT << "Wrapper: 64bit " << is64Bit;
693 DOUT << " AM "; DEBUG(AM.dump()); DOUT << "\n";
Rafael Espindolab2157762009-04-12 23:00:38 +0000694
Rafael Espindola49a168d2009-04-12 21:55:03 +0000695 // Under X86-64 non-small code model, GV (and friends) are 64-bits.
Rafael Espindolab2157762009-04-12 23:00:38 +0000696 if (is64Bit && (TM.getCodeModel() != CodeModel::Small))
Rafael Espindola49a168d2009-04-12 21:55:03 +0000697 return true;
Rafael Espindolab2157762009-04-12 23:00:38 +0000698
699 // Base and index reg must be 0 in order to use rip as base.
700 bool canUsePICRel = !AM.Base.Reg.getNode() && !AM.IndexReg.getNode();
Dan Gohmanc5534622009-06-03 20:20:00 +0000701 if (is64Bit && !canUsePICRel && SymbolicAddressesAreRIPRel)
Rafael Espindolab2157762009-04-12 23:00:38 +0000702 return true;
703
Rafael Espindola49a168d2009-04-12 21:55:03 +0000704 if (AM.hasSymbolicDisplacement())
705 return true;
706 // If value is available in a register both base and index components have
707 // been picked, we can't fit the result available in the register in the
708 // addressing mode. Duplicate GlobalAddress or ConstantPool as displacement.
709
710 SDValue N0 = N.getOperand(0);
711 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) {
712 uint64_t Offset = G->getOffset();
713 if (!is64Bit || isInt32(AM.Disp + Offset)) {
714 GlobalValue *GV = G->getGlobal();
Dan Gohmanc5534622009-06-03 20:20:00 +0000715 bool isRIPRel = SymbolicAddressesAreRIPRel;
Rafael Espindola7ff5bff2009-04-13 13:02:49 +0000716 if (N0.getOpcode() == llvm::ISD::TargetGlobalTLSAddress) {
717 TLSModel::Model model =
718 getTLSModel (GV, TM.getRelocationModel());
719 if (is64Bit && model == TLSModel::InitialExec)
720 isRIPRel = true;
721 }
Rafael Espindola49a168d2009-04-12 21:55:03 +0000722 AM.GV = GV;
723 AM.Disp += Offset;
Rafael Espindola7ff5bff2009-04-13 13:02:49 +0000724 AM.isRIPRel = isRIPRel;
Chris Lattnerb8afeb92009-06-26 05:51:45 +0000725 AM.SymbolFlags = G->getTargetFlags();
Rafael Espindola49a168d2009-04-12 21:55:03 +0000726 return false;
727 }
728 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) {
729 uint64_t Offset = CP->getOffset();
730 if (!is64Bit || isInt32(AM.Disp + Offset)) {
731 AM.CP = CP->getConstVal();
732 AM.Align = CP->getAlignment();
733 AM.Disp += Offset;
Dan Gohmanc5534622009-06-03 20:20:00 +0000734 AM.isRIPRel = SymbolicAddressesAreRIPRel;
Chris Lattner0b0deab2009-06-26 05:56:49 +0000735 AM.SymbolFlags = CP->getTargetFlags();
Rafael Espindola49a168d2009-04-12 21:55:03 +0000736 return false;
737 }
Chris Lattnerb8afeb92009-06-26 05:51:45 +0000738 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) {
Rafael Espindola49a168d2009-04-12 21:55:03 +0000739 AM.ES = S->getSymbol();
Dan Gohmanc5534622009-06-03 20:20:00 +0000740 AM.isRIPRel = SymbolicAddressesAreRIPRel;
Chris Lattnerb8afeb92009-06-26 05:51:45 +0000741 AM.SymbolFlags = S->getTargetFlags();
Rafael Espindola49a168d2009-04-12 21:55:03 +0000742 return false;
743 } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) {
744 AM.JT = J->getIndex();
Dan Gohmanc5534622009-06-03 20:20:00 +0000745 AM.isRIPRel = SymbolicAddressesAreRIPRel;
Chris Lattnerb8afeb92009-06-26 05:51:45 +0000746 AM.SymbolFlags = J->getTargetFlags();
Rafael Espindola49a168d2009-04-12 21:55:03 +0000747 return false;
748 }
749
750 return true;
751}
752
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000753/// MatchAddress - Add the specified node to the specified addressing mode,
754/// returning true if it cannot be done. This just pattern matches for the
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000755/// addressing mode.
Dan Gohman475871a2008-07-27 21:46:04 +0000756bool X86DAGToDAGISel::MatchAddress(SDValue N, X86ISelAddressMode &AM,
Rafael Espindola523249f2009-03-31 16:16:57 +0000757 unsigned Depth) {
Dan Gohman6520e202008-10-18 02:06:02 +0000758 bool is64Bit = Subtarget->is64Bit();
Dale Johannesen6f38cb62009-02-07 19:59:05 +0000759 DebugLoc dl = N.getDebugLoc();
Evan Chengda43bcf2008-09-24 00:05:32 +0000760 DOUT << "MatchAddress: "; DEBUG(AM.dump());
Dan Gohmanbadb2d22007-08-13 20:03:06 +0000761 // Limit recursion.
762 if (Depth > 5)
Rafael Espindola523249f2009-03-31 16:16:57 +0000763 return MatchAddressBase(N, AM);
Anton Korobeynikov33bf8c42007-03-28 18:36:33 +0000764
Evan Cheng25ab6902006-09-08 06:48:29 +0000765 // RIP relative addressing: %rip + 32-bit displacement!
766 if (AM.isRIPRel) {
767 if (!AM.ES && AM.JT != -1 && N.getOpcode() == ISD::Constant) {
Dan Gohman27cae7b2008-11-11 15:52:29 +0000768 uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue();
Dan Gohman6520e202008-10-18 02:06:02 +0000769 if (!is64Bit || isInt32(AM.Disp + Val)) {
Evan Cheng25ab6902006-09-08 06:48:29 +0000770 AM.Disp += Val;
771 return false;
772 }
773 }
774 return true;
775 }
776
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000777 switch (N.getOpcode()) {
778 default: break;
Evan Cheng25ab6902006-09-08 06:48:29 +0000779 case ISD::Constant: {
Dan Gohman27cae7b2008-11-11 15:52:29 +0000780 uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue();
Dan Gohman6520e202008-10-18 02:06:02 +0000781 if (!is64Bit || isInt32(AM.Disp + Val)) {
Evan Cheng25ab6902006-09-08 06:48:29 +0000782 AM.Disp += Val;
783 return false;
784 }
785 break;
786 }
Evan Cheng51a9ed92006-02-25 10:09:08 +0000787
Rafael Espindola094fad32009-04-08 21:14:34 +0000788 case X86ISD::SegmentBaseAddress:
789 if (!MatchSegmentBaseAddress(N, AM))
790 return false;
791 break;
792
Rafael Espindola49a168d2009-04-12 21:55:03 +0000793 case X86ISD::Wrapper:
794 if (!MatchWrapper(N, AM))
795 return false;
Evan Cheng51a9ed92006-02-25 10:09:08 +0000796 break;
797
Rafael Espindola094fad32009-04-08 21:14:34 +0000798 case ISD::LOAD:
799 if (!MatchLoad(N, AM))
800 return false;
801 break;
802
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000803 case ISD::FrameIndex:
Gabor Greif93c53e52008-08-31 15:37:04 +0000804 if (AM.BaseType == X86ISelAddressMode::RegBase
805 && AM.Base.Reg.getNode() == 0) {
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000806 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
807 AM.Base.FrameIndex = cast<FrameIndexSDNode>(N)->getIndex();
808 return false;
809 }
810 break;
Evan Chengec693f72005-12-08 02:01:35 +0000811
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000812 case ISD::SHL:
Dan Gohman8be6bbe2008-11-05 04:14:16 +0000813 if (AM.IndexReg.getNode() != 0 || AM.Scale != 1 || AM.isRIPRel)
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000814 break;
815
Gabor Greif93c53e52008-08-31 15:37:04 +0000816 if (ConstantSDNode
817 *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +0000818 unsigned Val = CN->getZExtValue();
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000819 if (Val == 1 || Val == 2 || Val == 3) {
820 AM.Scale = 1 << Val;
Gabor Greifba36cb52008-08-28 21:40:38 +0000821 SDValue ShVal = N.getNode()->getOperand(0);
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000822
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000823 // Okay, we know that we have a scale by now. However, if the scaled
824 // value is an add of something and a constant, we can fold the
825 // constant into the disp field here.
Gabor Greifba36cb52008-08-28 21:40:38 +0000826 if (ShVal.getNode()->getOpcode() == ISD::ADD && ShVal.hasOneUse() &&
827 isa<ConstantSDNode>(ShVal.getNode()->getOperand(1))) {
828 AM.IndexReg = ShVal.getNode()->getOperand(0);
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000829 ConstantSDNode *AddVal =
Gabor Greifba36cb52008-08-28 21:40:38 +0000830 cast<ConstantSDNode>(ShVal.getNode()->getOperand(1));
Evan Cheng8e278262009-01-17 07:09:27 +0000831 uint64_t Disp = AM.Disp + (AddVal->getSExtValue() << Val);
Dan Gohman6520e202008-10-18 02:06:02 +0000832 if (!is64Bit || isInt32(Disp))
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000833 AM.Disp = Disp;
834 else
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000835 AM.IndexReg = ShVal;
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000836 } else {
837 AM.IndexReg = ShVal;
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000838 }
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000839 return false;
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000840 }
841 break;
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000842 }
Evan Chengec693f72005-12-08 02:01:35 +0000843
Dan Gohman83688052007-10-22 20:22:24 +0000844 case ISD::SMUL_LOHI:
845 case ISD::UMUL_LOHI:
846 // A mul_lohi where we need the low part can be folded as a plain multiply.
Gabor Greif99a6cb92008-08-26 22:36:50 +0000847 if (N.getResNo() != 0) break;
Dan Gohman83688052007-10-22 20:22:24 +0000848 // FALL THROUGH
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000849 case ISD::MUL:
Evan Cheng73f24c92009-03-30 21:36:47 +0000850 case X86ISD::MUL_IMM:
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000851 // X*[3,5,9] -> X+X*[2,4,8]
Dan Gohman8be6bbe2008-11-05 04:14:16 +0000852 if (AM.BaseType == X86ISelAddressMode::RegBase &&
Gabor Greifba36cb52008-08-28 21:40:38 +0000853 AM.Base.Reg.getNode() == 0 &&
854 AM.IndexReg.getNode() == 0 &&
Evan Chengbe3bf422008-02-07 08:53:49 +0000855 !AM.isRIPRel) {
Gabor Greif93c53e52008-08-31 15:37:04 +0000856 if (ConstantSDNode
857 *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1)))
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +0000858 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
859 CN->getZExtValue() == 9) {
860 AM.Scale = unsigned(CN->getZExtValue())-1;
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000861
Gabor Greifba36cb52008-08-28 21:40:38 +0000862 SDValue MulVal = N.getNode()->getOperand(0);
Dan Gohman475871a2008-07-27 21:46:04 +0000863 SDValue Reg;
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000864
865 // Okay, we know that we have a scale by now. However, if the scaled
866 // value is an add of something and a constant, we can fold the
867 // constant into the disp field here.
Gabor Greifba36cb52008-08-28 21:40:38 +0000868 if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() &&
869 isa<ConstantSDNode>(MulVal.getNode()->getOperand(1))) {
870 Reg = MulVal.getNode()->getOperand(0);
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000871 ConstantSDNode *AddVal =
Gabor Greifba36cb52008-08-28 21:40:38 +0000872 cast<ConstantSDNode>(MulVal.getNode()->getOperand(1));
Evan Cheng8e278262009-01-17 07:09:27 +0000873 uint64_t Disp = AM.Disp + AddVal->getSExtValue() *
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +0000874 CN->getZExtValue();
Dan Gohman6520e202008-10-18 02:06:02 +0000875 if (!is64Bit || isInt32(Disp))
Evan Cheng25ab6902006-09-08 06:48:29 +0000876 AM.Disp = Disp;
877 else
Gabor Greifba36cb52008-08-28 21:40:38 +0000878 Reg = N.getNode()->getOperand(0);
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000879 } else {
Gabor Greifba36cb52008-08-28 21:40:38 +0000880 Reg = N.getNode()->getOperand(0);
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000881 }
882
883 AM.IndexReg = AM.Base.Reg = Reg;
884 return false;
885 }
Chris Lattner62412262007-02-04 20:18:17 +0000886 }
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000887 break;
888
Dan Gohman3cd90a12009-05-11 18:02:53 +0000889 case ISD::SUB: {
890 // Given A-B, if A can be completely folded into the address and
891 // the index field with the index field unused, use -B as the index.
892 // This is a win if a has multiple parts that can be folded into
893 // the address. Also, this saves a mov if the base register has
894 // other uses, since it avoids a two-address sub instruction, however
895 // it costs an additional mov if the index register has other uses.
896
897 // Test if the LHS of the sub can be folded.
898 X86ISelAddressMode Backup = AM;
899 if (MatchAddress(N.getNode()->getOperand(0), AM, Depth+1)) {
900 AM = Backup;
901 break;
902 }
903 // Test if the index field is free for use.
904 if (AM.IndexReg.getNode() || AM.isRIPRel) {
905 AM = Backup;
906 break;
907 }
908 int Cost = 0;
909 SDValue RHS = N.getNode()->getOperand(1);
910 // If the RHS involves a register with multiple uses, this
911 // transformation incurs an extra mov, due to the neg instruction
912 // clobbering its operand.
913 if (!RHS.getNode()->hasOneUse() ||
914 RHS.getNode()->getOpcode() == ISD::CopyFromReg ||
915 RHS.getNode()->getOpcode() == ISD::TRUNCATE ||
916 RHS.getNode()->getOpcode() == ISD::ANY_EXTEND ||
917 (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND &&
918 RHS.getNode()->getOperand(0).getValueType() == MVT::i32))
919 ++Cost;
920 // If the base is a register with multiple uses, this
921 // transformation may save a mov.
922 if ((AM.BaseType == X86ISelAddressMode::RegBase &&
923 AM.Base.Reg.getNode() &&
924 !AM.Base.Reg.getNode()->hasOneUse()) ||
925 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
926 --Cost;
927 // If the folded LHS was interesting, this transformation saves
928 // address arithmetic.
929 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
930 ((AM.Disp != 0) && (Backup.Disp == 0)) +
931 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
932 --Cost;
933 // If it doesn't look like it may be an overall win, don't do it.
934 if (Cost >= 0) {
935 AM = Backup;
936 break;
937 }
938
939 // Ok, the transformation is legal and appears profitable. Go for it.
940 SDValue Zero = CurDAG->getConstant(0, N.getValueType());
941 SDValue Neg = CurDAG->getNode(ISD::SUB, dl, N.getValueType(), Zero, RHS);
942 AM.IndexReg = Neg;
943 AM.Scale = 1;
944
945 // Insert the new nodes into the topological ordering.
946 if (Zero.getNode()->getNodeId() == -1 ||
947 Zero.getNode()->getNodeId() > N.getNode()->getNodeId()) {
948 CurDAG->RepositionNode(N.getNode(), Zero.getNode());
949 Zero.getNode()->setNodeId(N.getNode()->getNodeId());
950 }
951 if (Neg.getNode()->getNodeId() == -1 ||
952 Neg.getNode()->getNodeId() > N.getNode()->getNodeId()) {
953 CurDAG->RepositionNode(N.getNode(), Neg.getNode());
954 Neg.getNode()->setNodeId(N.getNode()->getNodeId());
955 }
956 return false;
957 }
958
Evan Cheng8e278262009-01-17 07:09:27 +0000959 case ISD::ADD: {
960 X86ISelAddressMode Backup = AM;
Rafael Espindola523249f2009-03-31 16:16:57 +0000961 if (!MatchAddress(N.getNode()->getOperand(0), AM, Depth+1) &&
962 !MatchAddress(N.getNode()->getOperand(1), AM, Depth+1))
Evan Cheng8e278262009-01-17 07:09:27 +0000963 return false;
964 AM = Backup;
Rafael Espindola523249f2009-03-31 16:16:57 +0000965 if (!MatchAddress(N.getNode()->getOperand(1), AM, Depth+1) &&
966 !MatchAddress(N.getNode()->getOperand(0), AM, Depth+1))
Evan Cheng8e278262009-01-17 07:09:27 +0000967 return false;
968 AM = Backup;
Dan Gohman77502c92009-03-13 02:25:09 +0000969
970 // If we couldn't fold both operands into the address at the same time,
971 // see if we can just put each operand into a register and fold at least
972 // the add.
973 if (AM.BaseType == X86ISelAddressMode::RegBase &&
974 !AM.Base.Reg.getNode() &&
975 !AM.IndexReg.getNode() &&
976 !AM.isRIPRel) {
977 AM.Base.Reg = N.getNode()->getOperand(0);
978 AM.IndexReg = N.getNode()->getOperand(1);
979 AM.Scale = 1;
980 return false;
981 }
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +0000982 break;
Evan Cheng8e278262009-01-17 07:09:27 +0000983 }
Evan Chenge6ad27e2006-05-30 06:59:36 +0000984
Chris Lattner62412262007-02-04 20:18:17 +0000985 case ISD::OR:
986 // Handle "X | C" as "X + C" iff X is known to have C bits clear.
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000987 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
988 X86ISelAddressMode Backup = AM;
Dan Gohman27cae7b2008-11-11 15:52:29 +0000989 uint64_t Offset = CN->getSExtValue();
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000990 // Start with the LHS as an addr mode.
Rafael Espindola523249f2009-03-31 16:16:57 +0000991 if (!MatchAddress(N.getOperand(0), AM, Depth+1) &&
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000992 // Address could not have picked a GV address for the displacement.
993 AM.GV == NULL &&
994 // On x86-64, the resultant disp must fit in 32-bits.
Dan Gohman27cae7b2008-11-11 15:52:29 +0000995 (!is64Bit || isInt32(AM.Disp + Offset)) &&
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000996 // Check to see if the LHS & C is zero.
Dan Gohman2e68b6f2008-02-25 21:11:39 +0000997 CurDAG->MaskedValueIsZero(N.getOperand(0), CN->getAPIntValue())) {
Dan Gohman27cae7b2008-11-11 15:52:29 +0000998 AM.Disp += Offset;
Chris Lattner5aaddaa2007-12-08 07:22:58 +0000999 return false;
Evan Chenge6ad27e2006-05-30 06:59:36 +00001000 }
Chris Lattner5aaddaa2007-12-08 07:22:58 +00001001 AM = Backup;
Evan Chenge6ad27e2006-05-30 06:59:36 +00001002 }
1003 break;
Evan Cheng1314b002007-12-13 00:43:27 +00001004
1005 case ISD::AND: {
Dan Gohman21e3dfb2009-04-13 16:09:41 +00001006 // Perform some heroic transforms on an and of a constant-count shift
1007 // with a constant to enable use of the scaled offset field.
1008
Dan Gohman475871a2008-07-27 21:46:04 +00001009 SDValue Shift = N.getOperand(0);
Dan Gohman21e3dfb2009-04-13 16:09:41 +00001010 if (Shift.getNumOperands() != 2) break;
Dan Gohman8be6bbe2008-11-05 04:14:16 +00001011
Evan Cheng1314b002007-12-13 00:43:27 +00001012 // Scale must not be used already.
Gabor Greifba36cb52008-08-28 21:40:38 +00001013 if (AM.IndexReg.getNode() != 0 || AM.Scale != 1) break;
Evan Chengbe3bf422008-02-07 08:53:49 +00001014
1015 // Not when RIP is used as the base.
1016 if (AM.isRIPRel) break;
Dan Gohman21e3dfb2009-04-13 16:09:41 +00001017
1018 SDValue X = Shift.getOperand(0);
Evan Cheng1314b002007-12-13 00:43:27 +00001019 ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N.getOperand(1));
1020 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(Shift.getOperand(1));
1021 if (!C1 || !C2) break;
1022
Dan Gohman21e3dfb2009-04-13 16:09:41 +00001023 // Handle "(X >> (8-C1)) & C2" as "(X >> 8) & 0xff)" if safe. This
1024 // allows us to convert the shift and and into an h-register extract and
1025 // a scaled index.
1026 if (Shift.getOpcode() == ISD::SRL && Shift.hasOneUse()) {
1027 unsigned ScaleLog = 8 - C1->getZExtValue();
Rafael Espindola7c366832009-04-16 12:34:53 +00001028 if (ScaleLog > 0 && ScaleLog < 4 &&
Dan Gohman21e3dfb2009-04-13 16:09:41 +00001029 C2->getZExtValue() == (UINT64_C(0xff) << ScaleLog)) {
1030 SDValue Eight = CurDAG->getConstant(8, MVT::i8);
1031 SDValue Mask = CurDAG->getConstant(0xff, N.getValueType());
1032 SDValue Srl = CurDAG->getNode(ISD::SRL, dl, N.getValueType(),
1033 X, Eight);
1034 SDValue And = CurDAG->getNode(ISD::AND, dl, N.getValueType(),
1035 Srl, Mask);
Dan Gohman62ad1382009-04-14 22:45:05 +00001036 SDValue ShlCount = CurDAG->getConstant(ScaleLog, MVT::i8);
1037 SDValue Shl = CurDAG->getNode(ISD::SHL, dl, N.getValueType(),
1038 And, ShlCount);
Dan Gohman21e3dfb2009-04-13 16:09:41 +00001039
1040 // Insert the new nodes into the topological ordering.
1041 if (Eight.getNode()->getNodeId() == -1 ||
1042 Eight.getNode()->getNodeId() > X.getNode()->getNodeId()) {
1043 CurDAG->RepositionNode(X.getNode(), Eight.getNode());
1044 Eight.getNode()->setNodeId(X.getNode()->getNodeId());
1045 }
1046 if (Mask.getNode()->getNodeId() == -1 ||
1047 Mask.getNode()->getNodeId() > X.getNode()->getNodeId()) {
1048 CurDAG->RepositionNode(X.getNode(), Mask.getNode());
1049 Mask.getNode()->setNodeId(X.getNode()->getNodeId());
1050 }
1051 if (Srl.getNode()->getNodeId() == -1 ||
1052 Srl.getNode()->getNodeId() > Shift.getNode()->getNodeId()) {
1053 CurDAG->RepositionNode(Shift.getNode(), Srl.getNode());
1054 Srl.getNode()->setNodeId(Shift.getNode()->getNodeId());
1055 }
1056 if (And.getNode()->getNodeId() == -1 ||
1057 And.getNode()->getNodeId() > N.getNode()->getNodeId()) {
1058 CurDAG->RepositionNode(N.getNode(), And.getNode());
1059 And.getNode()->setNodeId(N.getNode()->getNodeId());
1060 }
Dan Gohman62ad1382009-04-14 22:45:05 +00001061 if (ShlCount.getNode()->getNodeId() == -1 ||
1062 ShlCount.getNode()->getNodeId() > X.getNode()->getNodeId()) {
1063 CurDAG->RepositionNode(X.getNode(), ShlCount.getNode());
1064 ShlCount.getNode()->setNodeId(N.getNode()->getNodeId());
1065 }
1066 if (Shl.getNode()->getNodeId() == -1 ||
1067 Shl.getNode()->getNodeId() > N.getNode()->getNodeId()) {
1068 CurDAG->RepositionNode(N.getNode(), Shl.getNode());
1069 Shl.getNode()->setNodeId(N.getNode()->getNodeId());
1070 }
1071 CurDAG->ReplaceAllUsesWith(N, Shl);
Dan Gohman21e3dfb2009-04-13 16:09:41 +00001072 AM.IndexReg = And;
1073 AM.Scale = (1 << ScaleLog);
1074 return false;
1075 }
1076 }
1077
1078 // Handle "(X << C1) & C2" as "(X & (C2>>C1)) << C1" if safe and if this
1079 // allows us to fold the shift into this addressing mode.
1080 if (Shift.getOpcode() != ISD::SHL) break;
1081
Evan Cheng1314b002007-12-13 00:43:27 +00001082 // Not likely to be profitable if either the AND or SHIFT node has more
1083 // than one use (unless all uses are for address computation). Besides,
1084 // isel mechanism requires their node ids to be reused.
1085 if (!N.hasOneUse() || !Shift.hasOneUse())
1086 break;
1087
1088 // Verify that the shift amount is something we can fold.
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00001089 unsigned ShiftCst = C1->getZExtValue();
Evan Cheng1314b002007-12-13 00:43:27 +00001090 if (ShiftCst != 1 && ShiftCst != 2 && ShiftCst != 3)
1091 break;
1092
1093 // Get the new AND mask, this folds to a constant.
Dale Johannesend8392542009-02-03 21:48:12 +00001094 SDValue NewANDMask = CurDAG->getNode(ISD::SRL, dl, N.getValueType(),
Evan Cheng552e3be2008-10-14 17:15:39 +00001095 SDValue(C2, 0), SDValue(C1, 0));
Dale Johannesend8392542009-02-03 21:48:12 +00001096 SDValue NewAND = CurDAG->getNode(ISD::AND, dl, N.getValueType(), X,
1097 NewANDMask);
1098 SDValue NewSHIFT = CurDAG->getNode(ISD::SHL, dl, N.getValueType(),
Dan Gohman7b8e9642008-10-13 20:52:04 +00001099 NewAND, SDValue(C1, 0));
Dan Gohman8be6bbe2008-11-05 04:14:16 +00001100
1101 // Insert the new nodes into the topological ordering.
1102 if (C1->getNodeId() > X.getNode()->getNodeId()) {
1103 CurDAG->RepositionNode(X.getNode(), C1);
1104 C1->setNodeId(X.getNode()->getNodeId());
1105 }
1106 if (NewANDMask.getNode()->getNodeId() == -1 ||
1107 NewANDMask.getNode()->getNodeId() > X.getNode()->getNodeId()) {
1108 CurDAG->RepositionNode(X.getNode(), NewANDMask.getNode());
1109 NewANDMask.getNode()->setNodeId(X.getNode()->getNodeId());
1110 }
1111 if (NewAND.getNode()->getNodeId() == -1 ||
1112 NewAND.getNode()->getNodeId() > Shift.getNode()->getNodeId()) {
1113 CurDAG->RepositionNode(Shift.getNode(), NewAND.getNode());
1114 NewAND.getNode()->setNodeId(Shift.getNode()->getNodeId());
1115 }
1116 if (NewSHIFT.getNode()->getNodeId() == -1 ||
1117 NewSHIFT.getNode()->getNodeId() > N.getNode()->getNodeId()) {
1118 CurDAG->RepositionNode(N.getNode(), NewSHIFT.getNode());
1119 NewSHIFT.getNode()->setNodeId(N.getNode()->getNodeId());
1120 }
1121
Dan Gohman7b8e9642008-10-13 20:52:04 +00001122 CurDAG->ReplaceAllUsesWith(N, NewSHIFT);
Evan Cheng1314b002007-12-13 00:43:27 +00001123
1124 AM.Scale = 1 << ShiftCst;
1125 AM.IndexReg = NewAND;
1126 return false;
1127 }
Evan Chenge6ad27e2006-05-30 06:59:36 +00001128 }
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +00001129
Rafael Espindola523249f2009-03-31 16:16:57 +00001130 return MatchAddressBase(N, AM);
Dan Gohmanbadb2d22007-08-13 20:03:06 +00001131}
1132
1133/// MatchAddressBase - Helper for MatchAddress. Add the specified node to the
1134/// specified addressing mode without any further recursion.
Rafael Espindola523249f2009-03-31 16:16:57 +00001135bool X86DAGToDAGISel::MatchAddressBase(SDValue N, X86ISelAddressMode &AM) {
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +00001136 // Is the base register already occupied?
Gabor Greifba36cb52008-08-28 21:40:38 +00001137 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base.Reg.getNode()) {
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +00001138 // If so, check to see if the scale index register is set.
Gabor Greifba36cb52008-08-28 21:40:38 +00001139 if (AM.IndexReg.getNode() == 0 && !AM.isRIPRel) {
Chris Lattnerf9ce9fb2005-11-19 02:11:08 +00001140 AM.IndexReg = N;
1141 AM.Scale = 1;
1142 return false;
1143 }
1144
1145 // Otherwise, we cannot select it.
1146 return true;
1147 }
1148
1149 // Default, generate it as a register.
1150 AM.BaseType = X86ISelAddressMode::RegBase;
1151 AM.Base.Reg = N;
1152 return false;
1153}
1154
Evan Chengec693f72005-12-08 02:01:35 +00001155/// SelectAddr - returns true if it is able pattern match an addressing mode.
1156/// It returns the operands which make up the maximal addressing mode it can
1157/// match by reference.
Dan Gohman475871a2008-07-27 21:46:04 +00001158bool X86DAGToDAGISel::SelectAddr(SDValue Op, SDValue N, SDValue &Base,
1159 SDValue &Scale, SDValue &Index,
Rafael Espindola094fad32009-04-08 21:14:34 +00001160 SDValue &Disp, SDValue &Segment) {
Evan Chengec693f72005-12-08 02:01:35 +00001161 X86ISelAddressMode AM;
Evan Cheng4d952322009-03-31 01:13:53 +00001162 bool Done = false;
1163 if (AvoidDupAddrCompute && !N.hasOneUse()) {
1164 unsigned Opcode = N.getOpcode();
1165 if (Opcode != ISD::Constant && Opcode != ISD::FrameIndex &&
1166 Opcode != X86ISD::Wrapper) {
1167 // If we are able to fold N into addressing mode, then we'll allow it even
1168 // if N has multiple uses. In general, addressing computation is used as
1169 // addresses by all of its uses. But watch out for CopyToReg uses, that
1170 // means the address computation is liveout. It will be computed by a LEA
1171 // so we want to avoid computing the address twice.
1172 for (SDNode::use_iterator UI = N.getNode()->use_begin(),
1173 UE = N.getNode()->use_end(); UI != UE; ++UI) {
1174 if (UI->getOpcode() == ISD::CopyToReg) {
Rafael Espindola523249f2009-03-31 16:16:57 +00001175 MatchAddressBase(N, AM);
Evan Cheng4d952322009-03-31 01:13:53 +00001176 Done = true;
1177 break;
1178 }
1179 }
1180 }
1181 }
1182
1183 if (!Done && MatchAddress(N, AM))
Evan Cheng8700e142006-01-11 06:09:51 +00001184 return false;
Evan Chengec693f72005-12-08 02:01:35 +00001185
Duncan Sands83ec4b62008-06-06 12:08:01 +00001186 MVT VT = N.getValueType();
Evan Cheng8700e142006-01-11 06:09:51 +00001187 if (AM.BaseType == X86ISelAddressMode::RegBase) {
Gabor Greifba36cb52008-08-28 21:40:38 +00001188 if (!AM.Base.Reg.getNode())
Evan Cheng25ab6902006-09-08 06:48:29 +00001189 AM.Base.Reg = CurDAG->getRegister(0, VT);
Evan Chengec693f72005-12-08 02:01:35 +00001190 }
Evan Cheng8700e142006-01-11 06:09:51 +00001191
Gabor Greifba36cb52008-08-28 21:40:38 +00001192 if (!AM.IndexReg.getNode())
Evan Cheng25ab6902006-09-08 06:48:29 +00001193 AM.IndexReg = CurDAG->getRegister(0, VT);
Evan Cheng8700e142006-01-11 06:09:51 +00001194
Rafael Espindola094fad32009-04-08 21:14:34 +00001195 getAddressOperands(AM, Base, Scale, Index, Disp, Segment);
Evan Cheng8700e142006-01-11 06:09:51 +00001196 return true;
Evan Chengec693f72005-12-08 02:01:35 +00001197}
1198
Chris Lattner3a7cd952006-10-07 21:55:32 +00001199/// SelectScalarSSELoad - Match a scalar SSE load. In particular, we want to
1200/// match a load whose top elements are either undef or zeros. The load flavor
1201/// is derived from the type of N, which is either v4f32 or v2f64.
Dan Gohman475871a2008-07-27 21:46:04 +00001202bool X86DAGToDAGISel::SelectScalarSSELoad(SDValue Op, SDValue Pred,
1203 SDValue N, SDValue &Base,
1204 SDValue &Scale, SDValue &Index,
Rafael Espindola094fad32009-04-08 21:14:34 +00001205 SDValue &Disp, SDValue &Segment,
1206 SDValue &InChain,
Dan Gohman475871a2008-07-27 21:46:04 +00001207 SDValue &OutChain) {
Chris Lattner3a7cd952006-10-07 21:55:32 +00001208 if (N.getOpcode() == ISD::SCALAR_TO_VECTOR) {
Chris Lattner4fe4f252006-10-11 22:09:58 +00001209 InChain = N.getOperand(0).getValue(1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001210 if (ISD::isNON_EXTLoad(InChain.getNode()) &&
Evan Cheng07e4b002006-10-16 06:34:55 +00001211 InChain.getValue(0).hasOneUse() &&
Evan Chengd6373bc2006-11-10 21:23:04 +00001212 N.hasOneUse() &&
Evan Cheng884c70c2008-11-27 00:49:46 +00001213 IsLegalAndProfitableToFold(N.getNode(), Pred.getNode(), Op.getNode())) {
Evan Cheng82a91642006-10-11 21:06:01 +00001214 LoadSDNode *LD = cast<LoadSDNode>(InChain);
Rafael Espindola094fad32009-04-08 21:14:34 +00001215 if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp, Segment))
Chris Lattner3a7cd952006-10-07 21:55:32 +00001216 return false;
Evan Cheng82a91642006-10-11 21:06:01 +00001217 OutChain = LD->getChain();
Chris Lattner3a7cd952006-10-07 21:55:32 +00001218 return true;
1219 }
1220 }
Chris Lattner4fe4f252006-10-11 22:09:58 +00001221
1222 // Also handle the case where we explicitly require zeros in the top
Chris Lattner3a7cd952006-10-07 21:55:32 +00001223 // elements. This is a vector shuffle from the zero vector.
Gabor Greifba36cb52008-08-28 21:40:38 +00001224 if (N.getOpcode() == X86ISD::VZEXT_MOVL && N.getNode()->hasOneUse() &&
Chris Lattner8a594482007-11-25 00:24:49 +00001225 // Check to see if the top elements are all zeros (or bitcast of zeros).
Evan Cheng7e2ff772008-05-08 00:57:18 +00001226 N.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR &&
Gabor Greifba36cb52008-08-28 21:40:38 +00001227 N.getOperand(0).getNode()->hasOneUse() &&
1228 ISD::isNON_EXTLoad(N.getOperand(0).getOperand(0).getNode()) &&
Evan Cheng7e2ff772008-05-08 00:57:18 +00001229 N.getOperand(0).getOperand(0).hasOneUse()) {
1230 // Okay, this is a zero extending load. Fold it.
1231 LoadSDNode *LD = cast<LoadSDNode>(N.getOperand(0).getOperand(0));
Rafael Espindola094fad32009-04-08 21:14:34 +00001232 if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp, Segment))
Evan Cheng7e2ff772008-05-08 00:57:18 +00001233 return false;
1234 OutChain = LD->getChain();
Dan Gohman475871a2008-07-27 21:46:04 +00001235 InChain = SDValue(LD, 1);
Evan Cheng7e2ff772008-05-08 00:57:18 +00001236 return true;
Chris Lattner4fe4f252006-10-11 22:09:58 +00001237 }
Chris Lattner3a7cd952006-10-07 21:55:32 +00001238 return false;
1239}
1240
1241
Evan Cheng51a9ed92006-02-25 10:09:08 +00001242/// SelectLEAAddr - it calls SelectAddr and determines if the maximal addressing
1243/// mode it matches can be cost effectively emitted as an LEA instruction.
Dan Gohman475871a2008-07-27 21:46:04 +00001244bool X86DAGToDAGISel::SelectLEAAddr(SDValue Op, SDValue N,
1245 SDValue &Base, SDValue &Scale,
1246 SDValue &Index, SDValue &Disp) {
Evan Cheng51a9ed92006-02-25 10:09:08 +00001247 X86ISelAddressMode AM;
Rafael Espindoladbcfb302009-04-10 10:09:34 +00001248
1249 // Set AM.Segment to prevent MatchAddress from using one. LEA doesn't support
1250 // segments.
1251 SDValue Copy = AM.Segment;
1252 SDValue T = CurDAG->getRegister(0, MVT::i32);
1253 AM.Segment = T;
Evan Cheng51a9ed92006-02-25 10:09:08 +00001254 if (MatchAddress(N, AM))
1255 return false;
Rafael Espindoladbcfb302009-04-10 10:09:34 +00001256 assert (T == AM.Segment);
1257 AM.Segment = Copy;
Rafael Espindola094fad32009-04-08 21:14:34 +00001258
Duncan Sands83ec4b62008-06-06 12:08:01 +00001259 MVT VT = N.getValueType();
Evan Cheng51a9ed92006-02-25 10:09:08 +00001260 unsigned Complexity = 0;
1261 if (AM.BaseType == X86ISelAddressMode::RegBase)
Gabor Greifba36cb52008-08-28 21:40:38 +00001262 if (AM.Base.Reg.getNode())
Evan Cheng51a9ed92006-02-25 10:09:08 +00001263 Complexity = 1;
1264 else
Evan Cheng25ab6902006-09-08 06:48:29 +00001265 AM.Base.Reg = CurDAG->getRegister(0, VT);
Evan Cheng51a9ed92006-02-25 10:09:08 +00001266 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
1267 Complexity = 4;
1268
Gabor Greifba36cb52008-08-28 21:40:38 +00001269 if (AM.IndexReg.getNode())
Evan Cheng51a9ed92006-02-25 10:09:08 +00001270 Complexity++;
1271 else
Evan Cheng25ab6902006-09-08 06:48:29 +00001272 AM.IndexReg = CurDAG->getRegister(0, VT);
Evan Cheng51a9ed92006-02-25 10:09:08 +00001273
Chris Lattnera16b7cb2007-03-20 06:08:29 +00001274 // Don't match just leal(,%reg,2). It's cheaper to do addl %reg, %reg, or with
1275 // a simple shift.
1276 if (AM.Scale > 1)
Evan Cheng8c03fe42006-02-28 21:13:57 +00001277 Complexity++;
Evan Cheng51a9ed92006-02-25 10:09:08 +00001278
1279 // FIXME: We are artificially lowering the criteria to turn ADD %reg, $GA
1280 // to a LEA. This is determined with some expermentation but is by no means
1281 // optimal (especially for code size consideration). LEA is nice because of
1282 // its three-address nature. Tweak the cost function again when we can run
1283 // convertToThreeAddress() at register allocation time.
Dan Gohman2d0a1cc2009-02-07 00:43:41 +00001284 if (AM.hasSymbolicDisplacement()) {
Evan Cheng25ab6902006-09-08 06:48:29 +00001285 // For X86-64, we should always use lea to materialize RIP relative
1286 // addresses.
Evan Cheng953fa042006-12-05 22:03:40 +00001287 if (Subtarget->is64Bit())
Evan Cheng25ab6902006-09-08 06:48:29 +00001288 Complexity = 4;
1289 else
1290 Complexity += 2;
1291 }
Evan Cheng51a9ed92006-02-25 10:09:08 +00001292
Gabor Greifba36cb52008-08-28 21:40:38 +00001293 if (AM.Disp && (AM.Base.Reg.getNode() || AM.IndexReg.getNode()))
Evan Cheng51a9ed92006-02-25 10:09:08 +00001294 Complexity++;
1295
1296 if (Complexity > 2) {
Rafael Espindola094fad32009-04-08 21:14:34 +00001297 SDValue Segment;
1298 getAddressOperands(AM, Base, Scale, Index, Disp, Segment);
Evan Cheng51a9ed92006-02-25 10:09:08 +00001299 return true;
1300 }
Evan Cheng51a9ed92006-02-25 10:09:08 +00001301 return false;
1302}
1303
Chris Lattner5c0b16d2009-06-20 20:38:48 +00001304/// SelectTLSADDRAddr - This is only run on TargetGlobalTLSAddress nodes.
1305bool X86DAGToDAGISel::SelectTLSADDRAddr(SDValue Op, SDValue N, SDValue &Base,
1306 SDValue &Scale, SDValue &Index,
1307 SDValue &Disp) {
1308 assert(Op.getOpcode() == X86ISD::TLSADDR);
1309 assert(N.getOpcode() == ISD::TargetGlobalTLSAddress);
1310 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
1311
1312 X86ISelAddressMode AM;
1313 AM.GV = GA->getGlobal();
1314 AM.Disp += GA->getOffset();
1315 AM.Base.Reg = CurDAG->getRegister(0, N.getValueType());
1316
1317 if (N.getValueType() == MVT::i32) {
1318 AM.Scale = 1;
1319 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
1320 } else {
1321 AM.IndexReg = CurDAG->getRegister(0, MVT::i64);
1322 }
1323
1324 SDValue Segment;
1325 getAddressOperands(AM, Base, Scale, Index, Disp, Segment);
1326 return true;
1327}
1328
1329
Dan Gohman475871a2008-07-27 21:46:04 +00001330bool X86DAGToDAGISel::TryFoldLoad(SDValue P, SDValue N,
1331 SDValue &Base, SDValue &Scale,
Rafael Espindola094fad32009-04-08 21:14:34 +00001332 SDValue &Index, SDValue &Disp,
1333 SDValue &Segment) {
Gabor Greifba36cb52008-08-28 21:40:38 +00001334 if (ISD::isNON_EXTLoad(N.getNode()) &&
Evan Cheng5e351682006-02-06 06:02:33 +00001335 N.hasOneUse() &&
Evan Cheng884c70c2008-11-27 00:49:46 +00001336 IsLegalAndProfitableToFold(N.getNode(), P.getNode(), P.getNode()))
Rafael Espindola094fad32009-04-08 21:14:34 +00001337 return SelectAddr(P, N.getOperand(1), Base, Scale, Index, Disp, Segment);
Evan Cheng0114e942006-01-06 20:36:21 +00001338 return false;
1339}
1340
Dan Gohman8b746962008-09-23 18:22:58 +00001341/// getGlobalBaseReg - Return an SDNode that returns the value of
1342/// the global base register. Output instructions required to
1343/// initialize the global base register, if necessary.
Evan Cheng7ccced62006-02-18 00:15:05 +00001344///
Evan Cheng9ade2182006-08-26 05:34:46 +00001345SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
Dan Gohman57c3dac2008-09-30 00:58:23 +00001346 MachineFunction *MF = CurBB->getParent();
Dan Gohmanc5534622009-06-03 20:20:00 +00001347 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
Gabor Greifba36cb52008-08-28 21:40:38 +00001348 return CurDAG->getRegister(GlobalBaseReg, TLI.getPointerTy()).getNode();
Evan Cheng7ccced62006-02-18 00:15:05 +00001349}
1350
Evan Chengb245d922006-05-20 01:36:52 +00001351static SDNode *FindCallStartFromCall(SDNode *Node) {
1352 if (Node->getOpcode() == ISD::CALLSEQ_START) return Node;
1353 assert(Node->getOperand(0).getValueType() == MVT::Other &&
1354 "Node doesn't have a token chain argument!");
Gabor Greifba36cb52008-08-28 21:40:38 +00001355 return FindCallStartFromCall(Node->getOperand(0).getNode());
Evan Chengb245d922006-05-20 01:36:52 +00001356}
1357
Dale Johannesen48c1bc22008-10-02 18:53:47 +00001358SDNode *X86DAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) {
1359 SDValue Chain = Node->getOperand(0);
1360 SDValue In1 = Node->getOperand(1);
1361 SDValue In2L = Node->getOperand(2);
1362 SDValue In2H = Node->getOperand(3);
Rafael Espindola094fad32009-04-08 21:14:34 +00001363 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
1364 if (!SelectAddr(In1, In1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4))
Dale Johannesen48c1bc22008-10-02 18:53:47 +00001365 return NULL;
Dale Johannesen1b54c7f2008-10-03 19:41:08 +00001366 SDValue LSI = Node->getOperand(4); // MemOperand
Rafael Espindola094fad32009-04-08 21:14:34 +00001367 const SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, In2L, In2H, LSI, Chain};
Rafael Espindolae4d5d342009-03-27 15:45:05 +00001368 return CurDAG->getTargetNode(Opc, Node->getDebugLoc(),
1369 MVT::i32, MVT::i32, MVT::Other, Ops,
Rafael Espindolaa0a4f072009-03-28 19:02:18 +00001370 array_lengthof(Ops));
Dale Johannesen48c1bc22008-10-02 18:53:47 +00001371}
Christopher Lambc59e5212007-08-10 21:48:46 +00001372
Dan Gohman475871a2008-07-27 21:46:04 +00001373SDNode *X86DAGToDAGISel::Select(SDValue N) {
Gabor Greifba36cb52008-08-28 21:40:38 +00001374 SDNode *Node = N.getNode();
Duncan Sands83ec4b62008-06-06 12:08:01 +00001375 MVT NVT = Node->getValueType(0);
Evan Cheng0114e942006-01-06 20:36:21 +00001376 unsigned Opc, MOpc;
1377 unsigned Opcode = Node->getOpcode();
Dale Johannesend8392542009-02-03 21:48:12 +00001378 DebugLoc dl = Node->getDebugLoc();
1379
Evan Chengf597dc72006-02-10 22:24:32 +00001380#ifndef NDEBUG
Bill Wendling6345d752006-11-17 07:52:03 +00001381 DOUT << std::string(Indent, ' ') << "Selecting: ";
Evan Chengf597dc72006-02-10 22:24:32 +00001382 DEBUG(Node->dump(CurDAG));
Bill Wendling6345d752006-11-17 07:52:03 +00001383 DOUT << "\n";
Evan Cheng23addc02006-02-10 22:46:26 +00001384 Indent += 2;
Evan Chengf597dc72006-02-10 22:24:32 +00001385#endif
1386
Dan Gohmane8be6c62008-07-17 19:10:17 +00001387 if (Node->isMachineOpcode()) {
Evan Chengf597dc72006-02-10 22:24:32 +00001388#ifndef NDEBUG
Bill Wendling6345d752006-11-17 07:52:03 +00001389 DOUT << std::string(Indent-2, ' ') << "== ";
Evan Chengf597dc72006-02-10 22:24:32 +00001390 DEBUG(Node->dump(CurDAG));
Bill Wendling6345d752006-11-17 07:52:03 +00001391 DOUT << "\n";
Evan Cheng23addc02006-02-10 22:46:26 +00001392 Indent -= 2;
Evan Chengf597dc72006-02-10 22:24:32 +00001393#endif
Evan Cheng64a752f2006-08-11 09:08:15 +00001394 return NULL; // Already selected.
Evan Cheng34167212006-02-09 00:37:58 +00001395 }
Evan Cheng38262ca2006-01-11 22:15:18 +00001396
Evan Cheng0114e942006-01-06 20:36:21 +00001397 switch (Opcode) {
Chris Lattnerc961eea2005-11-16 01:54:32 +00001398 default: break;
Evan Cheng020d2e82006-02-23 20:41:18 +00001399 case X86ISD::GlobalBaseReg:
Evan Cheng9ade2182006-08-26 05:34:46 +00001400 return getGlobalBaseReg();
Evan Cheng020d2e82006-02-23 20:41:18 +00001401
Dale Johannesen48c1bc22008-10-02 18:53:47 +00001402 case X86ISD::ATOMOR64_DAG:
1403 return SelectAtomic64(Node, X86::ATOMOR6432);
1404 case X86ISD::ATOMXOR64_DAG:
1405 return SelectAtomic64(Node, X86::ATOMXOR6432);
1406 case X86ISD::ATOMADD64_DAG:
1407 return SelectAtomic64(Node, X86::ATOMADD6432);
1408 case X86ISD::ATOMSUB64_DAG:
1409 return SelectAtomic64(Node, X86::ATOMSUB6432);
1410 case X86ISD::ATOMNAND64_DAG:
1411 return SelectAtomic64(Node, X86::ATOMNAND6432);
1412 case X86ISD::ATOMAND64_DAG:
1413 return SelectAtomic64(Node, X86::ATOMAND6432);
Dale Johannesen880ae362008-10-03 22:25:52 +00001414 case X86ISD::ATOMSWAP64_DAG:
1415 return SelectAtomic64(Node, X86::ATOMSWAP6432);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00001416
Dan Gohman525178c2007-10-08 18:33:35 +00001417 case ISD::SMUL_LOHI:
1418 case ISD::UMUL_LOHI: {
Dan Gohman475871a2008-07-27 21:46:04 +00001419 SDValue N0 = Node->getOperand(0);
1420 SDValue N1 = Node->getOperand(1);
Dan Gohman525178c2007-10-08 18:33:35 +00001421
Dan Gohman525178c2007-10-08 18:33:35 +00001422 bool isSigned = Opcode == ISD::SMUL_LOHI;
1423 if (!isSigned)
Duncan Sands83ec4b62008-06-06 12:08:01 +00001424 switch (NVT.getSimpleVT()) {
Evan Cheng0114e942006-01-06 20:36:21 +00001425 default: assert(0 && "Unsupported VT!");
1426 case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m; break;
1427 case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m; break;
1428 case MVT::i32: Opc = X86::MUL32r; MOpc = X86::MUL32m; break;
Evan Cheng25ab6902006-09-08 06:48:29 +00001429 case MVT::i64: Opc = X86::MUL64r; MOpc = X86::MUL64m; break;
Evan Cheng0114e942006-01-06 20:36:21 +00001430 }
1431 else
Duncan Sands83ec4b62008-06-06 12:08:01 +00001432 switch (NVT.getSimpleVT()) {
Evan Cheng0114e942006-01-06 20:36:21 +00001433 default: assert(0 && "Unsupported VT!");
1434 case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m; break;
1435 case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m; break;
1436 case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m; break;
Evan Cheng25ab6902006-09-08 06:48:29 +00001437 case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m; break;
Evan Cheng0114e942006-01-06 20:36:21 +00001438 }
1439
1440 unsigned LoReg, HiReg;
Duncan Sands83ec4b62008-06-06 12:08:01 +00001441 switch (NVT.getSimpleVT()) {
Evan Cheng0114e942006-01-06 20:36:21 +00001442 default: assert(0 && "Unsupported VT!");
1443 case MVT::i8: LoReg = X86::AL; HiReg = X86::AH; break;
1444 case MVT::i16: LoReg = X86::AX; HiReg = X86::DX; break;
1445 case MVT::i32: LoReg = X86::EAX; HiReg = X86::EDX; break;
Evan Cheng25ab6902006-09-08 06:48:29 +00001446 case MVT::i64: LoReg = X86::RAX; HiReg = X86::RDX; break;
Evan Cheng0114e942006-01-06 20:36:21 +00001447 }
1448
Rafael Espindola094fad32009-04-08 21:14:34 +00001449 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
1450 bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
Dan Gohman525178c2007-10-08 18:33:35 +00001451 // multiplty is commmutative
Evan Cheng948f3432006-01-06 23:19:29 +00001452 if (!foldedLoad) {
Rafael Espindola094fad32009-04-08 21:14:34 +00001453 foldedLoad = TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
Evan Cheng7afa1662007-08-02 05:48:35 +00001454 if (foldedLoad)
1455 std::swap(N0, N1);
Evan Cheng948f3432006-01-06 23:19:29 +00001456 }
1457
Dale Johannesendd64c412009-02-04 00:33:20 +00001458 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
Dan Gohman475871a2008-07-27 21:46:04 +00001459 N0, SDValue()).getValue(1);
Evan Cheng0114e942006-01-06 20:36:21 +00001460
1461 if (foldedLoad) {
Rafael Espindola094fad32009-04-08 21:14:34 +00001462 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
1463 InFlag };
Evan Cheng7e9b26f2006-02-09 07:17:49 +00001464 SDNode *CNode =
Rafael Espindolae4d5d342009-03-27 15:45:05 +00001465 CurDAG->getTargetNode(MOpc, dl, MVT::Other, MVT::Flag, Ops,
Rafael Espindolaa0a4f072009-03-28 19:02:18 +00001466 array_lengthof(Ops));
Dan Gohman475871a2008-07-27 21:46:04 +00001467 InFlag = SDValue(CNode, 1);
Dan Gohman525178c2007-10-08 18:33:35 +00001468 // Update the chain.
Dan Gohman475871a2008-07-27 21:46:04 +00001469 ReplaceUses(N1.getValue(1), SDValue(CNode, 0));
Evan Cheng0114e942006-01-06 20:36:21 +00001470 } else {
Evan Cheng7e9b26f2006-02-09 07:17:49 +00001471 InFlag =
Dale Johannesend8392542009-02-03 21:48:12 +00001472 SDValue(CurDAG->getTargetNode(Opc, dl, MVT::Flag, N1, InFlag), 0);
Evan Cheng0114e942006-01-06 20:36:21 +00001473 }
1474
Dan Gohman525178c2007-10-08 18:33:35 +00001475 // Copy the low half of the result, if it is needed.
1476 if (!N.getValue(0).use_empty()) {
Dale Johannesendd64c412009-02-04 00:33:20 +00001477 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
Dan Gohman525178c2007-10-08 18:33:35 +00001478 LoReg, NVT, InFlag);
1479 InFlag = Result.getValue(2);
1480 ReplaceUses(N.getValue(0), Result);
1481#ifndef NDEBUG
1482 DOUT << std::string(Indent-2, ' ') << "=> ";
Gabor Greifba36cb52008-08-28 21:40:38 +00001483 DEBUG(Result.getNode()->dump(CurDAG));
Dan Gohman525178c2007-10-08 18:33:35 +00001484 DOUT << "\n";
1485#endif
Evan Chengf7ef26e2007-08-09 21:59:35 +00001486 }
Dan Gohman525178c2007-10-08 18:33:35 +00001487 // Copy the high half of the result, if it is needed.
1488 if (!N.getValue(1).use_empty()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001489 SDValue Result;
Dan Gohman525178c2007-10-08 18:33:35 +00001490 if (HiReg == X86::AH && Subtarget->is64Bit()) {
1491 // Prevent use of AH in a REX instruction by referencing AX instead.
1492 // Shift it down 8 bits.
Dale Johannesendd64c412009-02-04 00:33:20 +00001493 Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
Dan Gohman525178c2007-10-08 18:33:35 +00001494 X86::AX, MVT::i16, InFlag);
1495 InFlag = Result.getValue(2);
Dale Johannesend8392542009-02-03 21:48:12 +00001496 Result = SDValue(CurDAG->getTargetNode(X86::SHR16ri, dl, MVT::i16,
1497 Result,
Gabor Greif93c53e52008-08-31 15:37:04 +00001498 CurDAG->getTargetConstant(8, MVT::i8)), 0);
Dan Gohman525178c2007-10-08 18:33:35 +00001499 // Then truncate it down to i8.
Dan Gohman3cd0aa32009-04-13 15:14:03 +00001500 SDValue SRIdx = CurDAG->getTargetConstant(X86::SUBREG_8BIT, MVT::i32);
Dale Johannesend8392542009-02-03 21:48:12 +00001501 Result = SDValue(CurDAG->getTargetNode(X86::EXTRACT_SUBREG, dl,
Dan Gohman525178c2007-10-08 18:33:35 +00001502 MVT::i8, Result, SRIdx), 0);
1503 } else {
Dale Johannesendd64c412009-02-04 00:33:20 +00001504 Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
Dan Gohman525178c2007-10-08 18:33:35 +00001505 HiReg, NVT, InFlag);
1506 InFlag = Result.getValue(2);
1507 }
1508 ReplaceUses(N.getValue(1), Result);
1509#ifndef NDEBUG
1510 DOUT << std::string(Indent-2, ' ') << "=> ";
Gabor Greifba36cb52008-08-28 21:40:38 +00001511 DEBUG(Result.getNode()->dump(CurDAG));
Dan Gohman525178c2007-10-08 18:33:35 +00001512 DOUT << "\n";
1513#endif
1514 }
Evan Cheng34167212006-02-09 00:37:58 +00001515
Evan Chengf597dc72006-02-10 22:24:32 +00001516#ifndef NDEBUG
Evan Cheng23addc02006-02-10 22:46:26 +00001517 Indent -= 2;
Evan Chengf597dc72006-02-10 22:24:32 +00001518#endif
Dan Gohman525178c2007-10-08 18:33:35 +00001519
Evan Cheng64a752f2006-08-11 09:08:15 +00001520 return NULL;
Evan Cheng948f3432006-01-06 23:19:29 +00001521 }
Evan Cheng7ccced62006-02-18 00:15:05 +00001522
Dan Gohman525178c2007-10-08 18:33:35 +00001523 case ISD::SDIVREM:
1524 case ISD::UDIVREM: {
Dan Gohman475871a2008-07-27 21:46:04 +00001525 SDValue N0 = Node->getOperand(0);
1526 SDValue N1 = Node->getOperand(1);
Dan Gohman525178c2007-10-08 18:33:35 +00001527
1528 bool isSigned = Opcode == ISD::SDIVREM;
Evan Cheng948f3432006-01-06 23:19:29 +00001529 if (!isSigned)
Duncan Sands83ec4b62008-06-06 12:08:01 +00001530 switch (NVT.getSimpleVT()) {
Evan Cheng948f3432006-01-06 23:19:29 +00001531 default: assert(0 && "Unsupported VT!");
1532 case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m; break;
1533 case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m; break;
1534 case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m; break;
Evan Cheng25ab6902006-09-08 06:48:29 +00001535 case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m; break;
Evan Cheng948f3432006-01-06 23:19:29 +00001536 }
1537 else
Duncan Sands83ec4b62008-06-06 12:08:01 +00001538 switch (NVT.getSimpleVT()) {
Evan Cheng948f3432006-01-06 23:19:29 +00001539 default: assert(0 && "Unsupported VT!");
1540 case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m; break;
1541 case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m; break;
1542 case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m; break;
Evan Cheng25ab6902006-09-08 06:48:29 +00001543 case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m; break;
Evan Cheng948f3432006-01-06 23:19:29 +00001544 }
1545
1546 unsigned LoReg, HiReg;
1547 unsigned ClrOpcode, SExtOpcode;
Duncan Sands83ec4b62008-06-06 12:08:01 +00001548 switch (NVT.getSimpleVT()) {
Evan Cheng948f3432006-01-06 23:19:29 +00001549 default: assert(0 && "Unsupported VT!");
1550 case MVT::i8:
1551 LoReg = X86::AL; HiReg = X86::AH;
Evan Chengb1409ce2006-11-17 22:10:14 +00001552 ClrOpcode = 0;
Evan Cheng948f3432006-01-06 23:19:29 +00001553 SExtOpcode = X86::CBW;
1554 break;
1555 case MVT::i16:
1556 LoReg = X86::AX; HiReg = X86::DX;
Evan Chengaede9b92006-06-02 21:20:34 +00001557 ClrOpcode = X86::MOV16r0;
Evan Cheng948f3432006-01-06 23:19:29 +00001558 SExtOpcode = X86::CWD;
1559 break;
1560 case MVT::i32:
1561 LoReg = X86::EAX; HiReg = X86::EDX;
Evan Chengaede9b92006-06-02 21:20:34 +00001562 ClrOpcode = X86::MOV32r0;
Evan Cheng948f3432006-01-06 23:19:29 +00001563 SExtOpcode = X86::CDQ;
1564 break;
Evan Cheng25ab6902006-09-08 06:48:29 +00001565 case MVT::i64:
1566 LoReg = X86::RAX; HiReg = X86::RDX;
1567 ClrOpcode = X86::MOV64r0;
1568 SExtOpcode = X86::CQO;
1569 break;
Evan Cheng948f3432006-01-06 23:19:29 +00001570 }
1571
Rafael Espindola094fad32009-04-08 21:14:34 +00001572 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
1573 bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
Dan Gohman1ef4d8f2009-01-21 14:50:16 +00001574 bool signBitIsZero = CurDAG->SignBitIsZero(N0);
Dan Gohman525178c2007-10-08 18:33:35 +00001575
Dan Gohman475871a2008-07-27 21:46:04 +00001576 SDValue InFlag;
Dan Gohman1ef4d8f2009-01-21 14:50:16 +00001577 if (NVT == MVT::i8 && (!isSigned || signBitIsZero)) {
Evan Chengb1409ce2006-11-17 22:10:14 +00001578 // Special case for div8, just use a move with zero extension to AX to
1579 // clear the upper 8 bits (AH).
Rafael Espindola094fad32009-04-08 21:14:34 +00001580 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Move, Chain;
1581 if (TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
1582 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) };
Evan Chengb1409ce2006-11-17 22:10:14 +00001583 Move =
Dale Johannesend8392542009-02-03 21:48:12 +00001584 SDValue(CurDAG->getTargetNode(X86::MOVZX16rm8, dl, MVT::i16,
Rafael Espindolae4d5d342009-03-27 15:45:05 +00001585 MVT::Other, Ops,
Rafael Espindolaa0a4f072009-03-28 19:02:18 +00001586 array_lengthof(Ops)), 0);
Evan Chengb1409ce2006-11-17 22:10:14 +00001587 Chain = Move.getValue(1);
1588 ReplaceUses(N0.getValue(1), Chain);
1589 } else {
Evan Chengb1409ce2006-11-17 22:10:14 +00001590 Move =
Dale Johannesend8392542009-02-03 21:48:12 +00001591 SDValue(CurDAG->getTargetNode(X86::MOVZX16rr8, dl, MVT::i16, N0),0);
Evan Chengb1409ce2006-11-17 22:10:14 +00001592 Chain = CurDAG->getEntryNode();
1593 }
Dale Johannesendd64c412009-02-04 00:33:20 +00001594 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX, Move, SDValue());
Evan Cheng948f3432006-01-06 23:19:29 +00001595 InFlag = Chain.getValue(1);
Evan Chengb1409ce2006-11-17 22:10:14 +00001596 } else {
Evan Chengb1409ce2006-11-17 22:10:14 +00001597 InFlag =
Dale Johannesendd64c412009-02-04 00:33:20 +00001598 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
Dan Gohman475871a2008-07-27 21:46:04 +00001599 LoReg, N0, SDValue()).getValue(1);
Dan Gohman1ef4d8f2009-01-21 14:50:16 +00001600 if (isSigned && !signBitIsZero) {
Evan Chengb1409ce2006-11-17 22:10:14 +00001601 // Sign extend the low part into the high part.
1602 InFlag =
Dale Johannesend8392542009-02-03 21:48:12 +00001603 SDValue(CurDAG->getTargetNode(SExtOpcode, dl, MVT::Flag, InFlag),0);
Evan Chengb1409ce2006-11-17 22:10:14 +00001604 } else {
1605 // Zero out the high part, effectively zero extending the input.
Dale Johannesend8392542009-02-03 21:48:12 +00001606 SDValue ClrNode = SDValue(CurDAG->getTargetNode(ClrOpcode, dl, NVT),
1607 0);
Dale Johannesendd64c412009-02-04 00:33:20 +00001608 InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, HiReg,
Dan Gohman525178c2007-10-08 18:33:35 +00001609 ClrNode, InFlag).getValue(1);
Evan Chengb1409ce2006-11-17 22:10:14 +00001610 }
Evan Cheng948f3432006-01-06 23:19:29 +00001611 }
1612
1613 if (foldedLoad) {
Rafael Espindola094fad32009-04-08 21:14:34 +00001614 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
1615 InFlag };
Evan Cheng7e9b26f2006-02-09 07:17:49 +00001616 SDNode *CNode =
Rafael Espindolae4d5d342009-03-27 15:45:05 +00001617 CurDAG->getTargetNode(MOpc, dl, MVT::Other, MVT::Flag, Ops,
Rafael Espindolaa0a4f072009-03-28 19:02:18 +00001618 array_lengthof(Ops));
Dan Gohman475871a2008-07-27 21:46:04 +00001619 InFlag = SDValue(CNode, 1);
Dan Gohman525178c2007-10-08 18:33:35 +00001620 // Update the chain.
Dan Gohman475871a2008-07-27 21:46:04 +00001621 ReplaceUses(N1.getValue(1), SDValue(CNode, 0));
Evan Cheng948f3432006-01-06 23:19:29 +00001622 } else {
Evan Cheng7e9b26f2006-02-09 07:17:49 +00001623 InFlag =
Dale Johannesend8392542009-02-03 21:48:12 +00001624 SDValue(CurDAG->getTargetNode(Opc, dl, MVT::Flag, N1, InFlag), 0);
Evan Cheng948f3432006-01-06 23:19:29 +00001625 }
1626
Dan Gohmana37c9f72007-09-25 18:23:27 +00001627 // Copy the division (low) result, if it is needed.
1628 if (!N.getValue(0).use_empty()) {
Dale Johannesendd64c412009-02-04 00:33:20 +00001629 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
Dan Gohman525178c2007-10-08 18:33:35 +00001630 LoReg, NVT, InFlag);
Dan Gohmana37c9f72007-09-25 18:23:27 +00001631 InFlag = Result.getValue(2);
1632 ReplaceUses(N.getValue(0), Result);
1633#ifndef NDEBUG
1634 DOUT << std::string(Indent-2, ' ') << "=> ";
Gabor Greifba36cb52008-08-28 21:40:38 +00001635 DEBUG(Result.getNode()->dump(CurDAG));
Dan Gohmana37c9f72007-09-25 18:23:27 +00001636 DOUT << "\n";
1637#endif
Evan Chengf7ef26e2007-08-09 21:59:35 +00001638 }
Dan Gohmana37c9f72007-09-25 18:23:27 +00001639 // Copy the remainder (high) result, if it is needed.
1640 if (!N.getValue(1).use_empty()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001641 SDValue Result;
Dan Gohmana37c9f72007-09-25 18:23:27 +00001642 if (HiReg == X86::AH && Subtarget->is64Bit()) {
1643 // Prevent use of AH in a REX instruction by referencing AX instead.
1644 // Shift it down 8 bits.
Dale Johannesendd64c412009-02-04 00:33:20 +00001645 Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
Dan Gohman525178c2007-10-08 18:33:35 +00001646 X86::AX, MVT::i16, InFlag);
Dan Gohmana37c9f72007-09-25 18:23:27 +00001647 InFlag = Result.getValue(2);
Dale Johannesend8392542009-02-03 21:48:12 +00001648 Result = SDValue(CurDAG->getTargetNode(X86::SHR16ri, dl, MVT::i16,
1649 Result,
1650 CurDAG->getTargetConstant(8, MVT::i8)),
1651 0);
Dan Gohmana37c9f72007-09-25 18:23:27 +00001652 // Then truncate it down to i8.
Dan Gohman3cd0aa32009-04-13 15:14:03 +00001653 SDValue SRIdx = CurDAG->getTargetConstant(X86::SUBREG_8BIT, MVT::i32);
Dale Johannesend8392542009-02-03 21:48:12 +00001654 Result = SDValue(CurDAG->getTargetNode(X86::EXTRACT_SUBREG, dl,
Dan Gohmana37c9f72007-09-25 18:23:27 +00001655 MVT::i8, Result, SRIdx), 0);
1656 } else {
Dale Johannesendd64c412009-02-04 00:33:20 +00001657 Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
Dan Gohman525178c2007-10-08 18:33:35 +00001658 HiReg, NVT, InFlag);
Dan Gohmana37c9f72007-09-25 18:23:27 +00001659 InFlag = Result.getValue(2);
1660 }
1661 ReplaceUses(N.getValue(1), Result);
1662#ifndef NDEBUG
1663 DOUT << std::string(Indent-2, ' ') << "=> ";
Gabor Greifba36cb52008-08-28 21:40:38 +00001664 DEBUG(Result.getNode()->dump(CurDAG));
Dan Gohmana37c9f72007-09-25 18:23:27 +00001665 DOUT << "\n";
1666#endif
1667 }
Evan Chengf597dc72006-02-10 22:24:32 +00001668
1669#ifndef NDEBUG
Evan Cheng23addc02006-02-10 22:46:26 +00001670 Indent -= 2;
Evan Chengf597dc72006-02-10 22:24:32 +00001671#endif
Evan Cheng64a752f2006-08-11 09:08:15 +00001672
1673 return NULL;
Evan Cheng0114e942006-01-06 20:36:21 +00001674 }
Christopher Lamba1eb1552007-08-10 22:22:41 +00001675
Evan Cheng851bc042008-06-17 02:01:22 +00001676 case ISD::DECLARE: {
1677 // Handle DECLARE nodes here because the second operand may have been
1678 // wrapped in X86ISD::Wrapper.
Dan Gohman475871a2008-07-27 21:46:04 +00001679 SDValue Chain = Node->getOperand(0);
1680 SDValue N1 = Node->getOperand(1);
1681 SDValue N2 = Node->getOperand(2);
Evan Cheng6bb14ca2008-12-10 21:49:05 +00001682 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N1);
Chris Lattner1823c922009-02-12 17:33:11 +00001683
1684 // FIXME: We need to handle this for VLAs.
1685 if (!FINode) {
1686 ReplaceUses(N.getValue(0), Chain);
1687 return NULL;
1688 }
1689
Evan Chengfab83872008-06-18 02:48:27 +00001690 if (N2.getOpcode() == ISD::ADD &&
1691 N2.getOperand(0).getOpcode() == X86ISD::GlobalBaseReg)
1692 N2 = N2.getOperand(1);
Chris Lattner1823c922009-02-12 17:33:11 +00001693
1694 // If N2 is not Wrapper(decriptor) then the llvm.declare is mangled
1695 // somehow, just ignore it.
1696 if (N2.getOpcode() != X86ISD::Wrapper) {
1697 ReplaceUses(N.getValue(0), Chain);
1698 return NULL;
1699 }
Evan Chengf2accb52009-01-10 03:33:22 +00001700 GlobalAddressSDNode *GVNode =
1701 dyn_cast<GlobalAddressSDNode>(N2.getOperand(0));
Chris Lattner1823c922009-02-12 17:33:11 +00001702 if (GVNode == 0) {
1703 ReplaceUses(N.getValue(0), Chain);
1704 return NULL;
1705 }
Evan Cheng6bb14ca2008-12-10 21:49:05 +00001706 SDValue Tmp1 = CurDAG->getTargetFrameIndex(FINode->getIndex(),
1707 TLI.getPointerTy());
1708 SDValue Tmp2 = CurDAG->getTargetGlobalAddress(GVNode->getGlobal(),
1709 TLI.getPointerTy());
1710 SDValue Ops[] = { Tmp1, Tmp2, Chain };
Dale Johannesend8392542009-02-03 21:48:12 +00001711 return CurDAG->getTargetNode(TargetInstrInfo::DECLARE, dl,
Rafael Espindolae4d5d342009-03-27 15:45:05 +00001712 MVT::Other, Ops,
Rafael Espindolaa0a4f072009-03-28 19:02:18 +00001713 array_lengthof(Ops));
Evan Cheng851bc042008-06-17 02:01:22 +00001714 }
Chris Lattnerc961eea2005-11-16 01:54:32 +00001715 }
1716
Evan Cheng9ade2182006-08-26 05:34:46 +00001717 SDNode *ResNode = SelectCode(N);
Evan Cheng64a752f2006-08-11 09:08:15 +00001718
Evan Chengf597dc72006-02-10 22:24:32 +00001719#ifndef NDEBUG
Bill Wendling6345d752006-11-17 07:52:03 +00001720 DOUT << std::string(Indent-2, ' ') << "=> ";
Gabor Greifba36cb52008-08-28 21:40:38 +00001721 if (ResNode == NULL || ResNode == N.getNode())
1722 DEBUG(N.getNode()->dump(CurDAG));
Evan Cheng9ade2182006-08-26 05:34:46 +00001723 else
1724 DEBUG(ResNode->dump(CurDAG));
Bill Wendling6345d752006-11-17 07:52:03 +00001725 DOUT << "\n";
Evan Cheng23addc02006-02-10 22:46:26 +00001726 Indent -= 2;
Evan Chengf597dc72006-02-10 22:24:32 +00001727#endif
Evan Cheng64a752f2006-08-11 09:08:15 +00001728
1729 return ResNode;
Chris Lattnerc961eea2005-11-16 01:54:32 +00001730}
1731
Chris Lattnerc0bad572006-06-08 18:03:49 +00001732bool X86DAGToDAGISel::
Dan Gohman475871a2008-07-27 21:46:04 +00001733SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
Dan Gohmanf350b272008-08-23 02:25:05 +00001734 std::vector<SDValue> &OutOps) {
Rafael Espindola094fad32009-04-08 21:14:34 +00001735 SDValue Op0, Op1, Op2, Op3, Op4;
Chris Lattnerc0bad572006-06-08 18:03:49 +00001736 switch (ConstraintCode) {
1737 case 'o': // offsetable ??
1738 case 'v': // not offsetable ??
1739 default: return true;
1740 case 'm': // memory
Rafael Espindola094fad32009-04-08 21:14:34 +00001741 if (!SelectAddr(Op, Op, Op0, Op1, Op2, Op3, Op4))
Chris Lattnerc0bad572006-06-08 18:03:49 +00001742 return true;
1743 break;
1744 }
1745
Evan Cheng04699902006-08-26 01:05:16 +00001746 OutOps.push_back(Op0);
1747 OutOps.push_back(Op1);
1748 OutOps.push_back(Op2);
1749 OutOps.push_back(Op3);
Rafael Espindola094fad32009-04-08 21:14:34 +00001750 OutOps.push_back(Op4);
Chris Lattnerc0bad572006-06-08 18:03:49 +00001751 return false;
1752}
1753
Chris Lattnerc961eea2005-11-16 01:54:32 +00001754/// createX86ISelDag - This pass converts a legalized DAG into a
1755/// X86-specific DAG, ready for instruction scheduling.
1756///
Bill Wendling98a366d2009-04-29 23:29:43 +00001757FunctionPass *llvm::createX86ISelDag(X86TargetMachine &TM,
1758 llvm::CodeGenOpt::Level OptLevel) {
Bill Wendlingbe8cc2a2009-04-29 00:15:41 +00001759 return new X86DAGToDAGISel(TM, OptLevel);
Chris Lattnerc961eea2005-11-16 01:54:32 +00001760}