blob: 397afb2b0243d80eefd4993ab8785052f63e8ece [file] [log] [blame]
Chris Lattnera58f5592006-05-23 23:20:42 +00001//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
Chris Lattner76ac0682005-11-15 00:40:23 +00002//
3// The LLVM Compiler Infrastructure
4//
5// This file was developed by Chris Lattner and is distributed under
6// the University of Illinois Open Source License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the interfaces that X86 uses to lower LLVM code into a
11// selection DAG.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
Evan Cheng911c68d2006-01-16 21:21:29 +000016#include "X86InstrBuilder.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000017#include "X86ISelLowering.h"
Evan Chengdc614c12006-06-06 23:30:24 +000018#include "X86MachineFunctionInfo.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000019#include "X86TargetMachine.h"
20#include "llvm/CallingConv.h"
Evan Cheng72d5c252006-01-31 22:28:30 +000021#include "llvm/Constants.h"
Evan Cheng88decde2006-04-28 21:29:37 +000022#include "llvm/DerivedTypes.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000023#include "llvm/Function.h"
Evan Cheng78038292006-04-05 23:38:46 +000024#include "llvm/Intrinsics.h"
Evan Chengaf598d22006-03-13 23:18:16 +000025#include "llvm/ADT/VectorExtras.h"
26#include "llvm/Analysis/ScalarEvolutionExpressions.h"
Chris Lattnerdc3adc82007-02-27 04:43:02 +000027#include "llvm/CodeGen/CallingConvLower.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000028#include "llvm/CodeGen/MachineFrameInfo.h"
Evan Cheng339edad2006-01-11 00:33:36 +000029#include "llvm/CodeGen/MachineFunction.h"
30#include "llvm/CodeGen/MachineInstrBuilder.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000031#include "llvm/CodeGen/SelectionDAG.h"
32#include "llvm/CodeGen/SSARegMap.h"
Evan Cheng2dd217b2006-01-31 03:14:29 +000033#include "llvm/Support/MathExtras.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000034#include "llvm/Target/TargetOptions.h"
Evan Cheng8c5766e2006-10-04 18:33:38 +000035#include "llvm/Support/CommandLine.h"
Chris Lattnerf6a69662006-10-31 19:42:44 +000036#include "llvm/ADT/StringExtras.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000037using namespace llvm;
38
39// FIXME: temporary.
Chris Lattner76ac0682005-11-15 00:40:23 +000040static cl::opt<bool> EnableFastCC("enable-x86-fastcc", cl::Hidden,
41 cl::desc("Enable fastcc on X86"));
Chris Lattner76ac0682005-11-15 00:40:23 +000042X86TargetLowering::X86TargetLowering(TargetMachine &TM)
43 : TargetLowering(TM) {
Evan Chengcde9e302006-01-27 08:10:46 +000044 Subtarget = &TM.getSubtarget<X86Subtarget>();
45 X86ScalarSSE = Subtarget->hasSSE2();
Evan Cheng11b0a5d2006-09-08 06:48:29 +000046 X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP;
Evan Chengcde9e302006-01-27 08:10:46 +000047
Chris Lattner76ac0682005-11-15 00:40:23 +000048 // Set up the TargetLowering object.
49
50 // X86 is weird, it always uses i8 for shift amounts and setcc results.
51 setShiftAmountType(MVT::i8);
52 setSetCCResultType(MVT::i8);
53 setSetCCResultContents(ZeroOrOneSetCCResult);
Evan Cheng83eeefb2006-01-25 09:15:17 +000054 setSchedulingPreference(SchedulingForRegPressure);
Chris Lattner76ac0682005-11-15 00:40:23 +000055 setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0
Evan Cheng11b0a5d2006-09-08 06:48:29 +000056 setStackPointerRegisterToSaveRestore(X86StackPtr);
Evan Cheng20931a72006-03-16 21:47:42 +000057
Anton Korobeynikov3b7c2572006-12-10 23:12:42 +000058 if (Subtarget->isTargetDarwin()) {
Evan Chengb09a56f2006-03-17 20:31:41 +000059 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp.
Anton Korobeynikov3b7c2572006-12-10 23:12:42 +000060 setUseUnderscoreSetJmp(false);
61 setUseUnderscoreLongJmp(false);
Anton Korobeynikov4efbbc92007-01-03 11:43:14 +000062 } else if (Subtarget->isTargetMingw()) {
Anton Korobeynikov3b7c2572006-12-10 23:12:42 +000063 // MS runtime is weird: it exports _setjmp, but longjmp!
64 setUseUnderscoreSetJmp(true);
65 setUseUnderscoreLongJmp(false);
66 } else {
67 setUseUnderscoreSetJmp(true);
68 setUseUnderscoreLongJmp(true);
69 }
70
Evan Cheng20931a72006-03-16 21:47:42 +000071 // Add legal addressing mode scale values.
72 addLegalAddressScale(8);
73 addLegalAddressScale(4);
74 addLegalAddressScale(2);
75 // Enter the ones which require both scale + index last. These are more
76 // expensive.
77 addLegalAddressScale(9);
78 addLegalAddressScale(5);
79 addLegalAddressScale(3);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +000080
Chris Lattner76ac0682005-11-15 00:40:23 +000081 // Set up the register classes.
Evan Cheng9fee4422006-05-16 07:21:53 +000082 addRegisterClass(MVT::i8, X86::GR8RegisterClass);
83 addRegisterClass(MVT::i16, X86::GR16RegisterClass);
84 addRegisterClass(MVT::i32, X86::GR32RegisterClass);
Evan Cheng11b0a5d2006-09-08 06:48:29 +000085 if (Subtarget->is64Bit())
86 addRegisterClass(MVT::i64, X86::GR64RegisterClass);
Chris Lattner76ac0682005-11-15 00:40:23 +000087
Evan Cheng5d9fd972006-10-04 00:56:09 +000088 setLoadXAction(ISD::SEXTLOAD, MVT::i1, Expand);
89
Chris Lattner76ac0682005-11-15 00:40:23 +000090 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
91 // operation.
92 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote);
93 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote);
94 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote);
Evan Cheng0d5b69f2006-01-17 02:32:49 +000095
Evan Cheng11b0a5d2006-09-08 06:48:29 +000096 if (Subtarget->is64Bit()) {
97 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand);
Evan Cheng0d5b69f2006-01-17 02:32:49 +000098 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote);
Evan Cheng11b0a5d2006-09-08 06:48:29 +000099 } else {
100 if (X86ScalarSSE)
101 // If SSE i64 SINT_TO_FP is not available, expand i32 UINT_TO_FP.
102 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand);
103 else
104 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote);
105 }
Chris Lattner76ac0682005-11-15 00:40:23 +0000106
107 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
108 // this operation.
109 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote);
110 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote);
Nate Begeman7e5496d2006-02-17 00:03:04 +0000111 // SSE has no i16 to fp conversion, only i32
Evan Cheng08390f62006-01-30 22:13:22 +0000112 if (X86ScalarSSE)
Evan Cheng08390f62006-01-30 22:13:22 +0000113 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote);
Evan Cheng593bea72006-02-17 07:01:52 +0000114 else {
115 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom);
116 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom);
117 }
Chris Lattner76ac0682005-11-15 00:40:23 +0000118
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000119 if (!Subtarget->is64Bit()) {
120 // Custom lower SINT_TO_FP and FP_TO_SINT from/to i64 in 32-bit mode.
121 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom);
122 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom);
123 }
Evan Cheng5b97fcf2006-01-30 08:02:57 +0000124
Evan Cheng08390f62006-01-30 22:13:22 +0000125 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
126 // this operation.
127 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote);
128 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote);
129
130 if (X86ScalarSSE) {
131 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote);
132 } else {
Chris Lattner76ac0682005-11-15 00:40:23 +0000133 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom);
Evan Cheng08390f62006-01-30 22:13:22 +0000134 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom);
Chris Lattner76ac0682005-11-15 00:40:23 +0000135 }
136
137 // Handle FP_TO_UINT by promoting the destination to a larger signed
138 // conversion.
139 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote);
140 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote);
141 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote);
142
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000143 if (Subtarget->is64Bit()) {
144 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000145 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000146 } else {
147 if (X86ScalarSSE && !Subtarget->hasSSE3())
148 // Expand FP_TO_UINT into a select.
149 // FIXME: We would like to use a Custom expander here eventually to do
150 // the optimal thing for SSE vs. the default expansion in the legalizer.
151 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand);
152 else
153 // With SSE3 we can use fisttpll to convert to a signed i64.
154 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote);
155 }
Chris Lattner76ac0682005-11-15 00:40:23 +0000156
Chris Lattner55c17f92006-12-05 18:22:22 +0000157 // TODO: when we have SSE, these could be more efficient, by using movd/movq.
Chris Lattnerc20b7e82006-12-05 18:45:06 +0000158 if (!X86ScalarSSE) {
159 setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand);
160 setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand);
161 }
Chris Lattner30107e62005-12-23 05:15:23 +0000162
Evan Cheng0d41d192006-10-30 08:02:39 +0000163 setOperationAction(ISD::BR_JT , MVT::Other, Expand);
Evan Cheng593bea72006-02-17 07:01:52 +0000164 setOperationAction(ISD::BRCOND , MVT::Other, Custom);
Nate Begeman7e7f4392006-02-01 07:19:44 +0000165 setOperationAction(ISD::BR_CC , MVT::Other, Expand);
166 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000167 setOperationAction(ISD::MEMMOVE , MVT::Other, Expand);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000168 if (Subtarget->is64Bit())
169 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000170 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Expand);
Chris Lattner32257332005-12-07 17:59:14 +0000171 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000172 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
173 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000174 setOperationAction(ISD::FREM , MVT::f64 , Expand);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000175
Chris Lattner76ac0682005-11-15 00:40:23 +0000176 setOperationAction(ISD::CTPOP , MVT::i8 , Expand);
177 setOperationAction(ISD::CTTZ , MVT::i8 , Expand);
178 setOperationAction(ISD::CTLZ , MVT::i8 , Expand);
179 setOperationAction(ISD::CTPOP , MVT::i16 , Expand);
180 setOperationAction(ISD::CTTZ , MVT::i16 , Expand);
181 setOperationAction(ISD::CTLZ , MVT::i16 , Expand);
182 setOperationAction(ISD::CTPOP , MVT::i32 , Expand);
183 setOperationAction(ISD::CTTZ , MVT::i32 , Expand);
184 setOperationAction(ISD::CTLZ , MVT::i32 , Expand);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000185 if (Subtarget->is64Bit()) {
186 setOperationAction(ISD::CTPOP , MVT::i64 , Expand);
187 setOperationAction(ISD::CTTZ , MVT::i64 , Expand);
188 setOperationAction(ISD::CTLZ , MVT::i64 , Expand);
189 }
190
Andrew Lenharth0bf68ae2005-11-20 21:41:10 +0000191 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
Nate Begeman2fba8a32006-01-14 03:14:10 +0000192 setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
Nate Begeman1b8121b2006-01-11 21:21:00 +0000193
Chris Lattner76ac0682005-11-15 00:40:23 +0000194 // These should be promoted to a larger select which is supported.
195 setOperationAction(ISD::SELECT , MVT::i1 , Promote);
196 setOperationAction(ISD::SELECT , MVT::i8 , Promote);
Nate Begeman7e5496d2006-02-17 00:03:04 +0000197 // X86 wants to expand cmov itself.
Evan Cheng593bea72006-02-17 07:01:52 +0000198 setOperationAction(ISD::SELECT , MVT::i16 , Custom);
199 setOperationAction(ISD::SELECT , MVT::i32 , Custom);
200 setOperationAction(ISD::SELECT , MVT::f32 , Custom);
201 setOperationAction(ISD::SELECT , MVT::f64 , Custom);
202 setOperationAction(ISD::SETCC , MVT::i8 , Custom);
203 setOperationAction(ISD::SETCC , MVT::i16 , Custom);
204 setOperationAction(ISD::SETCC , MVT::i32 , Custom);
205 setOperationAction(ISD::SETCC , MVT::f32 , Custom);
206 setOperationAction(ISD::SETCC , MVT::f64 , Custom);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000207 if (Subtarget->is64Bit()) {
208 setOperationAction(ISD::SELECT , MVT::i64 , Custom);
209 setOperationAction(ISD::SETCC , MVT::i64 , Custom);
210 }
Nate Begeman7e5496d2006-02-17 00:03:04 +0000211 // X86 ret instruction may pop stack.
Evan Cheng593bea72006-02-17 07:01:52 +0000212 setOperationAction(ISD::RET , MVT::Other, Custom);
Nate Begeman7e5496d2006-02-17 00:03:04 +0000213 // Darwin ABI issue.
Evan Cheng5588de92006-02-18 00:15:05 +0000214 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom);
Nate Begeman4ca2ea52006-04-22 18:53:45 +0000215 setOperationAction(ISD::JumpTable , MVT::i32 , Custom);
Evan Cheng593bea72006-02-17 07:01:52 +0000216 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom);
Evan Chenge0ed6ec2006-02-23 20:41:18 +0000217 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000218 if (Subtarget->is64Bit()) {
219 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom);
220 setOperationAction(ISD::JumpTable , MVT::i64 , Custom);
221 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom);
222 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom);
223 }
Nate Begeman7e5496d2006-02-17 00:03:04 +0000224 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86)
Evan Cheng593bea72006-02-17 07:01:52 +0000225 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom);
226 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom);
227 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom);
Nate Begeman7e5496d2006-02-17 00:03:04 +0000228 // X86 wants to expand memset / memcpy itself.
Evan Cheng593bea72006-02-17 07:01:52 +0000229 setOperationAction(ISD::MEMSET , MVT::Other, Custom);
230 setOperationAction(ISD::MEMCPY , MVT::Other, Custom);
Chris Lattner76ac0682005-11-15 00:40:23 +0000231
Chris Lattner9c415362005-11-29 06:16:21 +0000232 // We don't have line number support yet.
233 setOperationAction(ISD::LOCATION, MVT::Other, Expand);
Jim Laskeydeeafa02006-01-05 01:47:43 +0000234 setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand);
Evan Cheng30d7b702006-03-07 02:02:57 +0000235 // FIXME - use subtarget debug flags
Anton Korobeynikovaa4c0f92006-10-31 08:31:24 +0000236 if (!Subtarget->isTargetDarwin() &&
237 !Subtarget->isTargetELF() &&
Anton Korobeynikov4efbbc92007-01-03 11:43:14 +0000238 !Subtarget->isTargetCygMing())
Jim Laskeyf9e54452007-01-26 14:34:52 +0000239 setOperationAction(ISD::LABEL, MVT::Other, Expand);
Chris Lattner9c415362005-11-29 06:16:21 +0000240
Nate Begemane74795c2006-01-25 18:21:52 +0000241 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
242 setOperationAction(ISD::VASTART , MVT::Other, Custom);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000243
Nate Begemane74795c2006-01-25 18:21:52 +0000244 // Use the default implementation.
245 setOperationAction(ISD::VAARG , MVT::Other, Expand);
246 setOperationAction(ISD::VACOPY , MVT::Other, Expand);
247 setOperationAction(ISD::VAEND , MVT::Other, Expand);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000248 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
Chris Lattner78c358d2006-01-15 09:00:21 +0000249 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000250 if (Subtarget->is64Bit())
251 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
Chris Lattner78c358d2006-01-15 09:00:21 +0000252 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Expand);
Chris Lattner8e2f52e2006-01-13 02:42:53 +0000253
Chris Lattner76ac0682005-11-15 00:40:23 +0000254 if (X86ScalarSSE) {
255 // Set up the FP register classes.
Evan Cheng84dc9b52006-01-12 08:27:59 +0000256 addRegisterClass(MVT::f32, X86::FR32RegisterClass);
257 addRegisterClass(MVT::f64, X86::FR64RegisterClass);
Chris Lattner76ac0682005-11-15 00:40:23 +0000258
Evan Cheng72d5c252006-01-31 22:28:30 +0000259 // Use ANDPD to simulate FABS.
260 setOperationAction(ISD::FABS , MVT::f64, Custom);
261 setOperationAction(ISD::FABS , MVT::f32, Custom);
262
263 // Use XORP to simulate FNEG.
264 setOperationAction(ISD::FNEG , MVT::f64, Custom);
265 setOperationAction(ISD::FNEG , MVT::f32, Custom);
266
Evan Cheng4363e882007-01-05 07:55:56 +0000267 // Use ANDPD and ORPD to simulate FCOPYSIGN.
268 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
269 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
270
Evan Chengd8fba3a2006-02-02 00:28:23 +0000271 // We don't support sin/cos/fmod
Chris Lattner76ac0682005-11-15 00:40:23 +0000272 setOperationAction(ISD::FSIN , MVT::f64, Expand);
273 setOperationAction(ISD::FCOS , MVT::f64, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000274 setOperationAction(ISD::FREM , MVT::f64, Expand);
275 setOperationAction(ISD::FSIN , MVT::f32, Expand);
276 setOperationAction(ISD::FCOS , MVT::f32, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000277 setOperationAction(ISD::FREM , MVT::f32, Expand);
278
Chris Lattner61c9a8e2006-01-29 06:26:08 +0000279 // Expand FP immediates into loads from the stack, except for the special
280 // cases we handle.
281 setOperationAction(ISD::ConstantFP, MVT::f64, Expand);
282 setOperationAction(ISD::ConstantFP, MVT::f32, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000283 addLegalFPImmediate(+0.0); // xorps / xorpd
284 } else {
285 // Set up the FP register classes.
286 addRegisterClass(MVT::f64, X86::RFPRegisterClass);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000287
Evan Cheng4363e882007-01-05 07:55:56 +0000288 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
289 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
290 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000291
Chris Lattner76ac0682005-11-15 00:40:23 +0000292 if (!UnsafeFPMath) {
293 setOperationAction(ISD::FSIN , MVT::f64 , Expand);
294 setOperationAction(ISD::FCOS , MVT::f64 , Expand);
295 }
296
Chris Lattner61c9a8e2006-01-29 06:26:08 +0000297 setOperationAction(ISD::ConstantFP, MVT::f64, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000298 addLegalFPImmediate(+0.0); // FLD0
299 addLegalFPImmediate(+1.0); // FLD1
300 addLegalFPImmediate(-0.0); // FLD0/FCHS
301 addLegalFPImmediate(-1.0); // FLD1/FCHS
302 }
Evan Cheng9e252e32006-02-22 02:26:30 +0000303
Evan Cheng19264272006-03-01 01:11:20 +0000304 // First set operation action for all vector types to expand. Then we
305 // will selectively turn on ones that can be effectively codegen'd.
306 for (unsigned VT = (unsigned)MVT::Vector + 1;
307 VT != (unsigned)MVT::LAST_VALUETYPE; VT++) {
308 setOperationAction(ISD::ADD , (MVT::ValueType)VT, Expand);
309 setOperationAction(ISD::SUB , (MVT::ValueType)VT, Expand);
Evan Chengbf3df772006-10-27 18:49:08 +0000310 setOperationAction(ISD::FADD, (MVT::ValueType)VT, Expand);
311 setOperationAction(ISD::FSUB, (MVT::ValueType)VT, Expand);
Evan Cheng19264272006-03-01 01:11:20 +0000312 setOperationAction(ISD::MUL , (MVT::ValueType)VT, Expand);
Evan Chengbf3df772006-10-27 18:49:08 +0000313 setOperationAction(ISD::FMUL, (MVT::ValueType)VT, Expand);
314 setOperationAction(ISD::SDIV, (MVT::ValueType)VT, Expand);
315 setOperationAction(ISD::UDIV, (MVT::ValueType)VT, Expand);
316 setOperationAction(ISD::FDIV, (MVT::ValueType)VT, Expand);
317 setOperationAction(ISD::SREM, (MVT::ValueType)VT, Expand);
318 setOperationAction(ISD::UREM, (MVT::ValueType)VT, Expand);
Evan Cheng19264272006-03-01 01:11:20 +0000319 setOperationAction(ISD::LOAD, (MVT::ValueType)VT, Expand);
Evan Chengcbffa462006-03-31 19:22:53 +0000320 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::ValueType)VT, Expand);
Chris Lattner00f46832006-03-21 20:51:05 +0000321 setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::ValueType)VT, Expand);
Evan Chengcbffa462006-03-31 19:22:53 +0000322 setOperationAction(ISD::INSERT_VECTOR_ELT, (MVT::ValueType)VT, Expand);
Evan Cheng19264272006-03-01 01:11:20 +0000323 }
324
Evan Chengbc047222006-03-22 19:22:18 +0000325 if (Subtarget->hasMMX()) {
Evan Cheng9e252e32006-02-22 02:26:30 +0000326 addRegisterClass(MVT::v8i8, X86::VR64RegisterClass);
327 addRegisterClass(MVT::v4i16, X86::VR64RegisterClass);
328 addRegisterClass(MVT::v2i32, X86::VR64RegisterClass);
329
Evan Cheng19264272006-03-01 01:11:20 +0000330 // FIXME: add MMX packed arithmetics
Evan Chengd5e905d2006-03-21 23:01:21 +0000331 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Expand);
332 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Expand);
333 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Expand);
Evan Cheng9e252e32006-02-22 02:26:30 +0000334 }
335
Evan Chengbc047222006-03-22 19:22:18 +0000336 if (Subtarget->hasSSE1()) {
Evan Cheng9e252e32006-02-22 02:26:30 +0000337 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass);
338
Evan Chengbf3df772006-10-27 18:49:08 +0000339 setOperationAction(ISD::FADD, MVT::v4f32, Legal);
340 setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
341 setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
342 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
Evan Cheng617a6a82006-04-10 07:23:14 +0000343 setOperationAction(ISD::LOAD, MVT::v4f32, Legal);
344 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
345 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
Evan Chengebf10062006-04-03 20:53:28 +0000346 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
Evan Cheng617a6a82006-04-10 07:23:14 +0000347 setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
Evan Cheng9e252e32006-02-22 02:26:30 +0000348 }
349
Evan Chengbc047222006-03-22 19:22:18 +0000350 if (Subtarget->hasSSE2()) {
Evan Cheng9e252e32006-02-22 02:26:30 +0000351 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass);
352 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass);
353 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass);
354 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass);
355 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass);
356
Evan Cheng617a6a82006-04-10 07:23:14 +0000357 setOperationAction(ISD::ADD, MVT::v16i8, Legal);
358 setOperationAction(ISD::ADD, MVT::v8i16, Legal);
359 setOperationAction(ISD::ADD, MVT::v4i32, Legal);
Evan Cheng617a6a82006-04-10 07:23:14 +0000360 setOperationAction(ISD::SUB, MVT::v16i8, Legal);
361 setOperationAction(ISD::SUB, MVT::v8i16, Legal);
362 setOperationAction(ISD::SUB, MVT::v4i32, Legal);
Evan Chenge4f97cc2006-04-13 05:10:25 +0000363 setOperationAction(ISD::MUL, MVT::v8i16, Legal);
Evan Chengbf3df772006-10-27 18:49:08 +0000364 setOperationAction(ISD::FADD, MVT::v2f64, Legal);
365 setOperationAction(ISD::FSUB, MVT::v2f64, Legal);
366 setOperationAction(ISD::FMUL, MVT::v2f64, Legal);
367 setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
Evan Cheng92232302006-04-12 21:21:57 +0000368
Evan Cheng617a6a82006-04-10 07:23:14 +0000369 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom);
370 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom);
Evan Chengcbffa462006-03-31 19:22:53 +0000371 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
Evan Cheng6e5e2052006-04-17 22:04:06 +0000372 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
373 // Implement v4f32 insert_vector_elt in terms of SSE2 v8i16 ones.
374 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
Evan Cheng617a6a82006-04-10 07:23:14 +0000375
Evan Cheng92232302006-04-12 21:21:57 +0000376 // Custom lower build_vector, vector_shuffle, and extract_vector_elt.
377 for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) {
378 setOperationAction(ISD::BUILD_VECTOR, (MVT::ValueType)VT, Custom);
379 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::ValueType)VT, Custom);
380 setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::ValueType)VT, Custom);
381 }
382 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom);
383 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom);
384 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom);
385 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom);
386 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom);
387 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom);
388
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000389 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64.
Evan Cheng92232302006-04-12 21:21:57 +0000390 for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) {
391 setOperationAction(ISD::AND, (MVT::ValueType)VT, Promote);
392 AddPromotedToType (ISD::AND, (MVT::ValueType)VT, MVT::v2i64);
393 setOperationAction(ISD::OR, (MVT::ValueType)VT, Promote);
394 AddPromotedToType (ISD::OR, (MVT::ValueType)VT, MVT::v2i64);
395 setOperationAction(ISD::XOR, (MVT::ValueType)VT, Promote);
396 AddPromotedToType (ISD::XOR, (MVT::ValueType)VT, MVT::v2i64);
Evan Chenge2157c62006-04-12 17:12:36 +0000397 setOperationAction(ISD::LOAD, (MVT::ValueType)VT, Promote);
398 AddPromotedToType (ISD::LOAD, (MVT::ValueType)VT, MVT::v2i64);
Evan Cheng92232302006-04-12 21:21:57 +0000399 setOperationAction(ISD::SELECT, (MVT::ValueType)VT, Promote);
400 AddPromotedToType (ISD::SELECT, (MVT::ValueType)VT, MVT::v2i64);
Evan Cheng617a6a82006-04-10 07:23:14 +0000401 }
Evan Cheng92232302006-04-12 21:21:57 +0000402
403 // Custom lower v2i64 and v2f64 selects.
404 setOperationAction(ISD::LOAD, MVT::v2f64, Legal);
Evan Chenge2157c62006-04-12 17:12:36 +0000405 setOperationAction(ISD::LOAD, MVT::v2i64, Legal);
Evan Cheng617a6a82006-04-10 07:23:14 +0000406 setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
Evan Cheng92232302006-04-12 21:21:57 +0000407 setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
Evan Cheng9e252e32006-02-22 02:26:30 +0000408 }
409
Evan Cheng78038292006-04-05 23:38:46 +0000410 // We want to custom lower some of our intrinsics.
411 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
412
Evan Cheng5987cfb2006-07-07 08:33:52 +0000413 // We have target-specific dag combine patterns for the following nodes:
414 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
Chris Lattner9259b1e2006-10-04 06:57:07 +0000415 setTargetDAGCombine(ISD::SELECT);
Evan Cheng5987cfb2006-07-07 08:33:52 +0000416
Chris Lattner76ac0682005-11-15 00:40:23 +0000417 computeRegisterProperties();
418
Evan Cheng6a374562006-02-14 08:25:08 +0000419 // FIXME: These should be based on subtarget info. Plus, the values should
420 // be smaller when we are in optimizing for size mode.
Evan Cheng4b40a422006-02-14 08:38:30 +0000421 maxStoresPerMemset = 16; // For %llvm.memset -> sequence of stores
422 maxStoresPerMemcpy = 16; // For %llvm.memcpy -> sequence of stores
423 maxStoresPerMemmove = 16; // For %llvm.memmove -> sequence of stores
Chris Lattner76ac0682005-11-15 00:40:23 +0000424 allowUnalignedMemoryAccesses = true; // x86 supports it!
425}
426
Chris Lattner3c763092007-02-25 08:29:00 +0000427
428//===----------------------------------------------------------------------===//
429// Return Value Calling Convention Implementation
430//===----------------------------------------------------------------------===//
431
Chris Lattnerba3d2732007-02-28 04:55:35 +0000432#include "X86GenCallingConv.inc"
Chris Lattnerc9eed392007-02-27 05:28:59 +0000433
Chris Lattner2fc0d702007-02-25 09:12:39 +0000434/// LowerRET - Lower an ISD::RET node.
435SDOperand X86TargetLowering::LowerRET(SDOperand Op, SelectionDAG &DAG) {
436 assert((Op.getNumOperands() & 1) == 1 && "ISD::RET should have odd # args");
437
Chris Lattnerc9eed392007-02-27 05:28:59 +0000438 SmallVector<CCValAssign, 16> RVLocs;
439 unsigned CC = DAG.getMachineFunction().getFunction()->getCallingConv();
440 CCState CCInfo(CC, getTargetMachine(), RVLocs);
Chris Lattner2fc0d702007-02-25 09:12:39 +0000441
442 // Determine which register each value should be copied into.
Chris Lattnerc9eed392007-02-27 05:28:59 +0000443 for (unsigned i = 0; i != Op.getNumOperands() / 2; ++i) {
Chris Lattnerba3d2732007-02-28 04:55:35 +0000444 MVT::ValueType VT = Op.getOperand(i*2+1).getValueType();
445 if (RetCC_X86(i, VT, VT, CCValAssign::Full,
446 cast<ConstantSDNode>(Op.getOperand(i*2+2))->getValue(),
447 CCInfo))
Chris Lattnerc9eed392007-02-27 05:28:59 +0000448 assert(0 && "Unhandled result type!");
449 }
Chris Lattner2fc0d702007-02-25 09:12:39 +0000450
451 // If this is the first return lowered for this function, add the regs to the
452 // liveout set for the function.
453 if (DAG.getMachineFunction().liveout_empty()) {
Chris Lattnerc9eed392007-02-27 05:28:59 +0000454 for (unsigned i = 0; i != RVLocs.size(); ++i)
455 if (RVLocs[i].isRegLoc())
456 DAG.getMachineFunction().addLiveOut(RVLocs[i].getLocReg());
Chris Lattner2fc0d702007-02-25 09:12:39 +0000457 }
458
459 SDOperand Chain = Op.getOperand(0);
460 SDOperand Flag;
461
462 // Copy the result values into the output registers.
Chris Lattnerc9eed392007-02-27 05:28:59 +0000463 if (RVLocs.size() != 1 || !RVLocs[0].isRegLoc() ||
464 RVLocs[0].getLocReg() != X86::ST0) {
465 for (unsigned i = 0; i != RVLocs.size(); ++i) {
466 CCValAssign &VA = RVLocs[i];
467 assert(VA.isRegLoc() && "Can only return in registers!");
468 Chain = DAG.getCopyToReg(Chain, VA.getLocReg(), Op.getOperand(i*2+1),
469 Flag);
Chris Lattner2fc0d702007-02-25 09:12:39 +0000470 Flag = Chain.getValue(1);
471 }
472 } else {
473 // We need to handle a destination of ST0 specially, because it isn't really
474 // a register.
475 SDOperand Value = Op.getOperand(1);
476
477 // If this is an FP return with ScalarSSE, we need to move the value from
478 // an XMM register onto the fp-stack.
479 if (X86ScalarSSE) {
480 SDOperand MemLoc;
481
482 // If this is a load into a scalarsse value, don't store the loaded value
483 // back to the stack, only to reload it: just replace the scalar-sse load.
484 if (ISD::isNON_EXTLoad(Value.Val) &&
485 (Chain == Value.getValue(1) || Chain == Value.getOperand(0))) {
486 Chain = Value.getOperand(0);
487 MemLoc = Value.getOperand(1);
488 } else {
489 // Spill the value to memory and reload it into top of stack.
Chris Lattnerc9eed392007-02-27 05:28:59 +0000490 unsigned Size = MVT::getSizeInBits(RVLocs[0].getValVT())/8;
Chris Lattner2fc0d702007-02-25 09:12:39 +0000491 MachineFunction &MF = DAG.getMachineFunction();
492 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
493 MemLoc = DAG.getFrameIndex(SSFI, getPointerTy());
494 Chain = DAG.getStore(Op.getOperand(0), Value, MemLoc, NULL, 0);
495 }
496 SDVTList Tys = DAG.getVTList(MVT::f64, MVT::Other);
Chris Lattnerc9eed392007-02-27 05:28:59 +0000497 SDOperand Ops[] = {Chain, MemLoc, DAG.getValueType(RVLocs[0].getValVT())};
Chris Lattner2fc0d702007-02-25 09:12:39 +0000498 Value = DAG.getNode(X86ISD::FLD, Tys, Ops, 3);
499 Chain = Value.getValue(1);
500 }
501
502 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
503 SDOperand Ops[] = { Chain, Value };
504 Chain = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops, 2);
505 Flag = Chain.getValue(1);
506 }
507
508 SDOperand BytesToPop = DAG.getConstant(getBytesToPopOnReturn(), MVT::i16);
509 if (Flag.Val)
510 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, Chain, BytesToPop, Flag);
511 else
512 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, Chain, BytesToPop);
513}
514
515
Chris Lattner0cd99602007-02-25 08:59:22 +0000516/// LowerCallResult - Lower the result values of an ISD::CALL into the
517/// appropriate copies out of appropriate physical registers. This assumes that
518/// Chain/InFlag are the input chain/flag to use, and that TheCall is the call
519/// being lowered. The returns a SDNode with the same number of values as the
520/// ISD::CALL.
521SDNode *X86TargetLowering::
522LowerCallResult(SDOperand Chain, SDOperand InFlag, SDNode *TheCall,
523 unsigned CallingConv, SelectionDAG &DAG) {
524 SmallVector<SDOperand, 8> ResultVals;
525
Chris Lattnerc9eed392007-02-27 05:28:59 +0000526 SmallVector<CCValAssign, 16> RVLocs;
527 CCState CCInfo(CallingConv, getTargetMachine(), RVLocs);
Chris Lattner0cd99602007-02-25 08:59:22 +0000528
Chris Lattnerc9eed392007-02-27 05:28:59 +0000529 for (unsigned i = 0, e = TheCall->getNumValues() - 1; i != e; ++i) {
Chris Lattnerba3d2732007-02-28 04:55:35 +0000530 MVT::ValueType VT = TheCall->getValueType(i);
531 if (RetCC_X86(i, VT, VT, CCValAssign::Full, 0, CCInfo))
Chris Lattnerc9eed392007-02-27 05:28:59 +0000532 assert(0 && "Unhandled result type!");
533 }
Chris Lattner0cd99602007-02-25 08:59:22 +0000534
535 // Copy all of the result registers out of their specified physreg.
Chris Lattnerc9eed392007-02-27 05:28:59 +0000536 if (RVLocs.size() != 1 || RVLocs[0].getLocReg() != X86::ST0) {
537 for (unsigned i = 0; i != RVLocs.size(); ++i) {
538 Chain = DAG.getCopyFromReg(Chain, RVLocs[i].getLocReg(),
539 RVLocs[i].getValVT(), InFlag).getValue(1);
Chris Lattner0cd99602007-02-25 08:59:22 +0000540 InFlag = Chain.getValue(2);
541 ResultVals.push_back(Chain.getValue(0));
542 }
543 } else {
544 // Copies from the FP stack are special, as ST0 isn't a valid register
545 // before the fp stackifier runs.
546
547 // Copy ST0 into an RFP register with FP_GET_RESULT.
548 SDVTList Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag);
549 SDOperand GROps[] = { Chain, InFlag };
550 SDOperand RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, GROps, 2);
551 Chain = RetVal.getValue(1);
552 InFlag = RetVal.getValue(2);
553
554 // If we are using ScalarSSE, store ST(0) to the stack and reload it into
555 // an XMM register.
556 if (X86ScalarSSE) {
557 // FIXME: Currently the FST is flagged to the FP_GET_RESULT. This
558 // shouldn't be necessary except that RFP cannot be live across
559 // multiple blocks. When stackifier is fixed, they can be uncoupled.
560 MachineFunction &MF = DAG.getMachineFunction();
561 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8);
562 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
563 SDOperand Ops[] = {
Chris Lattnerc9eed392007-02-27 05:28:59 +0000564 Chain, RetVal, StackSlot, DAG.getValueType(RVLocs[0].getValVT()), InFlag
Chris Lattner0cd99602007-02-25 08:59:22 +0000565 };
566 Chain = DAG.getNode(X86ISD::FST, MVT::Other, Ops, 5);
Chris Lattnerc9eed392007-02-27 05:28:59 +0000567 RetVal = DAG.getLoad(RVLocs[0].getValVT(), Chain, StackSlot, NULL, 0);
Chris Lattner0cd99602007-02-25 08:59:22 +0000568 Chain = RetVal.getValue(1);
569 }
570
Chris Lattnerc9eed392007-02-27 05:28:59 +0000571 if (RVLocs[0].getValVT() == MVT::f32 && !X86ScalarSSE)
Chris Lattner0cd99602007-02-25 08:59:22 +0000572 // FIXME: we would really like to remember that this FP_ROUND
573 // operation is okay to eliminate if we allow excess FP precision.
574 RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal);
575 ResultVals.push_back(RetVal);
576 }
577
578 // Merge everything together with a MERGE_VALUES node.
579 ResultVals.push_back(Chain);
580 return DAG.getNode(ISD::MERGE_VALUES, TheCall->getVTList(),
581 &ResultVals[0], ResultVals.size()).Val;
Chris Lattner3c763092007-02-25 08:29:00 +0000582}
583
584
Chris Lattner76ac0682005-11-15 00:40:23 +0000585//===----------------------------------------------------------------------===//
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000586// C & StdCall Calling Convention implementation
Chris Lattner76ac0682005-11-15 00:40:23 +0000587//===----------------------------------------------------------------------===//
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000588// StdCall calling convention seems to be standard for many Windows' API
589// routines and around. It differs from C calling convention just a little:
590// callee should clean up the stack, not caller. Symbols should be also
591// decorated in some fancy way :) It doesn't support any vector arguments.
Chris Lattner76ac0682005-11-15 00:40:23 +0000592
Evan Cheng24eb3f42006-04-27 05:35:28 +0000593/// AddLiveIn - This helper function adds the specified physical register to the
594/// MachineFunction as a live in value. It also creates a corresponding virtual
595/// register for it.
596static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg,
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000597 const TargetRegisterClass *RC) {
Evan Cheng24eb3f42006-04-27 05:35:28 +0000598 assert(RC->contains(PReg) && "Not the correct regclass!");
599 unsigned VReg = MF.getSSARegMap()->createVirtualRegister(RC);
600 MF.addLiveIn(PReg, VReg);
601 return VReg;
602}
603
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000604SDOperand X86TargetLowering::LowerCCCArguments(SDOperand Op, SelectionDAG &DAG,
605 bool isStdCall) {
Evan Cheng17e734f2006-05-23 21:06:34 +0000606 unsigned NumArgs = Op.Val->getNumValues() - 1;
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000607 MachineFunction &MF = DAG.getMachineFunction();
608 MachineFrameInfo *MFI = MF.getFrameInfo();
Evan Cheng17e734f2006-05-23 21:06:34 +0000609 SDOperand Root = Op.getOperand(0);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000610 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0;
Chris Lattner76ac0682005-11-15 00:40:23 +0000611
Chris Lattnerb9db2252007-02-28 05:46:49 +0000612 SmallVector<CCValAssign, 16> ArgLocs;
613 CCState CCInfo(MF.getFunction()->getCallingConv(), getTargetMachine(),
614 ArgLocs);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000615
Chris Lattnerb9db2252007-02-28 05:46:49 +0000616 for (unsigned i = 0; i != NumArgs; ++i) {
617 MVT::ValueType ArgVT = Op.getValue(i).getValueType();
618 unsigned ArgFlags = cast<ConstantSDNode>(Op.getOperand(3+i))->getValue();
619 if (CC_X86_32_C(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo))
620 assert(0 && "Unhandled argument type!");
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000621 }
622
Chris Lattnerb9db2252007-02-28 05:46:49 +0000623 SmallVector<SDOperand, 8> ArgValues;
624 unsigned LastVal = ~0U;
625 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
626 CCValAssign &VA = ArgLocs[i];
627 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later
628 // places.
629 assert(VA.getValNo() != LastVal &&
630 "Don't support value assigned to multiple locs yet");
631 LastVal = VA.getValNo();
632
633 if (VA.isRegLoc()) {
634 MVT::ValueType RegVT = VA.getLocVT();
635 TargetRegisterClass *RC;
636 if (RegVT == MVT::i32)
637 RC = X86::GR32RegisterClass;
638 else {
639 assert(MVT::isVector(RegVT));
640 RC = X86::VR128RegisterClass;
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000641 }
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000642
Chris Lattnerb9db2252007-02-28 05:46:49 +0000643 SDOperand ArgValue = DAG.getCopyFromReg(Root, VA.getLocReg(), RegVT);
644 AddLiveIn(DAG.getMachineFunction(), VA.getLocReg(), RC);
645
646 // If this is an 8 or 16-bit value, it is really passed promoted to 32
647 // bits. Insert an assert[sz]ext to capture this, then truncate to the
648 // right size.
649 if (VA.getLocInfo() == CCValAssign::SExt)
650 ArgValue = DAG.getNode(ISD::AssertSext, RegVT, ArgValue,
651 DAG.getValueType(VA.getValVT()));
652 else if (VA.getLocInfo() == CCValAssign::ZExt)
653 ArgValue = DAG.getNode(ISD::AssertZext, RegVT, ArgValue,
654 DAG.getValueType(VA.getValVT()));
655
656 if (VA.getLocInfo() != CCValAssign::Full)
657 ArgValue = DAG.getNode(ISD::TRUNCATE, VA.getValVT(), ArgValue);
658
659 ArgValues.push_back(ArgValue);
660 } else {
661 assert(VA.isMemLoc());
662
663 // Create the nodes corresponding to a load from this parameter slot.
664 int FI = MFI->CreateFixedObject(MVT::getSizeInBits(VA.getValVT())/8,
665 VA.getLocMemOffset());
666 SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy());
667 ArgValues.push_back(DAG.getLoad(VA.getValVT(), Root, FIN, NULL, 0));
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000668 }
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000669 }
Chris Lattnerb9db2252007-02-28 05:46:49 +0000670
671 unsigned StackSize = CCInfo.getNextStackOffset();
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000672
Evan Cheng17e734f2006-05-23 21:06:34 +0000673 ArgValues.push_back(Root);
674
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000675 // If the function takes variable number of arguments, make a frame index for
676 // the start of the first vararg value... for expansion of llvm.va_start.
Evan Cheng7068a932006-05-23 21:08:24 +0000677 if (isVarArg)
Chris Lattnerb9db2252007-02-28 05:46:49 +0000678 VarArgsFrameIndex = MFI->CreateFixedObject(1, StackSize);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000679
680 if (isStdCall && !isVarArg) {
Chris Lattnerb9db2252007-02-28 05:46:49 +0000681 BytesToPopOnReturn = StackSize; // Callee pops everything..
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000682 BytesCallerReserves = 0;
683 } else {
Chris Lattnerb9db2252007-02-28 05:46:49 +0000684 BytesToPopOnReturn = 0; // Callee pops hidden struct pointer.
685
686 // If this is an sret function, the return should pop the hidden pointer.
687 if (NumArgs && (cast<ConstantSDNode>(Op.getOperand(3))->getValue() & 4))
688 BytesToPopOnReturn = 4;
689
690 BytesCallerReserves = StackSize;
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000691 }
692
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000693 RegSaveFrameIndex = 0xAAAAAAA; // X86-64 only.
694 ReturnAddrIndex = 0; // No return address slot generated yet.
Evan Cheng17e734f2006-05-23 21:06:34 +0000695
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000696 MF.getInfo<X86FunctionInfo>()->setBytesToPopOnReturn(BytesToPopOnReturn);
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000697
Evan Cheng17e734f2006-05-23 21:06:34 +0000698 // Return the new list of results.
Chris Lattner35a08552007-02-25 07:10:00 +0000699 return DAG.getNode(ISD::MERGE_VALUES, Op.Val->getVTList(),
Chris Lattner29478082007-02-26 07:50:02 +0000700 &ArgValues[0], ArgValues.size()).getValue(Op.ResNo);
Chris Lattner76ac0682005-11-15 00:40:23 +0000701}
702
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000703SDOperand X86TargetLowering::LowerCCCCallTo(SDOperand Op, SelectionDAG &DAG,
Chris Lattner7802f3e2007-02-25 09:06:15 +0000704 unsigned CC) {
Evan Cheng2a330942006-05-25 00:59:30 +0000705 SDOperand Chain = Op.getOperand(0);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000706 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0;
Evan Cheng2a330942006-05-25 00:59:30 +0000707 bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0;
708 SDOperand Callee = Op.getOperand(4);
Evan Cheng2a330942006-05-25 00:59:30 +0000709 unsigned NumOps = (Op.getNumOperands() - 5) / 2;
Chris Lattner76ac0682005-11-15 00:40:23 +0000710
Chris Lattnerbe799592007-02-28 05:31:48 +0000711 SmallVector<CCValAssign, 16> ArgLocs;
712 CCState CCInfo(CC, getTargetMachine(), ArgLocs);
713
714 for (unsigned i = 0; i != NumOps; ++i) {
715 MVT::ValueType ArgVT = Op.getOperand(5+2*i).getValueType();
716 unsigned ArgFlags =cast<ConstantSDNode>(Op.getOperand(5+2*i+1))->getValue();
717 if (CC_X86_32_C(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo))
718 assert(0 && "Unhandled argument type!");
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000719 }
720
Chris Lattnerbe799592007-02-28 05:31:48 +0000721 // Get a count of how many bytes are to be pushed on the stack.
722 unsigned NumBytes = CCInfo.getNextStackOffset();
Chris Lattner76ac0682005-11-15 00:40:23 +0000723
Evan Cheng2a330942006-05-25 00:59:30 +0000724 Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy()));
Chris Lattner76ac0682005-11-15 00:40:23 +0000725
Chris Lattner35a08552007-02-25 07:10:00 +0000726 SmallVector<std::pair<unsigned, SDOperand>, 8> RegsToPass;
727 SmallVector<SDOperand, 8> MemOpChains;
Evan Cheng2a330942006-05-25 00:59:30 +0000728
Chris Lattnerbe799592007-02-28 05:31:48 +0000729 SDOperand StackPtr;
Chris Lattnerbe799592007-02-28 05:31:48 +0000730
731 // Walk the register/memloc assignments, inserting copies/loads.
732 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
733 CCValAssign &VA = ArgLocs[i];
734 SDOperand Arg = Op.getOperand(5+2*VA.getValNo());
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000735
Chris Lattnerbe799592007-02-28 05:31:48 +0000736 // Promote the value if needed.
737 switch (VA.getLocInfo()) {
738 default: assert(0 && "Unknown loc info!");
739 case CCValAssign::Full: break;
740 case CCValAssign::SExt:
741 Arg = DAG.getNode(ISD::SIGN_EXTEND, VA.getLocVT(), Arg);
742 break;
743 case CCValAssign::ZExt:
744 Arg = DAG.getNode(ISD::ZERO_EXTEND, VA.getLocVT(), Arg);
745 break;
746 case CCValAssign::AExt:
747 Arg = DAG.getNode(ISD::ANY_EXTEND, VA.getLocVT(), Arg);
748 break;
Evan Cheng5ee96892006-05-25 18:56:34 +0000749 }
Chris Lattnerbe799592007-02-28 05:31:48 +0000750
751 if (VA.isRegLoc()) {
752 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
753 } else {
754 assert(VA.isMemLoc());
755 if (StackPtr.Val == 0)
756 StackPtr = DAG.getRegister(getStackPtrReg(), getPointerTy());
757 SDOperand PtrOff = DAG.getConstant(VA.getLocMemOffset(), getPointerTy());
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000758 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
759 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
Chris Lattner76ac0682005-11-15 00:40:23 +0000760 }
Chris Lattner76ac0682005-11-15 00:40:23 +0000761 }
762
Chris Lattner5958b172007-02-28 05:39:26 +0000763 // If the first argument is an sret pointer, remember it.
764 bool isSRet = NumOps &&(cast<ConstantSDNode>(Op.getOperand(6))->getValue()&4);
765
Evan Cheng2a330942006-05-25 00:59:30 +0000766 if (!MemOpChains.empty())
Chris Lattnerc24a1d32006-08-08 02:23:42 +0000767 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
768 &MemOpChains[0], MemOpChains.size());
Chris Lattner76ac0682005-11-15 00:40:23 +0000769
Evan Cheng88decde2006-04-28 21:29:37 +0000770 // Build a sequence of copy-to-reg nodes chained together with token chain
771 // and flag operands which copy the outgoing args into registers.
772 SDOperand InFlag;
Evan Cheng2a330942006-05-25 00:59:30 +0000773 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
774 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second,
775 InFlag);
Evan Cheng88decde2006-04-28 21:29:37 +0000776 InFlag = Chain.getValue(1);
777 }
778
Evan Cheng84a041e2007-02-21 21:18:14 +0000779 // ELF / PIC requires GOT in the EBX register before function calls via PLT
780 // GOT pointer.
Evan Cheng1281dc32007-01-22 21:34:25 +0000781 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
782 Subtarget->isPICStyleGOT()) {
Anton Korobeynikova0554d92007-01-12 19:20:47 +0000783 Chain = DAG.getCopyToReg(Chain, X86::EBX,
784 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
785 InFlag);
786 InFlag = Chain.getValue(1);
787 }
788
Evan Cheng2a330942006-05-25 00:59:30 +0000789 // If the callee is a GlobalAddress node (quite common, every direct call is)
790 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
Anton Korobeynikov37d080b2006-11-20 10:46:14 +0000791 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
Anton Korobeynikov430e68a12006-12-22 22:29:05 +0000792 // We should use extra load for direct calls to dllimported functions in
793 // non-JIT mode.
794 if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(),
795 getTargetMachine(), true))
Anton Korobeynikov37d080b2006-11-20 10:46:14 +0000796 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy());
797 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
Evan Cheng2a330942006-05-25 00:59:30 +0000798 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
799
Chris Lattnere56fef92007-02-25 06:40:16 +0000800 // Returns a chain & a flag for retval copy to use.
801 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Chris Lattner35a08552007-02-25 07:10:00 +0000802 SmallVector<SDOperand, 8> Ops;
Nate Begeman7e5496d2006-02-17 00:03:04 +0000803 Ops.push_back(Chain);
804 Ops.push_back(Callee);
Evan Chengca254862006-06-14 18:17:40 +0000805
806 // Add argument registers to the end of the list so that they are known live
807 // into the call.
808 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000809 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
Evan Chengca254862006-06-14 18:17:40 +0000810 RegsToPass[i].second.getValueType()));
Evan Cheng84a041e2007-02-21 21:18:14 +0000811
812 // Add an implicit use GOT pointer in EBX.
813 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
814 Subtarget->isPICStyleGOT())
815 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy()));
Anton Korobeynikova0554d92007-01-12 19:20:47 +0000816
Evan Cheng88decde2006-04-28 21:29:37 +0000817 if (InFlag.Val)
818 Ops.push_back(InFlag);
Evan Cheng45e190982006-01-05 00:27:02 +0000819
Evan Cheng2a330942006-05-25 00:59:30 +0000820 Chain = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
Chris Lattnerc24a1d32006-08-08 02:23:42 +0000821 NodeTys, &Ops[0], Ops.size());
Evan Cheng88decde2006-04-28 21:29:37 +0000822 InFlag = Chain.getValue(1);
Evan Cheng45e190982006-01-05 00:27:02 +0000823
Chris Lattner8be5be82006-05-23 18:50:38 +0000824 // Create the CALLSEQ_END node.
825 unsigned NumBytesForCalleeToPush = 0;
826
Chris Lattner7802f3e2007-02-25 09:06:15 +0000827 if (CC == CallingConv::X86_StdCall) {
828 if (isVarArg)
Chris Lattner5958b172007-02-28 05:39:26 +0000829 NumBytesForCalleeToPush = isSRet ? 4 : 0;
Chris Lattner7802f3e2007-02-25 09:06:15 +0000830 else
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000831 NumBytesForCalleeToPush = NumBytes;
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000832 } else {
833 // If this is is a call to a struct-return function, the callee
834 // pops the hidden struct pointer, so we have to push it back.
835 // This is common for Darwin/X86, Linux & Mingw32 targets.
Chris Lattner5958b172007-02-28 05:39:26 +0000836 NumBytesForCalleeToPush = isSRet ? 4 : 0;
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000837 }
838
Chris Lattnerd6b853ad2007-02-25 07:18:38 +0000839 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Nate Begeman7e5496d2006-02-17 00:03:04 +0000840 Ops.clear();
841 Ops.push_back(Chain);
842 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
Chris Lattner8be5be82006-05-23 18:50:38 +0000843 Ops.push_back(DAG.getConstant(NumBytesForCalleeToPush, getPointerTy()));
Nate Begeman7e5496d2006-02-17 00:03:04 +0000844 Ops.push_back(InFlag);
Chris Lattnerc24a1d32006-08-08 02:23:42 +0000845 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size());
Chris Lattner0cd99602007-02-25 08:59:22 +0000846 InFlag = Chain.getValue(1);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000847
Chris Lattner0cd99602007-02-25 08:59:22 +0000848 // Handle result values, copying them out of physregs into vregs that we
849 // return.
Chris Lattner7802f3e2007-02-25 09:06:15 +0000850 return SDOperand(LowerCallResult(Chain, InFlag, Op.Val, CC, DAG), Op.ResNo);
Chris Lattner76ac0682005-11-15 00:40:23 +0000851}
852
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000853
854//===----------------------------------------------------------------------===//
Chris Lattner3066bec2007-02-28 06:10:12 +0000855// FastCall Calling Convention implementation
Chris Lattner76ac0682005-11-15 00:40:23 +0000856//===----------------------------------------------------------------------===//
857//
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000858// The X86 'fastcall' calling convention passes up to two integer arguments in
859// registers (an appropriate portion of ECX/EDX), passes arguments in C order,
860// and requires that the callee pop its arguments off the stack (allowing proper
861// tail calls), and has the same return value conventions as C calling convs.
862//
863// This calling convention always arranges for the callee pop value to be 8n+4
864// bytes, which is needed for tail recursion elimination and stack alignment
865// reasons.
Evan Cheng17e734f2006-05-23 21:06:34 +0000866SDOperand
Chris Lattner3ed3be32007-02-28 06:05:16 +0000867X86TargetLowering::LowerFastCCArguments(SDOperand Op, SelectionDAG &DAG) {
Evan Cheng17e734f2006-05-23 21:06:34 +0000868 unsigned NumArgs = Op.Val->getNumValues()-1;
Chris Lattner76ac0682005-11-15 00:40:23 +0000869 MachineFunction &MF = DAG.getMachineFunction();
870 MachineFrameInfo *MFI = MF.getFrameInfo();
Evan Cheng17e734f2006-05-23 21:06:34 +0000871 SDOperand Root = Op.getOperand(0);
Chris Lattner76ac0682005-11-15 00:40:23 +0000872
Chris Lattner66e1d1d2007-02-28 06:21:19 +0000873 SmallVector<CCValAssign, 16> ArgLocs;
874 CCState CCInfo(MF.getFunction()->getCallingConv(), getTargetMachine(),
875 ArgLocs);
876
877 for (unsigned i = 0; i != NumArgs; ++i) {
878 MVT::ValueType ArgVT = Op.getValue(i).getValueType();
879 unsigned ArgFlags = cast<ConstantSDNode>(Op.getOperand(3+i))->getValue();
880 if (CC_X86_32_FastCall(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags,CCInfo))
881 assert(0 && "Unhandled argument type!");
Chris Lattner76ac0682005-11-15 00:40:23 +0000882 }
Chris Lattner66e1d1d2007-02-28 06:21:19 +0000883
884 SmallVector<SDOperand, 8> ArgValues;
885 unsigned LastVal = ~0U;
886 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
887 CCValAssign &VA = ArgLocs[i];
888 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later
889 // places.
890 assert(VA.getValNo() != LastVal &&
891 "Don't support value assigned to multiple locs yet");
892 LastVal = VA.getValNo();
893
894 if (VA.isRegLoc()) {
895 MVT::ValueType RegVT = VA.getLocVT();
896 TargetRegisterClass *RC;
897 if (RegVT == MVT::i32)
898 RC = X86::GR32RegisterClass;
899 else {
900 assert(MVT::isVector(RegVT));
901 RC = X86::VR128RegisterClass;
902 }
903
904 SDOperand ArgValue = DAG.getCopyFromReg(Root, VA.getLocReg(), RegVT);
905 AddLiveIn(DAG.getMachineFunction(), VA.getLocReg(), RC);
906
907 // If this is an 8 or 16-bit value, it is really passed promoted to 32
908 // bits. Insert an assert[sz]ext to capture this, then truncate to the
909 // right size.
910 if (VA.getLocInfo() == CCValAssign::SExt)
911 ArgValue = DAG.getNode(ISD::AssertSext, RegVT, ArgValue,
912 DAG.getValueType(VA.getValVT()));
913 else if (VA.getLocInfo() == CCValAssign::ZExt)
914 ArgValue = DAG.getNode(ISD::AssertZext, RegVT, ArgValue,
915 DAG.getValueType(VA.getValVT()));
916
917 if (VA.getLocInfo() != CCValAssign::Full)
918 ArgValue = DAG.getNode(ISD::TRUNCATE, VA.getValVT(), ArgValue);
919
920 ArgValues.push_back(ArgValue);
921 } else {
922 assert(VA.isMemLoc());
923
924 // Create the nodes corresponding to a load from this parameter slot.
925 int FI = MFI->CreateFixedObject(MVT::getSizeInBits(VA.getValVT())/8,
926 VA.getLocMemOffset());
927 SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy());
928 ArgValues.push_back(DAG.getLoad(VA.getValVT(), Root, FIN, NULL, 0));
929 }
930 }
931
Evan Cheng17e734f2006-05-23 21:06:34 +0000932 ArgValues.push_back(Root);
933
Chris Lattner66e1d1d2007-02-28 06:21:19 +0000934 unsigned StackSize = CCInfo.getNextStackOffset();
935
Chris Lattner76ac0682005-11-15 00:40:23 +0000936 // Make sure the instruction takes 8n+4 bytes to make sure the start of the
937 // arguments and the arguments after the retaddr has been pushed are aligned.
Chris Lattner66e1d1d2007-02-28 06:21:19 +0000938 if ((StackSize & 7) == 0)
939 StackSize += 4;
Chris Lattner76ac0682005-11-15 00:40:23 +0000940
941 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs.
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000942 RegSaveFrameIndex = 0xAAAAAAA; // X86-64 only.
Chris Lattner76ac0682005-11-15 00:40:23 +0000943 ReturnAddrIndex = 0; // No return address slot generated yet.
Chris Lattner66e1d1d2007-02-28 06:21:19 +0000944 BytesToPopOnReturn = StackSize; // Callee pops all stack arguments.
Chris Lattner76ac0682005-11-15 00:40:23 +0000945 BytesCallerReserves = 0;
946
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000947 MF.getInfo<X86FunctionInfo>()->setBytesToPopOnReturn(BytesToPopOnReturn);
948
Evan Cheng17e734f2006-05-23 21:06:34 +0000949 // Return the new list of results.
Chris Lattner35a08552007-02-25 07:10:00 +0000950 return DAG.getNode(ISD::MERGE_VALUES, Op.Val->getVTList(),
Chris Lattner29478082007-02-26 07:50:02 +0000951 &ArgValues[0], ArgValues.size()).getValue(Op.ResNo);
Chris Lattner76ac0682005-11-15 00:40:23 +0000952}
953
Chris Lattner104aa5d2006-09-26 03:57:53 +0000954SDOperand X86TargetLowering::LowerFastCCCallTo(SDOperand Op, SelectionDAG &DAG,
Chris Lattner7802f3e2007-02-25 09:06:15 +0000955 unsigned CC) {
Evan Cheng2a330942006-05-25 00:59:30 +0000956 SDOperand Chain = Op.getOperand(0);
Evan Cheng2a330942006-05-25 00:59:30 +0000957 bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0;
958 SDOperand Callee = Op.getOperand(4);
Evan Cheng2a330942006-05-25 00:59:30 +0000959 unsigned NumOps = (Op.getNumOperands() - 5) / 2;
960
Chris Lattner76ac0682005-11-15 00:40:23 +0000961 // Count how many bytes are to be pushed on the stack.
962 unsigned NumBytes = 0;
963
964 // Keep track of the number of integer regs passed so far. This can be either
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000965 // 0 (neither EAX/ECX or EDX used), 1 (EAX/ECX is used) or 2 (EAX/ECX and EDX
966 // are both used).
Chris Lattner76ac0682005-11-15 00:40:23 +0000967 unsigned NumIntRegs = 0;
Evan Cheng2a330942006-05-25 00:59:30 +0000968 unsigned NumXMMRegs = 0; // XMM regs used for parameter passing.
Chris Lattner76ac0682005-11-15 00:40:23 +0000969
Chris Lattner3ed3be32007-02-28 06:05:16 +0000970 static const unsigned GPRArgRegs[][2] = {
971 { X86::CL, X86::DL },
972 { X86::CX, X86::DX },
973 { X86::ECX, X86::EDX }
Evan Cheng2a330942006-05-25 00:59:30 +0000974 };
975 static const unsigned XMMArgRegs[] = {
Evan Chengbfb5ea62006-05-26 19:22:06 +0000976 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3
Evan Cheng2a330942006-05-25 00:59:30 +0000977 };
978
979 for (unsigned i = 0; i != NumOps; ++i) {
980 SDOperand Arg = Op.getOperand(5+2*i);
981
982 switch (Arg.getValueType()) {
Chris Lattner76ac0682005-11-15 00:40:23 +0000983 default: assert(0 && "Unknown value type!");
Chris Lattner76ac0682005-11-15 00:40:23 +0000984 case MVT::i8:
985 case MVT::i16:
Chris Lattner3ed3be32007-02-28 06:05:16 +0000986 case MVT::i32:
987 if (NumIntRegs < 2) {
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +0000988 ++NumIntRegs;
989 break;
Nick Lewyckyc68bbef2006-09-21 02:08:31 +0000990 } // Fall through
Chris Lattner76ac0682005-11-15 00:40:23 +0000991 case MVT::f32:
992 NumBytes += 4;
993 break;
Chris Lattner76ac0682005-11-15 00:40:23 +0000994 case MVT::f64:
995 NumBytes += 8;
996 break;
Evan Cheng2a330942006-05-25 00:59:30 +0000997 case MVT::v16i8:
998 case MVT::v8i16:
999 case MVT::v4i32:
1000 case MVT::v2i64:
1001 case MVT::v4f32:
Evan Cheng5ee96892006-05-25 18:56:34 +00001002 case MVT::v2f64:
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001003 if (NumXMMRegs < 4)
1004 NumXMMRegs++;
1005 else {
1006 // XMM arguments have to be aligned on 16-byte boundary.
1007 NumBytes = ((NumBytes + 15) / 16) * 16;
1008 NumBytes += 16;
1009 }
1010 break;
Chris Lattner76ac0682005-11-15 00:40:23 +00001011 }
Evan Cheng2a330942006-05-25 00:59:30 +00001012 }
Chris Lattner76ac0682005-11-15 00:40:23 +00001013
1014 // Make sure the instruction takes 8n+4 bytes to make sure the start of the
1015 // arguments and the arguments after the retaddr has been pushed are aligned.
1016 if ((NumBytes & 7) == 0)
1017 NumBytes += 4;
1018
Chris Lattner62c34842006-02-13 09:00:43 +00001019 Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy()));
Chris Lattner76ac0682005-11-15 00:40:23 +00001020
1021 // Arguments go on the stack in reverse order, as specified by the ABI.
1022 unsigned ArgOffset = 0;
Chris Lattner76ac0682005-11-15 00:40:23 +00001023 NumIntRegs = 0;
Chris Lattner35a08552007-02-25 07:10:00 +00001024 SmallVector<std::pair<unsigned, SDOperand>, 8> RegsToPass;
1025 SmallVector<SDOperand, 8> MemOpChains;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001026 SDOperand StackPtr = DAG.getRegister(X86StackPtr, getPointerTy());
Evan Cheng2a330942006-05-25 00:59:30 +00001027 for (unsigned i = 0; i != NumOps; ++i) {
1028 SDOperand Arg = Op.getOperand(5+2*i);
1029
1030 switch (Arg.getValueType()) {
Chris Lattner76ac0682005-11-15 00:40:23 +00001031 default: assert(0 && "Unexpected ValueType for argument!");
Chris Lattner76ac0682005-11-15 00:40:23 +00001032 case MVT::i8:
1033 case MVT::i16:
Chris Lattner3ed3be32007-02-28 06:05:16 +00001034 case MVT::i32:
1035 if (NumIntRegs < 2) {
1036 unsigned RegToUse =
1037 GPRArgRegs[Arg.getValueType()-MVT::i8][NumIntRegs];
1038 RegsToPass.push_back(std::make_pair(RegToUse, Arg));
1039 ++NumIntRegs;
1040 break;
1041 } // Fall through
Chris Lattner76ac0682005-11-15 00:40:23 +00001042 case MVT::f32: {
1043 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
Evan Cheng2a330942006-05-25 00:59:30 +00001044 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
Evan Chengab51cf22006-10-13 21:14:26 +00001045 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
Chris Lattner76ac0682005-11-15 00:40:23 +00001046 ArgOffset += 4;
1047 break;
1048 }
Evan Cheng2a330942006-05-25 00:59:30 +00001049 case MVT::f64: {
Chris Lattner76ac0682005-11-15 00:40:23 +00001050 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
Evan Cheng2a330942006-05-25 00:59:30 +00001051 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
Evan Chengab51cf22006-10-13 21:14:26 +00001052 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
Chris Lattner76ac0682005-11-15 00:40:23 +00001053 ArgOffset += 8;
1054 break;
1055 }
Evan Cheng2a330942006-05-25 00:59:30 +00001056 case MVT::v16i8:
1057 case MVT::v8i16:
1058 case MVT::v4i32:
1059 case MVT::v2i64:
1060 case MVT::v4f32:
Evan Cheng5ee96892006-05-25 18:56:34 +00001061 case MVT::v2f64:
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001062 if (NumXMMRegs < 4) {
1063 RegsToPass.push_back(std::make_pair(XMMArgRegs[NumXMMRegs], Arg));
1064 NumXMMRegs++;
1065 } else {
1066 // XMM arguments have to be aligned on 16-byte boundary.
1067 ArgOffset = ((ArgOffset + 15) / 16) * 16;
1068 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
1069 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
1070 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
1071 ArgOffset += 16;
1072 }
1073 break;
Evan Cheng2a330942006-05-25 00:59:30 +00001074 }
Chris Lattner76ac0682005-11-15 00:40:23 +00001075 }
Chris Lattner76ac0682005-11-15 00:40:23 +00001076
Evan Cheng2a330942006-05-25 00:59:30 +00001077 if (!MemOpChains.empty())
Chris Lattnerc24a1d32006-08-08 02:23:42 +00001078 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
1079 &MemOpChains[0], MemOpChains.size());
Chris Lattner76ac0682005-11-15 00:40:23 +00001080
Nate Begeman7e5496d2006-02-17 00:03:04 +00001081 // Build a sequence of copy-to-reg nodes chained together with token chain
1082 // and flag operands which copy the outgoing args into registers.
1083 SDOperand InFlag;
Evan Cheng2a330942006-05-25 00:59:30 +00001084 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
1085 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second,
1086 InFlag);
Nate Begeman7e5496d2006-02-17 00:03:04 +00001087 InFlag = Chain.getValue(1);
1088 }
1089
Evan Cheng2a330942006-05-25 00:59:30 +00001090 // If the callee is a GlobalAddress node (quite common, every direct call is)
1091 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
Anton Korobeynikov37d080b2006-11-20 10:46:14 +00001092 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
Anton Korobeynikov430e68a12006-12-22 22:29:05 +00001093 // We should use extra load for direct calls to dllimported functions in
1094 // non-JIT mode.
1095 if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(),
1096 getTargetMachine(), true))
Anton Korobeynikov37d080b2006-11-20 10:46:14 +00001097 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy());
1098 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
Evan Cheng2a330942006-05-25 00:59:30 +00001099 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
1100
Evan Cheng84a041e2007-02-21 21:18:14 +00001101 // ELF / PIC requires GOT in the EBX register before function calls via PLT
1102 // GOT pointer.
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001103 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
1104 Subtarget->isPICStyleGOT()) {
1105 Chain = DAG.getCopyToReg(Chain, X86::EBX,
1106 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
1107 InFlag);
1108 InFlag = Chain.getValue(1);
1109 }
1110
Chris Lattnere56fef92007-02-25 06:40:16 +00001111 // Returns a chain & a flag for retval copy to use.
1112 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Chris Lattner35a08552007-02-25 07:10:00 +00001113 SmallVector<SDOperand, 8> Ops;
Nate Begeman7e5496d2006-02-17 00:03:04 +00001114 Ops.push_back(Chain);
1115 Ops.push_back(Callee);
Evan Chengca254862006-06-14 18:17:40 +00001116
1117 // Add argument registers to the end of the list so that they are known live
1118 // into the call.
1119 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00001120 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
Evan Chengca254862006-06-14 18:17:40 +00001121 RegsToPass[i].second.getValueType()));
1122
Evan Cheng84a041e2007-02-21 21:18:14 +00001123 // Add an implicit use GOT pointer in EBX.
1124 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
1125 Subtarget->isPICStyleGOT())
1126 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy()));
1127
Nate Begeman7e5496d2006-02-17 00:03:04 +00001128 if (InFlag.Val)
1129 Ops.push_back(InFlag);
1130
1131 // FIXME: Do not generate X86ISD::TAILCALL for now.
Chris Lattner3d826992006-05-16 06:45:34 +00001132 Chain = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
Chris Lattnerc24a1d32006-08-08 02:23:42 +00001133 NodeTys, &Ops[0], Ops.size());
Nate Begeman7e5496d2006-02-17 00:03:04 +00001134 InFlag = Chain.getValue(1);
1135
Chris Lattnerd6b853ad2007-02-25 07:18:38 +00001136 // Returns a flag for retval copy to use.
1137 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Nate Begeman7e5496d2006-02-17 00:03:04 +00001138 Ops.clear();
1139 Ops.push_back(Chain);
Evan Cheng2a330942006-05-25 00:59:30 +00001140 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
1141 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
Nate Begeman7e5496d2006-02-17 00:03:04 +00001142 Ops.push_back(InFlag);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00001143 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size());
Chris Lattnerba474f52007-02-25 09:10:05 +00001144 InFlag = Chain.getValue(1);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00001145
Chris Lattnerba474f52007-02-25 09:10:05 +00001146 // Handle result values, copying them out of physregs into vregs that we
1147 // return.
1148 return SDOperand(LowerCallResult(Chain, InFlag, Op.Val, CC, DAG), Op.ResNo);
Chris Lattner76ac0682005-11-15 00:40:23 +00001149}
1150
Chris Lattner3066bec2007-02-28 06:10:12 +00001151
1152//===----------------------------------------------------------------------===//
1153// X86-64 C Calling Convention implementation
1154//===----------------------------------------------------------------------===//
1155
1156SDOperand
1157X86TargetLowering::LowerX86_64CCCArguments(SDOperand Op, SelectionDAG &DAG) {
1158 unsigned NumArgs = Op.Val->getNumValues() - 1;
1159 MachineFunction &MF = DAG.getMachineFunction();
1160 MachineFrameInfo *MFI = MF.getFrameInfo();
1161 SDOperand Root = Op.getOperand(0);
1162 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0;
1163
1164 static const unsigned GPR64ArgRegs[] = {
1165 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
1166 };
1167 static const unsigned XMMArgRegs[] = {
1168 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
1169 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
1170 };
1171
1172 SmallVector<CCValAssign, 16> ArgLocs;
1173 CCState CCInfo(MF.getFunction()->getCallingConv(), getTargetMachine(),
1174 ArgLocs);
1175
1176 for (unsigned i = 0; i != NumArgs; ++i) {
1177 MVT::ValueType ArgVT = Op.getValue(i).getValueType();
1178 unsigned ArgFlags = cast<ConstantSDNode>(Op.getOperand(3+i))->getValue();
1179 if (CC_X86_64_C(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo))
1180 assert(0 && "Unhandled argument type!");
1181 }
1182
1183 SmallVector<SDOperand, 8> ArgValues;
1184 unsigned LastVal = ~0U;
1185 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1186 CCValAssign &VA = ArgLocs[i];
1187 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later
1188 // places.
1189 assert(VA.getValNo() != LastVal &&
1190 "Don't support value assigned to multiple locs yet");
1191 LastVal = VA.getValNo();
1192
1193 if (VA.isRegLoc()) {
1194 MVT::ValueType RegVT = VA.getLocVT();
1195 TargetRegisterClass *RC;
1196 if (RegVT == MVT::i32)
1197 RC = X86::GR32RegisterClass;
1198 else if (RegVT == MVT::i64)
1199 RC = X86::GR64RegisterClass;
1200 else if (RegVT == MVT::f32)
1201 RC = X86::FR32RegisterClass;
1202 else if (RegVT == MVT::f64)
1203 RC = X86::FR64RegisterClass;
1204 else {
1205 assert(MVT::isVector(RegVT));
1206 RC = X86::VR128RegisterClass;
1207 }
1208
1209 SDOperand ArgValue = DAG.getCopyFromReg(Root, VA.getLocReg(), RegVT);
1210 AddLiveIn(DAG.getMachineFunction(), VA.getLocReg(), RC);
1211
1212 // If this is an 8 or 16-bit value, it is really passed promoted to 32
1213 // bits. Insert an assert[sz]ext to capture this, then truncate to the
1214 // right size.
1215 if (VA.getLocInfo() == CCValAssign::SExt)
1216 ArgValue = DAG.getNode(ISD::AssertSext, RegVT, ArgValue,
1217 DAG.getValueType(VA.getValVT()));
1218 else if (VA.getLocInfo() == CCValAssign::ZExt)
1219 ArgValue = DAG.getNode(ISD::AssertZext, RegVT, ArgValue,
1220 DAG.getValueType(VA.getValVT()));
1221
1222 if (VA.getLocInfo() != CCValAssign::Full)
1223 ArgValue = DAG.getNode(ISD::TRUNCATE, VA.getValVT(), ArgValue);
1224
1225 ArgValues.push_back(ArgValue);
1226 } else {
1227 assert(VA.isMemLoc());
1228
1229 // Create the nodes corresponding to a load from this parameter slot.
1230 int FI = MFI->CreateFixedObject(MVT::getSizeInBits(VA.getValVT())/8,
1231 VA.getLocMemOffset());
1232 SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy());
1233 ArgValues.push_back(DAG.getLoad(VA.getValVT(), Root, FIN, NULL, 0));
1234 }
1235 }
1236
1237 unsigned StackSize = CCInfo.getNextStackOffset();
1238
1239 // If the function takes variable number of arguments, make a frame index for
1240 // the start of the first vararg value... for expansion of llvm.va_start.
1241 if (isVarArg) {
1242 unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs, 6);
1243 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8);
1244
1245 // For X86-64, if there are vararg parameters that are passed via
1246 // registers, then we must store them to their spots on the stack so they
1247 // may be loaded by deferencing the result of va_next.
1248 VarArgsGPOffset = NumIntRegs * 8;
1249 VarArgsFPOffset = 6 * 8 + NumXMMRegs * 16;
1250 VarArgsFrameIndex = MFI->CreateFixedObject(1, StackSize);
1251 RegSaveFrameIndex = MFI->CreateStackObject(6 * 8 + 8 * 16, 16);
1252
1253 // Store the integer parameter registers.
1254 SmallVector<SDOperand, 8> MemOps;
1255 SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy());
1256 SDOperand FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN,
1257 DAG.getConstant(VarArgsGPOffset, getPointerTy()));
1258 for (; NumIntRegs != 6; ++NumIntRegs) {
1259 unsigned VReg = AddLiveIn(MF, GPR64ArgRegs[NumIntRegs],
1260 X86::GR64RegisterClass);
1261 SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::i64);
1262 SDOperand Store = DAG.getStore(Val.getValue(1), Val, FIN, NULL, 0);
1263 MemOps.push_back(Store);
1264 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
1265 DAG.getConstant(8, getPointerTy()));
1266 }
1267
1268 // Now store the XMM (fp + vector) parameter registers.
1269 FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN,
1270 DAG.getConstant(VarArgsFPOffset, getPointerTy()));
1271 for (; NumXMMRegs != 8; ++NumXMMRegs) {
1272 unsigned VReg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs],
1273 X86::VR128RegisterClass);
1274 SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::v4f32);
1275 SDOperand Store = DAG.getStore(Val.getValue(1), Val, FIN, NULL, 0);
1276 MemOps.push_back(Store);
1277 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
1278 DAG.getConstant(16, getPointerTy()));
1279 }
1280 if (!MemOps.empty())
1281 Root = DAG.getNode(ISD::TokenFactor, MVT::Other,
1282 &MemOps[0], MemOps.size());
1283 }
1284
1285 ArgValues.push_back(Root);
1286
1287 ReturnAddrIndex = 0; // No return address slot generated yet.
1288 BytesToPopOnReturn = 0; // Callee pops nothing.
1289 BytesCallerReserves = StackSize;
1290
1291 // Return the new list of results.
1292 return DAG.getNode(ISD::MERGE_VALUES, Op.Val->getVTList(),
1293 &ArgValues[0], ArgValues.size()).getValue(Op.ResNo);
1294}
1295
1296SDOperand
1297X86TargetLowering::LowerX86_64CCCCallTo(SDOperand Op, SelectionDAG &DAG,
1298 unsigned CC) {
1299 SDOperand Chain = Op.getOperand(0);
1300 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0;
1301 bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0;
1302 SDOperand Callee = Op.getOperand(4);
1303 unsigned NumOps = (Op.getNumOperands() - 5) / 2;
1304
1305 SmallVector<CCValAssign, 16> ArgLocs;
1306 CCState CCInfo(CC, getTargetMachine(), ArgLocs);
1307
1308 for (unsigned i = 0; i != NumOps; ++i) {
1309 MVT::ValueType ArgVT = Op.getOperand(5+2*i).getValueType();
1310 unsigned ArgFlags =cast<ConstantSDNode>(Op.getOperand(5+2*i+1))->getValue();
1311 if (CC_X86_64_C(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo))
1312 assert(0 && "Unhandled argument type!");
1313 }
1314
1315 // Get a count of how many bytes are to be pushed on the stack.
1316 unsigned NumBytes = CCInfo.getNextStackOffset();
1317 Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy()));
1318
1319 SmallVector<std::pair<unsigned, SDOperand>, 8> RegsToPass;
1320 SmallVector<SDOperand, 8> MemOpChains;
1321
1322 SDOperand StackPtr;
1323
1324 // Walk the register/memloc assignments, inserting copies/loads.
1325 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1326 CCValAssign &VA = ArgLocs[i];
1327 SDOperand Arg = Op.getOperand(5+2*VA.getValNo());
1328
1329 // Promote the value if needed.
1330 switch (VA.getLocInfo()) {
1331 default: assert(0 && "Unknown loc info!");
1332 case CCValAssign::Full: break;
1333 case CCValAssign::SExt:
1334 Arg = DAG.getNode(ISD::SIGN_EXTEND, VA.getLocVT(), Arg);
1335 break;
1336 case CCValAssign::ZExt:
1337 Arg = DAG.getNode(ISD::ZERO_EXTEND, VA.getLocVT(), Arg);
1338 break;
1339 case CCValAssign::AExt:
1340 Arg = DAG.getNode(ISD::ANY_EXTEND, VA.getLocVT(), Arg);
1341 break;
1342 }
1343
1344 if (VA.isRegLoc()) {
1345 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
1346 } else {
1347 assert(VA.isMemLoc());
1348 if (StackPtr.Val == 0)
1349 StackPtr = DAG.getRegister(getStackPtrReg(), getPointerTy());
1350 SDOperand PtrOff = DAG.getConstant(VA.getLocMemOffset(), getPointerTy());
1351 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
1352 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
1353 }
1354 }
1355
1356 if (!MemOpChains.empty())
1357 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
1358 &MemOpChains[0], MemOpChains.size());
1359
1360 // Build a sequence of copy-to-reg nodes chained together with token chain
1361 // and flag operands which copy the outgoing args into registers.
1362 SDOperand InFlag;
1363 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
1364 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second,
1365 InFlag);
1366 InFlag = Chain.getValue(1);
1367 }
1368
1369 if (isVarArg) {
1370 // From AMD64 ABI document:
1371 // For calls that may call functions that use varargs or stdargs
1372 // (prototype-less calls or calls to functions containing ellipsis (...) in
1373 // the declaration) %al is used as hidden argument to specify the number
1374 // of SSE registers used. The contents of %al do not need to match exactly
1375 // the number of registers, but must be an ubound on the number of SSE
1376 // registers used and is in the range 0 - 8 inclusive.
1377
1378 // Count the number of XMM registers allocated.
1379 static const unsigned XMMArgRegs[] = {
1380 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
1381 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
1382 };
1383 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8);
1384
1385 Chain = DAG.getCopyToReg(Chain, X86::AL,
1386 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag);
1387 InFlag = Chain.getValue(1);
1388 }
1389
1390 // If the callee is a GlobalAddress node (quite common, every direct call is)
1391 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
1392 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1393 // We should use extra load for direct calls to dllimported functions in
1394 // non-JIT mode.
1395 if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(),
1396 getTargetMachine(), true))
1397 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy());
1398 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
1399 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
1400
1401 // Returns a chain & a flag for retval copy to use.
1402 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
1403 SmallVector<SDOperand, 8> Ops;
1404 Ops.push_back(Chain);
1405 Ops.push_back(Callee);
1406
1407 // Add argument registers to the end of the list so that they are known live
1408 // into the call.
1409 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
1410 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
1411 RegsToPass[i].second.getValueType()));
1412
1413 if (InFlag.Val)
1414 Ops.push_back(InFlag);
1415
1416 // FIXME: Do not generate X86ISD::TAILCALL for now.
1417 Chain = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
1418 NodeTys, &Ops[0], Ops.size());
1419 InFlag = Chain.getValue(1);
1420
1421 // Returns a flag for retval copy to use.
1422 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
1423 Ops.clear();
1424 Ops.push_back(Chain);
1425 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
1426 Ops.push_back(DAG.getConstant(0, getPointerTy()));
1427 Ops.push_back(InFlag);
1428 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size());
1429 InFlag = Chain.getValue(1);
1430
1431 // Handle result values, copying them out of physregs into vregs that we
1432 // return.
1433 return SDOperand(LowerCallResult(Chain, InFlag, Op.Val, CC, DAG), Op.ResNo);
1434}
1435
1436
1437//===----------------------------------------------------------------------===//
1438// Other Lowering Hooks
1439//===----------------------------------------------------------------------===//
1440
1441
Chris Lattner76ac0682005-11-15 00:40:23 +00001442SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) {
1443 if (ReturnAddrIndex == 0) {
1444 // Set up a frame object for the return address.
1445 MachineFunction &MF = DAG.getMachineFunction();
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001446 if (Subtarget->is64Bit())
1447 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(8, -8);
1448 else
1449 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4);
Chris Lattner76ac0682005-11-15 00:40:23 +00001450 }
1451
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001452 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy());
Chris Lattner76ac0682005-11-15 00:40:23 +00001453}
1454
1455
1456
Evan Cheng45df7f82006-01-30 23:41:35 +00001457/// translateX86CC - do a one to one translation of a ISD::CondCode to the X86
1458/// specific condition code. It returns a false if it cannot do a direct
Chris Lattner7a627672006-09-13 03:22:10 +00001459/// translation. X86CC is the translated CondCode. LHS/RHS are modified as
1460/// needed.
Evan Cheng78038292006-04-05 23:38:46 +00001461static bool translateX86CC(ISD::CondCode SetCCOpcode, bool isFP,
Chris Lattner7a627672006-09-13 03:22:10 +00001462 unsigned &X86CC, SDOperand &LHS, SDOperand &RHS,
1463 SelectionDAG &DAG) {
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001464 X86CC = X86::COND_INVALID;
Evan Cheng172fce72006-01-06 00:43:03 +00001465 if (!isFP) {
Chris Lattner971e3392006-09-13 17:04:54 +00001466 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
1467 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) {
1468 // X > -1 -> X == 0, jump !sign.
1469 RHS = DAG.getConstant(0, RHS.getValueType());
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001470 X86CC = X86::COND_NS;
Chris Lattner971e3392006-09-13 17:04:54 +00001471 return true;
1472 } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) {
1473 // X < 0 -> X == 0, jump on sign.
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001474 X86CC = X86::COND_S;
Chris Lattner971e3392006-09-13 17:04:54 +00001475 return true;
1476 }
Chris Lattner7a627672006-09-13 03:22:10 +00001477 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00001478
Evan Cheng172fce72006-01-06 00:43:03 +00001479 switch (SetCCOpcode) {
1480 default: break;
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001481 case ISD::SETEQ: X86CC = X86::COND_E; break;
1482 case ISD::SETGT: X86CC = X86::COND_G; break;
1483 case ISD::SETGE: X86CC = X86::COND_GE; break;
1484 case ISD::SETLT: X86CC = X86::COND_L; break;
1485 case ISD::SETLE: X86CC = X86::COND_LE; break;
1486 case ISD::SETNE: X86CC = X86::COND_NE; break;
1487 case ISD::SETULT: X86CC = X86::COND_B; break;
1488 case ISD::SETUGT: X86CC = X86::COND_A; break;
1489 case ISD::SETULE: X86CC = X86::COND_BE; break;
1490 case ISD::SETUGE: X86CC = X86::COND_AE; break;
Evan Cheng172fce72006-01-06 00:43:03 +00001491 }
1492 } else {
1493 // On a floating point condition, the flags are set as follows:
1494 // ZF PF CF op
1495 // 0 | 0 | 0 | X > Y
1496 // 0 | 0 | 1 | X < Y
1497 // 1 | 0 | 0 | X == Y
1498 // 1 | 1 | 1 | unordered
Chris Lattner7a627672006-09-13 03:22:10 +00001499 bool Flip = false;
Evan Cheng172fce72006-01-06 00:43:03 +00001500 switch (SetCCOpcode) {
1501 default: break;
1502 case ISD::SETUEQ:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001503 case ISD::SETEQ: X86CC = X86::COND_E; break;
Evan Chengb3b41c42006-04-17 07:24:10 +00001504 case ISD::SETOLT: Flip = true; // Fallthrough
Evan Cheng172fce72006-01-06 00:43:03 +00001505 case ISD::SETOGT:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001506 case ISD::SETGT: X86CC = X86::COND_A; break;
Evan Chengb3b41c42006-04-17 07:24:10 +00001507 case ISD::SETOLE: Flip = true; // Fallthrough
Evan Cheng172fce72006-01-06 00:43:03 +00001508 case ISD::SETOGE:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001509 case ISD::SETGE: X86CC = X86::COND_AE; break;
Evan Chengb3b41c42006-04-17 07:24:10 +00001510 case ISD::SETUGT: Flip = true; // Fallthrough
Evan Cheng172fce72006-01-06 00:43:03 +00001511 case ISD::SETULT:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001512 case ISD::SETLT: X86CC = X86::COND_B; break;
Evan Chengb3b41c42006-04-17 07:24:10 +00001513 case ISD::SETUGE: Flip = true; // Fallthrough
Evan Cheng172fce72006-01-06 00:43:03 +00001514 case ISD::SETULE:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001515 case ISD::SETLE: X86CC = X86::COND_BE; break;
Evan Cheng172fce72006-01-06 00:43:03 +00001516 case ISD::SETONE:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001517 case ISD::SETNE: X86CC = X86::COND_NE; break;
1518 case ISD::SETUO: X86CC = X86::COND_P; break;
1519 case ISD::SETO: X86CC = X86::COND_NP; break;
Evan Cheng172fce72006-01-06 00:43:03 +00001520 }
Chris Lattner7a627672006-09-13 03:22:10 +00001521 if (Flip)
1522 std::swap(LHS, RHS);
Evan Cheng172fce72006-01-06 00:43:03 +00001523 }
Evan Cheng45df7f82006-01-30 23:41:35 +00001524
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001525 return X86CC != X86::COND_INVALID;
Evan Cheng172fce72006-01-06 00:43:03 +00001526}
1527
Evan Cheng339edad2006-01-11 00:33:36 +00001528/// hasFPCMov - is there a floating point cmov for the specific X86 condition
1529/// code. Current x86 isa includes the following FP cmov instructions:
Evan Cheng73a1ad92006-01-10 20:26:56 +00001530/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
Evan Cheng339edad2006-01-11 00:33:36 +00001531static bool hasFPCMov(unsigned X86CC) {
Evan Cheng73a1ad92006-01-10 20:26:56 +00001532 switch (X86CC) {
1533 default:
1534 return false;
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001535 case X86::COND_B:
1536 case X86::COND_BE:
1537 case X86::COND_E:
1538 case X86::COND_P:
1539 case X86::COND_A:
1540 case X86::COND_AE:
1541 case X86::COND_NE:
1542 case X86::COND_NP:
Evan Cheng73a1ad92006-01-10 20:26:56 +00001543 return true;
1544 }
1545}
1546
Evan Chengc995b452006-04-06 23:23:56 +00001547/// isUndefOrInRange - Op is either an undef node or a ConstantSDNode. Return
Evan Chengac847262006-04-07 21:53:05 +00001548/// true if Op is undef or if its value falls within the specified range (L, H].
Evan Chengc995b452006-04-06 23:23:56 +00001549static bool isUndefOrInRange(SDOperand Op, unsigned Low, unsigned Hi) {
1550 if (Op.getOpcode() == ISD::UNDEF)
1551 return true;
1552
1553 unsigned Val = cast<ConstantSDNode>(Op)->getValue();
Evan Chengac847262006-04-07 21:53:05 +00001554 return (Val >= Low && Val < Hi);
1555}
1556
1557/// isUndefOrEqual - Op is either an undef node or a ConstantSDNode. Return
1558/// true if Op is undef or if its value equal to the specified value.
1559static bool isUndefOrEqual(SDOperand Op, unsigned Val) {
1560 if (Op.getOpcode() == ISD::UNDEF)
1561 return true;
1562 return cast<ConstantSDNode>(Op)->getValue() == Val;
Evan Chengc995b452006-04-06 23:23:56 +00001563}
1564
Evan Cheng68ad48b2006-03-22 18:59:22 +00001565/// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand
1566/// specifies a shuffle of elements that is suitable for input to PSHUFD.
1567bool X86::isPSHUFDMask(SDNode *N) {
1568 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1569
1570 if (N->getNumOperands() != 4)
1571 return false;
1572
1573 // Check if the value doesn't reference the second vector.
Evan Chengb7fedff2006-03-29 23:07:14 +00001574 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
Evan Cheng99d72052006-03-31 00:30:29 +00001575 SDOperand Arg = N->getOperand(i);
1576 if (Arg.getOpcode() == ISD::UNDEF) continue;
1577 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
1578 if (cast<ConstantSDNode>(Arg)->getValue() >= 4)
Evan Chengb7fedff2006-03-29 23:07:14 +00001579 return false;
1580 }
1581
1582 return true;
1583}
1584
1585/// isPSHUFHWMask - Return true if the specified VECTOR_SHUFFLE operand
Evan Cheng59a63552006-04-05 01:47:37 +00001586/// specifies a shuffle of elements that is suitable for input to PSHUFHW.
Evan Chengb7fedff2006-03-29 23:07:14 +00001587bool X86::isPSHUFHWMask(SDNode *N) {
1588 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1589
1590 if (N->getNumOperands() != 8)
1591 return false;
1592
1593 // Lower quadword copied in order.
1594 for (unsigned i = 0; i != 4; ++i) {
Evan Cheng99d72052006-03-31 00:30:29 +00001595 SDOperand Arg = N->getOperand(i);
1596 if (Arg.getOpcode() == ISD::UNDEF) continue;
1597 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
1598 if (cast<ConstantSDNode>(Arg)->getValue() != i)
Evan Chengb7fedff2006-03-29 23:07:14 +00001599 return false;
1600 }
1601
1602 // Upper quadword shuffled.
1603 for (unsigned i = 4; i != 8; ++i) {
Evan Cheng99d72052006-03-31 00:30:29 +00001604 SDOperand Arg = N->getOperand(i);
1605 if (Arg.getOpcode() == ISD::UNDEF) continue;
1606 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
1607 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
Evan Chengb7fedff2006-03-29 23:07:14 +00001608 if (Val < 4 || Val > 7)
1609 return false;
1610 }
1611
1612 return true;
1613}
1614
1615/// isPSHUFLWMask - Return true if the specified VECTOR_SHUFFLE operand
Evan Cheng59a63552006-04-05 01:47:37 +00001616/// specifies a shuffle of elements that is suitable for input to PSHUFLW.
Evan Chengb7fedff2006-03-29 23:07:14 +00001617bool X86::isPSHUFLWMask(SDNode *N) {
1618 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1619
1620 if (N->getNumOperands() != 8)
1621 return false;
1622
1623 // Upper quadword copied in order.
Evan Chengac847262006-04-07 21:53:05 +00001624 for (unsigned i = 4; i != 8; ++i)
1625 if (!isUndefOrEqual(N->getOperand(i), i))
Evan Chengb7fedff2006-03-29 23:07:14 +00001626 return false;
Evan Chengb7fedff2006-03-29 23:07:14 +00001627
1628 // Lower quadword shuffled.
Evan Chengac847262006-04-07 21:53:05 +00001629 for (unsigned i = 0; i != 4; ++i)
1630 if (!isUndefOrInRange(N->getOperand(i), 0, 4))
Evan Chengb7fedff2006-03-29 23:07:14 +00001631 return false;
Evan Cheng68ad48b2006-03-22 18:59:22 +00001632
1633 return true;
1634}
1635
Evan Chengd27fb3e2006-03-24 01:18:28 +00001636/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand
1637/// specifies a shuffle of elements that is suitable for input to SHUFP*.
Chris Lattner35a08552007-02-25 07:10:00 +00001638static bool isSHUFPMask(const SDOperand *Elems, unsigned NumElems) {
Evan Cheng60f0b892006-04-20 08:58:49 +00001639 if (NumElems != 2 && NumElems != 4) return false;
Evan Chengd27fb3e2006-03-24 01:18:28 +00001640
Evan Cheng60f0b892006-04-20 08:58:49 +00001641 unsigned Half = NumElems / 2;
1642 for (unsigned i = 0; i < Half; ++i)
Chris Lattner35a08552007-02-25 07:10:00 +00001643 if (!isUndefOrInRange(Elems[i], 0, NumElems))
Evan Cheng60f0b892006-04-20 08:58:49 +00001644 return false;
1645 for (unsigned i = Half; i < NumElems; ++i)
Chris Lattner35a08552007-02-25 07:10:00 +00001646 if (!isUndefOrInRange(Elems[i], NumElems, NumElems*2))
Evan Cheng60f0b892006-04-20 08:58:49 +00001647 return false;
Evan Chengd27fb3e2006-03-24 01:18:28 +00001648
1649 return true;
1650}
1651
Evan Cheng60f0b892006-04-20 08:58:49 +00001652bool X86::isSHUFPMask(SDNode *N) {
1653 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00001654 return ::isSHUFPMask(N->op_begin(), N->getNumOperands());
Evan Cheng60f0b892006-04-20 08:58:49 +00001655}
1656
1657/// isCommutedSHUFP - Returns true if the shuffle mask is except
1658/// the reverse of what x86 shuffles want. x86 shuffles requires the lower
1659/// half elements to come from vector 1 (which would equal the dest.) and
1660/// the upper half to come from vector 2.
Chris Lattner35a08552007-02-25 07:10:00 +00001661static bool isCommutedSHUFP(const SDOperand *Ops, unsigned NumOps) {
1662 if (NumOps != 2 && NumOps != 4) return false;
Evan Cheng60f0b892006-04-20 08:58:49 +00001663
Chris Lattner35a08552007-02-25 07:10:00 +00001664 unsigned Half = NumOps / 2;
Evan Cheng60f0b892006-04-20 08:58:49 +00001665 for (unsigned i = 0; i < Half; ++i)
Chris Lattner35a08552007-02-25 07:10:00 +00001666 if (!isUndefOrInRange(Ops[i], NumOps, NumOps*2))
Evan Cheng60f0b892006-04-20 08:58:49 +00001667 return false;
Chris Lattner35a08552007-02-25 07:10:00 +00001668 for (unsigned i = Half; i < NumOps; ++i)
1669 if (!isUndefOrInRange(Ops[i], 0, NumOps))
Evan Cheng60f0b892006-04-20 08:58:49 +00001670 return false;
1671 return true;
1672}
1673
1674static bool isCommutedSHUFP(SDNode *N) {
1675 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00001676 return isCommutedSHUFP(N->op_begin(), N->getNumOperands());
Evan Cheng60f0b892006-04-20 08:58:49 +00001677}
1678
Evan Cheng2595a682006-03-24 02:58:06 +00001679/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand
1680/// specifies a shuffle of elements that is suitable for input to MOVHLPS.
1681bool X86::isMOVHLPSMask(SDNode *N) {
1682 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1683
Evan Cheng1a194a52006-03-28 06:50:32 +00001684 if (N->getNumOperands() != 4)
Evan Cheng2595a682006-03-24 02:58:06 +00001685 return false;
1686
Evan Cheng1a194a52006-03-28 06:50:32 +00001687 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3
Evan Chengac847262006-04-07 21:53:05 +00001688 return isUndefOrEqual(N->getOperand(0), 6) &&
1689 isUndefOrEqual(N->getOperand(1), 7) &&
1690 isUndefOrEqual(N->getOperand(2), 2) &&
1691 isUndefOrEqual(N->getOperand(3), 3);
Evan Cheng1a194a52006-03-28 06:50:32 +00001692}
1693
Evan Cheng922e1912006-11-07 22:14:24 +00001694/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form
1695/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef,
1696/// <2, 3, 2, 3>
1697bool X86::isMOVHLPS_v_undef_Mask(SDNode *N) {
1698 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1699
1700 if (N->getNumOperands() != 4)
1701 return false;
1702
1703 // Expect bit0 == 2, bit1 == 3, bit2 == 2, bit3 == 3
1704 return isUndefOrEqual(N->getOperand(0), 2) &&
1705 isUndefOrEqual(N->getOperand(1), 3) &&
1706 isUndefOrEqual(N->getOperand(2), 2) &&
1707 isUndefOrEqual(N->getOperand(3), 3);
1708}
1709
Evan Chengc995b452006-04-06 23:23:56 +00001710/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand
1711/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}.
1712bool X86::isMOVLPMask(SDNode *N) {
1713 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1714
1715 unsigned NumElems = N->getNumOperands();
1716 if (NumElems != 2 && NumElems != 4)
1717 return false;
1718
Evan Chengac847262006-04-07 21:53:05 +00001719 for (unsigned i = 0; i < NumElems/2; ++i)
1720 if (!isUndefOrEqual(N->getOperand(i), i + NumElems))
1721 return false;
Evan Chengc995b452006-04-06 23:23:56 +00001722
Evan Chengac847262006-04-07 21:53:05 +00001723 for (unsigned i = NumElems/2; i < NumElems; ++i)
1724 if (!isUndefOrEqual(N->getOperand(i), i))
1725 return false;
Evan Chengc995b452006-04-06 23:23:56 +00001726
1727 return true;
1728}
1729
1730/// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand
Evan Cheng7855e4d2006-04-19 20:35:22 +00001731/// specifies a shuffle of elements that is suitable for input to MOVHP{S|D}
1732/// and MOVLHPS.
Evan Chengc995b452006-04-06 23:23:56 +00001733bool X86::isMOVHPMask(SDNode *N) {
1734 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1735
1736 unsigned NumElems = N->getNumOperands();
1737 if (NumElems != 2 && NumElems != 4)
1738 return false;
1739
Evan Chengac847262006-04-07 21:53:05 +00001740 for (unsigned i = 0; i < NumElems/2; ++i)
1741 if (!isUndefOrEqual(N->getOperand(i), i))
1742 return false;
Evan Chengc995b452006-04-06 23:23:56 +00001743
1744 for (unsigned i = 0; i < NumElems/2; ++i) {
1745 SDOperand Arg = N->getOperand(i + NumElems/2);
Evan Chengac847262006-04-07 21:53:05 +00001746 if (!isUndefOrEqual(Arg, i + NumElems))
1747 return false;
Evan Chengc995b452006-04-06 23:23:56 +00001748 }
1749
1750 return true;
1751}
1752
Evan Cheng5df75882006-03-28 00:39:58 +00001753/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand
1754/// specifies a shuffle of elements that is suitable for input to UNPCKL.
Chris Lattner35a08552007-02-25 07:10:00 +00001755bool static isUNPCKLMask(const SDOperand *Elts, unsigned NumElts,
1756 bool V2IsSplat = false) {
1757 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
Evan Cheng5df75882006-03-28 00:39:58 +00001758 return false;
1759
Chris Lattner35a08552007-02-25 07:10:00 +00001760 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) {
1761 SDOperand BitI = Elts[i];
1762 SDOperand BitI1 = Elts[i+1];
Evan Chengac847262006-04-07 21:53:05 +00001763 if (!isUndefOrEqual(BitI, j))
1764 return false;
Evan Cheng60f0b892006-04-20 08:58:49 +00001765 if (V2IsSplat) {
Chris Lattner35a08552007-02-25 07:10:00 +00001766 if (isUndefOrEqual(BitI1, NumElts))
Evan Cheng60f0b892006-04-20 08:58:49 +00001767 return false;
1768 } else {
Chris Lattner35a08552007-02-25 07:10:00 +00001769 if (!isUndefOrEqual(BitI1, j + NumElts))
Evan Cheng60f0b892006-04-20 08:58:49 +00001770 return false;
1771 }
Evan Cheng5df75882006-03-28 00:39:58 +00001772 }
1773
1774 return true;
1775}
1776
Evan Cheng60f0b892006-04-20 08:58:49 +00001777bool X86::isUNPCKLMask(SDNode *N, bool V2IsSplat) {
1778 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00001779 return ::isUNPCKLMask(N->op_begin(), N->getNumOperands(), V2IsSplat);
Evan Cheng60f0b892006-04-20 08:58:49 +00001780}
1781
Evan Cheng2bc32802006-03-28 02:43:26 +00001782/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand
1783/// specifies a shuffle of elements that is suitable for input to UNPCKH.
Chris Lattner35a08552007-02-25 07:10:00 +00001784bool static isUNPCKHMask(const SDOperand *Elts, unsigned NumElts,
1785 bool V2IsSplat = false) {
1786 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
Evan Cheng2bc32802006-03-28 02:43:26 +00001787 return false;
1788
Chris Lattner35a08552007-02-25 07:10:00 +00001789 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) {
1790 SDOperand BitI = Elts[i];
1791 SDOperand BitI1 = Elts[i+1];
1792 if (!isUndefOrEqual(BitI, j + NumElts/2))
Evan Chengac847262006-04-07 21:53:05 +00001793 return false;
Evan Cheng60f0b892006-04-20 08:58:49 +00001794 if (V2IsSplat) {
Chris Lattner35a08552007-02-25 07:10:00 +00001795 if (isUndefOrEqual(BitI1, NumElts))
Evan Cheng60f0b892006-04-20 08:58:49 +00001796 return false;
1797 } else {
Chris Lattner35a08552007-02-25 07:10:00 +00001798 if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts))
Evan Cheng60f0b892006-04-20 08:58:49 +00001799 return false;
1800 }
Evan Cheng2bc32802006-03-28 02:43:26 +00001801 }
1802
1803 return true;
1804}
1805
Evan Cheng60f0b892006-04-20 08:58:49 +00001806bool X86::isUNPCKHMask(SDNode *N, bool V2IsSplat) {
1807 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00001808 return ::isUNPCKHMask(N->op_begin(), N->getNumOperands(), V2IsSplat);
Evan Cheng60f0b892006-04-20 08:58:49 +00001809}
1810
Evan Chengf3b52c82006-04-05 07:20:06 +00001811/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form
1812/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef,
1813/// <0, 0, 1, 1>
1814bool X86::isUNPCKL_v_undef_Mask(SDNode *N) {
1815 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1816
1817 unsigned NumElems = N->getNumOperands();
1818 if (NumElems != 4 && NumElems != 8 && NumElems != 16)
1819 return false;
1820
1821 for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) {
1822 SDOperand BitI = N->getOperand(i);
1823 SDOperand BitI1 = N->getOperand(i+1);
1824
Evan Chengac847262006-04-07 21:53:05 +00001825 if (!isUndefOrEqual(BitI, j))
1826 return false;
1827 if (!isUndefOrEqual(BitI1, j))
1828 return false;
Evan Chengf3b52c82006-04-05 07:20:06 +00001829 }
1830
1831 return true;
1832}
1833
Evan Chenge8b51802006-04-21 01:05:10 +00001834/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand
1835/// specifies a shuffle of elements that is suitable for input to MOVSS,
1836/// MOVSD, and MOVD, i.e. setting the lowest element.
Chris Lattner35a08552007-02-25 07:10:00 +00001837static bool isMOVLMask(const SDOperand *Elts, unsigned NumElts) {
1838 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
Evan Cheng12ba3e22006-04-11 00:19:04 +00001839 return false;
1840
Chris Lattner35a08552007-02-25 07:10:00 +00001841 if (!isUndefOrEqual(Elts[0], NumElts))
Evan Cheng12ba3e22006-04-11 00:19:04 +00001842 return false;
1843
Chris Lattner35a08552007-02-25 07:10:00 +00001844 for (unsigned i = 1; i < NumElts; ++i) {
1845 if (!isUndefOrEqual(Elts[i], i))
Evan Cheng12ba3e22006-04-11 00:19:04 +00001846 return false;
1847 }
1848
1849 return true;
1850}
Evan Chengf3b52c82006-04-05 07:20:06 +00001851
Evan Chenge8b51802006-04-21 01:05:10 +00001852bool X86::isMOVLMask(SDNode *N) {
Evan Cheng60f0b892006-04-20 08:58:49 +00001853 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00001854 return ::isMOVLMask(N->op_begin(), N->getNumOperands());
Evan Cheng60f0b892006-04-20 08:58:49 +00001855}
1856
Evan Chenge8b51802006-04-21 01:05:10 +00001857/// isCommutedMOVL - Returns true if the shuffle mask is except the reverse
1858/// of what x86 movss want. X86 movs requires the lowest element to be lowest
Evan Cheng60f0b892006-04-20 08:58:49 +00001859/// element of vector 2 and the other elements to come from vector 1 in order.
Chris Lattner35a08552007-02-25 07:10:00 +00001860static bool isCommutedMOVL(const SDOperand *Ops, unsigned NumOps,
1861 bool V2IsSplat = false,
Evan Cheng89c5d042006-09-08 01:50:06 +00001862 bool V2IsUndef = false) {
Chris Lattner35a08552007-02-25 07:10:00 +00001863 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16)
Evan Cheng60f0b892006-04-20 08:58:49 +00001864 return false;
1865
1866 if (!isUndefOrEqual(Ops[0], 0))
1867 return false;
1868
Chris Lattner35a08552007-02-25 07:10:00 +00001869 for (unsigned i = 1; i < NumOps; ++i) {
Evan Cheng60f0b892006-04-20 08:58:49 +00001870 SDOperand Arg = Ops[i];
Chris Lattner35a08552007-02-25 07:10:00 +00001871 if (!(isUndefOrEqual(Arg, i+NumOps) ||
1872 (V2IsUndef && isUndefOrInRange(Arg, NumOps, NumOps*2)) ||
1873 (V2IsSplat && isUndefOrEqual(Arg, NumOps))))
Evan Cheng89c5d042006-09-08 01:50:06 +00001874 return false;
Evan Cheng60f0b892006-04-20 08:58:49 +00001875 }
1876
1877 return true;
1878}
1879
Evan Cheng89c5d042006-09-08 01:50:06 +00001880static bool isCommutedMOVL(SDNode *N, bool V2IsSplat = false,
1881 bool V2IsUndef = false) {
Evan Cheng60f0b892006-04-20 08:58:49 +00001882 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00001883 return isCommutedMOVL(N->op_begin(), N->getNumOperands(),
1884 V2IsSplat, V2IsUndef);
Evan Cheng60f0b892006-04-20 08:58:49 +00001885}
1886
Evan Cheng5d247f82006-04-14 21:59:03 +00001887/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand
1888/// specifies a shuffle of elements that is suitable for input to MOVSHDUP.
1889bool X86::isMOVSHDUPMask(SDNode *N) {
1890 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1891
1892 if (N->getNumOperands() != 4)
1893 return false;
1894
1895 // Expect 1, 1, 3, 3
1896 for (unsigned i = 0; i < 2; ++i) {
1897 SDOperand Arg = N->getOperand(i);
1898 if (Arg.getOpcode() == ISD::UNDEF) continue;
1899 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
1900 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
1901 if (Val != 1) return false;
1902 }
Evan Cheng6222cf22006-04-15 05:37:34 +00001903
1904 bool HasHi = false;
Evan Cheng5d247f82006-04-14 21:59:03 +00001905 for (unsigned i = 2; i < 4; ++i) {
1906 SDOperand Arg = N->getOperand(i);
1907 if (Arg.getOpcode() == ISD::UNDEF) continue;
1908 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
1909 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
1910 if (Val != 3) return false;
Evan Cheng6222cf22006-04-15 05:37:34 +00001911 HasHi = true;
Evan Cheng5d247f82006-04-14 21:59:03 +00001912 }
Evan Cheng65bb7202006-04-15 03:13:24 +00001913
Evan Cheng6222cf22006-04-15 05:37:34 +00001914 // Don't use movshdup if it can be done with a shufps.
1915 return HasHi;
Evan Cheng5d247f82006-04-14 21:59:03 +00001916}
1917
1918/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand
1919/// specifies a shuffle of elements that is suitable for input to MOVSLDUP.
1920bool X86::isMOVSLDUPMask(SDNode *N) {
1921 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1922
1923 if (N->getNumOperands() != 4)
1924 return false;
1925
1926 // Expect 0, 0, 2, 2
1927 for (unsigned i = 0; i < 2; ++i) {
1928 SDOperand Arg = N->getOperand(i);
1929 if (Arg.getOpcode() == ISD::UNDEF) continue;
1930 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
1931 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
1932 if (Val != 0) return false;
1933 }
Evan Cheng6222cf22006-04-15 05:37:34 +00001934
1935 bool HasHi = false;
Evan Cheng5d247f82006-04-14 21:59:03 +00001936 for (unsigned i = 2; i < 4; ++i) {
1937 SDOperand Arg = N->getOperand(i);
1938 if (Arg.getOpcode() == ISD::UNDEF) continue;
1939 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
1940 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
1941 if (Val != 2) return false;
Evan Cheng6222cf22006-04-15 05:37:34 +00001942 HasHi = true;
Evan Cheng5d247f82006-04-14 21:59:03 +00001943 }
Evan Cheng65bb7202006-04-15 03:13:24 +00001944
Evan Cheng6222cf22006-04-15 05:37:34 +00001945 // Don't use movshdup if it can be done with a shufps.
1946 return HasHi;
Evan Cheng5d247f82006-04-14 21:59:03 +00001947}
1948
Evan Chengd097e672006-03-22 02:53:00 +00001949/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies
1950/// a splat of a single element.
Evan Cheng5022b342006-04-17 20:43:08 +00001951static bool isSplatMask(SDNode *N) {
Evan Chengd097e672006-03-22 02:53:00 +00001952 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1953
Evan Chengd097e672006-03-22 02:53:00 +00001954 // This is a splat operation if each element of the permute is the same, and
1955 // if the value doesn't reference the second vector.
Evan Cheng4a1b0d32006-04-19 23:28:59 +00001956 unsigned NumElems = N->getNumOperands();
1957 SDOperand ElementBase;
1958 unsigned i = 0;
1959 for (; i != NumElems; ++i) {
1960 SDOperand Elt = N->getOperand(i);
Reid Spencerde46e482006-11-02 20:25:50 +00001961 if (isa<ConstantSDNode>(Elt)) {
Evan Cheng4a1b0d32006-04-19 23:28:59 +00001962 ElementBase = Elt;
1963 break;
1964 }
1965 }
1966
1967 if (!ElementBase.Val)
1968 return false;
1969
1970 for (; i != NumElems; ++i) {
Evan Cheng99d72052006-03-31 00:30:29 +00001971 SDOperand Arg = N->getOperand(i);
1972 if (Arg.getOpcode() == ISD::UNDEF) continue;
1973 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
Evan Cheng4a1b0d32006-04-19 23:28:59 +00001974 if (Arg != ElementBase) return false;
Evan Chengd097e672006-03-22 02:53:00 +00001975 }
1976
1977 // Make sure it is a splat of the first vector operand.
Evan Cheng4a1b0d32006-04-19 23:28:59 +00001978 return cast<ConstantSDNode>(ElementBase)->getValue() < NumElems;
Evan Chengd097e672006-03-22 02:53:00 +00001979}
1980
Evan Cheng5022b342006-04-17 20:43:08 +00001981/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies
1982/// a splat of a single element and it's a 2 or 4 element mask.
1983bool X86::isSplatMask(SDNode *N) {
1984 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1985
Evan Cheng4a1b0d32006-04-19 23:28:59 +00001986 // We can only splat 64-bit, and 32-bit quantities with a single instruction.
Evan Cheng5022b342006-04-17 20:43:08 +00001987 if (N->getNumOperands() != 4 && N->getNumOperands() != 2)
1988 return false;
1989 return ::isSplatMask(N);
1990}
1991
Evan Chenge056dd52006-10-27 21:08:32 +00001992/// isSplatLoMask - Return true if the specified VECTOR_SHUFFLE operand
1993/// specifies a splat of zero element.
1994bool X86::isSplatLoMask(SDNode *N) {
1995 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1996
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00001997 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
Evan Chenge056dd52006-10-27 21:08:32 +00001998 if (!isUndefOrEqual(N->getOperand(i), 0))
1999 return false;
2000 return true;
2001}
2002
Evan Cheng8fdbdf22006-03-22 08:01:21 +00002003/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle
2004/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP*
2005/// instructions.
2006unsigned X86::getShuffleSHUFImmediate(SDNode *N) {
Evan Chengd097e672006-03-22 02:53:00 +00002007 unsigned NumOperands = N->getNumOperands();
2008 unsigned Shift = (NumOperands == 4) ? 2 : 1;
2009 unsigned Mask = 0;
Evan Cheng8160fd32006-03-28 23:41:33 +00002010 for (unsigned i = 0; i < NumOperands; ++i) {
Evan Cheng99d72052006-03-31 00:30:29 +00002011 unsigned Val = 0;
2012 SDOperand Arg = N->getOperand(NumOperands-i-1);
2013 if (Arg.getOpcode() != ISD::UNDEF)
2014 Val = cast<ConstantSDNode>(Arg)->getValue();
Evan Chengd27fb3e2006-03-24 01:18:28 +00002015 if (Val >= NumOperands) Val -= NumOperands;
Evan Cheng8fdbdf22006-03-22 08:01:21 +00002016 Mask |= Val;
Evan Cheng8160fd32006-03-28 23:41:33 +00002017 if (i != NumOperands - 1)
2018 Mask <<= Shift;
2019 }
Evan Cheng8fdbdf22006-03-22 08:01:21 +00002020
2021 return Mask;
2022}
2023
Evan Chengb7fedff2006-03-29 23:07:14 +00002024/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle
2025/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW
2026/// instructions.
2027unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) {
2028 unsigned Mask = 0;
2029 // 8 nodes, but we only care about the last 4.
2030 for (unsigned i = 7; i >= 4; --i) {
Evan Cheng99d72052006-03-31 00:30:29 +00002031 unsigned Val = 0;
2032 SDOperand Arg = N->getOperand(i);
2033 if (Arg.getOpcode() != ISD::UNDEF)
2034 Val = cast<ConstantSDNode>(Arg)->getValue();
Evan Chengb7fedff2006-03-29 23:07:14 +00002035 Mask |= (Val - 4);
2036 if (i != 4)
2037 Mask <<= 2;
2038 }
2039
2040 return Mask;
2041}
2042
2043/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle
2044/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW
2045/// instructions.
2046unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) {
2047 unsigned Mask = 0;
2048 // 8 nodes, but we only care about the first 4.
2049 for (int i = 3; i >= 0; --i) {
Evan Cheng99d72052006-03-31 00:30:29 +00002050 unsigned Val = 0;
2051 SDOperand Arg = N->getOperand(i);
2052 if (Arg.getOpcode() != ISD::UNDEF)
2053 Val = cast<ConstantSDNode>(Arg)->getValue();
Evan Chengb7fedff2006-03-29 23:07:14 +00002054 Mask |= Val;
2055 if (i != 0)
2056 Mask <<= 2;
2057 }
2058
2059 return Mask;
2060}
2061
Evan Cheng59a63552006-04-05 01:47:37 +00002062/// isPSHUFHW_PSHUFLWMask - true if the specified VECTOR_SHUFFLE operand
2063/// specifies a 8 element shuffle that can be broken into a pair of
2064/// PSHUFHW and PSHUFLW.
2065static bool isPSHUFHW_PSHUFLWMask(SDNode *N) {
2066 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2067
2068 if (N->getNumOperands() != 8)
2069 return false;
2070
2071 // Lower quadword shuffled.
2072 for (unsigned i = 0; i != 4; ++i) {
2073 SDOperand Arg = N->getOperand(i);
2074 if (Arg.getOpcode() == ISD::UNDEF) continue;
2075 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2076 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2077 if (Val > 4)
2078 return false;
2079 }
2080
2081 // Upper quadword shuffled.
2082 for (unsigned i = 4; i != 8; ++i) {
2083 SDOperand Arg = N->getOperand(i);
2084 if (Arg.getOpcode() == ISD::UNDEF) continue;
2085 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2086 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2087 if (Val < 4 || Val > 7)
2088 return false;
2089 }
2090
2091 return true;
2092}
2093
Evan Chengc995b452006-04-06 23:23:56 +00002094/// CommuteVectorShuffle - Swap vector_shuffle operandsas well as
2095/// values in ther permute mask.
Evan Chengc415c5b2006-10-25 21:49:50 +00002096static SDOperand CommuteVectorShuffle(SDOperand Op, SDOperand &V1,
2097 SDOperand &V2, SDOperand &Mask,
2098 SelectionDAG &DAG) {
Evan Chengc995b452006-04-06 23:23:56 +00002099 MVT::ValueType VT = Op.getValueType();
2100 MVT::ValueType MaskVT = Mask.getValueType();
2101 MVT::ValueType EltVT = MVT::getVectorBaseType(MaskVT);
2102 unsigned NumElems = Mask.getNumOperands();
Chris Lattner35a08552007-02-25 07:10:00 +00002103 SmallVector<SDOperand, 8> MaskVec;
Evan Chengc995b452006-04-06 23:23:56 +00002104
2105 for (unsigned i = 0; i != NumElems; ++i) {
2106 SDOperand Arg = Mask.getOperand(i);
Evan Chenga3caaee2006-04-19 22:48:17 +00002107 if (Arg.getOpcode() == ISD::UNDEF) {
2108 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT));
2109 continue;
2110 }
Evan Chengc995b452006-04-06 23:23:56 +00002111 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2112 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2113 if (Val < NumElems)
2114 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT));
2115 else
2116 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT));
2117 }
2118
Evan Chengc415c5b2006-10-25 21:49:50 +00002119 std::swap(V1, V2);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002120 Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size());
Evan Chengc415c5b2006-10-25 21:49:50 +00002121 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask);
Evan Chengc995b452006-04-06 23:23:56 +00002122}
2123
Evan Cheng7855e4d2006-04-19 20:35:22 +00002124/// ShouldXformToMOVHLPS - Return true if the node should be transformed to
2125/// match movhlps. The lower half elements should come from upper half of
2126/// V1 (and in order), and the upper half elements should come from the upper
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00002127/// half of V2 (and in order).
Evan Cheng7855e4d2006-04-19 20:35:22 +00002128static bool ShouldXformToMOVHLPS(SDNode *Mask) {
2129 unsigned NumElems = Mask->getNumOperands();
2130 if (NumElems != 4)
2131 return false;
2132 for (unsigned i = 0, e = 2; i != e; ++i)
2133 if (!isUndefOrEqual(Mask->getOperand(i), i+2))
2134 return false;
2135 for (unsigned i = 2; i != 4; ++i)
2136 if (!isUndefOrEqual(Mask->getOperand(i), i+4))
2137 return false;
2138 return true;
2139}
2140
Evan Chengc995b452006-04-06 23:23:56 +00002141/// isScalarLoadToVector - Returns true if the node is a scalar load that
2142/// is promoted to a vector.
Evan Cheng7855e4d2006-04-19 20:35:22 +00002143static inline bool isScalarLoadToVector(SDNode *N) {
2144 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2145 N = N->getOperand(0).Val;
Evan Chenge71fe34d2006-10-09 20:57:25 +00002146 return ISD::isNON_EXTLoad(N);
Evan Chengc995b452006-04-06 23:23:56 +00002147 }
2148 return false;
2149}
2150
Evan Cheng7855e4d2006-04-19 20:35:22 +00002151/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to
2152/// match movlp{s|d}. The lower half elements should come from lower half of
2153/// V1 (and in order), and the upper half elements should come from the upper
2154/// half of V2 (and in order). And since V1 will become the source of the
2155/// MOVLP, it must be either a vector load or a scalar load to vector.
Evan Chenge646abb2006-10-09 21:39:25 +00002156static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, SDNode *Mask) {
Evan Chenge71fe34d2006-10-09 20:57:25 +00002157 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1))
Evan Cheng7855e4d2006-04-19 20:35:22 +00002158 return false;
Evan Chenge646abb2006-10-09 21:39:25 +00002159 // Is V2 is a vector load, don't do this transformation. We will try to use
2160 // load folding shufps op.
2161 if (ISD::isNON_EXTLoad(V2))
2162 return false;
Evan Chengc995b452006-04-06 23:23:56 +00002163
Evan Cheng7855e4d2006-04-19 20:35:22 +00002164 unsigned NumElems = Mask->getNumOperands();
2165 if (NumElems != 2 && NumElems != 4)
2166 return false;
2167 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
2168 if (!isUndefOrEqual(Mask->getOperand(i), i))
2169 return false;
2170 for (unsigned i = NumElems/2; i != NumElems; ++i)
2171 if (!isUndefOrEqual(Mask->getOperand(i), i+NumElems))
2172 return false;
2173 return true;
Evan Chengc995b452006-04-06 23:23:56 +00002174}
2175
Evan Cheng60f0b892006-04-20 08:58:49 +00002176/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are
2177/// all the same.
2178static bool isSplatVector(SDNode *N) {
2179 if (N->getOpcode() != ISD::BUILD_VECTOR)
2180 return false;
Evan Chengc995b452006-04-06 23:23:56 +00002181
Evan Cheng60f0b892006-04-20 08:58:49 +00002182 SDOperand SplatValue = N->getOperand(0);
2183 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
2184 if (N->getOperand(i) != SplatValue)
Evan Chengc995b452006-04-06 23:23:56 +00002185 return false;
2186 return true;
2187}
2188
Evan Cheng89c5d042006-09-08 01:50:06 +00002189/// isUndefShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved
2190/// to an undef.
2191static bool isUndefShuffle(SDNode *N) {
2192 if (N->getOpcode() != ISD::BUILD_VECTOR)
2193 return false;
2194
2195 SDOperand V1 = N->getOperand(0);
2196 SDOperand V2 = N->getOperand(1);
2197 SDOperand Mask = N->getOperand(2);
2198 unsigned NumElems = Mask.getNumOperands();
2199 for (unsigned i = 0; i != NumElems; ++i) {
2200 SDOperand Arg = Mask.getOperand(i);
2201 if (Arg.getOpcode() != ISD::UNDEF) {
2202 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2203 if (Val < NumElems && V1.getOpcode() != ISD::UNDEF)
2204 return false;
2205 else if (Val >= NumElems && V2.getOpcode() != ISD::UNDEF)
2206 return false;
2207 }
2208 }
2209 return true;
2210}
2211
Evan Cheng60f0b892006-04-20 08:58:49 +00002212/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements
2213/// that point to V2 points to its first element.
2214static SDOperand NormalizeMask(SDOperand Mask, SelectionDAG &DAG) {
2215 assert(Mask.getOpcode() == ISD::BUILD_VECTOR);
2216
2217 bool Changed = false;
Chris Lattner35a08552007-02-25 07:10:00 +00002218 SmallVector<SDOperand, 8> MaskVec;
Evan Cheng60f0b892006-04-20 08:58:49 +00002219 unsigned NumElems = Mask.getNumOperands();
2220 for (unsigned i = 0; i != NumElems; ++i) {
2221 SDOperand Arg = Mask.getOperand(i);
2222 if (Arg.getOpcode() != ISD::UNDEF) {
2223 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2224 if (Val > NumElems) {
2225 Arg = DAG.getConstant(NumElems, Arg.getValueType());
2226 Changed = true;
2227 }
2228 }
2229 MaskVec.push_back(Arg);
2230 }
2231
2232 if (Changed)
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002233 Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(),
2234 &MaskVec[0], MaskVec.size());
Evan Cheng60f0b892006-04-20 08:58:49 +00002235 return Mask;
2236}
2237
Evan Chenge8b51802006-04-21 01:05:10 +00002238/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd
2239/// operation of specified width.
2240static SDOperand getMOVLMask(unsigned NumElems, SelectionDAG &DAG) {
Evan Cheng60f0b892006-04-20 08:58:49 +00002241 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2242 MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT);
2243
Chris Lattner35a08552007-02-25 07:10:00 +00002244 SmallVector<SDOperand, 8> MaskVec;
Evan Cheng60f0b892006-04-20 08:58:49 +00002245 MaskVec.push_back(DAG.getConstant(NumElems, BaseVT));
2246 for (unsigned i = 1; i != NumElems; ++i)
2247 MaskVec.push_back(DAG.getConstant(i, BaseVT));
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002248 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size());
Evan Cheng60f0b892006-04-20 08:58:49 +00002249}
2250
Evan Cheng5022b342006-04-17 20:43:08 +00002251/// getUnpacklMask - Returns a vector_shuffle mask for an unpackl operation
2252/// of specified width.
2253static SDOperand getUnpacklMask(unsigned NumElems, SelectionDAG &DAG) {
2254 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2255 MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002256 SmallVector<SDOperand, 8> MaskVec;
Evan Cheng5022b342006-04-17 20:43:08 +00002257 for (unsigned i = 0, e = NumElems/2; i != e; ++i) {
2258 MaskVec.push_back(DAG.getConstant(i, BaseVT));
2259 MaskVec.push_back(DAG.getConstant(i + NumElems, BaseVT));
2260 }
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002261 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size());
Evan Cheng5022b342006-04-17 20:43:08 +00002262}
2263
Evan Cheng60f0b892006-04-20 08:58:49 +00002264/// getUnpackhMask - Returns a vector_shuffle mask for an unpackh operation
2265/// of specified width.
2266static SDOperand getUnpackhMask(unsigned NumElems, SelectionDAG &DAG) {
2267 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2268 MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT);
2269 unsigned Half = NumElems/2;
Chris Lattner35a08552007-02-25 07:10:00 +00002270 SmallVector<SDOperand, 8> MaskVec;
Evan Cheng60f0b892006-04-20 08:58:49 +00002271 for (unsigned i = 0; i != Half; ++i) {
2272 MaskVec.push_back(DAG.getConstant(i + Half, BaseVT));
2273 MaskVec.push_back(DAG.getConstant(i + NumElems + Half, BaseVT));
2274 }
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002275 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size());
Evan Cheng60f0b892006-04-20 08:58:49 +00002276}
2277
Evan Chenge8b51802006-04-21 01:05:10 +00002278/// getZeroVector - Returns a vector of specified type with all zero elements.
2279///
2280static SDOperand getZeroVector(MVT::ValueType VT, SelectionDAG &DAG) {
2281 assert(MVT::isVector(VT) && "Expected a vector type");
2282 unsigned NumElems = getVectorNumElements(VT);
2283 MVT::ValueType EVT = MVT::getVectorBaseType(VT);
2284 bool isFP = MVT::isFloatingPoint(EVT);
2285 SDOperand Zero = isFP ? DAG.getConstantFP(0.0, EVT) : DAG.getConstant(0, EVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002286 SmallVector<SDOperand, 8> ZeroVec(NumElems, Zero);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002287 return DAG.getNode(ISD::BUILD_VECTOR, VT, &ZeroVec[0], ZeroVec.size());
Evan Chenge8b51802006-04-21 01:05:10 +00002288}
2289
Evan Cheng5022b342006-04-17 20:43:08 +00002290/// PromoteSplat - Promote a splat of v8i16 or v16i8 to v4i32.
2291///
2292static SDOperand PromoteSplat(SDOperand Op, SelectionDAG &DAG) {
2293 SDOperand V1 = Op.getOperand(0);
Evan Chenge8b51802006-04-21 01:05:10 +00002294 SDOperand Mask = Op.getOperand(2);
Evan Cheng5022b342006-04-17 20:43:08 +00002295 MVT::ValueType VT = Op.getValueType();
Evan Chenge8b51802006-04-21 01:05:10 +00002296 unsigned NumElems = Mask.getNumOperands();
2297 Mask = getUnpacklMask(NumElems, DAG);
Evan Cheng5022b342006-04-17 20:43:08 +00002298 while (NumElems != 4) {
Evan Chenge8b51802006-04-21 01:05:10 +00002299 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask);
Evan Cheng5022b342006-04-17 20:43:08 +00002300 NumElems >>= 1;
2301 }
2302 V1 = DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, V1);
2303
2304 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4);
Evan Chenge8b51802006-04-21 01:05:10 +00002305 Mask = getZeroVector(MaskVT, DAG);
Evan Cheng5022b342006-04-17 20:43:08 +00002306 SDOperand Shuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v4i32, V1,
Evan Chenge8b51802006-04-21 01:05:10 +00002307 DAG.getNode(ISD::UNDEF, MVT::v4i32), Mask);
Evan Cheng5022b342006-04-17 20:43:08 +00002308 return DAG.getNode(ISD::BIT_CONVERT, VT, Shuffle);
2309}
2310
Evan Chenge8b51802006-04-21 01:05:10 +00002311/// isZeroNode - Returns true if Elt is a constant zero or a floating point
2312/// constant +0.0.
2313static inline bool isZeroNode(SDOperand Elt) {
2314 return ((isa<ConstantSDNode>(Elt) &&
2315 cast<ConstantSDNode>(Elt)->getValue() == 0) ||
2316 (isa<ConstantFPSDNode>(Elt) &&
2317 cast<ConstantFPSDNode>(Elt)->isExactlyValue(0.0)));
2318}
2319
Evan Cheng14215c32006-04-21 23:03:30 +00002320/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified
2321/// vector and zero or undef vector.
2322static SDOperand getShuffleVectorZeroOrUndef(SDOperand V2, MVT::ValueType VT,
Evan Chenge8b51802006-04-21 01:05:10 +00002323 unsigned NumElems, unsigned Idx,
Evan Cheng14215c32006-04-21 23:03:30 +00002324 bool isZero, SelectionDAG &DAG) {
2325 SDOperand V1 = isZero ? getZeroVector(VT, DAG) : DAG.getNode(ISD::UNDEF, VT);
Evan Chenge8b51802006-04-21 01:05:10 +00002326 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2327 MVT::ValueType EVT = MVT::getVectorBaseType(MaskVT);
2328 SDOperand Zero = DAG.getConstant(0, EVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002329 SmallVector<SDOperand, 8> MaskVec(NumElems, Zero);
Evan Chenge8b51802006-04-21 01:05:10 +00002330 MaskVec[Idx] = DAG.getConstant(NumElems, EVT);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002331 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2332 &MaskVec[0], MaskVec.size());
Evan Cheng14215c32006-04-21 23:03:30 +00002333 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask);
Evan Chenge8b51802006-04-21 01:05:10 +00002334}
2335
Evan Chengb0461082006-04-24 18:01:45 +00002336/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8.
2337///
2338static SDOperand LowerBuildVectorv16i8(SDOperand Op, unsigned NonZeros,
2339 unsigned NumNonZero, unsigned NumZero,
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002340 SelectionDAG &DAG, TargetLowering &TLI) {
Evan Chengb0461082006-04-24 18:01:45 +00002341 if (NumNonZero > 8)
2342 return SDOperand();
2343
2344 SDOperand V(0, 0);
2345 bool First = true;
2346 for (unsigned i = 0; i < 16; ++i) {
2347 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0;
2348 if (ThisIsNonZero && First) {
2349 if (NumZero)
2350 V = getZeroVector(MVT::v8i16, DAG);
2351 else
2352 V = DAG.getNode(ISD::UNDEF, MVT::v8i16);
2353 First = false;
2354 }
2355
2356 if ((i & 1) != 0) {
2357 SDOperand ThisElt(0, 0), LastElt(0, 0);
2358 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0;
2359 if (LastIsNonZero) {
2360 LastElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i-1));
2361 }
2362 if (ThisIsNonZero) {
2363 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i));
2364 ThisElt = DAG.getNode(ISD::SHL, MVT::i16,
2365 ThisElt, DAG.getConstant(8, MVT::i8));
2366 if (LastIsNonZero)
2367 ThisElt = DAG.getNode(ISD::OR, MVT::i16, ThisElt, LastElt);
2368 } else
2369 ThisElt = LastElt;
2370
2371 if (ThisElt.Val)
2372 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, ThisElt,
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002373 DAG.getConstant(i/2, TLI.getPointerTy()));
Evan Chengb0461082006-04-24 18:01:45 +00002374 }
2375 }
2376
2377 return DAG.getNode(ISD::BIT_CONVERT, MVT::v16i8, V);
2378}
2379
2380/// LowerBuildVectorv16i8 - Custom lower build_vector of v8i16.
2381///
2382static SDOperand LowerBuildVectorv8i16(SDOperand Op, unsigned NonZeros,
2383 unsigned NumNonZero, unsigned NumZero,
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002384 SelectionDAG &DAG, TargetLowering &TLI) {
Evan Chengb0461082006-04-24 18:01:45 +00002385 if (NumNonZero > 4)
2386 return SDOperand();
2387
2388 SDOperand V(0, 0);
2389 bool First = true;
2390 for (unsigned i = 0; i < 8; ++i) {
2391 bool isNonZero = (NonZeros & (1 << i)) != 0;
2392 if (isNonZero) {
2393 if (First) {
2394 if (NumZero)
2395 V = getZeroVector(MVT::v8i16, DAG);
2396 else
2397 V = DAG.getNode(ISD::UNDEF, MVT::v8i16);
2398 First = false;
2399 }
2400 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, Op.getOperand(i),
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002401 DAG.getConstant(i, TLI.getPointerTy()));
Evan Chengb0461082006-04-24 18:01:45 +00002402 }
2403 }
2404
2405 return V;
2406}
2407
Evan Chenga9467aa2006-04-25 20:13:52 +00002408SDOperand
2409X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
2410 // All zero's are handled with pxor.
2411 if (ISD::isBuildVectorAllZeros(Op.Val))
2412 return Op;
2413
2414 // All one's are handled with pcmpeqd.
2415 if (ISD::isBuildVectorAllOnes(Op.Val))
2416 return Op;
2417
2418 MVT::ValueType VT = Op.getValueType();
2419 MVT::ValueType EVT = MVT::getVectorBaseType(VT);
2420 unsigned EVTBits = MVT::getSizeInBits(EVT);
2421
2422 unsigned NumElems = Op.getNumOperands();
2423 unsigned NumZero = 0;
2424 unsigned NumNonZero = 0;
2425 unsigned NonZeros = 0;
2426 std::set<SDOperand> Values;
2427 for (unsigned i = 0; i < NumElems; ++i) {
2428 SDOperand Elt = Op.getOperand(i);
2429 if (Elt.getOpcode() != ISD::UNDEF) {
2430 Values.insert(Elt);
2431 if (isZeroNode(Elt))
2432 NumZero++;
2433 else {
2434 NonZeros |= (1 << i);
2435 NumNonZero++;
2436 }
2437 }
2438 }
2439
2440 if (NumNonZero == 0)
2441 // Must be a mix of zero and undef. Return a zero vector.
2442 return getZeroVector(VT, DAG);
2443
2444 // Splat is obviously ok. Let legalizer expand it to a shuffle.
2445 if (Values.size() == 1)
2446 return SDOperand();
2447
2448 // Special case for single non-zero element.
Evan Cheng798b3062006-10-25 20:48:19 +00002449 if (NumNonZero == 1) {
Evan Chenga9467aa2006-04-25 20:13:52 +00002450 unsigned Idx = CountTrailingZeros_32(NonZeros);
2451 SDOperand Item = Op.getOperand(Idx);
2452 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item);
2453 if (Idx == 0)
2454 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector.
2455 return getShuffleVectorZeroOrUndef(Item, VT, NumElems, Idx,
2456 NumZero > 0, DAG);
2457
2458 if (EVTBits == 32) {
2459 // Turn it into a shuffle of zero and zero-extended scalar to vector.
2460 Item = getShuffleVectorZeroOrUndef(Item, VT, NumElems, 0, NumZero > 0,
2461 DAG);
2462 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2463 MVT::ValueType MaskEVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002464 SmallVector<SDOperand, 8> MaskVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00002465 for (unsigned i = 0; i < NumElems; i++)
2466 MaskVec.push_back(DAG.getConstant((i == Idx) ? 0 : 1, MaskEVT));
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002467 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2468 &MaskVec[0], MaskVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00002469 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Item,
2470 DAG.getNode(ISD::UNDEF, VT), Mask);
2471 }
2472 }
2473
Evan Cheng8c5766e2006-10-04 18:33:38 +00002474 // Let legalizer expand 2-wide build_vector's.
Evan Chenga9467aa2006-04-25 20:13:52 +00002475 if (EVTBits == 64)
2476 return SDOperand();
2477
2478 // If element VT is < 32 bits, convert it to inserts into a zero vector.
2479 if (EVTBits == 8) {
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002480 SDOperand V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG,
2481 *this);
Evan Chenga9467aa2006-04-25 20:13:52 +00002482 if (V.Val) return V;
2483 }
2484
2485 if (EVTBits == 16) {
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002486 SDOperand V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG,
2487 *this);
Evan Chenga9467aa2006-04-25 20:13:52 +00002488 if (V.Val) return V;
2489 }
2490
2491 // If element VT is == 32 bits, turn it into a number of shuffles.
Chris Lattner35a08552007-02-25 07:10:00 +00002492 SmallVector<SDOperand, 8> V;
2493 V.resize(NumElems);
Evan Chenga9467aa2006-04-25 20:13:52 +00002494 if (NumElems == 4 && NumZero > 0) {
2495 for (unsigned i = 0; i < 4; ++i) {
2496 bool isZero = !(NonZeros & (1 << i));
2497 if (isZero)
2498 V[i] = getZeroVector(VT, DAG);
2499 else
2500 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i));
2501 }
2502
2503 for (unsigned i = 0; i < 2; ++i) {
2504 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) {
2505 default: break;
2506 case 0:
2507 V[i] = V[i*2]; // Must be a zero vector.
2508 break;
2509 case 1:
2510 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2+1], V[i*2],
2511 getMOVLMask(NumElems, DAG));
2512 break;
2513 case 2:
2514 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1],
2515 getMOVLMask(NumElems, DAG));
2516 break;
2517 case 3:
2518 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1],
2519 getUnpacklMask(NumElems, DAG));
2520 break;
2521 }
2522 }
2523
Evan Cheng9fee4422006-05-16 07:21:53 +00002524 // Take advantage of the fact GR32 to VR128 scalar_to_vector (i.e. movd)
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00002525 // clears the upper bits.
Evan Chenga9467aa2006-04-25 20:13:52 +00002526 // FIXME: we can do the same for v4f32 case when we know both parts of
2527 // the lower half come from scalar_to_vector (loadf32). We should do
2528 // that in post legalizer dag combiner with target specific hooks.
Evan Cheng798b3062006-10-25 20:48:19 +00002529 if (MVT::isInteger(EVT) && (NonZeros & (0x3 << 2)) == 0)
Evan Chenga9467aa2006-04-25 20:13:52 +00002530 return V[0];
2531 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2532 MVT::ValueType EVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002533 SmallVector<SDOperand, 8> MaskVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00002534 bool Reverse = (NonZeros & 0x3) == 2;
2535 for (unsigned i = 0; i < 2; ++i)
2536 if (Reverse)
2537 MaskVec.push_back(DAG.getConstant(1-i, EVT));
2538 else
2539 MaskVec.push_back(DAG.getConstant(i, EVT));
2540 Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2;
2541 for (unsigned i = 0; i < 2; ++i)
2542 if (Reverse)
2543 MaskVec.push_back(DAG.getConstant(1-i+NumElems, EVT));
2544 else
2545 MaskVec.push_back(DAG.getConstant(i+NumElems, EVT));
Chris Lattnered728e82006-08-11 17:38:39 +00002546 SDOperand ShufMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2547 &MaskVec[0], MaskVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00002548 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[0], V[1], ShufMask);
2549 }
2550
2551 if (Values.size() > 2) {
2552 // Expand into a number of unpckl*.
2553 // e.g. for v4f32
2554 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0>
2555 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1>
2556 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0>
2557 SDOperand UnpckMask = getUnpacklMask(NumElems, DAG);
2558 for (unsigned i = 0; i < NumElems; ++i)
2559 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i));
2560 NumElems >>= 1;
2561 while (NumElems != 0) {
2562 for (unsigned i = 0; i < NumElems; ++i)
2563 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i], V[i + NumElems],
2564 UnpckMask);
2565 NumElems >>= 1;
2566 }
2567 return V[0];
2568 }
2569
2570 return SDOperand();
2571}
2572
2573SDOperand
2574X86TargetLowering::LowerVECTOR_SHUFFLE(SDOperand Op, SelectionDAG &DAG) {
2575 SDOperand V1 = Op.getOperand(0);
2576 SDOperand V2 = Op.getOperand(1);
2577 SDOperand PermMask = Op.getOperand(2);
2578 MVT::ValueType VT = Op.getValueType();
2579 unsigned NumElems = PermMask.getNumOperands();
2580 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF;
2581 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
Evan Cheng949bcc92006-10-16 06:36:00 +00002582 bool V1IsSplat = false;
2583 bool V2IsSplat = false;
Evan Chenga9467aa2006-04-25 20:13:52 +00002584
Evan Cheng89c5d042006-09-08 01:50:06 +00002585 if (isUndefShuffle(Op.Val))
2586 return DAG.getNode(ISD::UNDEF, VT);
2587
Evan Chenga9467aa2006-04-25 20:13:52 +00002588 if (isSplatMask(PermMask.Val)) {
2589 if (NumElems <= 4) return Op;
2590 // Promote it to a v4i32 splat.
Evan Cheng798b3062006-10-25 20:48:19 +00002591 return PromoteSplat(Op, DAG);
Evan Chenga9467aa2006-04-25 20:13:52 +00002592 }
2593
Evan Cheng798b3062006-10-25 20:48:19 +00002594 if (X86::isMOVLMask(PermMask.Val))
2595 return (V1IsUndef) ? V2 : Op;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00002596
Evan Cheng798b3062006-10-25 20:48:19 +00002597 if (X86::isMOVSHDUPMask(PermMask.Val) ||
2598 X86::isMOVSLDUPMask(PermMask.Val) ||
2599 X86::isMOVHLPSMask(PermMask.Val) ||
2600 X86::isMOVHPMask(PermMask.Val) ||
2601 X86::isMOVLPMask(PermMask.Val))
2602 return Op;
Evan Chenga9467aa2006-04-25 20:13:52 +00002603
Evan Cheng798b3062006-10-25 20:48:19 +00002604 if (ShouldXformToMOVHLPS(PermMask.Val) ||
2605 ShouldXformToMOVLP(V1.Val, V2.Val, PermMask.Val))
Evan Chengc415c5b2006-10-25 21:49:50 +00002606 return CommuteVectorShuffle(Op, V1, V2, PermMask, DAG);
Evan Chenga9467aa2006-04-25 20:13:52 +00002607
Evan Chengc415c5b2006-10-25 21:49:50 +00002608 bool Commuted = false;
Evan Cheng798b3062006-10-25 20:48:19 +00002609 V1IsSplat = isSplatVector(V1.Val);
2610 V2IsSplat = isSplatVector(V2.Val);
2611 if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) {
Evan Chengc415c5b2006-10-25 21:49:50 +00002612 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG);
Evan Cheng798b3062006-10-25 20:48:19 +00002613 std::swap(V1IsSplat, V2IsSplat);
2614 std::swap(V1IsUndef, V2IsUndef);
Evan Chengc415c5b2006-10-25 21:49:50 +00002615 Commuted = true;
Evan Cheng798b3062006-10-25 20:48:19 +00002616 }
2617
2618 if (isCommutedMOVL(PermMask.Val, V2IsSplat, V2IsUndef)) {
2619 if (V2IsUndef) return V1;
Evan Chengc415c5b2006-10-25 21:49:50 +00002620 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG);
Evan Cheng798b3062006-10-25 20:48:19 +00002621 if (V2IsSplat) {
2622 // V2 is a splat, so the mask may be malformed. That is, it may point
2623 // to any V2 element. The instruction selectior won't like this. Get
2624 // a corrected mask and commute to form a proper MOVS{S|D}.
2625 SDOperand NewMask = getMOVLMask(NumElems, DAG);
2626 if (NewMask.Val != PermMask.Val)
2627 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask);
Evan Chenga9467aa2006-04-25 20:13:52 +00002628 }
Evan Cheng798b3062006-10-25 20:48:19 +00002629 return Op;
Evan Cheng949bcc92006-10-16 06:36:00 +00002630 }
Evan Chenga9467aa2006-04-25 20:13:52 +00002631
Evan Cheng949bcc92006-10-16 06:36:00 +00002632 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) ||
2633 X86::isUNPCKLMask(PermMask.Val) ||
2634 X86::isUNPCKHMask(PermMask.Val))
2635 return Op;
Evan Cheng8c5766e2006-10-04 18:33:38 +00002636
Evan Cheng798b3062006-10-25 20:48:19 +00002637 if (V2IsSplat) {
2638 // Normalize mask so all entries that point to V2 points to its first
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00002639 // element then try to match unpck{h|l} again. If match, return a
Evan Cheng798b3062006-10-25 20:48:19 +00002640 // new vector_shuffle with the corrected mask.
2641 SDOperand NewMask = NormalizeMask(PermMask, DAG);
2642 if (NewMask.Val != PermMask.Val) {
2643 if (X86::isUNPCKLMask(PermMask.Val, true)) {
2644 SDOperand NewMask = getUnpacklMask(NumElems, DAG);
2645 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask);
2646 } else if (X86::isUNPCKHMask(PermMask.Val, true)) {
2647 SDOperand NewMask = getUnpackhMask(NumElems, DAG);
2648 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask);
Evan Chenga9467aa2006-04-25 20:13:52 +00002649 }
2650 }
2651 }
2652
2653 // Normalize the node to match x86 shuffle ops if needed
Evan Chengc415c5b2006-10-25 21:49:50 +00002654 if (V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(PermMask.Val))
2655 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG);
2656
2657 if (Commuted) {
2658 // Commute is back and try unpck* again.
2659 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG);
2660 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) ||
2661 X86::isUNPCKLMask(PermMask.Val) ||
2662 X86::isUNPCKHMask(PermMask.Val))
2663 return Op;
2664 }
Evan Chenga9467aa2006-04-25 20:13:52 +00002665
2666 // If VT is integer, try PSHUF* first, then SHUFP*.
2667 if (MVT::isInteger(VT)) {
2668 if (X86::isPSHUFDMask(PermMask.Val) ||
2669 X86::isPSHUFHWMask(PermMask.Val) ||
2670 X86::isPSHUFLWMask(PermMask.Val)) {
2671 if (V2.getOpcode() != ISD::UNDEF)
2672 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1,
2673 DAG.getNode(ISD::UNDEF, V1.getValueType()),PermMask);
2674 return Op;
2675 }
2676
2677 if (X86::isSHUFPMask(PermMask.Val))
2678 return Op;
2679
2680 // Handle v8i16 shuffle high / low shuffle node pair.
2681 if (VT == MVT::v8i16 && isPSHUFHW_PSHUFLWMask(PermMask.Val)) {
2682 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2683 MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002684 SmallVector<SDOperand, 8> MaskVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00002685 for (unsigned i = 0; i != 4; ++i)
2686 MaskVec.push_back(PermMask.getOperand(i));
2687 for (unsigned i = 4; i != 8; ++i)
2688 MaskVec.push_back(DAG.getConstant(i, BaseVT));
Chris Lattnered728e82006-08-11 17:38:39 +00002689 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2690 &MaskVec[0], MaskVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00002691 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask);
2692 MaskVec.clear();
2693 for (unsigned i = 0; i != 4; ++i)
2694 MaskVec.push_back(DAG.getConstant(i, BaseVT));
2695 for (unsigned i = 4; i != 8; ++i)
2696 MaskVec.push_back(PermMask.getOperand(i));
Chris Lattnered728e82006-08-11 17:38:39 +00002697 Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0],MaskVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00002698 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask);
2699 }
2700 } else {
2701 // Floating point cases in the other order.
2702 if (X86::isSHUFPMask(PermMask.Val))
2703 return Op;
2704 if (X86::isPSHUFDMask(PermMask.Val) ||
2705 X86::isPSHUFHWMask(PermMask.Val) ||
2706 X86::isPSHUFLWMask(PermMask.Val)) {
2707 if (V2.getOpcode() != ISD::UNDEF)
2708 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1,
2709 DAG.getNode(ISD::UNDEF, V1.getValueType()),PermMask);
2710 return Op;
2711 }
2712 }
2713
2714 if (NumElems == 4) {
Evan Chenga9467aa2006-04-25 20:13:52 +00002715 MVT::ValueType MaskVT = PermMask.getValueType();
2716 MVT::ValueType MaskEVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002717 SmallVector<std::pair<int, int>, 8> Locs;
Evan Cheng3cd43622006-04-28 07:03:38 +00002718 Locs.reserve(NumElems);
Chris Lattner35a08552007-02-25 07:10:00 +00002719 SmallVector<SDOperand, 8> Mask1(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT));
2720 SmallVector<SDOperand, 8> Mask2(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT));
Evan Cheng3cd43622006-04-28 07:03:38 +00002721 unsigned NumHi = 0;
2722 unsigned NumLo = 0;
2723 // If no more than two elements come from either vector. This can be
2724 // implemented with two shuffles. First shuffle gather the elements.
2725 // The second shuffle, which takes the first shuffle as both of its
2726 // vector operands, put the elements into the right order.
2727 for (unsigned i = 0; i != NumElems; ++i) {
2728 SDOperand Elt = PermMask.getOperand(i);
2729 if (Elt.getOpcode() == ISD::UNDEF) {
2730 Locs[i] = std::make_pair(-1, -1);
2731 } else {
2732 unsigned Val = cast<ConstantSDNode>(Elt)->getValue();
2733 if (Val < NumElems) {
2734 Locs[i] = std::make_pair(0, NumLo);
2735 Mask1[NumLo] = Elt;
2736 NumLo++;
2737 } else {
2738 Locs[i] = std::make_pair(1, NumHi);
2739 if (2+NumHi < NumElems)
2740 Mask1[2+NumHi] = Elt;
2741 NumHi++;
2742 }
2743 }
2744 }
2745 if (NumLo <= 2 && NumHi <= 2) {
2746 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2,
Chris Lattnered728e82006-08-11 17:38:39 +00002747 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2748 &Mask1[0], Mask1.size()));
Evan Cheng3cd43622006-04-28 07:03:38 +00002749 for (unsigned i = 0; i != NumElems; ++i) {
2750 if (Locs[i].first == -1)
2751 continue;
2752 else {
2753 unsigned Idx = (i < NumElems/2) ? 0 : NumElems;
2754 Idx += Locs[i].first * (NumElems/2) + Locs[i].second;
2755 Mask2[i] = DAG.getConstant(Idx, MaskEVT);
2756 }
2757 }
2758
2759 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1,
Chris Lattnered728e82006-08-11 17:38:39 +00002760 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2761 &Mask2[0], Mask2.size()));
Evan Cheng3cd43622006-04-28 07:03:38 +00002762 }
2763
2764 // Break it into (shuffle shuffle_hi, shuffle_lo).
2765 Locs.clear();
Chris Lattner35a08552007-02-25 07:10:00 +00002766 SmallVector<SDOperand,8> LoMask(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT));
2767 SmallVector<SDOperand,8> HiMask(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT));
2768 SmallVector<SDOperand,8> *MaskPtr = &LoMask;
Evan Chenga9467aa2006-04-25 20:13:52 +00002769 unsigned MaskIdx = 0;
2770 unsigned LoIdx = 0;
2771 unsigned HiIdx = NumElems/2;
2772 for (unsigned i = 0; i != NumElems; ++i) {
2773 if (i == NumElems/2) {
2774 MaskPtr = &HiMask;
2775 MaskIdx = 1;
2776 LoIdx = 0;
2777 HiIdx = NumElems/2;
2778 }
2779 SDOperand Elt = PermMask.getOperand(i);
2780 if (Elt.getOpcode() == ISD::UNDEF) {
2781 Locs[i] = std::make_pair(-1, -1);
2782 } else if (cast<ConstantSDNode>(Elt)->getValue() < NumElems) {
2783 Locs[i] = std::make_pair(MaskIdx, LoIdx);
2784 (*MaskPtr)[LoIdx] = Elt;
2785 LoIdx++;
2786 } else {
2787 Locs[i] = std::make_pair(MaskIdx, HiIdx);
2788 (*MaskPtr)[HiIdx] = Elt;
2789 HiIdx++;
2790 }
2791 }
2792
Chris Lattner3d826992006-05-16 06:45:34 +00002793 SDOperand LoShuffle =
2794 DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2,
Chris Lattnered728e82006-08-11 17:38:39 +00002795 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2796 &LoMask[0], LoMask.size()));
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00002797 SDOperand HiShuffle =
Chris Lattner3d826992006-05-16 06:45:34 +00002798 DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2,
Chris Lattnered728e82006-08-11 17:38:39 +00002799 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2800 &HiMask[0], HiMask.size()));
Chris Lattner35a08552007-02-25 07:10:00 +00002801 SmallVector<SDOperand, 8> MaskOps;
Evan Chenga9467aa2006-04-25 20:13:52 +00002802 for (unsigned i = 0; i != NumElems; ++i) {
2803 if (Locs[i].first == -1) {
2804 MaskOps.push_back(DAG.getNode(ISD::UNDEF, MaskEVT));
2805 } else {
2806 unsigned Idx = Locs[i].first * NumElems + Locs[i].second;
2807 MaskOps.push_back(DAG.getConstant(Idx, MaskEVT));
2808 }
2809 }
2810 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, LoShuffle, HiShuffle,
Chris Lattnered728e82006-08-11 17:38:39 +00002811 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2812 &MaskOps[0], MaskOps.size()));
Evan Chenga9467aa2006-04-25 20:13:52 +00002813 }
2814
2815 return SDOperand();
2816}
2817
2818SDOperand
2819X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) {
2820 if (!isa<ConstantSDNode>(Op.getOperand(1)))
2821 return SDOperand();
2822
2823 MVT::ValueType VT = Op.getValueType();
2824 // TODO: handle v16i8.
2825 if (MVT::getSizeInBits(VT) == 16) {
2826 // Transform it so it match pextrw which produces a 32-bit result.
2827 MVT::ValueType EVT = (MVT::ValueType)(VT+1);
2828 SDOperand Extract = DAG.getNode(X86ISD::PEXTRW, EVT,
2829 Op.getOperand(0), Op.getOperand(1));
2830 SDOperand Assert = DAG.getNode(ISD::AssertZext, EVT, Extract,
2831 DAG.getValueType(VT));
2832 return DAG.getNode(ISD::TRUNCATE, VT, Assert);
2833 } else if (MVT::getSizeInBits(VT) == 32) {
2834 SDOperand Vec = Op.getOperand(0);
2835 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue();
2836 if (Idx == 0)
2837 return Op;
Evan Chenga9467aa2006-04-25 20:13:52 +00002838 // SHUFPS the element to the lowest double word, then movss.
2839 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4);
Chris Lattner35a08552007-02-25 07:10:00 +00002840 SmallVector<SDOperand, 8> IdxVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00002841 IdxVec.push_back(DAG.getConstant(Idx, MVT::getVectorBaseType(MaskVT)));
2842 IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT)));
2843 IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT)));
2844 IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT)));
Chris Lattnered728e82006-08-11 17:38:39 +00002845 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2846 &IdxVec[0], IdxVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00002847 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(),
Evan Cheng922e1912006-11-07 22:14:24 +00002848 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask);
Evan Chenga9467aa2006-04-25 20:13:52 +00002849 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec,
Evan Chengde7156f2006-06-15 08:14:54 +00002850 DAG.getConstant(0, getPointerTy()));
Evan Chenga9467aa2006-04-25 20:13:52 +00002851 } else if (MVT::getSizeInBits(VT) == 64) {
2852 SDOperand Vec = Op.getOperand(0);
2853 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue();
2854 if (Idx == 0)
2855 return Op;
2856
2857 // UNPCKHPD the element to the lowest double word, then movsd.
2858 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored
2859 // to a f64mem, the whole operation is folded into a single MOVHPDmr.
2860 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4);
Chris Lattner35a08552007-02-25 07:10:00 +00002861 SmallVector<SDOperand, 8> IdxVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00002862 IdxVec.push_back(DAG.getConstant(1, MVT::getVectorBaseType(MaskVT)));
2863 IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT)));
Chris Lattnered728e82006-08-11 17:38:39 +00002864 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2865 &IdxVec[0], IdxVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00002866 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(),
2867 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask);
2868 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec,
Evan Chengde7156f2006-06-15 08:14:54 +00002869 DAG.getConstant(0, getPointerTy()));
Evan Chenga9467aa2006-04-25 20:13:52 +00002870 }
2871
2872 return SDOperand();
2873}
2874
2875SDOperand
2876X86TargetLowering::LowerINSERT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) {
Evan Cheng9fee4422006-05-16 07:21:53 +00002877 // Transform it so it match pinsrw which expects a 16-bit value in a GR32
Evan Chenga9467aa2006-04-25 20:13:52 +00002878 // as its second argument.
2879 MVT::ValueType VT = Op.getValueType();
2880 MVT::ValueType BaseVT = MVT::getVectorBaseType(VT);
2881 SDOperand N0 = Op.getOperand(0);
2882 SDOperand N1 = Op.getOperand(1);
2883 SDOperand N2 = Op.getOperand(2);
2884 if (MVT::getSizeInBits(BaseVT) == 16) {
2885 if (N1.getValueType() != MVT::i32)
2886 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1);
2887 if (N2.getValueType() != MVT::i32)
2888 N2 = DAG.getConstant(cast<ConstantSDNode>(N2)->getValue(), MVT::i32);
2889 return DAG.getNode(X86ISD::PINSRW, VT, N0, N1, N2);
2890 } else if (MVT::getSizeInBits(BaseVT) == 32) {
2891 unsigned Idx = cast<ConstantSDNode>(N2)->getValue();
2892 if (Idx == 0) {
2893 // Use a movss.
2894 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, N1);
2895 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4);
2896 MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002897 SmallVector<SDOperand, 8> MaskVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00002898 MaskVec.push_back(DAG.getConstant(4, BaseVT));
2899 for (unsigned i = 1; i <= 3; ++i)
2900 MaskVec.push_back(DAG.getConstant(i, BaseVT));
2901 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, N0, N1,
Chris Lattnered728e82006-08-11 17:38:39 +00002902 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2903 &MaskVec[0], MaskVec.size()));
Evan Chenga9467aa2006-04-25 20:13:52 +00002904 } else {
2905 // Use two pinsrw instructions to insert a 32 bit value.
2906 Idx <<= 1;
2907 if (MVT::isFloatingPoint(N1.getValueType())) {
Evan Chenge71fe34d2006-10-09 20:57:25 +00002908 if (ISD::isNON_EXTLoad(N1.Val)) {
Evan Cheng9fee4422006-05-16 07:21:53 +00002909 // Just load directly from f32mem to GR32.
Evan Chenge71fe34d2006-10-09 20:57:25 +00002910 LoadSDNode *LD = cast<LoadSDNode>(N1);
2911 N1 = DAG.getLoad(MVT::i32, LD->getChain(), LD->getBasePtr(),
2912 LD->getSrcValue(), LD->getSrcValueOffset());
Evan Chenga9467aa2006-04-25 20:13:52 +00002913 } else {
2914 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v4f32, N1);
2915 N1 = DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, N1);
2916 N1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, N1,
Evan Chengde7156f2006-06-15 08:14:54 +00002917 DAG.getConstant(0, getPointerTy()));
Evan Chenga9467aa2006-04-25 20:13:52 +00002918 }
2919 }
2920 N0 = DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, N0);
2921 N0 = DAG.getNode(X86ISD::PINSRW, MVT::v8i16, N0, N1,
Evan Chengde7156f2006-06-15 08:14:54 +00002922 DAG.getConstant(Idx, getPointerTy()));
Evan Chenga9467aa2006-04-25 20:13:52 +00002923 N1 = DAG.getNode(ISD::SRL, MVT::i32, N1, DAG.getConstant(16, MVT::i8));
2924 N0 = DAG.getNode(X86ISD::PINSRW, MVT::v8i16, N0, N1,
Evan Chengde7156f2006-06-15 08:14:54 +00002925 DAG.getConstant(Idx+1, getPointerTy()));
Evan Chenga9467aa2006-04-25 20:13:52 +00002926 return DAG.getNode(ISD::BIT_CONVERT, VT, N0);
2927 }
2928 }
2929
2930 return SDOperand();
2931}
2932
2933SDOperand
2934X86TargetLowering::LowerSCALAR_TO_VECTOR(SDOperand Op, SelectionDAG &DAG) {
2935 SDOperand AnyExt = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, Op.getOperand(0));
2936 return DAG.getNode(X86ISD::S2VEC, Op.getValueType(), AnyExt);
2937}
2938
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00002939// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
Evan Chenga9467aa2006-04-25 20:13:52 +00002940// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is
2941// one of the above mentioned nodes. It has to be wrapped because otherwise
2942// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
2943// be used to form addressing mode. These wrapped nodes will be selected
2944// into MOV32ri.
2945SDOperand
2946X86TargetLowering::LowerConstantPool(SDOperand Op, SelectionDAG &DAG) {
2947 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
Evan Cheng0b169222006-11-29 23:19:46 +00002948 SDOperand Result = DAG.getTargetConstantPool(CP->getConstVal(),
2949 getPointerTy(),
2950 CP->getAlignment());
Evan Cheng62cdc3f2006-12-05 04:01:03 +00002951 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result);
Anton Korobeynikova0554d92007-01-12 19:20:47 +00002952 // With PIC, the address is actually $g + Offset.
2953 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
2954 !Subtarget->isPICStyleRIPRel()) {
2955 Result = DAG.getNode(ISD::ADD, getPointerTy(),
2956 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
2957 Result);
Evan Chenga9467aa2006-04-25 20:13:52 +00002958 }
2959
2960 return Result;
2961}
2962
2963SDOperand
2964X86TargetLowering::LowerGlobalAddress(SDOperand Op, SelectionDAG &DAG) {
2965 GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
Evan Cheng0b169222006-11-29 23:19:46 +00002966 SDOperand Result = DAG.getTargetGlobalAddress(GV, getPointerTy());
Evan Cheng62cdc3f2006-12-05 04:01:03 +00002967 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result);
Anton Korobeynikova0554d92007-01-12 19:20:47 +00002968 // With PIC, the address is actually $g + Offset.
2969 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
2970 !Subtarget->isPICStyleRIPRel()) {
2971 Result = DAG.getNode(ISD::ADD, getPointerTy(),
2972 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
2973 Result);
Evan Chenga9467aa2006-04-25 20:13:52 +00002974 }
Anton Korobeynikov430e68a12006-12-22 22:29:05 +00002975
2976 // For Darwin & Mingw32, external and weak symbols are indirect, so we want to
2977 // load the value at address GV, not the value of GV itself. This means that
2978 // the GlobalAddress must be in the base or index register of the address, not
2979 // the GV offset field. Platform check is inside GVRequiresExtraLoad() call
Anton Korobeynikova0554d92007-01-12 19:20:47 +00002980 // The same applies for external symbols during PIC codegen
Anton Korobeynikov430e68a12006-12-22 22:29:05 +00002981 if (Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false))
2982 Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), Result, NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00002983
2984 return Result;
2985}
2986
2987SDOperand
2988X86TargetLowering::LowerExternalSymbol(SDOperand Op, SelectionDAG &DAG) {
2989 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol();
Evan Cheng0b169222006-11-29 23:19:46 +00002990 SDOperand Result = DAG.getTargetExternalSymbol(Sym, getPointerTy());
Evan Cheng62cdc3f2006-12-05 04:01:03 +00002991 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result);
Anton Korobeynikova0554d92007-01-12 19:20:47 +00002992 // With PIC, the address is actually $g + Offset.
2993 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
2994 !Subtarget->isPICStyleRIPRel()) {
2995 Result = DAG.getNode(ISD::ADD, getPointerTy(),
2996 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
2997 Result);
2998 }
2999
3000 return Result;
3001}
3002
3003SDOperand X86TargetLowering::LowerJumpTable(SDOperand Op, SelectionDAG &DAG) {
3004 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
3005 SDOperand Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy());
3006 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result);
3007 // With PIC, the address is actually $g + Offset.
3008 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
3009 !Subtarget->isPICStyleRIPRel()) {
3010 Result = DAG.getNode(ISD::ADD, getPointerTy(),
3011 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
3012 Result);
Evan Chenga9467aa2006-04-25 20:13:52 +00003013 }
3014
3015 return Result;
3016}
3017
3018SDOperand X86TargetLowering::LowerShift(SDOperand Op, SelectionDAG &DAG) {
Evan Cheng9c249c32006-01-09 18:33:28 +00003019 assert(Op.getNumOperands() == 3 && Op.getValueType() == MVT::i32 &&
3020 "Not an i64 shift!");
3021 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
3022 SDOperand ShOpLo = Op.getOperand(0);
3023 SDOperand ShOpHi = Op.getOperand(1);
3024 SDOperand ShAmt = Op.getOperand(2);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003025 SDOperand Tmp1 = isSRA ?
3026 DAG.getNode(ISD::SRA, MVT::i32, ShOpHi, DAG.getConstant(31, MVT::i8)) :
3027 DAG.getConstant(0, MVT::i32);
Evan Cheng9c249c32006-01-09 18:33:28 +00003028
3029 SDOperand Tmp2, Tmp3;
3030 if (Op.getOpcode() == ISD::SHL_PARTS) {
3031 Tmp2 = DAG.getNode(X86ISD::SHLD, MVT::i32, ShOpHi, ShOpLo, ShAmt);
3032 Tmp3 = DAG.getNode(ISD::SHL, MVT::i32, ShOpLo, ShAmt);
3033 } else {
3034 Tmp2 = DAG.getNode(X86ISD::SHRD, MVT::i32, ShOpLo, ShOpHi, ShAmt);
Evan Cheng267ba592006-01-19 01:46:14 +00003035 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, MVT::i32, ShOpHi, ShAmt);
Evan Cheng9c249c32006-01-09 18:33:28 +00003036 }
3037
Evan Cheng4259a0f2006-09-11 02:19:56 +00003038 const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag);
3039 SDOperand AndNode = DAG.getNode(ISD::AND, MVT::i8, ShAmt,
3040 DAG.getConstant(32, MVT::i8));
3041 SDOperand COps[]={DAG.getEntryNode(), AndNode, DAG.getConstant(0, MVT::i8)};
3042 SDOperand InFlag = DAG.getNode(X86ISD::CMP, VTs, 2, COps, 3).getValue(1);
Evan Cheng9c249c32006-01-09 18:33:28 +00003043
3044 SDOperand Hi, Lo;
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003045 SDOperand CC = DAG.getConstant(X86::COND_NE, MVT::i8);
Evan Cheng9c249c32006-01-09 18:33:28 +00003046
Evan Cheng4259a0f2006-09-11 02:19:56 +00003047 VTs = DAG.getNodeValueTypes(MVT::i32, MVT::Flag);
3048 SmallVector<SDOperand, 4> Ops;
Evan Cheng9c249c32006-01-09 18:33:28 +00003049 if (Op.getOpcode() == ISD::SHL_PARTS) {
3050 Ops.push_back(Tmp2);
3051 Ops.push_back(Tmp3);
3052 Ops.push_back(CC);
3053 Ops.push_back(InFlag);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003054 Hi = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size());
Evan Cheng9c249c32006-01-09 18:33:28 +00003055 InFlag = Hi.getValue(1);
3056
3057 Ops.clear();
3058 Ops.push_back(Tmp3);
3059 Ops.push_back(Tmp1);
3060 Ops.push_back(CC);
3061 Ops.push_back(InFlag);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003062 Lo = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size());
Evan Cheng9c249c32006-01-09 18:33:28 +00003063 } else {
3064 Ops.push_back(Tmp2);
3065 Ops.push_back(Tmp3);
3066 Ops.push_back(CC);
Evan Cheng12181af2006-01-09 22:29:54 +00003067 Ops.push_back(InFlag);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003068 Lo = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size());
Evan Cheng9c249c32006-01-09 18:33:28 +00003069 InFlag = Lo.getValue(1);
3070
3071 Ops.clear();
3072 Ops.push_back(Tmp3);
3073 Ops.push_back(Tmp1);
3074 Ops.push_back(CC);
3075 Ops.push_back(InFlag);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003076 Hi = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size());
Evan Cheng9c249c32006-01-09 18:33:28 +00003077 }
3078
Evan Cheng4259a0f2006-09-11 02:19:56 +00003079 VTs = DAG.getNodeValueTypes(MVT::i32, MVT::i32);
Evan Cheng9c249c32006-01-09 18:33:28 +00003080 Ops.clear();
3081 Ops.push_back(Lo);
3082 Ops.push_back(Hi);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003083 return DAG.getNode(ISD::MERGE_VALUES, VTs, 2, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003084}
Evan Cheng6305e502006-01-12 22:54:21 +00003085
Evan Chenga9467aa2006-04-25 20:13:52 +00003086SDOperand X86TargetLowering::LowerSINT_TO_FP(SDOperand Op, SelectionDAG &DAG) {
3087 assert(Op.getOperand(0).getValueType() <= MVT::i64 &&
3088 Op.getOperand(0).getValueType() >= MVT::i16 &&
3089 "Unknown SINT_TO_FP to lower!");
3090
3091 SDOperand Result;
3092 MVT::ValueType SrcVT = Op.getOperand(0).getValueType();
3093 unsigned Size = MVT::getSizeInBits(SrcVT)/8;
3094 MachineFunction &MF = DAG.getMachineFunction();
3095 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
3096 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
Evan Chengdf9ac472006-10-05 23:01:46 +00003097 SDOperand Chain = DAG.getStore(DAG.getEntryNode(), Op.getOperand(0),
Evan Chengab51cf22006-10-13 21:14:26 +00003098 StackSlot, NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003099
3100 // Build the FILD
Chris Lattner35a08552007-02-25 07:10:00 +00003101 SDVTList Tys;
3102 if (X86ScalarSSE)
3103 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag);
3104 else
3105 Tys = DAG.getVTList(MVT::f64, MVT::Other);
3106 SmallVector<SDOperand, 8> Ops;
Evan Chenga9467aa2006-04-25 20:13:52 +00003107 Ops.push_back(Chain);
3108 Ops.push_back(StackSlot);
3109 Ops.push_back(DAG.getValueType(SrcVT));
3110 Result = DAG.getNode(X86ScalarSSE ? X86ISD::FILD_FLAG :X86ISD::FILD,
Chris Lattnerc24a1d32006-08-08 02:23:42 +00003111 Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003112
3113 if (X86ScalarSSE) {
3114 Chain = Result.getValue(1);
3115 SDOperand InFlag = Result.getValue(2);
3116
3117 // FIXME: Currently the FST is flagged to the FILD_FLAG. This
3118 // shouldn't be necessary except that RFP cannot be live across
3119 // multiple blocks. When stackifier is fixed, they can be uncoupled.
Chris Lattner76ac0682005-11-15 00:40:23 +00003120 MachineFunction &MF = DAG.getMachineFunction();
Evan Chenga9467aa2006-04-25 20:13:52 +00003121 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8);
Chris Lattner76ac0682005-11-15 00:40:23 +00003122 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
Chris Lattner35a08552007-02-25 07:10:00 +00003123 Tys = DAG.getVTList(MVT::Other);
3124 SmallVector<SDOperand, 8> Ops;
Evan Cheng6305e502006-01-12 22:54:21 +00003125 Ops.push_back(Chain);
Evan Chenga9467aa2006-04-25 20:13:52 +00003126 Ops.push_back(Result);
Chris Lattner76ac0682005-11-15 00:40:23 +00003127 Ops.push_back(StackSlot);
Evan Chenga9467aa2006-04-25 20:13:52 +00003128 Ops.push_back(DAG.getValueType(Op.getValueType()));
3129 Ops.push_back(InFlag);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00003130 Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size());
Evan Chenge71fe34d2006-10-09 20:57:25 +00003131 Result = DAG.getLoad(Op.getValueType(), Chain, StackSlot, NULL, 0);
Chris Lattner76ac0682005-11-15 00:40:23 +00003132 }
Chris Lattner76ac0682005-11-15 00:40:23 +00003133
Evan Chenga9467aa2006-04-25 20:13:52 +00003134 return Result;
3135}
3136
3137SDOperand X86TargetLowering::LowerFP_TO_SINT(SDOperand Op, SelectionDAG &DAG) {
3138 assert(Op.getValueType() <= MVT::i64 && Op.getValueType() >= MVT::i16 &&
3139 "Unknown FP_TO_SINT to lower!");
3140 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary
3141 // stack slot.
3142 MachineFunction &MF = DAG.getMachineFunction();
3143 unsigned MemSize = MVT::getSizeInBits(Op.getValueType())/8;
3144 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize);
3145 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
3146
3147 unsigned Opc;
3148 switch (Op.getValueType()) {
Chris Lattner76ac0682005-11-15 00:40:23 +00003149 default: assert(0 && "Invalid FP_TO_SINT to lower!");
3150 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
3151 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
3152 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
Evan Chenga9467aa2006-04-25 20:13:52 +00003153 }
Chris Lattner76ac0682005-11-15 00:40:23 +00003154
Evan Chenga9467aa2006-04-25 20:13:52 +00003155 SDOperand Chain = DAG.getEntryNode();
3156 SDOperand Value = Op.getOperand(0);
3157 if (X86ScalarSSE) {
3158 assert(Op.getValueType() == MVT::i64 && "Invalid FP_TO_SINT to lower!");
Evan Chengab51cf22006-10-13 21:14:26 +00003159 Chain = DAG.getStore(Chain, Value, StackSlot, NULL, 0);
Chris Lattner35a08552007-02-25 07:10:00 +00003160 SDVTList Tys = DAG.getVTList(MVT::f64, MVT::Other);
3161 SDOperand Ops[] = {
3162 Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType())
3163 };
3164 Value = DAG.getNode(X86ISD::FLD, Tys, Ops, 3);
Evan Chenga9467aa2006-04-25 20:13:52 +00003165 Chain = Value.getValue(1);
3166 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize);
3167 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
3168 }
Chris Lattner76ac0682005-11-15 00:40:23 +00003169
Evan Chenga9467aa2006-04-25 20:13:52 +00003170 // Build the FP_TO_INT*_IN_MEM
Chris Lattner35a08552007-02-25 07:10:00 +00003171 SDOperand Ops[] = { Chain, Value, StackSlot };
3172 SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops, 3);
Evan Cheng172fce72006-01-06 00:43:03 +00003173
Evan Chenga9467aa2006-04-25 20:13:52 +00003174 // Load the result.
Evan Chenge71fe34d2006-10-09 20:57:25 +00003175 return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003176}
3177
3178SDOperand X86TargetLowering::LowerFABS(SDOperand Op, SelectionDAG &DAG) {
3179 MVT::ValueType VT = Op.getValueType();
3180 const Type *OpNTy = MVT::getTypeForValueType(VT);
3181 std::vector<Constant*> CV;
3182 if (VT == MVT::f64) {
3183 CV.push_back(ConstantFP::get(OpNTy, BitsToDouble(~(1ULL << 63))));
3184 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3185 } else {
3186 CV.push_back(ConstantFP::get(OpNTy, BitsToFloat(~(1U << 31))));
3187 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3188 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3189 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3190 }
3191 Constant *CS = ConstantStruct::get(CV);
3192 SDOperand CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4);
Chris Lattner35a08552007-02-25 07:10:00 +00003193 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
Evan Chengbd1c5a82006-08-11 09:08:15 +00003194 SmallVector<SDOperand, 3> Ops;
3195 Ops.push_back(DAG.getEntryNode());
3196 Ops.push_back(CPIdx);
3197 Ops.push_back(DAG.getSrcValue(NULL));
3198 SDOperand Mask = DAG.getNode(X86ISD::LOAD_PACK, Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003199 return DAG.getNode(X86ISD::FAND, VT, Op.getOperand(0), Mask);
3200}
3201
3202SDOperand X86TargetLowering::LowerFNEG(SDOperand Op, SelectionDAG &DAG) {
3203 MVT::ValueType VT = Op.getValueType();
3204 const Type *OpNTy = MVT::getTypeForValueType(VT);
3205 std::vector<Constant*> CV;
3206 if (VT == MVT::f64) {
3207 CV.push_back(ConstantFP::get(OpNTy, BitsToDouble(1ULL << 63)));
3208 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3209 } else {
3210 CV.push_back(ConstantFP::get(OpNTy, BitsToFloat(1U << 31)));
3211 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3212 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3213 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3214 }
3215 Constant *CS = ConstantStruct::get(CV);
3216 SDOperand CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4);
Chris Lattner35a08552007-02-25 07:10:00 +00003217 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
Evan Chengbd1c5a82006-08-11 09:08:15 +00003218 SmallVector<SDOperand, 3> Ops;
3219 Ops.push_back(DAG.getEntryNode());
3220 Ops.push_back(CPIdx);
3221 Ops.push_back(DAG.getSrcValue(NULL));
3222 SDOperand Mask = DAG.getNode(X86ISD::LOAD_PACK, Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003223 return DAG.getNode(X86ISD::FXOR, VT, Op.getOperand(0), Mask);
3224}
3225
Evan Cheng4363e882007-01-05 07:55:56 +00003226SDOperand X86TargetLowering::LowerFCOPYSIGN(SDOperand Op, SelectionDAG &DAG) {
Evan Cheng82241c82007-01-05 21:37:56 +00003227 SDOperand Op0 = Op.getOperand(0);
3228 SDOperand Op1 = Op.getOperand(1);
Evan Cheng4363e882007-01-05 07:55:56 +00003229 MVT::ValueType VT = Op.getValueType();
Evan Cheng82241c82007-01-05 21:37:56 +00003230 MVT::ValueType SrcVT = Op1.getValueType();
Evan Cheng4363e882007-01-05 07:55:56 +00003231 const Type *SrcTy = MVT::getTypeForValueType(SrcVT);
Evan Cheng82241c82007-01-05 21:37:56 +00003232
3233 // If second operand is smaller, extend it first.
3234 if (MVT::getSizeInBits(SrcVT) < MVT::getSizeInBits(VT)) {
3235 Op1 = DAG.getNode(ISD::FP_EXTEND, VT, Op1);
3236 SrcVT = VT;
3237 }
3238
Evan Cheng4363e882007-01-05 07:55:56 +00003239 // First get the sign bit of second operand.
3240 std::vector<Constant*> CV;
3241 if (SrcVT == MVT::f64) {
3242 CV.push_back(ConstantFP::get(SrcTy, BitsToDouble(1ULL << 63)));
3243 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3244 } else {
3245 CV.push_back(ConstantFP::get(SrcTy, BitsToFloat(1U << 31)));
3246 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3247 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3248 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3249 }
3250 Constant *CS = ConstantStruct::get(CV);
3251 SDOperand CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4);
Chris Lattnere56fef92007-02-25 06:40:16 +00003252 SDVTList Tys = DAG.getVTList(SrcVT, MVT::Other);
Evan Cheng4363e882007-01-05 07:55:56 +00003253 SmallVector<SDOperand, 3> Ops;
3254 Ops.push_back(DAG.getEntryNode());
3255 Ops.push_back(CPIdx);
3256 Ops.push_back(DAG.getSrcValue(NULL));
Evan Cheng82241c82007-01-05 21:37:56 +00003257 SDOperand Mask1 = DAG.getNode(X86ISD::LOAD_PACK, Tys, &Ops[0], Ops.size());
3258 SDOperand SignBit = DAG.getNode(X86ISD::FAND, SrcVT, Op1, Mask1);
Evan Cheng4363e882007-01-05 07:55:56 +00003259
3260 // Shift sign bit right or left if the two operands have different types.
3261 if (MVT::getSizeInBits(SrcVT) > MVT::getSizeInBits(VT)) {
3262 // Op0 is MVT::f32, Op1 is MVT::f64.
3263 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2f64, SignBit);
3264 SignBit = DAG.getNode(X86ISD::FSRL, MVT::v2f64, SignBit,
3265 DAG.getConstant(32, MVT::i32));
3266 SignBit = DAG.getNode(ISD::BIT_CONVERT, MVT::v4f32, SignBit);
3267 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::f32, SignBit,
3268 DAG.getConstant(0, getPointerTy()));
Evan Cheng4363e882007-01-05 07:55:56 +00003269 }
3270
Evan Cheng82241c82007-01-05 21:37:56 +00003271 // Clear first operand sign bit.
3272 CV.clear();
3273 if (VT == MVT::f64) {
3274 CV.push_back(ConstantFP::get(SrcTy, BitsToDouble(~(1ULL << 63))));
3275 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3276 } else {
3277 CV.push_back(ConstantFP::get(SrcTy, BitsToFloat(~(1U << 31))));
3278 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3279 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3280 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3281 }
3282 CS = ConstantStruct::get(CV);
3283 CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4);
Chris Lattnere56fef92007-02-25 06:40:16 +00003284 Tys = DAG.getVTList(VT, MVT::Other);
Evan Cheng82241c82007-01-05 21:37:56 +00003285 Ops.clear();
3286 Ops.push_back(DAG.getEntryNode());
3287 Ops.push_back(CPIdx);
3288 Ops.push_back(DAG.getSrcValue(NULL));
3289 SDOperand Mask2 = DAG.getNode(X86ISD::LOAD_PACK, Tys, &Ops[0], Ops.size());
3290 SDOperand Val = DAG.getNode(X86ISD::FAND, VT, Op0, Mask2);
3291
3292 // Or the value with the sign bit.
3293 return DAG.getNode(X86ISD::FOR, VT, Val, SignBit);
Evan Cheng4363e882007-01-05 07:55:56 +00003294}
3295
Evan Cheng4259a0f2006-09-11 02:19:56 +00003296SDOperand X86TargetLowering::LowerSETCC(SDOperand Op, SelectionDAG &DAG,
3297 SDOperand Chain) {
Evan Chenga9467aa2006-04-25 20:13:52 +00003298 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer");
3299 SDOperand Cond;
Evan Cheng4259a0f2006-09-11 02:19:56 +00003300 SDOperand Op0 = Op.getOperand(0);
3301 SDOperand Op1 = Op.getOperand(1);
Evan Chenga9467aa2006-04-25 20:13:52 +00003302 SDOperand CC = Op.getOperand(2);
3303 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
Evan Cheng694810c2006-10-12 19:12:56 +00003304 const MVT::ValueType *VTs1 = DAG.getNodeValueTypes(MVT::Other, MVT::Flag);
3305 const MVT::ValueType *VTs2 = DAG.getNodeValueTypes(MVT::i8, MVT::Flag);
Evan Chenga9467aa2006-04-25 20:13:52 +00003306 bool isFP = MVT::isFloatingPoint(Op.getOperand(1).getValueType());
Evan Chenga9467aa2006-04-25 20:13:52 +00003307 unsigned X86CC;
Evan Chenga9467aa2006-04-25 20:13:52 +00003308
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003309 if (translateX86CC(cast<CondCodeSDNode>(CC)->get(), isFP, X86CC,
Chris Lattner7a627672006-09-13 03:22:10 +00003310 Op0, Op1, DAG)) {
Evan Cheng4259a0f2006-09-11 02:19:56 +00003311 SDOperand Ops1[] = { Chain, Op0, Op1 };
Evan Cheng694810c2006-10-12 19:12:56 +00003312 Cond = DAG.getNode(X86ISD::CMP, VTs1, 2, Ops1, 3).getValue(1);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003313 SDOperand Ops2[] = { DAG.getConstant(X86CC, MVT::i8), Cond };
Evan Cheng694810c2006-10-12 19:12:56 +00003314 return DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops2, 2);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003315 }
3316
3317 assert(isFP && "Illegal integer SetCC!");
3318
3319 SDOperand COps[] = { Chain, Op0, Op1 };
Evan Cheng694810c2006-10-12 19:12:56 +00003320 Cond = DAG.getNode(X86ISD::CMP, VTs1, 2, COps, 3).getValue(1);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003321
3322 switch (SetCCOpcode) {
3323 default: assert(false && "Illegal floating point SetCC!");
3324 case ISD::SETOEQ: { // !PF & ZF
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003325 SDOperand Ops1[] = { DAG.getConstant(X86::COND_NP, MVT::i8), Cond };
Evan Cheng694810c2006-10-12 19:12:56 +00003326 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops1, 2);
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003327 SDOperand Ops2[] = { DAG.getConstant(X86::COND_E, MVT::i8),
Evan Cheng4259a0f2006-09-11 02:19:56 +00003328 Tmp1.getValue(1) };
Evan Cheng694810c2006-10-12 19:12:56 +00003329 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops2, 2);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003330 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
3331 }
3332 case ISD::SETUNE: { // PF | !ZF
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003333 SDOperand Ops1[] = { DAG.getConstant(X86::COND_P, MVT::i8), Cond };
Evan Cheng694810c2006-10-12 19:12:56 +00003334 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops1, 2);
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003335 SDOperand Ops2[] = { DAG.getConstant(X86::COND_NE, MVT::i8),
Evan Cheng4259a0f2006-09-11 02:19:56 +00003336 Tmp1.getValue(1) };
Evan Cheng694810c2006-10-12 19:12:56 +00003337 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops2, 2);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003338 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
3339 }
Evan Chengc1583db2005-12-21 20:21:51 +00003340 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003341}
Evan Cheng45df7f82006-01-30 23:41:35 +00003342
Evan Chenga9467aa2006-04-25 20:13:52 +00003343SDOperand X86TargetLowering::LowerSELECT(SDOperand Op, SelectionDAG &DAG) {
Evan Cheng4259a0f2006-09-11 02:19:56 +00003344 bool addTest = true;
3345 SDOperand Chain = DAG.getEntryNode();
3346 SDOperand Cond = Op.getOperand(0);
3347 SDOperand CC;
3348 const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag);
Evan Cheng944d1e92006-01-26 02:13:10 +00003349
Evan Cheng4259a0f2006-09-11 02:19:56 +00003350 if (Cond.getOpcode() == ISD::SETCC)
3351 Cond = LowerSETCC(Cond, DAG, Chain);
3352
3353 if (Cond.getOpcode() == X86ISD::SETCC) {
3354 CC = Cond.getOperand(0);
3355
Evan Chenga9467aa2006-04-25 20:13:52 +00003356 // If condition flag is set by a X86ISD::CMP, then make a copy of it
Evan Cheng4259a0f2006-09-11 02:19:56 +00003357 // (since flag operand cannot be shared). Use it as the condition setting
3358 // operand in place of the X86ISD::SETCC.
3359 // If the X86ISD::SETCC has more than one use, then perhaps it's better
Evan Chenga9467aa2006-04-25 20:13:52 +00003360 // to use a test instead of duplicating the X86ISD::CMP (for register
Evan Cheng4259a0f2006-09-11 02:19:56 +00003361 // pressure reason)?
3362 SDOperand Cmp = Cond.getOperand(1);
3363 unsigned Opc = Cmp.getOpcode();
3364 bool IllegalFPCMov = !X86ScalarSSE &&
3365 MVT::isFloatingPoint(Op.getValueType()) &&
3366 !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended());
3367 if ((Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI) &&
3368 !IllegalFPCMov) {
3369 SDOperand Ops[] = { Chain, Cmp.getOperand(1), Cmp.getOperand(2) };
3370 Cond = DAG.getNode(Opc, VTs, 2, Ops, 3);
3371 addTest = false;
3372 }
3373 }
Evan Cheng73a1ad92006-01-10 20:26:56 +00003374
Evan Chenga9467aa2006-04-25 20:13:52 +00003375 if (addTest) {
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003376 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003377 SDOperand Ops[] = { Chain, Cond, DAG.getConstant(0, MVT::i8) };
3378 Cond = DAG.getNode(X86ISD::CMP, VTs, 2, Ops, 3);
Evan Cheng225a4d02005-12-17 01:21:05 +00003379 }
Evan Cheng45df7f82006-01-30 23:41:35 +00003380
Evan Cheng4259a0f2006-09-11 02:19:56 +00003381 VTs = DAG.getNodeValueTypes(Op.getValueType(), MVT::Flag);
3382 SmallVector<SDOperand, 4> Ops;
Evan Chenga9467aa2006-04-25 20:13:52 +00003383 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
3384 // condition is true.
3385 Ops.push_back(Op.getOperand(2));
3386 Ops.push_back(Op.getOperand(1));
3387 Ops.push_back(CC);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003388 Ops.push_back(Cond.getValue(1));
3389 return DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003390}
Evan Cheng944d1e92006-01-26 02:13:10 +00003391
Evan Chenga9467aa2006-04-25 20:13:52 +00003392SDOperand X86TargetLowering::LowerBRCOND(SDOperand Op, SelectionDAG &DAG) {
Evan Cheng4259a0f2006-09-11 02:19:56 +00003393 bool addTest = true;
3394 SDOperand Chain = Op.getOperand(0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003395 SDOperand Cond = Op.getOperand(1);
3396 SDOperand Dest = Op.getOperand(2);
3397 SDOperand CC;
Evan Cheng4259a0f2006-09-11 02:19:56 +00003398 const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag);
3399
Evan Chenga9467aa2006-04-25 20:13:52 +00003400 if (Cond.getOpcode() == ISD::SETCC)
Evan Cheng4259a0f2006-09-11 02:19:56 +00003401 Cond = LowerSETCC(Cond, DAG, Chain);
Evan Chenga9467aa2006-04-25 20:13:52 +00003402
3403 if (Cond.getOpcode() == X86ISD::SETCC) {
Evan Cheng4259a0f2006-09-11 02:19:56 +00003404 CC = Cond.getOperand(0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003405
Evan Cheng4259a0f2006-09-11 02:19:56 +00003406 // If condition flag is set by a X86ISD::CMP, then make a copy of it
3407 // (since flag operand cannot be shared). Use it as the condition setting
3408 // operand in place of the X86ISD::SETCC.
3409 // If the X86ISD::SETCC has more than one use, then perhaps it's better
3410 // to use a test instead of duplicating the X86ISD::CMP (for register
3411 // pressure reason)?
3412 SDOperand Cmp = Cond.getOperand(1);
3413 unsigned Opc = Cmp.getOpcode();
3414 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI) {
3415 SDOperand Ops[] = { Chain, Cmp.getOperand(1), Cmp.getOperand(2) };
3416 Cond = DAG.getNode(Opc, VTs, 2, Ops, 3);
3417 addTest = false;
3418 }
3419 }
Evan Chengfb22e862006-01-13 01:03:02 +00003420
Evan Chenga9467aa2006-04-25 20:13:52 +00003421 if (addTest) {
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003422 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003423 SDOperand Ops[] = { Chain, Cond, DAG.getConstant(0, MVT::i8) };
3424 Cond = DAG.getNode(X86ISD::CMP, VTs, 2, Ops, 3);
Evan Cheng6fc31042005-12-19 23:12:38 +00003425 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003426 return DAG.getNode(X86ISD::BRCOND, Op.getValueType(),
Evan Cheng4259a0f2006-09-11 02:19:56 +00003427 Cond, Op.getOperand(2), CC, Cond.getValue(1));
Evan Chenga9467aa2006-04-25 20:13:52 +00003428}
Evan Chengae986f12006-01-11 22:15:48 +00003429
Evan Cheng2a330942006-05-25 00:59:30 +00003430SDOperand X86TargetLowering::LowerCALL(SDOperand Op, SelectionDAG &DAG) {
3431 unsigned CallingConv= cast<ConstantSDNode>(Op.getOperand(1))->getValue();
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003432
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003433 if (Subtarget->is64Bit())
Chris Lattner7802f3e2007-02-25 09:06:15 +00003434 return LowerX86_64CCCCallTo(Op, DAG, CallingConv);
Evan Cheng2a330942006-05-25 00:59:30 +00003435 else
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003436 switch (CallingConv) {
Chris Lattnerfc360392006-09-27 18:29:38 +00003437 default:
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003438 assert(0 && "Unsupported calling convention");
Chris Lattnerfc360392006-09-27 18:29:38 +00003439 case CallingConv::Fast:
Chris Lattner3ed3be32007-02-28 06:05:16 +00003440 // TODO: Implement fastcc
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003441 // Falls through
Chris Lattnerfc360392006-09-27 18:29:38 +00003442 case CallingConv::C:
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003443 case CallingConv::X86_StdCall:
Chris Lattner7802f3e2007-02-25 09:06:15 +00003444 return LowerCCCCallTo(Op, DAG, CallingConv);
Chris Lattnerfc360392006-09-27 18:29:38 +00003445 case CallingConv::X86_FastCall:
Chris Lattner7802f3e2007-02-25 09:06:15 +00003446 return LowerFastCCCallTo(Op, DAG, CallingConv);
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003447 }
Evan Cheng2a330942006-05-25 00:59:30 +00003448}
3449
Evan Chenge0bcfbe2006-04-26 01:20:17 +00003450SDOperand
3451X86TargetLowering::LowerFORMAL_ARGUMENTS(SDOperand Op, SelectionDAG &DAG) {
Evan Chengdc614c12006-06-06 23:30:24 +00003452 MachineFunction &MF = DAG.getMachineFunction();
3453 const Function* Fn = MF.getFunction();
3454 if (Fn->hasExternalLinkage() &&
Anton Korobeynikov4efbbc92007-01-03 11:43:14 +00003455 Subtarget->isTargetCygMing() &&
Evan Cheng0e14a562006-06-09 06:24:42 +00003456 Fn->getName() == "main")
Evan Chengdc614c12006-06-06 23:30:24 +00003457 MF.getInfo<X86FunctionInfo>()->setForceFramePointer(true);
3458
Evan Cheng17e734f2006-05-23 21:06:34 +00003459 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue();
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003460 if (Subtarget->is64Bit())
3461 return LowerX86_64CCCArguments(Op, DAG);
Evan Cheng17e734f2006-05-23 21:06:34 +00003462 else
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003463 switch(CC) {
Chris Lattnerfc360392006-09-27 18:29:38 +00003464 default:
3465 assert(0 && "Unsupported calling convention");
3466 case CallingConv::Fast:
Chris Lattner3ed3be32007-02-28 06:05:16 +00003467 // TODO: implement fastcc.
3468
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003469 // Falls through
Chris Lattnerfc360392006-09-27 18:29:38 +00003470 case CallingConv::C:
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003471 return LowerCCCArguments(Op, DAG);
Chris Lattnerfc360392006-09-27 18:29:38 +00003472 case CallingConv::X86_StdCall:
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003473 MF.getInfo<X86FunctionInfo>()->setDecorationStyle(StdCall);
Anton Korobeynikov037c8672007-01-28 13:31:35 +00003474 return LowerCCCArguments(Op, DAG, true);
Chris Lattnerfc360392006-09-27 18:29:38 +00003475 case CallingConv::X86_FastCall:
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003476 MF.getInfo<X86FunctionInfo>()->setDecorationStyle(FastCall);
Chris Lattner3ed3be32007-02-28 06:05:16 +00003477 return LowerFastCCArguments(Op, DAG);
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003478 }
Evan Chenge0bcfbe2006-04-26 01:20:17 +00003479}
3480
Evan Chenga9467aa2006-04-25 20:13:52 +00003481SDOperand X86TargetLowering::LowerMEMSET(SDOperand Op, SelectionDAG &DAG) {
3482 SDOperand InFlag(0, 0);
3483 SDOperand Chain = Op.getOperand(0);
3484 unsigned Align =
3485 (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
3486 if (Align == 0) Align = 1;
3487
3488 ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3));
3489 // If not DWORD aligned, call memset if size is less than the threshold.
3490 // It knows how to align to the right boundary first.
3491 if ((Align & 3) != 0 ||
3492 (I && I->getValue() < Subtarget->getMinRepStrSizeThreshold())) {
3493 MVT::ValueType IntPtr = getPointerTy();
Owen Anderson20a631f2006-05-03 01:29:57 +00003494 const Type *IntPtrTy = getTargetData()->getIntPtrType();
Reid Spencere63b6512006-12-31 05:55:36 +00003495 TargetLowering::ArgListTy Args;
3496 TargetLowering::ArgListEntry Entry;
3497 Entry.Node = Op.getOperand(1);
3498 Entry.Ty = IntPtrTy;
3499 Entry.isSigned = false;
Anton Korobeynikov037c8672007-01-28 13:31:35 +00003500 Entry.isInReg = false;
3501 Entry.isSRet = false;
Reid Spencere63b6512006-12-31 05:55:36 +00003502 Args.push_back(Entry);
Reid Spencere87b5e92007-01-03 17:24:59 +00003503 // Extend the unsigned i8 argument to be an int value for the call.
Reid Spencere63b6512006-12-31 05:55:36 +00003504 Entry.Node = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Op.getOperand(2));
3505 Entry.Ty = IntPtrTy;
3506 Entry.isSigned = false;
Anton Korobeynikov037c8672007-01-28 13:31:35 +00003507 Entry.isInReg = false;
3508 Entry.isSRet = false;
Reid Spencere63b6512006-12-31 05:55:36 +00003509 Args.push_back(Entry);
3510 Entry.Node = Op.getOperand(3);
3511 Args.push_back(Entry);
Evan Chenga9467aa2006-04-25 20:13:52 +00003512 std::pair<SDOperand,SDOperand> CallResult =
Reid Spencere63b6512006-12-31 05:55:36 +00003513 LowerCallTo(Chain, Type::VoidTy, false, false, CallingConv::C, false,
Evan Chenga9467aa2006-04-25 20:13:52 +00003514 DAG.getExternalSymbol("memset", IntPtr), Args, DAG);
3515 return CallResult.second;
Evan Chengd5e905d2006-03-21 23:01:21 +00003516 }
Evan Chengd097e672006-03-22 02:53:00 +00003517
Evan Chenga9467aa2006-04-25 20:13:52 +00003518 MVT::ValueType AVT;
3519 SDOperand Count;
3520 ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3521 unsigned BytesLeft = 0;
3522 bool TwoRepStos = false;
3523 if (ValC) {
3524 unsigned ValReg;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003525 uint64_t Val = ValC->getValue() & 255;
Evan Chengc995b452006-04-06 23:23:56 +00003526
Evan Chenga9467aa2006-04-25 20:13:52 +00003527 // If the value is a constant, then we can potentially use larger sets.
3528 switch (Align & 3) {
3529 case 2: // WORD aligned
3530 AVT = MVT::i16;
Evan Chenga9467aa2006-04-25 20:13:52 +00003531 ValReg = X86::AX;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003532 Val = (Val << 8) | Val;
Evan Chenga9467aa2006-04-25 20:13:52 +00003533 break;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003534 case 0: // DWORD aligned
Evan Chenga9467aa2006-04-25 20:13:52 +00003535 AVT = MVT::i32;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003536 ValReg = X86::EAX;
Evan Chenga9467aa2006-04-25 20:13:52 +00003537 Val = (Val << 8) | Val;
3538 Val = (Val << 16) | Val;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003539 if (Subtarget->is64Bit() && ((Align & 0xF) == 0)) { // QWORD aligned
3540 AVT = MVT::i64;
3541 ValReg = X86::RAX;
3542 Val = (Val << 32) | Val;
3543 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003544 break;
3545 default: // Byte aligned
3546 AVT = MVT::i8;
Evan Chenga9467aa2006-04-25 20:13:52 +00003547 ValReg = X86::AL;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003548 Count = Op.getOperand(3);
Evan Chenga9467aa2006-04-25 20:13:52 +00003549 break;
Evan Chenga3caaee2006-04-19 22:48:17 +00003550 }
3551
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003552 if (AVT > MVT::i8) {
3553 if (I) {
3554 unsigned UBytes = MVT::getSizeInBits(AVT) / 8;
3555 Count = DAG.getConstant(I->getValue() / UBytes, getPointerTy());
3556 BytesLeft = I->getValue() % UBytes;
3557 } else {
3558 assert(AVT >= MVT::i32 &&
3559 "Do not use rep;stos if not at least DWORD aligned");
3560 Count = DAG.getNode(ISD::SRL, Op.getOperand(3).getValueType(),
3561 Op.getOperand(3), DAG.getConstant(2, MVT::i8));
3562 TwoRepStos = true;
3563 }
3564 }
3565
Evan Chenga9467aa2006-04-25 20:13:52 +00003566 Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT),
3567 InFlag);
3568 InFlag = Chain.getValue(1);
3569 } else {
3570 AVT = MVT::i8;
3571 Count = Op.getOperand(3);
3572 Chain = DAG.getCopyToReg(Chain, X86::AL, Op.getOperand(2), InFlag);
3573 InFlag = Chain.getValue(1);
Evan Chengd097e672006-03-22 02:53:00 +00003574 }
Evan Chengb0461082006-04-24 18:01:45 +00003575
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003576 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX,
3577 Count, InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00003578 InFlag = Chain.getValue(1);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003579 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI,
3580 Op.getOperand(1), InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00003581 InFlag = Chain.getValue(1);
Evan Cheng9b9cc4f2006-03-27 07:00:16 +00003582
Chris Lattnere56fef92007-02-25 06:40:16 +00003583 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Chris Lattner35a08552007-02-25 07:10:00 +00003584 SmallVector<SDOperand, 8> Ops;
Evan Chenga9467aa2006-04-25 20:13:52 +00003585 Ops.push_back(Chain);
3586 Ops.push_back(DAG.getValueType(AVT));
3587 Ops.push_back(InFlag);
Evan Cheng5c68bba2006-08-11 07:35:45 +00003588 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size());
Evan Chengb0461082006-04-24 18:01:45 +00003589
Evan Chenga9467aa2006-04-25 20:13:52 +00003590 if (TwoRepStos) {
3591 InFlag = Chain.getValue(1);
3592 Count = Op.getOperand(3);
3593 MVT::ValueType CVT = Count.getValueType();
3594 SDOperand Left = DAG.getNode(ISD::AND, CVT, Count,
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003595 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
3596 Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX,
3597 Left, InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00003598 InFlag = Chain.getValue(1);
Chris Lattnere56fef92007-02-25 06:40:16 +00003599 Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Evan Chenga9467aa2006-04-25 20:13:52 +00003600 Ops.clear();
3601 Ops.push_back(Chain);
3602 Ops.push_back(DAG.getValueType(MVT::i8));
3603 Ops.push_back(InFlag);
Evan Cheng5c68bba2006-08-11 07:35:45 +00003604 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003605 } else if (BytesLeft) {
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003606 // Issue stores for the last 1 - 7 bytes.
Evan Chenga9467aa2006-04-25 20:13:52 +00003607 SDOperand Value;
3608 unsigned Val = ValC->getValue() & 255;
3609 unsigned Offset = I->getValue() - BytesLeft;
3610 SDOperand DstAddr = Op.getOperand(1);
3611 MVT::ValueType AddrVT = DstAddr.getValueType();
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003612 if (BytesLeft >= 4) {
3613 Val = (Val << 8) | Val;
3614 Val = (Val << 16) | Val;
3615 Value = DAG.getConstant(Val, MVT::i32);
Evan Chengdf9ac472006-10-05 23:01:46 +00003616 Chain = DAG.getStore(Chain, Value,
3617 DAG.getNode(ISD::ADD, AddrVT, DstAddr,
3618 DAG.getConstant(Offset, AddrVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00003619 NULL, 0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003620 BytesLeft -= 4;
3621 Offset += 4;
3622 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003623 if (BytesLeft >= 2) {
3624 Value = DAG.getConstant((Val << 8) | Val, MVT::i16);
Evan Chengdf9ac472006-10-05 23:01:46 +00003625 Chain = DAG.getStore(Chain, Value,
3626 DAG.getNode(ISD::ADD, AddrVT, DstAddr,
3627 DAG.getConstant(Offset, AddrVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00003628 NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003629 BytesLeft -= 2;
3630 Offset += 2;
Evan Cheng082c8782006-03-24 07:29:27 +00003631 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003632 if (BytesLeft == 1) {
3633 Value = DAG.getConstant(Val, MVT::i8);
Evan Chengdf9ac472006-10-05 23:01:46 +00003634 Chain = DAG.getStore(Chain, Value,
3635 DAG.getNode(ISD::ADD, AddrVT, DstAddr,
3636 DAG.getConstant(Offset, AddrVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00003637 NULL, 0);
Evan Cheng14215c32006-04-21 23:03:30 +00003638 }
Evan Cheng082c8782006-03-24 07:29:27 +00003639 }
Evan Chengebf10062006-04-03 20:53:28 +00003640
Evan Chenga9467aa2006-04-25 20:13:52 +00003641 return Chain;
3642}
Evan Chengebf10062006-04-03 20:53:28 +00003643
Evan Chenga9467aa2006-04-25 20:13:52 +00003644SDOperand X86TargetLowering::LowerMEMCPY(SDOperand Op, SelectionDAG &DAG) {
3645 SDOperand Chain = Op.getOperand(0);
3646 unsigned Align =
3647 (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
3648 if (Align == 0) Align = 1;
Evan Chengebf10062006-04-03 20:53:28 +00003649
Evan Chenga9467aa2006-04-25 20:13:52 +00003650 ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3));
3651 // If not DWORD aligned, call memcpy if size is less than the threshold.
3652 // It knows how to align to the right boundary first.
3653 if ((Align & 3) != 0 ||
3654 (I && I->getValue() < Subtarget->getMinRepStrSizeThreshold())) {
3655 MVT::ValueType IntPtr = getPointerTy();
Reid Spencere63b6512006-12-31 05:55:36 +00003656 TargetLowering::ArgListTy Args;
3657 TargetLowering::ArgListEntry Entry;
Anton Korobeynikov037c8672007-01-28 13:31:35 +00003658 Entry.Ty = getTargetData()->getIntPtrType();
3659 Entry.isSigned = false;
3660 Entry.isInReg = false;
3661 Entry.isSRet = false;
Reid Spencere63b6512006-12-31 05:55:36 +00003662 Entry.Node = Op.getOperand(1); Args.push_back(Entry);
3663 Entry.Node = Op.getOperand(2); Args.push_back(Entry);
3664 Entry.Node = Op.getOperand(3); Args.push_back(Entry);
Evan Chenga9467aa2006-04-25 20:13:52 +00003665 std::pair<SDOperand,SDOperand> CallResult =
Reid Spencere63b6512006-12-31 05:55:36 +00003666 LowerCallTo(Chain, Type::VoidTy, false, false, CallingConv::C, false,
Evan Chenga9467aa2006-04-25 20:13:52 +00003667 DAG.getExternalSymbol("memcpy", IntPtr), Args, DAG);
3668 return CallResult.second;
Evan Chengcbffa462006-03-31 19:22:53 +00003669 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003670
3671 MVT::ValueType AVT;
3672 SDOperand Count;
3673 unsigned BytesLeft = 0;
3674 bool TwoRepMovs = false;
3675 switch (Align & 3) {
3676 case 2: // WORD aligned
3677 AVT = MVT::i16;
Evan Chenga9467aa2006-04-25 20:13:52 +00003678 break;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003679 case 0: // DWORD aligned
Evan Chenga9467aa2006-04-25 20:13:52 +00003680 AVT = MVT::i32;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003681 if (Subtarget->is64Bit() && ((Align & 0xF) == 0)) // QWORD aligned
3682 AVT = MVT::i64;
Evan Chenga9467aa2006-04-25 20:13:52 +00003683 break;
3684 default: // Byte aligned
3685 AVT = MVT::i8;
3686 Count = Op.getOperand(3);
3687 break;
3688 }
3689
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003690 if (AVT > MVT::i8) {
3691 if (I) {
3692 unsigned UBytes = MVT::getSizeInBits(AVT) / 8;
3693 Count = DAG.getConstant(I->getValue() / UBytes, getPointerTy());
3694 BytesLeft = I->getValue() % UBytes;
3695 } else {
3696 assert(AVT >= MVT::i32 &&
3697 "Do not use rep;movs if not at least DWORD aligned");
3698 Count = DAG.getNode(ISD::SRL, Op.getOperand(3).getValueType(),
3699 Op.getOperand(3), DAG.getConstant(2, MVT::i8));
3700 TwoRepMovs = true;
3701 }
3702 }
3703
Evan Chenga9467aa2006-04-25 20:13:52 +00003704 SDOperand InFlag(0, 0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003705 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX,
3706 Count, InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00003707 InFlag = Chain.getValue(1);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003708 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI,
3709 Op.getOperand(1), InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00003710 InFlag = Chain.getValue(1);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003711 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RSI : X86::ESI,
3712 Op.getOperand(2), InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00003713 InFlag = Chain.getValue(1);
3714
Chris Lattnere56fef92007-02-25 06:40:16 +00003715 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Chris Lattner35a08552007-02-25 07:10:00 +00003716 SmallVector<SDOperand, 8> Ops;
Evan Chenga9467aa2006-04-25 20:13:52 +00003717 Ops.push_back(Chain);
3718 Ops.push_back(DAG.getValueType(AVT));
3719 Ops.push_back(InFlag);
Evan Cheng5c68bba2006-08-11 07:35:45 +00003720 Chain = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003721
3722 if (TwoRepMovs) {
3723 InFlag = Chain.getValue(1);
3724 Count = Op.getOperand(3);
3725 MVT::ValueType CVT = Count.getValueType();
3726 SDOperand Left = DAG.getNode(ISD::AND, CVT, Count,
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003727 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
3728 Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX,
3729 Left, InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00003730 InFlag = Chain.getValue(1);
Chris Lattnere56fef92007-02-25 06:40:16 +00003731 Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Evan Chenga9467aa2006-04-25 20:13:52 +00003732 Ops.clear();
3733 Ops.push_back(Chain);
3734 Ops.push_back(DAG.getValueType(MVT::i8));
3735 Ops.push_back(InFlag);
Evan Cheng5c68bba2006-08-11 07:35:45 +00003736 Chain = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003737 } else if (BytesLeft) {
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003738 // Issue loads and stores for the last 1 - 7 bytes.
Evan Chenga9467aa2006-04-25 20:13:52 +00003739 unsigned Offset = I->getValue() - BytesLeft;
3740 SDOperand DstAddr = Op.getOperand(1);
3741 MVT::ValueType DstVT = DstAddr.getValueType();
3742 SDOperand SrcAddr = Op.getOperand(2);
3743 MVT::ValueType SrcVT = SrcAddr.getValueType();
3744 SDOperand Value;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003745 if (BytesLeft >= 4) {
3746 Value = DAG.getLoad(MVT::i32, Chain,
3747 DAG.getNode(ISD::ADD, SrcVT, SrcAddr,
3748 DAG.getConstant(Offset, SrcVT)),
Evan Chenge71fe34d2006-10-09 20:57:25 +00003749 NULL, 0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003750 Chain = Value.getValue(1);
Evan Chengdf9ac472006-10-05 23:01:46 +00003751 Chain = DAG.getStore(Chain, Value,
3752 DAG.getNode(ISD::ADD, DstVT, DstAddr,
3753 DAG.getConstant(Offset, DstVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00003754 NULL, 0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003755 BytesLeft -= 4;
3756 Offset += 4;
3757 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003758 if (BytesLeft >= 2) {
3759 Value = DAG.getLoad(MVT::i16, Chain,
3760 DAG.getNode(ISD::ADD, SrcVT, SrcAddr,
3761 DAG.getConstant(Offset, SrcVT)),
Evan Chenge71fe34d2006-10-09 20:57:25 +00003762 NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003763 Chain = Value.getValue(1);
Evan Chengdf9ac472006-10-05 23:01:46 +00003764 Chain = DAG.getStore(Chain, Value,
3765 DAG.getNode(ISD::ADD, DstVT, DstAddr,
3766 DAG.getConstant(Offset, DstVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00003767 NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003768 BytesLeft -= 2;
3769 Offset += 2;
Evan Chengcbffa462006-03-31 19:22:53 +00003770 }
3771
Evan Chenga9467aa2006-04-25 20:13:52 +00003772 if (BytesLeft == 1) {
3773 Value = DAG.getLoad(MVT::i8, Chain,
3774 DAG.getNode(ISD::ADD, SrcVT, SrcAddr,
3775 DAG.getConstant(Offset, SrcVT)),
Evan Chenge71fe34d2006-10-09 20:57:25 +00003776 NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003777 Chain = Value.getValue(1);
Evan Chengdf9ac472006-10-05 23:01:46 +00003778 Chain = DAG.getStore(Chain, Value,
3779 DAG.getNode(ISD::ADD, DstVT, DstAddr,
3780 DAG.getConstant(Offset, DstVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00003781 NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003782 }
Evan Chengcbffa462006-03-31 19:22:53 +00003783 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003784
3785 return Chain;
3786}
3787
3788SDOperand
3789X86TargetLowering::LowerREADCYCLCECOUNTER(SDOperand Op, SelectionDAG &DAG) {
Chris Lattnere56fef92007-02-25 06:40:16 +00003790 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Chris Lattner35a08552007-02-25 07:10:00 +00003791 SDOperand TheOp = Op.getOperand(0);
3792 SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheOp, 1);
Evan Cheng28a9e9b2006-11-29 08:28:13 +00003793 if (Subtarget->is64Bit()) {
3794 SDOperand Copy1 = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1));
3795 SDOperand Copy2 = DAG.getCopyFromReg(Copy1.getValue(1), X86::RDX,
3796 MVT::i64, Copy1.getValue(2));
3797 SDOperand Tmp = DAG.getNode(ISD::SHL, MVT::i64, Copy2,
3798 DAG.getConstant(32, MVT::i8));
Chris Lattner35a08552007-02-25 07:10:00 +00003799 SDOperand Ops[] = {
3800 DAG.getNode(ISD::OR, MVT::i64, Copy1, Tmp), Copy2.getValue(1)
3801 };
Chris Lattnere56fef92007-02-25 06:40:16 +00003802
3803 Tys = DAG.getVTList(MVT::i64, MVT::Other);
Chris Lattner35a08552007-02-25 07:10:00 +00003804 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops, 2);
Evan Cheng28a9e9b2006-11-29 08:28:13 +00003805 }
Chris Lattner35a08552007-02-25 07:10:00 +00003806
3807 SDOperand Copy1 = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1));
3808 SDOperand Copy2 = DAG.getCopyFromReg(Copy1.getValue(1), X86::EDX,
3809 MVT::i32, Copy1.getValue(2));
3810 SDOperand Ops[] = { Copy1, Copy2, Copy2.getValue(1) };
3811 Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
3812 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops, 3);
Evan Chenga9467aa2006-04-25 20:13:52 +00003813}
3814
3815SDOperand X86TargetLowering::LowerVASTART(SDOperand Op, SelectionDAG &DAG) {
Evan Chengab51cf22006-10-13 21:14:26 +00003816 SrcValueSDNode *SV = cast<SrcValueSDNode>(Op.getOperand(2));
3817
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003818 if (!Subtarget->is64Bit()) {
3819 // vastart just stores the address of the VarArgsFrameIndex slot into the
3820 // memory location argument.
3821 SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy());
Evan Chengab51cf22006-10-13 21:14:26 +00003822 return DAG.getStore(Op.getOperand(0), FR,Op.getOperand(1), SV->getValue(),
3823 SV->getOffset());
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003824 }
3825
3826 // __va_list_tag:
3827 // gp_offset (0 - 6 * 8)
3828 // fp_offset (48 - 48 + 8 * 16)
3829 // overflow_arg_area (point to parameters coming in memory).
3830 // reg_save_area
Chris Lattner35a08552007-02-25 07:10:00 +00003831 SmallVector<SDOperand, 8> MemOps;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003832 SDOperand FIN = Op.getOperand(1);
3833 // Store gp_offset
Evan Chengdf9ac472006-10-05 23:01:46 +00003834 SDOperand Store = DAG.getStore(Op.getOperand(0),
3835 DAG.getConstant(VarArgsGPOffset, MVT::i32),
Evan Chengab51cf22006-10-13 21:14:26 +00003836 FIN, SV->getValue(), SV->getOffset());
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003837 MemOps.push_back(Store);
3838
3839 // Store fp_offset
3840 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
3841 DAG.getConstant(4, getPointerTy()));
Evan Chengdf9ac472006-10-05 23:01:46 +00003842 Store = DAG.getStore(Op.getOperand(0),
3843 DAG.getConstant(VarArgsFPOffset, MVT::i32),
Evan Chengab51cf22006-10-13 21:14:26 +00003844 FIN, SV->getValue(), SV->getOffset());
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003845 MemOps.push_back(Store);
3846
3847 // Store ptr to overflow_arg_area
3848 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
3849 DAG.getConstant(4, getPointerTy()));
3850 SDOperand OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy());
Evan Chengab51cf22006-10-13 21:14:26 +00003851 Store = DAG.getStore(Op.getOperand(0), OVFIN, FIN, SV->getValue(),
3852 SV->getOffset());
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003853 MemOps.push_back(Store);
3854
3855 // Store ptr to reg_save_area.
3856 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
3857 DAG.getConstant(8, getPointerTy()));
3858 SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy());
Evan Chengab51cf22006-10-13 21:14:26 +00003859 Store = DAG.getStore(Op.getOperand(0), RSFIN, FIN, SV->getValue(),
3860 SV->getOffset());
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003861 MemOps.push_back(Store);
3862 return DAG.getNode(ISD::TokenFactor, MVT::Other, &MemOps[0], MemOps.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003863}
3864
3865SDOperand
3866X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDOperand Op, SelectionDAG &DAG) {
3867 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getValue();
3868 switch (IntNo) {
3869 default: return SDOperand(); // Don't custom lower most intrinsics.
Evan Cheng78038292006-04-05 23:38:46 +00003870 // Comparison intrinsics.
Evan Chenga9467aa2006-04-25 20:13:52 +00003871 case Intrinsic::x86_sse_comieq_ss:
3872 case Intrinsic::x86_sse_comilt_ss:
3873 case Intrinsic::x86_sse_comile_ss:
3874 case Intrinsic::x86_sse_comigt_ss:
3875 case Intrinsic::x86_sse_comige_ss:
3876 case Intrinsic::x86_sse_comineq_ss:
3877 case Intrinsic::x86_sse_ucomieq_ss:
3878 case Intrinsic::x86_sse_ucomilt_ss:
3879 case Intrinsic::x86_sse_ucomile_ss:
3880 case Intrinsic::x86_sse_ucomigt_ss:
3881 case Intrinsic::x86_sse_ucomige_ss:
3882 case Intrinsic::x86_sse_ucomineq_ss:
3883 case Intrinsic::x86_sse2_comieq_sd:
3884 case Intrinsic::x86_sse2_comilt_sd:
3885 case Intrinsic::x86_sse2_comile_sd:
3886 case Intrinsic::x86_sse2_comigt_sd:
3887 case Intrinsic::x86_sse2_comige_sd:
3888 case Intrinsic::x86_sse2_comineq_sd:
3889 case Intrinsic::x86_sse2_ucomieq_sd:
3890 case Intrinsic::x86_sse2_ucomilt_sd:
3891 case Intrinsic::x86_sse2_ucomile_sd:
3892 case Intrinsic::x86_sse2_ucomigt_sd:
3893 case Intrinsic::x86_sse2_ucomige_sd:
3894 case Intrinsic::x86_sse2_ucomineq_sd: {
3895 unsigned Opc = 0;
3896 ISD::CondCode CC = ISD::SETCC_INVALID;
3897 switch (IntNo) {
3898 default: break;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003899 case Intrinsic::x86_sse_comieq_ss:
3900 case Intrinsic::x86_sse2_comieq_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00003901 Opc = X86ISD::COMI;
3902 CC = ISD::SETEQ;
3903 break;
Evan Cheng78038292006-04-05 23:38:46 +00003904 case Intrinsic::x86_sse_comilt_ss:
Evan Cheng78038292006-04-05 23:38:46 +00003905 case Intrinsic::x86_sse2_comilt_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00003906 Opc = X86ISD::COMI;
3907 CC = ISD::SETLT;
3908 break;
3909 case Intrinsic::x86_sse_comile_ss:
Evan Cheng78038292006-04-05 23:38:46 +00003910 case Intrinsic::x86_sse2_comile_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00003911 Opc = X86ISD::COMI;
3912 CC = ISD::SETLE;
3913 break;
3914 case Intrinsic::x86_sse_comigt_ss:
Evan Cheng78038292006-04-05 23:38:46 +00003915 case Intrinsic::x86_sse2_comigt_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00003916 Opc = X86ISD::COMI;
3917 CC = ISD::SETGT;
3918 break;
3919 case Intrinsic::x86_sse_comige_ss:
Evan Cheng78038292006-04-05 23:38:46 +00003920 case Intrinsic::x86_sse2_comige_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00003921 Opc = X86ISD::COMI;
3922 CC = ISD::SETGE;
3923 break;
3924 case Intrinsic::x86_sse_comineq_ss:
Evan Cheng78038292006-04-05 23:38:46 +00003925 case Intrinsic::x86_sse2_comineq_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00003926 Opc = X86ISD::COMI;
3927 CC = ISD::SETNE;
3928 break;
3929 case Intrinsic::x86_sse_ucomieq_ss:
Evan Cheng78038292006-04-05 23:38:46 +00003930 case Intrinsic::x86_sse2_ucomieq_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00003931 Opc = X86ISD::UCOMI;
3932 CC = ISD::SETEQ;
3933 break;
3934 case Intrinsic::x86_sse_ucomilt_ss:
Evan Cheng78038292006-04-05 23:38:46 +00003935 case Intrinsic::x86_sse2_ucomilt_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00003936 Opc = X86ISD::UCOMI;
3937 CC = ISD::SETLT;
3938 break;
3939 case Intrinsic::x86_sse_ucomile_ss:
Evan Cheng78038292006-04-05 23:38:46 +00003940 case Intrinsic::x86_sse2_ucomile_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00003941 Opc = X86ISD::UCOMI;
3942 CC = ISD::SETLE;
3943 break;
3944 case Intrinsic::x86_sse_ucomigt_ss:
Evan Cheng78038292006-04-05 23:38:46 +00003945 case Intrinsic::x86_sse2_ucomigt_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00003946 Opc = X86ISD::UCOMI;
3947 CC = ISD::SETGT;
3948 break;
3949 case Intrinsic::x86_sse_ucomige_ss:
Evan Cheng78038292006-04-05 23:38:46 +00003950 case Intrinsic::x86_sse2_ucomige_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00003951 Opc = X86ISD::UCOMI;
3952 CC = ISD::SETGE;
3953 break;
3954 case Intrinsic::x86_sse_ucomineq_ss:
3955 case Intrinsic::x86_sse2_ucomineq_sd:
3956 Opc = X86ISD::UCOMI;
3957 CC = ISD::SETNE;
3958 break;
Evan Cheng78038292006-04-05 23:38:46 +00003959 }
Evan Cheng4259a0f2006-09-11 02:19:56 +00003960
Evan Chenga9467aa2006-04-25 20:13:52 +00003961 unsigned X86CC;
Chris Lattner7a627672006-09-13 03:22:10 +00003962 SDOperand LHS = Op.getOperand(1);
3963 SDOperand RHS = Op.getOperand(2);
3964 translateX86CC(CC, true, X86CC, LHS, RHS, DAG);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003965
3966 const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag);
Chris Lattner7a627672006-09-13 03:22:10 +00003967 SDOperand Ops1[] = { DAG.getEntryNode(), LHS, RHS };
Evan Cheng4259a0f2006-09-11 02:19:56 +00003968 SDOperand Cond = DAG.getNode(Opc, VTs, 2, Ops1, 3);
3969 VTs = DAG.getNodeValueTypes(MVT::i8, MVT::Flag);
3970 SDOperand Ops2[] = { DAG.getConstant(X86CC, MVT::i8), Cond };
3971 SDOperand SetCC = DAG.getNode(X86ISD::SETCC, VTs, 2, Ops2, 2);
Evan Chenga9467aa2006-04-25 20:13:52 +00003972 return DAG.getNode(ISD::ANY_EXTEND, MVT::i32, SetCC);
Evan Cheng78038292006-04-05 23:38:46 +00003973 }
Evan Cheng5c59d492005-12-23 07:31:11 +00003974 }
Chris Lattner76ac0682005-11-15 00:40:23 +00003975}
Evan Cheng6af02632005-12-20 06:22:03 +00003976
Nate Begemaneda59972007-01-29 22:58:52 +00003977SDOperand X86TargetLowering::LowerRETURNADDR(SDOperand Op, SelectionDAG &DAG) {
3978 // Depths > 0 not supported yet!
3979 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0)
3980 return SDOperand();
3981
3982 // Just load the return address
3983 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG);
3984 return DAG.getLoad(getPointerTy(), DAG.getEntryNode(), RetAddrFI, NULL, 0);
3985}
3986
3987SDOperand X86TargetLowering::LowerFRAMEADDR(SDOperand Op, SelectionDAG &DAG) {
3988 // Depths > 0 not supported yet!
3989 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0)
3990 return SDOperand();
3991
3992 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG);
3993 return DAG.getNode(ISD::SUB, getPointerTy(), RetAddrFI,
3994 DAG.getConstant(4, getPointerTy()));
3995}
3996
Evan Chenga9467aa2006-04-25 20:13:52 +00003997/// LowerOperation - Provide custom lowering hooks for some operations.
3998///
3999SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
4000 switch (Op.getOpcode()) {
4001 default: assert(0 && "Should not custom lower this!");
4002 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
4003 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
4004 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
4005 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
4006 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
4007 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
4008 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
4009 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG);
4010 case ISD::SHL_PARTS:
4011 case ISD::SRA_PARTS:
4012 case ISD::SRL_PARTS: return LowerShift(Op, DAG);
4013 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
4014 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG);
4015 case ISD::FABS: return LowerFABS(Op, DAG);
4016 case ISD::FNEG: return LowerFNEG(Op, DAG);
Evan Cheng4363e882007-01-05 07:55:56 +00004017 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
Evan Cheng4259a0f2006-09-11 02:19:56 +00004018 case ISD::SETCC: return LowerSETCC(Op, DAG, DAG.getEntryNode());
Evan Chenga9467aa2006-04-25 20:13:52 +00004019 case ISD::SELECT: return LowerSELECT(Op, DAG);
4020 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
4021 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
Evan Cheng2a330942006-05-25 00:59:30 +00004022 case ISD::CALL: return LowerCALL(Op, DAG);
Evan Chenga9467aa2006-04-25 20:13:52 +00004023 case ISD::RET: return LowerRET(Op, DAG);
Evan Chenge0bcfbe2006-04-26 01:20:17 +00004024 case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG);
Evan Chenga9467aa2006-04-25 20:13:52 +00004025 case ISD::MEMSET: return LowerMEMSET(Op, DAG);
4026 case ISD::MEMCPY: return LowerMEMCPY(Op, DAG);
4027 case ISD::READCYCLECOUNTER: return LowerREADCYCLCECOUNTER(Op, DAG);
4028 case ISD::VASTART: return LowerVASTART(Op, DAG);
4029 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
Nate Begemaneda59972007-01-29 22:58:52 +00004030 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
4031 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
Evan Chenga9467aa2006-04-25 20:13:52 +00004032 }
Jim Laskey3796abe2007-02-21 22:54:50 +00004033 return SDOperand();
Evan Chenga9467aa2006-04-25 20:13:52 +00004034}
4035
Evan Cheng6af02632005-12-20 06:22:03 +00004036const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
4037 switch (Opcode) {
4038 default: return NULL;
Evan Cheng9c249c32006-01-09 18:33:28 +00004039 case X86ISD::SHLD: return "X86ISD::SHLD";
4040 case X86ISD::SHRD: return "X86ISD::SHRD";
Evan Cheng2dd217b2006-01-31 03:14:29 +00004041 case X86ISD::FAND: return "X86ISD::FAND";
Evan Cheng4363e882007-01-05 07:55:56 +00004042 case X86ISD::FOR: return "X86ISD::FOR";
Evan Cheng72d5c252006-01-31 22:28:30 +00004043 case X86ISD::FXOR: return "X86ISD::FXOR";
Evan Cheng4363e882007-01-05 07:55:56 +00004044 case X86ISD::FSRL: return "X86ISD::FSRL";
Evan Cheng6305e502006-01-12 22:54:21 +00004045 case X86ISD::FILD: return "X86ISD::FILD";
Evan Cheng11613a52006-02-04 02:20:30 +00004046 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
Evan Cheng6af02632005-12-20 06:22:03 +00004047 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
4048 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
4049 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
Evan Chenga74ce622005-12-21 02:39:21 +00004050 case X86ISD::FLD: return "X86ISD::FLD";
Evan Cheng45e190982006-01-05 00:27:02 +00004051 case X86ISD::FST: return "X86ISD::FST";
4052 case X86ISD::FP_GET_RESULT: return "X86ISD::FP_GET_RESULT";
Evan Chenga74ce622005-12-21 02:39:21 +00004053 case X86ISD::FP_SET_RESULT: return "X86ISD::FP_SET_RESULT";
Evan Cheng6af02632005-12-20 06:22:03 +00004054 case X86ISD::CALL: return "X86ISD::CALL";
4055 case X86ISD::TAILCALL: return "X86ISD::TAILCALL";
4056 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG";
4057 case X86ISD::CMP: return "X86ISD::CMP";
Evan Cheng78038292006-04-05 23:38:46 +00004058 case X86ISD::COMI: return "X86ISD::COMI";
4059 case X86ISD::UCOMI: return "X86ISD::UCOMI";
Evan Chengc1583db2005-12-21 20:21:51 +00004060 case X86ISD::SETCC: return "X86ISD::SETCC";
Evan Cheng6af02632005-12-20 06:22:03 +00004061 case X86ISD::CMOV: return "X86ISD::CMOV";
4062 case X86ISD::BRCOND: return "X86ISD::BRCOND";
Evan Chenga74ce622005-12-21 02:39:21 +00004063 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
Evan Cheng084a1022006-03-04 01:12:00 +00004064 case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
4065 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
Evan Cheng72d5c252006-01-31 22:28:30 +00004066 case X86ISD::LOAD_PACK: return "X86ISD::LOAD_PACK";
Evan Cheng5987cfb2006-07-07 08:33:52 +00004067 case X86ISD::LOAD_UA: return "X86ISD::LOAD_UA";
Evan Cheng5588de92006-02-18 00:15:05 +00004068 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
Evan Chenge0ed6ec2006-02-23 20:41:18 +00004069 case X86ISD::Wrapper: return "X86ISD::Wrapper";
Evan Chenge7ee6a52006-03-24 23:15:12 +00004070 case X86ISD::S2VEC: return "X86ISD::S2VEC";
Evan Chengcbffa462006-03-31 19:22:53 +00004071 case X86ISD::PEXTRW: return "X86ISD::PEXTRW";
Evan Cheng5fd7c692006-03-31 21:55:24 +00004072 case X86ISD::PINSRW: return "X86ISD::PINSRW";
Evan Cheng49683ba2006-11-10 21:43:37 +00004073 case X86ISD::FMAX: return "X86ISD::FMAX";
4074 case X86ISD::FMIN: return "X86ISD::FMIN";
Evan Cheng6af02632005-12-20 06:22:03 +00004075 }
4076}
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004077
Evan Cheng02612422006-07-05 22:17:51 +00004078/// isLegalAddressImmediate - Return true if the integer value or
4079/// GlobalValue can be used as the offset of the target addressing mode.
4080bool X86TargetLowering::isLegalAddressImmediate(int64_t V) const {
4081 // X86 allows a sign-extended 32-bit immediate field.
4082 return (V > -(1LL << 32) && V < (1LL << 32)-1);
4083}
4084
4085bool X86TargetLowering::isLegalAddressImmediate(GlobalValue *GV) const {
Evan Cheng7a9238c2006-11-29 23:48:14 +00004086 // In 64-bit mode, GV is 64-bit so it won't fit in the 32-bit displacement
4087 // field unless we are in small code model.
4088 if (Subtarget->is64Bit() &&
4089 getTargetMachine().getCodeModel() != CodeModel::Small)
Evan Cheng02612422006-07-05 22:17:51 +00004090 return false;
Anton Korobeynikov430e68a12006-12-22 22:29:05 +00004091
4092 return (!Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false));
Evan Cheng02612422006-07-05 22:17:51 +00004093}
4094
4095/// isShuffleMaskLegal - Targets can use this to indicate that they only
4096/// support *some* VECTOR_SHUFFLE operations, those with specific masks.
4097/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
4098/// are assumed to be legal.
4099bool
4100X86TargetLowering::isShuffleMaskLegal(SDOperand Mask, MVT::ValueType VT) const {
4101 // Only do shuffles on 128-bit vector types for now.
4102 if (MVT::getSizeInBits(VT) == 64) return false;
4103 return (Mask.Val->getNumOperands() <= 4 ||
4104 isSplatMask(Mask.Val) ||
4105 isPSHUFHW_PSHUFLWMask(Mask.Val) ||
4106 X86::isUNPCKLMask(Mask.Val) ||
4107 X86::isUNPCKL_v_undef_Mask(Mask.Val) ||
4108 X86::isUNPCKHMask(Mask.Val));
4109}
4110
4111bool X86TargetLowering::isVectorClearMaskLegal(std::vector<SDOperand> &BVOps,
4112 MVT::ValueType EVT,
4113 SelectionDAG &DAG) const {
4114 unsigned NumElts = BVOps.size();
4115 // Only do shuffles on 128-bit vector types for now.
4116 if (MVT::getSizeInBits(EVT) * NumElts == 64) return false;
4117 if (NumElts == 2) return true;
4118 if (NumElts == 4) {
Chris Lattner35a08552007-02-25 07:10:00 +00004119 return (isMOVLMask(&BVOps[0], 4) ||
4120 isCommutedMOVL(&BVOps[0], 4, true) ||
4121 isSHUFPMask(&BVOps[0], 4) ||
4122 isCommutedSHUFP(&BVOps[0], 4));
Evan Cheng02612422006-07-05 22:17:51 +00004123 }
4124 return false;
4125}
4126
4127//===----------------------------------------------------------------------===//
4128// X86 Scheduler Hooks
4129//===----------------------------------------------------------------------===//
4130
4131MachineBasicBlock *
4132X86TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI,
4133 MachineBasicBlock *BB) {
Evan Cheng20350c42006-11-27 23:37:22 +00004134 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
Evan Cheng02612422006-07-05 22:17:51 +00004135 switch (MI->getOpcode()) {
4136 default: assert(false && "Unexpected instr type to insert");
4137 case X86::CMOV_FR32:
4138 case X86::CMOV_FR64:
4139 case X86::CMOV_V4F32:
4140 case X86::CMOV_V2F64:
4141 case X86::CMOV_V2I64: {
4142 // To "insert" a SELECT_CC instruction, we actually have to insert the
4143 // diamond control-flow pattern. The incoming instruction knows the
4144 // destination vreg to set, the condition code register to branch on, the
4145 // true/false values to select between, and a branch opcode to use.
4146 const BasicBlock *LLVM_BB = BB->getBasicBlock();
4147 ilist<MachineBasicBlock>::iterator It = BB;
4148 ++It;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004149
Evan Cheng02612422006-07-05 22:17:51 +00004150 // thisMBB:
4151 // ...
4152 // TrueVal = ...
4153 // cmpTY ccX, r1, r2
4154 // bCC copy1MBB
4155 // fallthrough --> copy0MBB
4156 MachineBasicBlock *thisMBB = BB;
4157 MachineBasicBlock *copy0MBB = new MachineBasicBlock(LLVM_BB);
4158 MachineBasicBlock *sinkMBB = new MachineBasicBlock(LLVM_BB);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004159 unsigned Opc =
Chris Lattnerc0fb5672006-10-20 17:42:20 +00004160 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm());
Evan Cheng20350c42006-11-27 23:37:22 +00004161 BuildMI(BB, TII->get(Opc)).addMBB(sinkMBB);
Evan Cheng02612422006-07-05 22:17:51 +00004162 MachineFunction *F = BB->getParent();
4163 F->getBasicBlockList().insert(It, copy0MBB);
4164 F->getBasicBlockList().insert(It, sinkMBB);
4165 // Update machine-CFG edges by first adding all successors of the current
4166 // block to the new block which will contain the Phi node for the select.
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004167 for(MachineBasicBlock::succ_iterator i = BB->succ_begin(),
Evan Cheng02612422006-07-05 22:17:51 +00004168 e = BB->succ_end(); i != e; ++i)
4169 sinkMBB->addSuccessor(*i);
4170 // Next, remove all successors of the current block, and add the true
4171 // and fallthrough blocks as its successors.
4172 while(!BB->succ_empty())
4173 BB->removeSuccessor(BB->succ_begin());
4174 BB->addSuccessor(copy0MBB);
4175 BB->addSuccessor(sinkMBB);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004176
Evan Cheng02612422006-07-05 22:17:51 +00004177 // copy0MBB:
4178 // %FalseValue = ...
4179 // # fallthrough to sinkMBB
4180 BB = copy0MBB;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004181
Evan Cheng02612422006-07-05 22:17:51 +00004182 // Update machine-CFG edges
4183 BB->addSuccessor(sinkMBB);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004184
Evan Cheng02612422006-07-05 22:17:51 +00004185 // sinkMBB:
4186 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
4187 // ...
4188 BB = sinkMBB;
Evan Cheng20350c42006-11-27 23:37:22 +00004189 BuildMI(BB, TII->get(X86::PHI), MI->getOperand(0).getReg())
Evan Cheng02612422006-07-05 22:17:51 +00004190 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
4191 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
4192
4193 delete MI; // The pseudo instruction is gone now.
4194 return BB;
4195 }
4196
4197 case X86::FP_TO_INT16_IN_MEM:
4198 case X86::FP_TO_INT32_IN_MEM:
4199 case X86::FP_TO_INT64_IN_MEM: {
4200 // Change the floating point control register to use "round towards zero"
4201 // mode when truncating to an integer value.
4202 MachineFunction *F = BB->getParent();
4203 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2);
Evan Cheng20350c42006-11-27 23:37:22 +00004204 addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx);
Evan Cheng02612422006-07-05 22:17:51 +00004205
4206 // Load the old value of the high byte of the control word...
4207 unsigned OldCW =
4208 F->getSSARegMap()->createVirtualRegister(X86::GR16RegisterClass);
Evan Cheng20350c42006-11-27 23:37:22 +00004209 addFrameReference(BuildMI(BB, TII->get(X86::MOV16rm), OldCW), CWFrameIdx);
Evan Cheng02612422006-07-05 22:17:51 +00004210
4211 // Set the high part to be round to zero...
Evan Cheng20350c42006-11-27 23:37:22 +00004212 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mi)), CWFrameIdx)
4213 .addImm(0xC7F);
Evan Cheng02612422006-07-05 22:17:51 +00004214
4215 // Reload the modified control word now...
Evan Cheng20350c42006-11-27 23:37:22 +00004216 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx);
Evan Cheng02612422006-07-05 22:17:51 +00004217
4218 // Restore the memory image of control word to original value
Evan Cheng20350c42006-11-27 23:37:22 +00004219 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mr)), CWFrameIdx)
4220 .addReg(OldCW);
Evan Cheng02612422006-07-05 22:17:51 +00004221
4222 // Get the X86 opcode to use.
4223 unsigned Opc;
4224 switch (MI->getOpcode()) {
4225 default: assert(0 && "illegal opcode!");
4226 case X86::FP_TO_INT16_IN_MEM: Opc = X86::FpIST16m; break;
4227 case X86::FP_TO_INT32_IN_MEM: Opc = X86::FpIST32m; break;
4228 case X86::FP_TO_INT64_IN_MEM: Opc = X86::FpIST64m; break;
4229 }
4230
4231 X86AddressMode AM;
4232 MachineOperand &Op = MI->getOperand(0);
4233 if (Op.isRegister()) {
4234 AM.BaseType = X86AddressMode::RegBase;
4235 AM.Base.Reg = Op.getReg();
4236 } else {
4237 AM.BaseType = X86AddressMode::FrameIndexBase;
4238 AM.Base.FrameIndex = Op.getFrameIndex();
4239 }
4240 Op = MI->getOperand(1);
4241 if (Op.isImmediate())
Chris Lattnerc0fb5672006-10-20 17:42:20 +00004242 AM.Scale = Op.getImm();
Evan Cheng02612422006-07-05 22:17:51 +00004243 Op = MI->getOperand(2);
4244 if (Op.isImmediate())
Chris Lattnerc0fb5672006-10-20 17:42:20 +00004245 AM.IndexReg = Op.getImm();
Evan Cheng02612422006-07-05 22:17:51 +00004246 Op = MI->getOperand(3);
4247 if (Op.isGlobalAddress()) {
4248 AM.GV = Op.getGlobal();
4249 } else {
Chris Lattnerc0fb5672006-10-20 17:42:20 +00004250 AM.Disp = Op.getImm();
Evan Cheng02612422006-07-05 22:17:51 +00004251 }
Evan Cheng20350c42006-11-27 23:37:22 +00004252 addFullAddress(BuildMI(BB, TII->get(Opc)), AM)
4253 .addReg(MI->getOperand(4).getReg());
Evan Cheng02612422006-07-05 22:17:51 +00004254
4255 // Reload the original control word now.
Evan Cheng20350c42006-11-27 23:37:22 +00004256 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx);
Evan Cheng02612422006-07-05 22:17:51 +00004257
4258 delete MI; // The pseudo instruction is gone now.
4259 return BB;
4260 }
4261 }
4262}
4263
4264//===----------------------------------------------------------------------===//
4265// X86 Optimization Hooks
4266//===----------------------------------------------------------------------===//
4267
Nate Begeman8a77efe2006-02-16 21:11:51 +00004268void X86TargetLowering::computeMaskedBitsForTargetNode(const SDOperand Op,
4269 uint64_t Mask,
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004270 uint64_t &KnownZero,
Nate Begeman8a77efe2006-02-16 21:11:51 +00004271 uint64_t &KnownOne,
4272 unsigned Depth) const {
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004273 unsigned Opc = Op.getOpcode();
Evan Cheng6d196db2006-04-05 06:11:20 +00004274 assert((Opc >= ISD::BUILTIN_OP_END ||
4275 Opc == ISD::INTRINSIC_WO_CHAIN ||
4276 Opc == ISD::INTRINSIC_W_CHAIN ||
4277 Opc == ISD::INTRINSIC_VOID) &&
4278 "Should use MaskedValueIsZero if you don't know whether Op"
4279 " is a target node!");
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004280
Evan Cheng6d196db2006-04-05 06:11:20 +00004281 KnownZero = KnownOne = 0; // Don't know anything.
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004282 switch (Opc) {
Evan Cheng6d196db2006-04-05 06:11:20 +00004283 default: break;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004284 case X86ISD::SETCC:
Nate Begeman8a77efe2006-02-16 21:11:51 +00004285 KnownZero |= (MVT::getIntVTBitMask(Op.getValueType()) ^ 1ULL);
4286 break;
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004287 }
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004288}
Chris Lattnerc642aa52006-01-31 19:43:35 +00004289
Evan Cheng5987cfb2006-07-07 08:33:52 +00004290/// getShuffleScalarElt - Returns the scalar element that will make up the ith
4291/// element of the result of the vector shuffle.
4292static SDOperand getShuffleScalarElt(SDNode *N, unsigned i, SelectionDAG &DAG) {
4293 MVT::ValueType VT = N->getValueType(0);
4294 SDOperand PermMask = N->getOperand(2);
4295 unsigned NumElems = PermMask.getNumOperands();
4296 SDOperand V = (i < NumElems) ? N->getOperand(0) : N->getOperand(1);
4297 i %= NumElems;
4298 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR) {
4299 return (i == 0)
4300 ? V.getOperand(0) : DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(VT));
4301 } else if (V.getOpcode() == ISD::VECTOR_SHUFFLE) {
4302 SDOperand Idx = PermMask.getOperand(i);
4303 if (Idx.getOpcode() == ISD::UNDEF)
4304 return DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(VT));
4305 return getShuffleScalarElt(V.Val,cast<ConstantSDNode>(Idx)->getValue(),DAG);
4306 }
4307 return SDOperand();
4308}
4309
4310/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the
4311/// node is a GlobalAddress + an offset.
4312static bool isGAPlusOffset(SDNode *N, GlobalValue* &GA, int64_t &Offset) {
Evan Chengae1cd752006-11-30 21:55:46 +00004313 unsigned Opc = N->getOpcode();
Evan Cheng62cdc3f2006-12-05 04:01:03 +00004314 if (Opc == X86ISD::Wrapper) {
Evan Cheng5987cfb2006-07-07 08:33:52 +00004315 if (dyn_cast<GlobalAddressSDNode>(N->getOperand(0))) {
4316 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal();
4317 return true;
4318 }
Evan Chengae1cd752006-11-30 21:55:46 +00004319 } else if (Opc == ISD::ADD) {
Evan Cheng5987cfb2006-07-07 08:33:52 +00004320 SDOperand N1 = N->getOperand(0);
4321 SDOperand N2 = N->getOperand(1);
4322 if (isGAPlusOffset(N1.Val, GA, Offset)) {
4323 ConstantSDNode *V = dyn_cast<ConstantSDNode>(N2);
4324 if (V) {
4325 Offset += V->getSignExtended();
4326 return true;
4327 }
4328 } else if (isGAPlusOffset(N2.Val, GA, Offset)) {
4329 ConstantSDNode *V = dyn_cast<ConstantSDNode>(N1);
4330 if (V) {
4331 Offset += V->getSignExtended();
4332 return true;
4333 }
4334 }
4335 }
4336 return false;
4337}
4338
4339/// isConsecutiveLoad - Returns true if N is loading from an address of Base
4340/// + Dist * Size.
4341static bool isConsecutiveLoad(SDNode *N, SDNode *Base, int Dist, int Size,
4342 MachineFrameInfo *MFI) {
4343 if (N->getOperand(0).Val != Base->getOperand(0).Val)
4344 return false;
4345
4346 SDOperand Loc = N->getOperand(1);
4347 SDOperand BaseLoc = Base->getOperand(1);
4348 if (Loc.getOpcode() == ISD::FrameIndex) {
4349 if (BaseLoc.getOpcode() != ISD::FrameIndex)
4350 return false;
4351 int FI = dyn_cast<FrameIndexSDNode>(Loc)->getIndex();
4352 int BFI = dyn_cast<FrameIndexSDNode>(BaseLoc)->getIndex();
4353 int FS = MFI->getObjectSize(FI);
4354 int BFS = MFI->getObjectSize(BFI);
4355 if (FS != BFS || FS != Size) return false;
4356 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Size);
4357 } else {
4358 GlobalValue *GV1 = NULL;
4359 GlobalValue *GV2 = NULL;
4360 int64_t Offset1 = 0;
4361 int64_t Offset2 = 0;
4362 bool isGA1 = isGAPlusOffset(Loc.Val, GV1, Offset1);
4363 bool isGA2 = isGAPlusOffset(BaseLoc.Val, GV2, Offset2);
4364 if (isGA1 && isGA2 && GV1 == GV2)
4365 return Offset1 == (Offset2 + Dist*Size);
4366 }
4367
4368 return false;
4369}
4370
Evan Cheng79cf9a52006-07-10 21:37:44 +00004371static bool isBaseAlignment16(SDNode *Base, MachineFrameInfo *MFI,
4372 const X86Subtarget *Subtarget) {
Evan Cheng5987cfb2006-07-07 08:33:52 +00004373 GlobalValue *GV;
4374 int64_t Offset;
4375 if (isGAPlusOffset(Base, GV, Offset))
4376 return (GV->getAlignment() >= 16 && (Offset % 16) == 0);
4377 else {
4378 assert(Base->getOpcode() == ISD::FrameIndex && "Unexpected base node!");
4379 int BFI = dyn_cast<FrameIndexSDNode>(Base)->getIndex();
Evan Cheng79cf9a52006-07-10 21:37:44 +00004380 if (BFI < 0)
4381 // Fixed objects do not specify alignment, however the offsets are known.
4382 return ((Subtarget->getStackAlignment() % 16) == 0 &&
4383 (MFI->getObjectOffset(BFI) % 16) == 0);
4384 else
4385 return MFI->getObjectAlignment(BFI) >= 16;
Evan Cheng5987cfb2006-07-07 08:33:52 +00004386 }
4387 return false;
4388}
4389
4390
4391/// PerformShuffleCombine - Combine a vector_shuffle that is equal to
4392/// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load
4393/// if the load addresses are consecutive, non-overlapping, and in the right
4394/// order.
Evan Cheng79cf9a52006-07-10 21:37:44 +00004395static SDOperand PerformShuffleCombine(SDNode *N, SelectionDAG &DAG,
4396 const X86Subtarget *Subtarget) {
Evan Cheng5987cfb2006-07-07 08:33:52 +00004397 MachineFunction &MF = DAG.getMachineFunction();
4398 MachineFrameInfo *MFI = MF.getFrameInfo();
4399 MVT::ValueType VT = N->getValueType(0);
4400 MVT::ValueType EVT = MVT::getVectorBaseType(VT);
4401 SDOperand PermMask = N->getOperand(2);
4402 int NumElems = (int)PermMask.getNumOperands();
4403 SDNode *Base = NULL;
4404 for (int i = 0; i < NumElems; ++i) {
4405 SDOperand Idx = PermMask.getOperand(i);
4406 if (Idx.getOpcode() == ISD::UNDEF) {
4407 if (!Base) return SDOperand();
4408 } else {
4409 SDOperand Arg =
4410 getShuffleScalarElt(N, cast<ConstantSDNode>(Idx)->getValue(), DAG);
Evan Chenge71fe34d2006-10-09 20:57:25 +00004411 if (!Arg.Val || !ISD::isNON_EXTLoad(Arg.Val))
Evan Cheng5987cfb2006-07-07 08:33:52 +00004412 return SDOperand();
4413 if (!Base)
4414 Base = Arg.Val;
4415 else if (!isConsecutiveLoad(Arg.Val, Base,
4416 i, MVT::getSizeInBits(EVT)/8,MFI))
4417 return SDOperand();
4418 }
4419 }
4420
Evan Cheng79cf9a52006-07-10 21:37:44 +00004421 bool isAlign16 = isBaseAlignment16(Base->getOperand(1).Val, MFI, Subtarget);
Evan Chenge71fe34d2006-10-09 20:57:25 +00004422 if (isAlign16) {
4423 LoadSDNode *LD = cast<LoadSDNode>(Base);
4424 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(),
4425 LD->getSrcValueOffset());
4426 } else {
Evan Cheng5987cfb2006-07-07 08:33:52 +00004427 // Just use movups, it's shorter.
Chris Lattnere56fef92007-02-25 06:40:16 +00004428 SDVTList Tys = DAG.getVTList(MVT::v4f32, MVT::Other);
Evan Chengbd1c5a82006-08-11 09:08:15 +00004429 SmallVector<SDOperand, 3> Ops;
4430 Ops.push_back(Base->getOperand(0));
4431 Ops.push_back(Base->getOperand(1));
4432 Ops.push_back(Base->getOperand(2));
Evan Cheng5987cfb2006-07-07 08:33:52 +00004433 return DAG.getNode(ISD::BIT_CONVERT, VT,
Evan Chengbd1c5a82006-08-11 09:08:15 +00004434 DAG.getNode(X86ISD::LOAD_UA, Tys, &Ops[0], Ops.size()));
Evan Cheng5c68bba2006-08-11 07:35:45 +00004435 }
Evan Cheng5987cfb2006-07-07 08:33:52 +00004436}
4437
Chris Lattner9259b1e2006-10-04 06:57:07 +00004438/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes.
4439static SDOperand PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
4440 const X86Subtarget *Subtarget) {
4441 SDOperand Cond = N->getOperand(0);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004442
Chris Lattner9259b1e2006-10-04 06:57:07 +00004443 // If we have SSE[12] support, try to form min/max nodes.
4444 if (Subtarget->hasSSE2() &&
4445 (N->getValueType(0) == MVT::f32 || N->getValueType(0) == MVT::f64)) {
4446 if (Cond.getOpcode() == ISD::SETCC) {
4447 // Get the LHS/RHS of the select.
4448 SDOperand LHS = N->getOperand(1);
4449 SDOperand RHS = N->getOperand(2);
4450 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004451
Evan Cheng49683ba2006-11-10 21:43:37 +00004452 unsigned Opcode = 0;
Chris Lattner9259b1e2006-10-04 06:57:07 +00004453 if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) {
Chris Lattnerf2ef2432006-10-05 04:11:26 +00004454 switch (CC) {
4455 default: break;
4456 case ISD::SETOLE: // (X <= Y) ? X : Y -> min
4457 case ISD::SETULE:
4458 case ISD::SETLE:
4459 if (!UnsafeFPMath) break;
4460 // FALL THROUGH.
4461 case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min
4462 case ISD::SETLT:
Evan Cheng49683ba2006-11-10 21:43:37 +00004463 Opcode = X86ISD::FMIN;
Chris Lattnerf2ef2432006-10-05 04:11:26 +00004464 break;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004465
Chris Lattnerf2ef2432006-10-05 04:11:26 +00004466 case ISD::SETOGT: // (X > Y) ? X : Y -> max
4467 case ISD::SETUGT:
4468 case ISD::SETGT:
4469 if (!UnsafeFPMath) break;
4470 // FALL THROUGH.
4471 case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max
4472 case ISD::SETGE:
Evan Cheng49683ba2006-11-10 21:43:37 +00004473 Opcode = X86ISD::FMAX;
Chris Lattnerf2ef2432006-10-05 04:11:26 +00004474 break;
4475 }
Chris Lattner9259b1e2006-10-04 06:57:07 +00004476 } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) {
Chris Lattnerf2ef2432006-10-05 04:11:26 +00004477 switch (CC) {
4478 default: break;
4479 case ISD::SETOGT: // (X > Y) ? Y : X -> min
4480 case ISD::SETUGT:
4481 case ISD::SETGT:
4482 if (!UnsafeFPMath) break;
4483 // FALL THROUGH.
4484 case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min
4485 case ISD::SETGE:
Evan Cheng49683ba2006-11-10 21:43:37 +00004486 Opcode = X86ISD::FMIN;
Chris Lattnerf2ef2432006-10-05 04:11:26 +00004487 break;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004488
Chris Lattnerf2ef2432006-10-05 04:11:26 +00004489 case ISD::SETOLE: // (X <= Y) ? Y : X -> max
4490 case ISD::SETULE:
4491 case ISD::SETLE:
4492 if (!UnsafeFPMath) break;
4493 // FALL THROUGH.
4494 case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max
4495 case ISD::SETLT:
Evan Cheng49683ba2006-11-10 21:43:37 +00004496 Opcode = X86ISD::FMAX;
Chris Lattnerf2ef2432006-10-05 04:11:26 +00004497 break;
4498 }
Chris Lattner9259b1e2006-10-04 06:57:07 +00004499 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004500
Evan Cheng49683ba2006-11-10 21:43:37 +00004501 if (Opcode)
4502 return DAG.getNode(Opcode, N->getValueType(0), LHS, RHS);
Chris Lattner9259b1e2006-10-04 06:57:07 +00004503 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004504
Chris Lattner9259b1e2006-10-04 06:57:07 +00004505 }
4506
4507 return SDOperand();
4508}
4509
4510
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004511SDOperand X86TargetLowering::PerformDAGCombine(SDNode *N,
Evan Cheng5987cfb2006-07-07 08:33:52 +00004512 DAGCombinerInfo &DCI) const {
Evan Cheng5987cfb2006-07-07 08:33:52 +00004513 SelectionDAG &DAG = DCI.DAG;
4514 switch (N->getOpcode()) {
4515 default: break;
4516 case ISD::VECTOR_SHUFFLE:
Evan Cheng79cf9a52006-07-10 21:37:44 +00004517 return PerformShuffleCombine(N, DAG, Subtarget);
Chris Lattner9259b1e2006-10-04 06:57:07 +00004518 case ISD::SELECT:
4519 return PerformSELECTCombine(N, DAG, Subtarget);
Evan Cheng5987cfb2006-07-07 08:33:52 +00004520 }
4521
4522 return SDOperand();
4523}
4524
Evan Cheng02612422006-07-05 22:17:51 +00004525//===----------------------------------------------------------------------===//
4526// X86 Inline Assembly Support
4527//===----------------------------------------------------------------------===//
4528
Chris Lattner298ef372006-07-11 02:54:03 +00004529/// getConstraintType - Given a constraint letter, return the type of
4530/// constraint it is for this target.
4531X86TargetLowering::ConstraintType
4532X86TargetLowering::getConstraintType(char ConstraintLetter) const {
4533 switch (ConstraintLetter) {
Chris Lattnerc8db1072006-07-12 16:59:49 +00004534 case 'A':
4535 case 'r':
4536 case 'R':
4537 case 'l':
4538 case 'q':
4539 case 'Q':
4540 case 'x':
4541 case 'Y':
4542 return C_RegisterClass;
Chris Lattner298ef372006-07-11 02:54:03 +00004543 default: return TargetLowering::getConstraintType(ConstraintLetter);
4544 }
4545}
4546
Chris Lattner44daa502006-10-31 20:13:11 +00004547/// isOperandValidForConstraint - Return the specified operand (possibly
4548/// modified) if the specified SDOperand is valid for the specified target
4549/// constraint letter, otherwise return null.
4550SDOperand X86TargetLowering::
4551isOperandValidForConstraint(SDOperand Op, char Constraint, SelectionDAG &DAG) {
4552 switch (Constraint) {
4553 default: break;
4554 case 'i':
4555 // Literal immediates are always ok.
4556 if (isa<ConstantSDNode>(Op)) return Op;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004557
Chris Lattner44daa502006-10-31 20:13:11 +00004558 // If we are in non-pic codegen mode, we allow the address of a global to
4559 // be used with 'i'.
4560 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
4561 if (getTargetMachine().getRelocationModel() == Reloc::PIC_)
4562 return SDOperand(0, 0);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004563
Chris Lattner44daa502006-10-31 20:13:11 +00004564 if (GA->getOpcode() != ISD::TargetGlobalAddress)
4565 Op = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0),
4566 GA->getOffset());
4567 return Op;
4568 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004569
Chris Lattner44daa502006-10-31 20:13:11 +00004570 // Otherwise, not valid for this mode.
4571 return SDOperand(0, 0);
4572 }
4573 return TargetLowering::isOperandValidForConstraint(Op, Constraint, DAG);
4574}
4575
4576
Chris Lattnerc642aa52006-01-31 19:43:35 +00004577std::vector<unsigned> X86TargetLowering::
Chris Lattner7ad77df2006-02-22 00:56:39 +00004578getRegClassForInlineAsmConstraint(const std::string &Constraint,
4579 MVT::ValueType VT) const {
Chris Lattnerc642aa52006-01-31 19:43:35 +00004580 if (Constraint.size() == 1) {
4581 // FIXME: not handling fp-stack yet!
4582 // FIXME: not handling MMX registers yet ('y' constraint).
4583 switch (Constraint[0]) { // GCC X86 Constraint Letters
Chris Lattner298ef372006-07-11 02:54:03 +00004584 default: break; // Unknown constraint letter
4585 case 'A': // EAX/EDX
4586 if (VT == MVT::i32 || VT == MVT::i64)
4587 return make_vector<unsigned>(X86::EAX, X86::EDX, 0);
4588 break;
Chris Lattnerc642aa52006-01-31 19:43:35 +00004589 case 'r': // GENERAL_REGS
4590 case 'R': // LEGACY_REGS
Chris Lattnerd139ddd2006-12-04 22:38:21 +00004591 if (VT == MVT::i64 && Subtarget->is64Bit())
4592 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX,
4593 X86::RSI, X86::RDI, X86::RBP, X86::RSP,
4594 X86::R8, X86::R9, X86::R10, X86::R11,
4595 X86::R12, X86::R13, X86::R14, X86::R15, 0);
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00004596 if (VT == MVT::i32)
4597 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX,
4598 X86::ESI, X86::EDI, X86::EBP, X86::ESP, 0);
4599 else if (VT == MVT::i16)
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004600 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX,
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00004601 X86::SI, X86::DI, X86::BP, X86::SP, 0);
4602 else if (VT == MVT::i8)
Chris Lattnera16201c2006-12-05 17:29:40 +00004603 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0);
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00004604 break;
Chris Lattnerc642aa52006-01-31 19:43:35 +00004605 case 'l': // INDEX_REGS
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00004606 if (VT == MVT::i32)
4607 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX,
4608 X86::ESI, X86::EDI, X86::EBP, 0);
4609 else if (VT == MVT::i16)
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004610 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX,
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00004611 X86::SI, X86::DI, X86::BP, 0);
4612 else if (VT == MVT::i8)
4613 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::DL, 0);
4614 break;
Chris Lattnerc642aa52006-01-31 19:43:35 +00004615 case 'q': // Q_REGS (GENERAL_REGS in 64-bit mode)
4616 case 'Q': // Q_REGS
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00004617 if (VT == MVT::i32)
4618 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0);
4619 else if (VT == MVT::i16)
4620 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0);
4621 else if (VT == MVT::i8)
4622 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::DL, 0);
4623 break;
Chris Lattnerc642aa52006-01-31 19:43:35 +00004624 case 'x': // SSE_REGS if SSE1 allowed
4625 if (Subtarget->hasSSE1())
4626 return make_vector<unsigned>(X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
4627 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7,
4628 0);
4629 return std::vector<unsigned>();
4630 case 'Y': // SSE_REGS if SSE2 allowed
4631 if (Subtarget->hasSSE2())
4632 return make_vector<unsigned>(X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
4633 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7,
4634 0);
4635 return std::vector<unsigned>();
4636 }
4637 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004638
Chris Lattner7ad77df2006-02-22 00:56:39 +00004639 return std::vector<unsigned>();
Chris Lattnerc642aa52006-01-31 19:43:35 +00004640}
Chris Lattner524129d2006-07-31 23:26:50 +00004641
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004642std::pair<unsigned, const TargetRegisterClass*>
Chris Lattner524129d2006-07-31 23:26:50 +00004643X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
4644 MVT::ValueType VT) const {
4645 // Use the default implementation in TargetLowering to convert the register
4646 // constraint into a member of a register class.
4647 std::pair<unsigned, const TargetRegisterClass*> Res;
4648 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
Chris Lattnerf6a69662006-10-31 19:42:44 +00004649
4650 // Not found as a standard register?
4651 if (Res.second == 0) {
4652 // GCC calls "st(0)" just plain "st".
4653 if (StringsEqualNoCase("{st}", Constraint)) {
4654 Res.first = X86::ST0;
4655 Res.second = X86::RSTRegisterClass;
4656 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004657
Chris Lattnerf6a69662006-10-31 19:42:44 +00004658 return Res;
4659 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004660
Chris Lattner524129d2006-07-31 23:26:50 +00004661 // Otherwise, check to see if this is a register class of the wrong value
4662 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to
4663 // turn into {ax},{dx}.
4664 if (Res.second->hasType(VT))
4665 return Res; // Correct type already, nothing to do.
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004666
Chris Lattner524129d2006-07-31 23:26:50 +00004667 // All of the single-register GCC register classes map their values onto
4668 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we
4669 // really want an 8-bit or 32-bit register, map to the appropriate register
4670 // class and return the appropriate register.
4671 if (Res.second != X86::GR16RegisterClass)
4672 return Res;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004673
Chris Lattner524129d2006-07-31 23:26:50 +00004674 if (VT == MVT::i8) {
4675 unsigned DestReg = 0;
4676 switch (Res.first) {
4677 default: break;
4678 case X86::AX: DestReg = X86::AL; break;
4679 case X86::DX: DestReg = X86::DL; break;
4680 case X86::CX: DestReg = X86::CL; break;
4681 case X86::BX: DestReg = X86::BL; break;
4682 }
4683 if (DestReg) {
4684 Res.first = DestReg;
4685 Res.second = Res.second = X86::GR8RegisterClass;
4686 }
4687 } else if (VT == MVT::i32) {
4688 unsigned DestReg = 0;
4689 switch (Res.first) {
4690 default: break;
4691 case X86::AX: DestReg = X86::EAX; break;
4692 case X86::DX: DestReg = X86::EDX; break;
4693 case X86::CX: DestReg = X86::ECX; break;
4694 case X86::BX: DestReg = X86::EBX; break;
4695 case X86::SI: DestReg = X86::ESI; break;
4696 case X86::DI: DestReg = X86::EDI; break;
4697 case X86::BP: DestReg = X86::EBP; break;
4698 case X86::SP: DestReg = X86::ESP; break;
4699 }
4700 if (DestReg) {
4701 Res.first = DestReg;
4702 Res.second = Res.second = X86::GR32RegisterClass;
4703 }
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004704 } else if (VT == MVT::i64) {
4705 unsigned DestReg = 0;
4706 switch (Res.first) {
4707 default: break;
4708 case X86::AX: DestReg = X86::RAX; break;
4709 case X86::DX: DestReg = X86::RDX; break;
4710 case X86::CX: DestReg = X86::RCX; break;
4711 case X86::BX: DestReg = X86::RBX; break;
4712 case X86::SI: DestReg = X86::RSI; break;
4713 case X86::DI: DestReg = X86::RDI; break;
4714 case X86::BP: DestReg = X86::RBP; break;
4715 case X86::SP: DestReg = X86::RSP; break;
4716 }
4717 if (DestReg) {
4718 Res.first = DestReg;
4719 Res.second = Res.second = X86::GR64RegisterClass;
4720 }
Chris Lattner524129d2006-07-31 23:26:50 +00004721 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004722
Chris Lattner524129d2006-07-31 23:26:50 +00004723 return Res;
4724}