blob: cfadec926567d94653ca8414cc5594d17ba549d4 [file] [log] [blame]
Chris Lattnera58f5592006-05-23 23:20:42 +00001//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
Chris Lattner76ac0682005-11-15 00:40:23 +00002//
3// The LLVM Compiler Infrastructure
4//
5// This file was developed by Chris Lattner and is distributed under
6// the University of Illinois Open Source License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the interfaces that X86 uses to lower LLVM code into a
11// selection DAG.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
Evan Cheng911c68d2006-01-16 21:21:29 +000016#include "X86InstrBuilder.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000017#include "X86ISelLowering.h"
Evan Chengdc614c12006-06-06 23:30:24 +000018#include "X86MachineFunctionInfo.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000019#include "X86TargetMachine.h"
20#include "llvm/CallingConv.h"
Evan Cheng72d5c252006-01-31 22:28:30 +000021#include "llvm/Constants.h"
Evan Cheng88decde2006-04-28 21:29:37 +000022#include "llvm/DerivedTypes.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000023#include "llvm/Function.h"
Evan Cheng78038292006-04-05 23:38:46 +000024#include "llvm/Intrinsics.h"
Evan Chengaf598d22006-03-13 23:18:16 +000025#include "llvm/ADT/VectorExtras.h"
26#include "llvm/Analysis/ScalarEvolutionExpressions.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000027#include "llvm/CodeGen/MachineFrameInfo.h"
Evan Cheng339edad2006-01-11 00:33:36 +000028#include "llvm/CodeGen/MachineFunction.h"
29#include "llvm/CodeGen/MachineInstrBuilder.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000030#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/SSARegMap.h"
Evan Cheng2dd217b2006-01-31 03:14:29 +000032#include "llvm/Support/MathExtras.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000033#include "llvm/Target/TargetOptions.h"
Evan Cheng8c5766e2006-10-04 18:33:38 +000034#include "llvm/Support/CommandLine.h"
Chris Lattnerf6a69662006-10-31 19:42:44 +000035#include "llvm/ADT/StringExtras.h"
Chris Lattner76ac0682005-11-15 00:40:23 +000036using namespace llvm;
37
38// FIXME: temporary.
Chris Lattner76ac0682005-11-15 00:40:23 +000039static cl::opt<bool> EnableFastCC("enable-x86-fastcc", cl::Hidden,
40 cl::desc("Enable fastcc on X86"));
Chris Lattner76ac0682005-11-15 00:40:23 +000041X86TargetLowering::X86TargetLowering(TargetMachine &TM)
42 : TargetLowering(TM) {
Evan Chengcde9e302006-01-27 08:10:46 +000043 Subtarget = &TM.getSubtarget<X86Subtarget>();
44 X86ScalarSSE = Subtarget->hasSSE2();
Evan Cheng11b0a5d2006-09-08 06:48:29 +000045 X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP;
Evan Chengcde9e302006-01-27 08:10:46 +000046
Chris Lattner76ac0682005-11-15 00:40:23 +000047 // Set up the TargetLowering object.
48
49 // X86 is weird, it always uses i8 for shift amounts and setcc results.
50 setShiftAmountType(MVT::i8);
51 setSetCCResultType(MVT::i8);
52 setSetCCResultContents(ZeroOrOneSetCCResult);
Evan Cheng83eeefb2006-01-25 09:15:17 +000053 setSchedulingPreference(SchedulingForRegPressure);
Chris Lattner76ac0682005-11-15 00:40:23 +000054 setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0
Evan Cheng11b0a5d2006-09-08 06:48:29 +000055 setStackPointerRegisterToSaveRestore(X86StackPtr);
Evan Cheng20931a72006-03-16 21:47:42 +000056
Anton Korobeynikov3b7c2572006-12-10 23:12:42 +000057 if (Subtarget->isTargetDarwin()) {
Evan Chengb09a56f2006-03-17 20:31:41 +000058 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp.
Anton Korobeynikov3b7c2572006-12-10 23:12:42 +000059 setUseUnderscoreSetJmp(false);
60 setUseUnderscoreLongJmp(false);
Anton Korobeynikov4efbbc92007-01-03 11:43:14 +000061 } else if (Subtarget->isTargetMingw()) {
Anton Korobeynikov3b7c2572006-12-10 23:12:42 +000062 // MS runtime is weird: it exports _setjmp, but longjmp!
63 setUseUnderscoreSetJmp(true);
64 setUseUnderscoreLongJmp(false);
65 } else {
66 setUseUnderscoreSetJmp(true);
67 setUseUnderscoreLongJmp(true);
68 }
69
Evan Cheng20931a72006-03-16 21:47:42 +000070 // Add legal addressing mode scale values.
71 addLegalAddressScale(8);
72 addLegalAddressScale(4);
73 addLegalAddressScale(2);
74 // Enter the ones which require both scale + index last. These are more
75 // expensive.
76 addLegalAddressScale(9);
77 addLegalAddressScale(5);
78 addLegalAddressScale(3);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +000079
Chris Lattner76ac0682005-11-15 00:40:23 +000080 // Set up the register classes.
Evan Cheng9fee4422006-05-16 07:21:53 +000081 addRegisterClass(MVT::i8, X86::GR8RegisterClass);
82 addRegisterClass(MVT::i16, X86::GR16RegisterClass);
83 addRegisterClass(MVT::i32, X86::GR32RegisterClass);
Evan Cheng11b0a5d2006-09-08 06:48:29 +000084 if (Subtarget->is64Bit())
85 addRegisterClass(MVT::i64, X86::GR64RegisterClass);
Chris Lattner76ac0682005-11-15 00:40:23 +000086
Evan Cheng5d9fd972006-10-04 00:56:09 +000087 setLoadXAction(ISD::SEXTLOAD, MVT::i1, Expand);
88
Chris Lattner76ac0682005-11-15 00:40:23 +000089 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
90 // operation.
91 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote);
92 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote);
93 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote);
Evan Cheng0d5b69f2006-01-17 02:32:49 +000094
Evan Cheng11b0a5d2006-09-08 06:48:29 +000095 if (Subtarget->is64Bit()) {
96 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand);
Evan Cheng0d5b69f2006-01-17 02:32:49 +000097 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote);
Evan Cheng11b0a5d2006-09-08 06:48:29 +000098 } else {
99 if (X86ScalarSSE)
100 // If SSE i64 SINT_TO_FP is not available, expand i32 UINT_TO_FP.
101 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand);
102 else
103 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote);
104 }
Chris Lattner76ac0682005-11-15 00:40:23 +0000105
106 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
107 // this operation.
108 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote);
109 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote);
Nate Begeman7e5496d2006-02-17 00:03:04 +0000110 // SSE has no i16 to fp conversion, only i32
Evan Cheng08390f62006-01-30 22:13:22 +0000111 if (X86ScalarSSE)
Evan Cheng08390f62006-01-30 22:13:22 +0000112 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote);
Evan Cheng593bea72006-02-17 07:01:52 +0000113 else {
114 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom);
115 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom);
116 }
Chris Lattner76ac0682005-11-15 00:40:23 +0000117
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000118 if (!Subtarget->is64Bit()) {
119 // Custom lower SINT_TO_FP and FP_TO_SINT from/to i64 in 32-bit mode.
120 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom);
121 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom);
122 }
Evan Cheng5b97fcf2006-01-30 08:02:57 +0000123
Evan Cheng08390f62006-01-30 22:13:22 +0000124 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
125 // this operation.
126 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote);
127 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote);
128
129 if (X86ScalarSSE) {
130 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote);
131 } else {
Chris Lattner76ac0682005-11-15 00:40:23 +0000132 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom);
Evan Cheng08390f62006-01-30 22:13:22 +0000133 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom);
Chris Lattner76ac0682005-11-15 00:40:23 +0000134 }
135
136 // Handle FP_TO_UINT by promoting the destination to a larger signed
137 // conversion.
138 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote);
139 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote);
140 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote);
141
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000142 if (Subtarget->is64Bit()) {
143 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000144 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000145 } else {
146 if (X86ScalarSSE && !Subtarget->hasSSE3())
147 // Expand FP_TO_UINT into a select.
148 // FIXME: We would like to use a Custom expander here eventually to do
149 // the optimal thing for SSE vs. the default expansion in the legalizer.
150 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand);
151 else
152 // With SSE3 we can use fisttpll to convert to a signed i64.
153 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote);
154 }
Chris Lattner76ac0682005-11-15 00:40:23 +0000155
Chris Lattner55c17f92006-12-05 18:22:22 +0000156 // TODO: when we have SSE, these could be more efficient, by using movd/movq.
Chris Lattnerc20b7e82006-12-05 18:45:06 +0000157 if (!X86ScalarSSE) {
158 setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand);
159 setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand);
160 }
Chris Lattner30107e62005-12-23 05:15:23 +0000161
Evan Cheng0d41d192006-10-30 08:02:39 +0000162 setOperationAction(ISD::BR_JT , MVT::Other, Expand);
Evan Cheng593bea72006-02-17 07:01:52 +0000163 setOperationAction(ISD::BRCOND , MVT::Other, Custom);
Nate Begeman7e7f4392006-02-01 07:19:44 +0000164 setOperationAction(ISD::BR_CC , MVT::Other, Expand);
165 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000166 setOperationAction(ISD::MEMMOVE , MVT::Other, Expand);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000167 if (Subtarget->is64Bit())
168 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000169 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Expand);
Chris Lattner32257332005-12-07 17:59:14 +0000170 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000171 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
172 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000173 setOperationAction(ISD::FREM , MVT::f64 , Expand);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000174
Chris Lattner76ac0682005-11-15 00:40:23 +0000175 setOperationAction(ISD::CTPOP , MVT::i8 , Expand);
176 setOperationAction(ISD::CTTZ , MVT::i8 , Expand);
177 setOperationAction(ISD::CTLZ , MVT::i8 , Expand);
178 setOperationAction(ISD::CTPOP , MVT::i16 , Expand);
179 setOperationAction(ISD::CTTZ , MVT::i16 , Expand);
180 setOperationAction(ISD::CTLZ , MVT::i16 , Expand);
181 setOperationAction(ISD::CTPOP , MVT::i32 , Expand);
182 setOperationAction(ISD::CTTZ , MVT::i32 , Expand);
183 setOperationAction(ISD::CTLZ , MVT::i32 , Expand);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000184 if (Subtarget->is64Bit()) {
185 setOperationAction(ISD::CTPOP , MVT::i64 , Expand);
186 setOperationAction(ISD::CTTZ , MVT::i64 , Expand);
187 setOperationAction(ISD::CTLZ , MVT::i64 , Expand);
188 }
189
Andrew Lenharth0bf68ae2005-11-20 21:41:10 +0000190 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
Nate Begeman2fba8a32006-01-14 03:14:10 +0000191 setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
Nate Begeman1b8121b2006-01-11 21:21:00 +0000192
Chris Lattner76ac0682005-11-15 00:40:23 +0000193 // These should be promoted to a larger select which is supported.
194 setOperationAction(ISD::SELECT , MVT::i1 , Promote);
195 setOperationAction(ISD::SELECT , MVT::i8 , Promote);
Nate Begeman7e5496d2006-02-17 00:03:04 +0000196 // X86 wants to expand cmov itself.
Evan Cheng593bea72006-02-17 07:01:52 +0000197 setOperationAction(ISD::SELECT , MVT::i16 , Custom);
198 setOperationAction(ISD::SELECT , MVT::i32 , Custom);
199 setOperationAction(ISD::SELECT , MVT::f32 , Custom);
200 setOperationAction(ISD::SELECT , MVT::f64 , Custom);
201 setOperationAction(ISD::SETCC , MVT::i8 , Custom);
202 setOperationAction(ISD::SETCC , MVT::i16 , Custom);
203 setOperationAction(ISD::SETCC , MVT::i32 , Custom);
204 setOperationAction(ISD::SETCC , MVT::f32 , Custom);
205 setOperationAction(ISD::SETCC , MVT::f64 , Custom);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000206 if (Subtarget->is64Bit()) {
207 setOperationAction(ISD::SELECT , MVT::i64 , Custom);
208 setOperationAction(ISD::SETCC , MVT::i64 , Custom);
209 }
Nate Begeman7e5496d2006-02-17 00:03:04 +0000210 // X86 ret instruction may pop stack.
Evan Cheng593bea72006-02-17 07:01:52 +0000211 setOperationAction(ISD::RET , MVT::Other, Custom);
Nate Begeman7e5496d2006-02-17 00:03:04 +0000212 // Darwin ABI issue.
Evan Cheng5588de92006-02-18 00:15:05 +0000213 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom);
Nate Begeman4ca2ea52006-04-22 18:53:45 +0000214 setOperationAction(ISD::JumpTable , MVT::i32 , Custom);
Evan Cheng593bea72006-02-17 07:01:52 +0000215 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom);
Evan Chenge0ed6ec2006-02-23 20:41:18 +0000216 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000217 if (Subtarget->is64Bit()) {
218 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom);
219 setOperationAction(ISD::JumpTable , MVT::i64 , Custom);
220 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom);
221 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom);
222 }
Nate Begeman7e5496d2006-02-17 00:03:04 +0000223 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86)
Evan Cheng593bea72006-02-17 07:01:52 +0000224 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom);
225 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom);
226 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom);
Nate Begeman7e5496d2006-02-17 00:03:04 +0000227 // X86 wants to expand memset / memcpy itself.
Evan Cheng593bea72006-02-17 07:01:52 +0000228 setOperationAction(ISD::MEMSET , MVT::Other, Custom);
229 setOperationAction(ISD::MEMCPY , MVT::Other, Custom);
Chris Lattner76ac0682005-11-15 00:40:23 +0000230
Chris Lattner9c415362005-11-29 06:16:21 +0000231 // We don't have line number support yet.
232 setOperationAction(ISD::LOCATION, MVT::Other, Expand);
Jim Laskeydeeafa02006-01-05 01:47:43 +0000233 setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand);
Evan Cheng30d7b702006-03-07 02:02:57 +0000234 // FIXME - use subtarget debug flags
Anton Korobeynikovaa4c0f92006-10-31 08:31:24 +0000235 if (!Subtarget->isTargetDarwin() &&
236 !Subtarget->isTargetELF() &&
Anton Korobeynikov4efbbc92007-01-03 11:43:14 +0000237 !Subtarget->isTargetCygMing())
Jim Laskeyf9e54452007-01-26 14:34:52 +0000238 setOperationAction(ISD::LABEL, MVT::Other, Expand);
Chris Lattner9c415362005-11-29 06:16:21 +0000239
Nate Begemane74795c2006-01-25 18:21:52 +0000240 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
241 setOperationAction(ISD::VASTART , MVT::Other, Custom);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000242
Nate Begemane74795c2006-01-25 18:21:52 +0000243 // Use the default implementation.
244 setOperationAction(ISD::VAARG , MVT::Other, Expand);
245 setOperationAction(ISD::VACOPY , MVT::Other, Expand);
246 setOperationAction(ISD::VAEND , MVT::Other, Expand);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000247 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
Chris Lattner78c358d2006-01-15 09:00:21 +0000248 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000249 if (Subtarget->is64Bit())
250 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
Chris Lattner78c358d2006-01-15 09:00:21 +0000251 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Expand);
Chris Lattner8e2f52e2006-01-13 02:42:53 +0000252
Chris Lattner76ac0682005-11-15 00:40:23 +0000253 if (X86ScalarSSE) {
254 // Set up the FP register classes.
Evan Cheng84dc9b52006-01-12 08:27:59 +0000255 addRegisterClass(MVT::f32, X86::FR32RegisterClass);
256 addRegisterClass(MVT::f64, X86::FR64RegisterClass);
Chris Lattner76ac0682005-11-15 00:40:23 +0000257
Evan Cheng72d5c252006-01-31 22:28:30 +0000258 // Use ANDPD to simulate FABS.
259 setOperationAction(ISD::FABS , MVT::f64, Custom);
260 setOperationAction(ISD::FABS , MVT::f32, Custom);
261
262 // Use XORP to simulate FNEG.
263 setOperationAction(ISD::FNEG , MVT::f64, Custom);
264 setOperationAction(ISD::FNEG , MVT::f32, Custom);
265
Evan Cheng4363e882007-01-05 07:55:56 +0000266 // Use ANDPD and ORPD to simulate FCOPYSIGN.
267 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
268 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
269
Evan Chengd8fba3a2006-02-02 00:28:23 +0000270 // We don't support sin/cos/fmod
Chris Lattner76ac0682005-11-15 00:40:23 +0000271 setOperationAction(ISD::FSIN , MVT::f64, Expand);
272 setOperationAction(ISD::FCOS , MVT::f64, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000273 setOperationAction(ISD::FREM , MVT::f64, Expand);
274 setOperationAction(ISD::FSIN , MVT::f32, Expand);
275 setOperationAction(ISD::FCOS , MVT::f32, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000276 setOperationAction(ISD::FREM , MVT::f32, Expand);
277
Chris Lattner61c9a8e2006-01-29 06:26:08 +0000278 // Expand FP immediates into loads from the stack, except for the special
279 // cases we handle.
280 setOperationAction(ISD::ConstantFP, MVT::f64, Expand);
281 setOperationAction(ISD::ConstantFP, MVT::f32, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000282 addLegalFPImmediate(+0.0); // xorps / xorpd
283 } else {
284 // Set up the FP register classes.
285 addRegisterClass(MVT::f64, X86::RFPRegisterClass);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000286
Evan Cheng4363e882007-01-05 07:55:56 +0000287 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
288 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
289 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000290
Chris Lattner76ac0682005-11-15 00:40:23 +0000291 if (!UnsafeFPMath) {
292 setOperationAction(ISD::FSIN , MVT::f64 , Expand);
293 setOperationAction(ISD::FCOS , MVT::f64 , Expand);
294 }
295
Chris Lattner61c9a8e2006-01-29 06:26:08 +0000296 setOperationAction(ISD::ConstantFP, MVT::f64, Expand);
Chris Lattner76ac0682005-11-15 00:40:23 +0000297 addLegalFPImmediate(+0.0); // FLD0
298 addLegalFPImmediate(+1.0); // FLD1
299 addLegalFPImmediate(-0.0); // FLD0/FCHS
300 addLegalFPImmediate(-1.0); // FLD1/FCHS
301 }
Evan Cheng9e252e32006-02-22 02:26:30 +0000302
Evan Cheng19264272006-03-01 01:11:20 +0000303 // First set operation action for all vector types to expand. Then we
304 // will selectively turn on ones that can be effectively codegen'd.
305 for (unsigned VT = (unsigned)MVT::Vector + 1;
306 VT != (unsigned)MVT::LAST_VALUETYPE; VT++) {
307 setOperationAction(ISD::ADD , (MVT::ValueType)VT, Expand);
308 setOperationAction(ISD::SUB , (MVT::ValueType)VT, Expand);
Evan Chengbf3df772006-10-27 18:49:08 +0000309 setOperationAction(ISD::FADD, (MVT::ValueType)VT, Expand);
310 setOperationAction(ISD::FSUB, (MVT::ValueType)VT, Expand);
Evan Cheng19264272006-03-01 01:11:20 +0000311 setOperationAction(ISD::MUL , (MVT::ValueType)VT, Expand);
Evan Chengbf3df772006-10-27 18:49:08 +0000312 setOperationAction(ISD::FMUL, (MVT::ValueType)VT, Expand);
313 setOperationAction(ISD::SDIV, (MVT::ValueType)VT, Expand);
314 setOperationAction(ISD::UDIV, (MVT::ValueType)VT, Expand);
315 setOperationAction(ISD::FDIV, (MVT::ValueType)VT, Expand);
316 setOperationAction(ISD::SREM, (MVT::ValueType)VT, Expand);
317 setOperationAction(ISD::UREM, (MVT::ValueType)VT, Expand);
Evan Cheng19264272006-03-01 01:11:20 +0000318 setOperationAction(ISD::LOAD, (MVT::ValueType)VT, Expand);
Evan Chengcbffa462006-03-31 19:22:53 +0000319 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::ValueType)VT, Expand);
Chris Lattner00f46832006-03-21 20:51:05 +0000320 setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::ValueType)VT, Expand);
Evan Chengcbffa462006-03-31 19:22:53 +0000321 setOperationAction(ISD::INSERT_VECTOR_ELT, (MVT::ValueType)VT, Expand);
Evan Cheng19264272006-03-01 01:11:20 +0000322 }
323
Evan Chengbc047222006-03-22 19:22:18 +0000324 if (Subtarget->hasMMX()) {
Evan Cheng9e252e32006-02-22 02:26:30 +0000325 addRegisterClass(MVT::v8i8, X86::VR64RegisterClass);
326 addRegisterClass(MVT::v4i16, X86::VR64RegisterClass);
327 addRegisterClass(MVT::v2i32, X86::VR64RegisterClass);
328
Evan Cheng19264272006-03-01 01:11:20 +0000329 // FIXME: add MMX packed arithmetics
Evan Chengd5e905d2006-03-21 23:01:21 +0000330 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Expand);
331 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Expand);
332 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Expand);
Evan Cheng9e252e32006-02-22 02:26:30 +0000333 }
334
Evan Chengbc047222006-03-22 19:22:18 +0000335 if (Subtarget->hasSSE1()) {
Evan Cheng9e252e32006-02-22 02:26:30 +0000336 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass);
337
Evan Chengbf3df772006-10-27 18:49:08 +0000338 setOperationAction(ISD::FADD, MVT::v4f32, Legal);
339 setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
340 setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
341 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
Evan Cheng617a6a82006-04-10 07:23:14 +0000342 setOperationAction(ISD::LOAD, MVT::v4f32, Legal);
343 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
344 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
Evan Chengebf10062006-04-03 20:53:28 +0000345 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
Evan Cheng617a6a82006-04-10 07:23:14 +0000346 setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
Evan Cheng9e252e32006-02-22 02:26:30 +0000347 }
348
Evan Chengbc047222006-03-22 19:22:18 +0000349 if (Subtarget->hasSSE2()) {
Evan Cheng9e252e32006-02-22 02:26:30 +0000350 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass);
351 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass);
352 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass);
353 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass);
354 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass);
355
Evan Cheng617a6a82006-04-10 07:23:14 +0000356 setOperationAction(ISD::ADD, MVT::v16i8, Legal);
357 setOperationAction(ISD::ADD, MVT::v8i16, Legal);
358 setOperationAction(ISD::ADD, MVT::v4i32, Legal);
Evan Cheng617a6a82006-04-10 07:23:14 +0000359 setOperationAction(ISD::SUB, MVT::v16i8, Legal);
360 setOperationAction(ISD::SUB, MVT::v8i16, Legal);
361 setOperationAction(ISD::SUB, MVT::v4i32, Legal);
Evan Chenge4f97cc2006-04-13 05:10:25 +0000362 setOperationAction(ISD::MUL, MVT::v8i16, Legal);
Evan Chengbf3df772006-10-27 18:49:08 +0000363 setOperationAction(ISD::FADD, MVT::v2f64, Legal);
364 setOperationAction(ISD::FSUB, MVT::v2f64, Legal);
365 setOperationAction(ISD::FMUL, MVT::v2f64, Legal);
366 setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
Evan Cheng92232302006-04-12 21:21:57 +0000367
Evan Cheng617a6a82006-04-10 07:23:14 +0000368 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom);
369 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom);
Evan Chengcbffa462006-03-31 19:22:53 +0000370 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
Evan Cheng6e5e2052006-04-17 22:04:06 +0000371 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
372 // Implement v4f32 insert_vector_elt in terms of SSE2 v8i16 ones.
373 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
Evan Cheng617a6a82006-04-10 07:23:14 +0000374
Evan Cheng92232302006-04-12 21:21:57 +0000375 // Custom lower build_vector, vector_shuffle, and extract_vector_elt.
376 for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) {
377 setOperationAction(ISD::BUILD_VECTOR, (MVT::ValueType)VT, Custom);
378 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::ValueType)VT, Custom);
379 setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::ValueType)VT, Custom);
380 }
381 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom);
382 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom);
383 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom);
384 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom);
385 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom);
386 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom);
387
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000388 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64.
Evan Cheng92232302006-04-12 21:21:57 +0000389 for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) {
390 setOperationAction(ISD::AND, (MVT::ValueType)VT, Promote);
391 AddPromotedToType (ISD::AND, (MVT::ValueType)VT, MVT::v2i64);
392 setOperationAction(ISD::OR, (MVT::ValueType)VT, Promote);
393 AddPromotedToType (ISD::OR, (MVT::ValueType)VT, MVT::v2i64);
394 setOperationAction(ISD::XOR, (MVT::ValueType)VT, Promote);
395 AddPromotedToType (ISD::XOR, (MVT::ValueType)VT, MVT::v2i64);
Evan Chenge2157c62006-04-12 17:12:36 +0000396 setOperationAction(ISD::LOAD, (MVT::ValueType)VT, Promote);
397 AddPromotedToType (ISD::LOAD, (MVT::ValueType)VT, MVT::v2i64);
Evan Cheng92232302006-04-12 21:21:57 +0000398 setOperationAction(ISD::SELECT, (MVT::ValueType)VT, Promote);
399 AddPromotedToType (ISD::SELECT, (MVT::ValueType)VT, MVT::v2i64);
Evan Cheng617a6a82006-04-10 07:23:14 +0000400 }
Evan Cheng92232302006-04-12 21:21:57 +0000401
402 // Custom lower v2i64 and v2f64 selects.
403 setOperationAction(ISD::LOAD, MVT::v2f64, Legal);
Evan Chenge2157c62006-04-12 17:12:36 +0000404 setOperationAction(ISD::LOAD, MVT::v2i64, Legal);
Evan Cheng617a6a82006-04-10 07:23:14 +0000405 setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
Evan Cheng92232302006-04-12 21:21:57 +0000406 setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
Evan Cheng9e252e32006-02-22 02:26:30 +0000407 }
408
Evan Cheng78038292006-04-05 23:38:46 +0000409 // We want to custom lower some of our intrinsics.
410 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
411
Evan Cheng5987cfb2006-07-07 08:33:52 +0000412 // We have target-specific dag combine patterns for the following nodes:
413 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
Chris Lattner9259b1e2006-10-04 06:57:07 +0000414 setTargetDAGCombine(ISD::SELECT);
Evan Cheng5987cfb2006-07-07 08:33:52 +0000415
Chris Lattner76ac0682005-11-15 00:40:23 +0000416 computeRegisterProperties();
417
Evan Cheng6a374562006-02-14 08:25:08 +0000418 // FIXME: These should be based on subtarget info. Plus, the values should
419 // be smaller when we are in optimizing for size mode.
Evan Cheng4b40a422006-02-14 08:38:30 +0000420 maxStoresPerMemset = 16; // For %llvm.memset -> sequence of stores
421 maxStoresPerMemcpy = 16; // For %llvm.memcpy -> sequence of stores
422 maxStoresPerMemmove = 16; // For %llvm.memmove -> sequence of stores
Chris Lattner76ac0682005-11-15 00:40:23 +0000423 allowUnalignedMemoryAccesses = true; // x86 supports it!
424}
425
Chris Lattner3c763092007-02-25 08:29:00 +0000426
427//===----------------------------------------------------------------------===//
428// Return Value Calling Convention Implementation
429//===----------------------------------------------------------------------===//
430
431/// GetRetValueLocs - If we are returning a set of values with the specified
432/// value types, determine the set of registers each one will land in. This
433/// sets one element of the ResultRegs array for each element in the VTs array.
434static void GetRetValueLocs(const MVT::ValueType *VTs, unsigned NumVTs,
435 unsigned *ResultRegs,
436 const X86Subtarget *Subtarget,
437 unsigned CallingConv) {
438 if (NumVTs == 0) return;
439
440 if (NumVTs == 2) {
441 ResultRegs[0] = VTs[0] == MVT::i64 ? X86::RAX : X86::EAX;
442 ResultRegs[1] = VTs[1] == MVT::i64 ? X86::RDX : X86::EDX;
443 return;
444 }
445
446 // Otherwise, NumVTs is 1.
447 MVT::ValueType ArgVT = VTs[0];
448
Chris Lattner0cd99602007-02-25 08:59:22 +0000449 unsigned Reg;
450 switch (ArgVT) {
451 case MVT::i8: Reg = X86::AL; break;
452 case MVT::i16: Reg = X86::AX; break;
453 case MVT::i32: Reg = X86::EAX; break;
454 case MVT::i64: Reg = X86::RAX; break;
455 case MVT::f32:
456 case MVT::f64:
457 if (Subtarget->is64Bit())
458 Reg = X86::XMM0; // FP values in X86-64 go in XMM0.
459 else
460 Reg = X86::ST0; // FP values in X86-32 go in ST0.
461 break;
462 default:
463 assert(MVT::isVector(ArgVT) && "Unknown return value type!");
464 Reg = X86::XMM0; // Int/FP vector result -> XMM0.
465 break;
Chris Lattner3c763092007-02-25 08:29:00 +0000466 }
Chris Lattner0cd99602007-02-25 08:59:22 +0000467 ResultRegs[0] = Reg;
468}
469
470/// LowerCallResult - Lower the result values of an ISD::CALL into the
471/// appropriate copies out of appropriate physical registers. This assumes that
472/// Chain/InFlag are the input chain/flag to use, and that TheCall is the call
473/// being lowered. The returns a SDNode with the same number of values as the
474/// ISD::CALL.
475SDNode *X86TargetLowering::
476LowerCallResult(SDOperand Chain, SDOperand InFlag, SDNode *TheCall,
477 unsigned CallingConv, SelectionDAG &DAG) {
478 SmallVector<SDOperand, 8> ResultVals;
479
480 // We support returning up to two registers.
481 MVT::ValueType VTs[2];
482 unsigned DestRegs[2];
483 unsigned NumRegs = TheCall->getNumValues() - 1;
484 assert(NumRegs <= 2 && "Can only return up to two regs!");
485
486 for (unsigned i = 0; i != NumRegs; ++i)
487 VTs[i] = TheCall->getValueType(i);
488
489 // Determine which register each value should be copied into.
490 GetRetValueLocs(VTs, NumRegs, DestRegs, Subtarget, CallingConv);
491
492 // Copy all of the result registers out of their specified physreg.
493 if (NumRegs != 1 || DestRegs[0] != X86::ST0) {
494 for (unsigned i = 0; i != NumRegs; ++i) {
495 Chain = DAG.getCopyFromReg(Chain, DestRegs[i], VTs[i],
496 InFlag).getValue(1);
497 InFlag = Chain.getValue(2);
498 ResultVals.push_back(Chain.getValue(0));
499 }
500 } else {
501 // Copies from the FP stack are special, as ST0 isn't a valid register
502 // before the fp stackifier runs.
503
504 // Copy ST0 into an RFP register with FP_GET_RESULT.
505 SDVTList Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag);
506 SDOperand GROps[] = { Chain, InFlag };
507 SDOperand RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, GROps, 2);
508 Chain = RetVal.getValue(1);
509 InFlag = RetVal.getValue(2);
510
511 // If we are using ScalarSSE, store ST(0) to the stack and reload it into
512 // an XMM register.
513 if (X86ScalarSSE) {
514 // FIXME: Currently the FST is flagged to the FP_GET_RESULT. This
515 // shouldn't be necessary except that RFP cannot be live across
516 // multiple blocks. When stackifier is fixed, they can be uncoupled.
517 MachineFunction &MF = DAG.getMachineFunction();
518 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8);
519 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
520 SDOperand Ops[] = {
521 Chain, RetVal, StackSlot, DAG.getValueType(VTs[0]), InFlag
522 };
523 Chain = DAG.getNode(X86ISD::FST, MVT::Other, Ops, 5);
524 RetVal = DAG.getLoad(VTs[0], Chain, StackSlot, NULL, 0);
525 Chain = RetVal.getValue(1);
526 }
527
528 if (VTs[0] == MVT::f32 && !X86ScalarSSE)
529 // FIXME: we would really like to remember that this FP_ROUND
530 // operation is okay to eliminate if we allow excess FP precision.
531 RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal);
532 ResultVals.push_back(RetVal);
533 }
534
535 // Merge everything together with a MERGE_VALUES node.
536 ResultVals.push_back(Chain);
537 return DAG.getNode(ISD::MERGE_VALUES, TheCall->getVTList(),
538 &ResultVals[0], ResultVals.size()).Val;
Chris Lattner3c763092007-02-25 08:29:00 +0000539}
540
541
Chris Lattner76ac0682005-11-15 00:40:23 +0000542//===----------------------------------------------------------------------===//
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000543// C & StdCall Calling Convention implementation
Chris Lattner76ac0682005-11-15 00:40:23 +0000544//===----------------------------------------------------------------------===//
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000545// StdCall calling convention seems to be standard for many Windows' API
546// routines and around. It differs from C calling convention just a little:
547// callee should clean up the stack, not caller. Symbols should be also
548// decorated in some fancy way :) It doesn't support any vector arguments.
Chris Lattner76ac0682005-11-15 00:40:23 +0000549
Evan Cheng24eb3f42006-04-27 05:35:28 +0000550/// AddLiveIn - This helper function adds the specified physical register to the
551/// MachineFunction as a live in value. It also creates a corresponding virtual
552/// register for it.
553static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg,
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000554 const TargetRegisterClass *RC) {
Evan Cheng24eb3f42006-04-27 05:35:28 +0000555 assert(RC->contains(PReg) && "Not the correct regclass!");
556 unsigned VReg = MF.getSSARegMap()->createVirtualRegister(RC);
557 MF.addLiveIn(PReg, VReg);
558 return VReg;
559}
560
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000561/// HowToPassArgument - Returns how an formal argument of the specified type
Evan Cheng89001ad2006-04-27 08:31:10 +0000562/// should be passed. If it is through stack, returns the size of the stack
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000563/// slot; if it is through integer or XMM register, returns the number of
564/// integer or XMM registers are needed.
Evan Cheng89001ad2006-04-27 08:31:10 +0000565static void
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000566HowToPassCallArgument(MVT::ValueType ObjectVT,
567 bool ArgInReg,
568 unsigned NumIntRegs, unsigned NumXMMRegs,
569 unsigned MaxNumIntRegs,
570 unsigned &ObjSize, unsigned &ObjIntRegs,
571 unsigned &ObjXMMRegs,
572 bool AllowVectors = true) {
573 ObjSize = 0;
574 ObjIntRegs = 0;
Evan Cheng2b2c1be2006-06-01 05:53:27 +0000575 ObjXMMRegs = 0;
Evan Cheng8aca43e2006-05-25 23:31:23 +0000576
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000577 if (MaxNumIntRegs>3) {
578 // We don't have too much registers on ia32! :)
579 MaxNumIntRegs = 3;
580 }
581
Evan Cheng48940d12006-04-27 01:32:22 +0000582 switch (ObjectVT) {
583 default: assert(0 && "Unhandled argument type!");
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000584 case MVT::i8:
585 if (ArgInReg && (NumIntRegs < MaxNumIntRegs))
586 ObjIntRegs = 1;
587 else
588 ObjSize = 1;
589 break;
590 case MVT::i16:
591 if (ArgInReg && (NumIntRegs < MaxNumIntRegs))
592 ObjIntRegs = 1;
593 else
594 ObjSize = 2;
595 break;
596 case MVT::i32:
597 if (ArgInReg && (NumIntRegs < MaxNumIntRegs))
598 ObjIntRegs = 1;
599 else
600 ObjSize = 4;
601 break;
602 case MVT::i64:
603 if (ArgInReg && (NumIntRegs+2 <= MaxNumIntRegs)) {
604 ObjIntRegs = 2;
605 } else if (ArgInReg && (NumIntRegs+1 <= MaxNumIntRegs)) {
606 ObjIntRegs = 1;
607 ObjSize = 4;
608 } else
609 ObjSize = 8;
610 case MVT::f32:
611 ObjSize = 4;
612 break;
613 case MVT::f64:
614 ObjSize = 8;
615 break;
Evan Cheng89001ad2006-04-27 08:31:10 +0000616 case MVT::v16i8:
617 case MVT::v8i16:
618 case MVT::v4i32:
619 case MVT::v2i64:
620 case MVT::v4f32:
621 case MVT::v2f64:
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000622 if (AllowVectors) {
623 if (NumXMMRegs < 4)
624 ObjXMMRegs = 1;
625 else
626 ObjSize = 16;
627 break;
628 } else
629 assert(0 && "Unhandled argument type [vector]!");
Evan Cheng48940d12006-04-27 01:32:22 +0000630 }
Evan Cheng48940d12006-04-27 01:32:22 +0000631}
632
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000633SDOperand X86TargetLowering::LowerCCCArguments(SDOperand Op, SelectionDAG &DAG,
634 bool isStdCall) {
Evan Cheng17e734f2006-05-23 21:06:34 +0000635 unsigned NumArgs = Op.Val->getNumValues() - 1;
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000636 MachineFunction &MF = DAG.getMachineFunction();
637 MachineFrameInfo *MFI = MF.getFrameInfo();
Evan Cheng17e734f2006-05-23 21:06:34 +0000638 SDOperand Root = Op.getOperand(0);
Chris Lattner35a08552007-02-25 07:10:00 +0000639 SmallVector<SDOperand, 8> ArgValues;
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000640 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0;
Chris Lattner76ac0682005-11-15 00:40:23 +0000641
Evan Cheng48940d12006-04-27 01:32:22 +0000642 // Add DAG nodes to load the arguments... On entry to a function on the X86,
643 // the stack frame looks like this:
644 //
645 // [ESP] -- return address
646 // [ESP + 4] -- first argument (leftmost lexically)
Evan Chengcbfb3d02006-05-26 18:37:16 +0000647 // [ESP + 8] -- second argument, if first argument is <= 4 bytes in size
Evan Cheng48940d12006-04-27 01:32:22 +0000648 // ...
649 //
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000650 unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot
651 unsigned NumSRetBytes= 0; // How much bytes on stack used for struct return
652 unsigned NumXMMRegs = 0; // XMM regs used for parameter passing.
653 unsigned NumIntRegs = 0; // Integer regs used for parameter passing
654
Evan Chengbfb5ea62006-05-26 19:22:06 +0000655 static const unsigned XMMArgRegs[] = {
656 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3
657 };
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000658 static const unsigned GPRArgRegs[][3] = {
659 { X86::AL, X86::DL, X86::CL },
660 { X86::AX, X86::DX, X86::CX },
661 { X86::EAX, X86::EDX, X86::ECX }
662 };
663 static const TargetRegisterClass* GPRClasses[3] = {
664 X86::GR8RegisterClass, X86::GR16RegisterClass, X86::GR32RegisterClass
665 };
666
667 // Handle regparm attribute
Chris Lattner35a08552007-02-25 07:10:00 +0000668 SmallVector<bool, 8> ArgInRegs(NumArgs, false);
669 SmallVector<bool, 8> SRetArgs(NumArgs, false);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000670 if (!isVarArg) {
671 for (unsigned i = 0; i<NumArgs; ++i) {
672 unsigned Flags = cast<ConstantSDNode>(Op.getOperand(3+i))->getValue();
673 ArgInRegs[i] = (Flags >> 1) & 1;
674 SRetArgs[i] = (Flags >> 2) & 1;
675 }
676 }
677
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000678 for (unsigned i = 0; i < NumArgs; ++i) {
Evan Cheng17e734f2006-05-23 21:06:34 +0000679 MVT::ValueType ObjectVT = Op.getValue(i).getValueType();
680 unsigned ArgIncrement = 4;
681 unsigned ObjSize = 0;
682 unsigned ObjXMMRegs = 0;
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000683 unsigned ObjIntRegs = 0;
684 unsigned Reg = 0;
685 SDOperand ArgValue;
686
687 HowToPassCallArgument(ObjectVT,
688 ArgInRegs[i],
689 NumIntRegs, NumXMMRegs, 3,
690 ObjSize, ObjIntRegs, ObjXMMRegs,
691 !isStdCall);
692
Evan Chenga01e7992006-05-26 18:39:59 +0000693 if (ObjSize > 4)
Evan Cheng17e734f2006-05-23 21:06:34 +0000694 ArgIncrement = ObjSize;
Evan Cheng48940d12006-04-27 01:32:22 +0000695
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000696 if (ObjIntRegs || ObjXMMRegs) {
697 switch (ObjectVT) {
698 default: assert(0 && "Unhandled argument type!");
699 case MVT::i8:
700 case MVT::i16:
701 case MVT::i32: {
702 unsigned RegToUse = GPRArgRegs[ObjectVT-MVT::i8][NumIntRegs];
703 Reg = AddLiveIn(MF, RegToUse, GPRClasses[ObjectVT-MVT::i8]);
704 ArgValue = DAG.getCopyFromReg(Root, Reg, ObjectVT);
705 break;
706 }
707 case MVT::v16i8:
708 case MVT::v8i16:
709 case MVT::v4i32:
710 case MVT::v2i64:
711 case MVT::v4f32:
712 case MVT::v2f64:
713 assert(!isStdCall && "Unhandled argument type!");
714 Reg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], X86::VR128RegisterClass);
715 ArgValue = DAG.getCopyFromReg(Root, Reg, ObjectVT);
716 break;
717 }
718 NumIntRegs += ObjIntRegs;
Evan Cheng17e734f2006-05-23 21:06:34 +0000719 NumXMMRegs += ObjXMMRegs;
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000720 }
721 if (ObjSize) {
Evan Chengb92f4182006-05-26 20:37:47 +0000722 // XMM arguments have to be aligned on 16-byte boundary.
723 if (ObjSize == 16)
724 ArgOffset = ((ArgOffset + 15) / 16) * 16;
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000725 // Create the SelectionDAG nodes corresponding to a load from this
726 // parameter.
Evan Cheng17e734f2006-05-23 21:06:34 +0000727 int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
728 SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy());
Evan Chenge71fe34d2006-10-09 20:57:25 +0000729 ArgValue = DAG.getLoad(Op.Val->getValueType(i), Root, FIN, NULL, 0);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000730
731 ArgOffset += ArgIncrement; // Move on to the next argument.
732 if (SRetArgs[i])
733 NumSRetBytes += ArgIncrement;
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000734 }
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000735
736 ArgValues.push_back(ArgValue);
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000737 }
738
Evan Cheng17e734f2006-05-23 21:06:34 +0000739 ArgValues.push_back(Root);
740
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000741 // If the function takes variable number of arguments, make a frame index for
742 // the start of the first vararg value... for expansion of llvm.va_start.
Evan Cheng7068a932006-05-23 21:08:24 +0000743 if (isVarArg)
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000744 VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000745
746 if (isStdCall && !isVarArg) {
747 BytesToPopOnReturn = ArgOffset; // Callee pops everything..
748 BytesCallerReserves = 0;
749 } else {
750 BytesToPopOnReturn = NumSRetBytes; // Callee pops hidden struct pointer.
751 BytesCallerReserves = ArgOffset;
752 }
753
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000754 RegSaveFrameIndex = 0xAAAAAAA; // X86-64 only.
755 ReturnAddrIndex = 0; // No return address slot generated yet.
Evan Cheng17e734f2006-05-23 21:06:34 +0000756
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000757
758 MF.getInfo<X86FunctionInfo>()->setBytesToPopOnReturn(BytesToPopOnReturn);
Evan Chenge0bcfbe2006-04-26 01:20:17 +0000759
Evan Cheng17e734f2006-05-23 21:06:34 +0000760 // Return the new list of results.
Chris Lattner35a08552007-02-25 07:10:00 +0000761 return DAG.getNode(ISD::MERGE_VALUES, Op.Val->getVTList(),
762 &ArgValues[0], ArgValues.size());
Chris Lattner76ac0682005-11-15 00:40:23 +0000763}
764
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000765SDOperand X86TargetLowering::LowerCCCCallTo(SDOperand Op, SelectionDAG &DAG,
Chris Lattner7802f3e2007-02-25 09:06:15 +0000766 unsigned CC) {
Evan Cheng2a330942006-05-25 00:59:30 +0000767 SDOperand Chain = Op.getOperand(0);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000768 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0;
Evan Cheng2a330942006-05-25 00:59:30 +0000769 bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0;
770 SDOperand Callee = Op.getOperand(4);
Evan Cheng2a330942006-05-25 00:59:30 +0000771 unsigned NumOps = (Op.getNumOperands() - 5) / 2;
Chris Lattner76ac0682005-11-15 00:40:23 +0000772
Evan Cheng2a330942006-05-25 00:59:30 +0000773 static const unsigned XMMArgRegs[] = {
Evan Chengbfb5ea62006-05-26 19:22:06 +0000774 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3
Evan Cheng2a330942006-05-25 00:59:30 +0000775 };
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000776 static const unsigned GPR32ArgRegs[] = {
777 X86::EAX, X86::EDX, X86::ECX
778 };
Evan Cheng88decde2006-04-28 21:29:37 +0000779
Evan Cheng2a330942006-05-25 00:59:30 +0000780 // Count how many bytes are to be pushed on the stack.
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000781 unsigned NumBytes = 0;
782 // Keep track of the number of integer regs passed so far.
783 unsigned NumIntRegs = 0;
784 // Keep track of the number of XMM regs passed so far.
785 unsigned NumXMMRegs = 0;
786 // How much bytes on stack used for struct return
787 unsigned NumSRetBytes= 0;
788
789 // Handle regparm attribute
Chris Lattner35a08552007-02-25 07:10:00 +0000790 SmallVector<bool, 8> ArgInRegs(NumOps, false);
791 SmallVector<bool, 8> SRetArgs(NumOps, false);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000792 for (unsigned i = 0; i<NumOps; ++i) {
793 unsigned Flags =
794 dyn_cast<ConstantSDNode>(Op.getOperand(5+2*i+1))->getValue();
795 ArgInRegs[i] = (Flags >> 1) & 1;
796 SRetArgs[i] = (Flags >> 2) & 1;
797 }
798
799 // Calculate stack frame size
Evan Cheng2a330942006-05-25 00:59:30 +0000800 for (unsigned i = 0; i != NumOps; ++i) {
801 SDOperand Arg = Op.getOperand(5+2*i);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000802 unsigned ArgIncrement = 4;
803 unsigned ObjSize = 0;
804 unsigned ObjIntRegs = 0;
805 unsigned ObjXMMRegs = 0;
Chris Lattner76ac0682005-11-15 00:40:23 +0000806
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000807 HowToPassCallArgument(Arg.getValueType(),
808 ArgInRegs[i],
809 NumIntRegs, NumXMMRegs, 3,
810 ObjSize, ObjIntRegs, ObjXMMRegs,
Chris Lattner7802f3e2007-02-25 09:06:15 +0000811 CC != CallingConv::X86_StdCall);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000812 if (ObjSize > 4)
813 ArgIncrement = ObjSize;
814
815 NumIntRegs += ObjIntRegs;
816 NumXMMRegs += ObjXMMRegs;
817 if (ObjSize) {
818 // XMM arguments have to be aligned on 16-byte boundary.
819 if (ObjSize == 16)
Evan Chengb92f4182006-05-26 20:37:47 +0000820 NumBytes = ((NumBytes + 15) / 16) * 16;
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000821 NumBytes += ArgIncrement;
Evan Cheng2a330942006-05-25 00:59:30 +0000822 }
Evan Cheng2a330942006-05-25 00:59:30 +0000823 }
Chris Lattner76ac0682005-11-15 00:40:23 +0000824
Evan Cheng2a330942006-05-25 00:59:30 +0000825 Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy()));
Chris Lattner76ac0682005-11-15 00:40:23 +0000826
Evan Cheng2a330942006-05-25 00:59:30 +0000827 // Arguments go on the stack in reverse order, as specified by the ABI.
828 unsigned ArgOffset = 0;
829 NumXMMRegs = 0;
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000830 NumIntRegs = 0;
Chris Lattner35a08552007-02-25 07:10:00 +0000831 SmallVector<std::pair<unsigned, SDOperand>, 8> RegsToPass;
832 SmallVector<SDOperand, 8> MemOpChains;
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000833 SDOperand StackPtr = DAG.getRegister(X86StackPtr, getPointerTy());
Evan Cheng2a330942006-05-25 00:59:30 +0000834 for (unsigned i = 0; i != NumOps; ++i) {
835 SDOperand Arg = Op.getOperand(5+2*i);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000836 unsigned ArgIncrement = 4;
837 unsigned ObjSize = 0;
838 unsigned ObjIntRegs = 0;
839 unsigned ObjXMMRegs = 0;
Evan Cheng2a330942006-05-25 00:59:30 +0000840
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000841 HowToPassCallArgument(Arg.getValueType(),
842 ArgInRegs[i],
843 NumIntRegs, NumXMMRegs, 3,
844 ObjSize, ObjIntRegs, ObjXMMRegs,
Chris Lattner7802f3e2007-02-25 09:06:15 +0000845 CC != CallingConv::X86_StdCall);
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000846
847 if (ObjSize > 4)
848 ArgIncrement = ObjSize;
849
850 if (Arg.getValueType() == MVT::i8 || Arg.getValueType() == MVT::i16) {
Evan Cheng2a330942006-05-25 00:59:30 +0000851 // Promote the integer to 32 bits. If the input type is signed use a
852 // sign extend, otherwise use a zero extend.
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000853 unsigned Flags = cast<ConstantSDNode>(Op.getOperand(5+2*i+1))->getValue();
854
855 unsigned ExtOp = (Flags & 1) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
Evan Cheng2a330942006-05-25 00:59:30 +0000856 Arg = DAG.getNode(ExtOp, MVT::i32, Arg);
Evan Cheng5ee96892006-05-25 18:56:34 +0000857 }
Evan Cheng2a330942006-05-25 00:59:30 +0000858
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000859 if (ObjIntRegs || ObjXMMRegs) {
860 switch (Arg.getValueType()) {
861 default: assert(0 && "Unhandled argument type!");
862 case MVT::i32:
863 RegsToPass.push_back(std::make_pair(GPR32ArgRegs[NumIntRegs], Arg));
864 break;
865 case MVT::v16i8:
866 case MVT::v8i16:
867 case MVT::v4i32:
868 case MVT::v2i64:
869 case MVT::v4f32:
870 case MVT::v2f64:
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000871 RegsToPass.push_back(std::make_pair(XMMArgRegs[NumXMMRegs], Arg));
872 break;
Evan Cheng88decde2006-04-28 21:29:37 +0000873 }
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000874
875 NumIntRegs += ObjIntRegs;
876 NumXMMRegs += ObjXMMRegs;
877 }
878 if (ObjSize) {
879 // XMM arguments have to be aligned on 16-byte boundary.
880 if (ObjSize == 16)
881 ArgOffset = ((ArgOffset + 15) / 16) * 16;
882
883 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
884 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
885 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
886
887 ArgOffset += ArgIncrement; // Move on to the next argument.
888 if (SRetArgs[i])
889 NumSRetBytes += ArgIncrement;
Chris Lattner76ac0682005-11-15 00:40:23 +0000890 }
Chris Lattner76ac0682005-11-15 00:40:23 +0000891 }
892
Anton Korobeynikov1b4e6012007-02-01 08:39:52 +0000893 // Sanity check: we haven't seen NumSRetBytes > 4
894 assert((NumSRetBytes<=4) &&
895 "Too much space for struct-return pointer requested");
896
Evan Cheng2a330942006-05-25 00:59:30 +0000897 if (!MemOpChains.empty())
Chris Lattnerc24a1d32006-08-08 02:23:42 +0000898 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
899 &MemOpChains[0], MemOpChains.size());
Chris Lattner76ac0682005-11-15 00:40:23 +0000900
Evan Cheng88decde2006-04-28 21:29:37 +0000901 // Build a sequence of copy-to-reg nodes chained together with token chain
902 // and flag operands which copy the outgoing args into registers.
903 SDOperand InFlag;
Evan Cheng2a330942006-05-25 00:59:30 +0000904 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
905 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second,
906 InFlag);
Evan Cheng88decde2006-04-28 21:29:37 +0000907 InFlag = Chain.getValue(1);
908 }
909
Evan Cheng84a041e2007-02-21 21:18:14 +0000910 // ELF / PIC requires GOT in the EBX register before function calls via PLT
911 // GOT pointer.
Evan Cheng1281dc32007-01-22 21:34:25 +0000912 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
913 Subtarget->isPICStyleGOT()) {
Anton Korobeynikova0554d92007-01-12 19:20:47 +0000914 Chain = DAG.getCopyToReg(Chain, X86::EBX,
915 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
916 InFlag);
917 InFlag = Chain.getValue(1);
918 }
919
Evan Cheng2a330942006-05-25 00:59:30 +0000920 // If the callee is a GlobalAddress node (quite common, every direct call is)
921 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
Anton Korobeynikov37d080b2006-11-20 10:46:14 +0000922 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
Anton Korobeynikov430e68a12006-12-22 22:29:05 +0000923 // We should use extra load for direct calls to dllimported functions in
924 // non-JIT mode.
925 if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(),
926 getTargetMachine(), true))
Anton Korobeynikov37d080b2006-11-20 10:46:14 +0000927 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy());
928 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
Evan Cheng2a330942006-05-25 00:59:30 +0000929 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
930
Chris Lattnere56fef92007-02-25 06:40:16 +0000931 // Returns a chain & a flag for retval copy to use.
932 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Chris Lattner35a08552007-02-25 07:10:00 +0000933 SmallVector<SDOperand, 8> Ops;
Nate Begeman7e5496d2006-02-17 00:03:04 +0000934 Ops.push_back(Chain);
935 Ops.push_back(Callee);
Evan Chengca254862006-06-14 18:17:40 +0000936
937 // Add argument registers to the end of the list so that they are known live
938 // into the call.
939 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000940 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
Evan Chengca254862006-06-14 18:17:40 +0000941 RegsToPass[i].second.getValueType()));
Evan Cheng84a041e2007-02-21 21:18:14 +0000942
943 // Add an implicit use GOT pointer in EBX.
944 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
945 Subtarget->isPICStyleGOT())
946 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy()));
Anton Korobeynikova0554d92007-01-12 19:20:47 +0000947
Evan Cheng88decde2006-04-28 21:29:37 +0000948 if (InFlag.Val)
949 Ops.push_back(InFlag);
Evan Cheng45e190982006-01-05 00:27:02 +0000950
Evan Cheng2a330942006-05-25 00:59:30 +0000951 Chain = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
Chris Lattnerc24a1d32006-08-08 02:23:42 +0000952 NodeTys, &Ops[0], Ops.size());
Evan Cheng88decde2006-04-28 21:29:37 +0000953 InFlag = Chain.getValue(1);
Evan Cheng45e190982006-01-05 00:27:02 +0000954
Chris Lattner8be5be82006-05-23 18:50:38 +0000955 // Create the CALLSEQ_END node.
956 unsigned NumBytesForCalleeToPush = 0;
957
Chris Lattner7802f3e2007-02-25 09:06:15 +0000958 if (CC == CallingConv::X86_StdCall) {
959 if (isVarArg)
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000960 NumBytesForCalleeToPush = NumSRetBytes;
Chris Lattner7802f3e2007-02-25 09:06:15 +0000961 else
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000962 NumBytesForCalleeToPush = NumBytes;
Anton Korobeynikov037c8672007-01-28 13:31:35 +0000963 } else {
964 // If this is is a call to a struct-return function, the callee
965 // pops the hidden struct pointer, so we have to push it back.
966 // This is common for Darwin/X86, Linux & Mingw32 targets.
967 NumBytesForCalleeToPush = NumSRetBytes;
968 }
969
Chris Lattnerd6b853ad2007-02-25 07:18:38 +0000970 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Nate Begeman7e5496d2006-02-17 00:03:04 +0000971 Ops.clear();
972 Ops.push_back(Chain);
973 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
Chris Lattner8be5be82006-05-23 18:50:38 +0000974 Ops.push_back(DAG.getConstant(NumBytesForCalleeToPush, getPointerTy()));
Nate Begeman7e5496d2006-02-17 00:03:04 +0000975 Ops.push_back(InFlag);
Chris Lattnerc24a1d32006-08-08 02:23:42 +0000976 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size());
Chris Lattner0cd99602007-02-25 08:59:22 +0000977 InFlag = Chain.getValue(1);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +0000978
Chris Lattner0cd99602007-02-25 08:59:22 +0000979 // Handle result values, copying them out of physregs into vregs that we
980 // return.
Chris Lattner7802f3e2007-02-25 09:06:15 +0000981 return SDOperand(LowerCallResult(Chain, InFlag, Op.Val, CC, DAG), Op.ResNo);
Chris Lattner76ac0682005-11-15 00:40:23 +0000982}
983
Evan Cheng11b0a5d2006-09-08 06:48:29 +0000984
985//===----------------------------------------------------------------------===//
986// X86-64 C Calling Convention implementation
987//===----------------------------------------------------------------------===//
988
989/// HowToPassX86_64CCCArgument - Returns how an formal argument of the specified
990/// type should be passed. If it is through stack, returns the size of the stack
991/// slot; if it is through integer or XMM register, returns the number of
992/// integer or XMM registers are needed.
993static void
994HowToPassX86_64CCCArgument(MVT::ValueType ObjectVT,
995 unsigned NumIntRegs, unsigned NumXMMRegs,
996 unsigned &ObjSize, unsigned &ObjIntRegs,
997 unsigned &ObjXMMRegs) {
998 ObjSize = 0;
999 ObjIntRegs = 0;
1000 ObjXMMRegs = 0;
1001
1002 switch (ObjectVT) {
1003 default: assert(0 && "Unhandled argument type!");
1004 case MVT::i8:
1005 case MVT::i16:
1006 case MVT::i32:
1007 case MVT::i64:
1008 if (NumIntRegs < 6)
1009 ObjIntRegs = 1;
1010 else {
1011 switch (ObjectVT) {
1012 default: break;
1013 case MVT::i8: ObjSize = 1; break;
1014 case MVT::i16: ObjSize = 2; break;
1015 case MVT::i32: ObjSize = 4; break;
1016 case MVT::i64: ObjSize = 8; break;
1017 }
1018 }
1019 break;
1020 case MVT::f32:
1021 case MVT::f64:
1022 case MVT::v16i8:
1023 case MVT::v8i16:
1024 case MVT::v4i32:
1025 case MVT::v2i64:
1026 case MVT::v4f32:
1027 case MVT::v2f64:
1028 if (NumXMMRegs < 8)
1029 ObjXMMRegs = 1;
1030 else {
1031 switch (ObjectVT) {
1032 default: break;
1033 case MVT::f32: ObjSize = 4; break;
1034 case MVT::f64: ObjSize = 8; break;
1035 case MVT::v16i8:
1036 case MVT::v8i16:
1037 case MVT::v4i32:
1038 case MVT::v2i64:
1039 case MVT::v4f32:
1040 case MVT::v2f64: ObjSize = 16; break;
1041 }
1042 break;
1043 }
1044 }
1045}
1046
1047SDOperand
1048X86TargetLowering::LowerX86_64CCCArguments(SDOperand Op, SelectionDAG &DAG) {
1049 unsigned NumArgs = Op.Val->getNumValues() - 1;
1050 MachineFunction &MF = DAG.getMachineFunction();
1051 MachineFrameInfo *MFI = MF.getFrameInfo();
1052 SDOperand Root = Op.getOperand(0);
1053 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0;
Chris Lattner35a08552007-02-25 07:10:00 +00001054 SmallVector<SDOperand, 8> ArgValues;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001055
1056 // Add DAG nodes to load the arguments... On entry to a function on the X86,
1057 // the stack frame looks like this:
1058 //
1059 // [RSP] -- return address
1060 // [RSP + 8] -- first nonreg argument (leftmost lexically)
1061 // [RSP +16] -- second nonreg argument, if 1st argument is <= 8 bytes in size
1062 // ...
1063 //
1064 unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot
1065 unsigned NumIntRegs = 0; // Int regs used for parameter passing.
1066 unsigned NumXMMRegs = 0; // XMM regs used for parameter passing.
1067
1068 static const unsigned GPR8ArgRegs[] = {
1069 X86::DIL, X86::SIL, X86::DL, X86::CL, X86::R8B, X86::R9B
1070 };
1071 static const unsigned GPR16ArgRegs[] = {
1072 X86::DI, X86::SI, X86::DX, X86::CX, X86::R8W, X86::R9W
1073 };
1074 static const unsigned GPR32ArgRegs[] = {
1075 X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D
1076 };
1077 static const unsigned GPR64ArgRegs[] = {
1078 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
1079 };
1080 static const unsigned XMMArgRegs[] = {
1081 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
1082 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
1083 };
1084
1085 for (unsigned i = 0; i < NumArgs; ++i) {
1086 MVT::ValueType ObjectVT = Op.getValue(i).getValueType();
1087 unsigned ArgIncrement = 8;
1088 unsigned ObjSize = 0;
1089 unsigned ObjIntRegs = 0;
1090 unsigned ObjXMMRegs = 0;
1091
1092 // FIXME: __int128 and long double support?
1093 HowToPassX86_64CCCArgument(ObjectVT, NumIntRegs, NumXMMRegs,
1094 ObjSize, ObjIntRegs, ObjXMMRegs);
1095 if (ObjSize > 8)
1096 ArgIncrement = ObjSize;
1097
1098 unsigned Reg = 0;
1099 SDOperand ArgValue;
1100 if (ObjIntRegs || ObjXMMRegs) {
1101 switch (ObjectVT) {
1102 default: assert(0 && "Unhandled argument type!");
1103 case MVT::i8:
1104 case MVT::i16:
1105 case MVT::i32:
1106 case MVT::i64: {
1107 TargetRegisterClass *RC = NULL;
1108 switch (ObjectVT) {
1109 default: break;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00001110 case MVT::i8:
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001111 RC = X86::GR8RegisterClass;
1112 Reg = GPR8ArgRegs[NumIntRegs];
1113 break;
1114 case MVT::i16:
1115 RC = X86::GR16RegisterClass;
1116 Reg = GPR16ArgRegs[NumIntRegs];
1117 break;
1118 case MVT::i32:
1119 RC = X86::GR32RegisterClass;
1120 Reg = GPR32ArgRegs[NumIntRegs];
1121 break;
1122 case MVT::i64:
1123 RC = X86::GR64RegisterClass;
1124 Reg = GPR64ArgRegs[NumIntRegs];
1125 break;
1126 }
1127 Reg = AddLiveIn(MF, Reg, RC);
1128 ArgValue = DAG.getCopyFromReg(Root, Reg, ObjectVT);
1129 break;
1130 }
1131 case MVT::f32:
1132 case MVT::f64:
1133 case MVT::v16i8:
1134 case MVT::v8i16:
1135 case MVT::v4i32:
1136 case MVT::v2i64:
1137 case MVT::v4f32:
1138 case MVT::v2f64: {
1139 TargetRegisterClass *RC= (ObjectVT == MVT::f32) ?
1140 X86::FR32RegisterClass : ((ObjectVT == MVT::f64) ?
1141 X86::FR64RegisterClass : X86::VR128RegisterClass);
1142 Reg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], RC);
1143 ArgValue = DAG.getCopyFromReg(Root, Reg, ObjectVT);
1144 break;
1145 }
1146 }
1147 NumIntRegs += ObjIntRegs;
1148 NumXMMRegs += ObjXMMRegs;
1149 } else if (ObjSize) {
1150 // XMM arguments have to be aligned on 16-byte boundary.
1151 if (ObjSize == 16)
1152 ArgOffset = ((ArgOffset + 15) / 16) * 16;
1153 // Create the SelectionDAG nodes corresponding to a load from this
1154 // parameter.
1155 int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
1156 SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy());
Evan Chenge71fe34d2006-10-09 20:57:25 +00001157 ArgValue = DAG.getLoad(Op.Val->getValueType(i), Root, FIN, NULL, 0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001158 ArgOffset += ArgIncrement; // Move on to the next argument.
1159 }
1160
1161 ArgValues.push_back(ArgValue);
1162 }
1163
1164 // If the function takes variable number of arguments, make a frame index for
1165 // the start of the first vararg value... for expansion of llvm.va_start.
1166 if (isVarArg) {
1167 // For X86-64, if there are vararg parameters that are passed via
1168 // registers, then we must store them to their spots on the stack so they
1169 // may be loaded by deferencing the result of va_next.
1170 VarArgsGPOffset = NumIntRegs * 8;
1171 VarArgsFPOffset = 6 * 8 + NumXMMRegs * 16;
1172 VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset);
1173 RegSaveFrameIndex = MFI->CreateStackObject(6 * 8 + 8 * 16, 16);
1174
1175 // Store the integer parameter registers.
Chris Lattner35a08552007-02-25 07:10:00 +00001176 SmallVector<SDOperand, 8> MemOps;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001177 SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy());
1178 SDOperand FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN,
1179 DAG.getConstant(VarArgsGPOffset, getPointerTy()));
1180 for (; NumIntRegs != 6; ++NumIntRegs) {
1181 unsigned VReg = AddLiveIn(MF, GPR64ArgRegs[NumIntRegs],
1182 X86::GR64RegisterClass);
1183 SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::i64);
Evan Chengab51cf22006-10-13 21:14:26 +00001184 SDOperand Store = DAG.getStore(Val.getValue(1), Val, FIN, NULL, 0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001185 MemOps.push_back(Store);
1186 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
1187 DAG.getConstant(8, getPointerTy()));
1188 }
1189
1190 // Now store the XMM (fp + vector) parameter registers.
1191 FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN,
1192 DAG.getConstant(VarArgsFPOffset, getPointerTy()));
1193 for (; NumXMMRegs != 8; ++NumXMMRegs) {
1194 unsigned VReg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs],
1195 X86::VR128RegisterClass);
1196 SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::v4f32);
Evan Chengab51cf22006-10-13 21:14:26 +00001197 SDOperand Store = DAG.getStore(Val.getValue(1), Val, FIN, NULL, 0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001198 MemOps.push_back(Store);
1199 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
1200 DAG.getConstant(16, getPointerTy()));
1201 }
1202 if (!MemOps.empty())
1203 Root = DAG.getNode(ISD::TokenFactor, MVT::Other,
1204 &MemOps[0], MemOps.size());
1205 }
1206
1207 ArgValues.push_back(Root);
1208
1209 ReturnAddrIndex = 0; // No return address slot generated yet.
1210 BytesToPopOnReturn = 0; // Callee pops nothing.
1211 BytesCallerReserves = ArgOffset;
1212
1213 // Return the new list of results.
Chris Lattner35a08552007-02-25 07:10:00 +00001214 return DAG.getNode(ISD::MERGE_VALUES, Op.Val->getVTList(),
1215 &ArgValues[0], ArgValues.size());
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001216}
1217
1218SDOperand
Chris Lattner7802f3e2007-02-25 09:06:15 +00001219X86TargetLowering::LowerX86_64CCCCallTo(SDOperand Op, SelectionDAG &DAG,
1220 unsigned CallingConv) {
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001221 SDOperand Chain = Op.getOperand(0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001222 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0;
1223 bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0;
1224 SDOperand Callee = Op.getOperand(4);
1225 MVT::ValueType RetVT= Op.Val->getValueType(0);
1226 unsigned NumOps = (Op.getNumOperands() - 5) / 2;
1227
1228 // Count how many bytes are to be pushed on the stack.
1229 unsigned NumBytes = 0;
1230 unsigned NumIntRegs = 0; // Int regs used for parameter passing.
1231 unsigned NumXMMRegs = 0; // XMM regs used for parameter passing.
1232
1233 static const unsigned GPR8ArgRegs[] = {
1234 X86::DIL, X86::SIL, X86::DL, X86::CL, X86::R8B, X86::R9B
1235 };
1236 static const unsigned GPR16ArgRegs[] = {
1237 X86::DI, X86::SI, X86::DX, X86::CX, X86::R8W, X86::R9W
1238 };
1239 static const unsigned GPR32ArgRegs[] = {
1240 X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D
1241 };
1242 static const unsigned GPR64ArgRegs[] = {
1243 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
1244 };
1245 static const unsigned XMMArgRegs[] = {
1246 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
1247 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
1248 };
1249
1250 for (unsigned i = 0; i != NumOps; ++i) {
1251 SDOperand Arg = Op.getOperand(5+2*i);
1252 MVT::ValueType ArgVT = Arg.getValueType();
1253
1254 switch (ArgVT) {
1255 default: assert(0 && "Unknown value type!");
1256 case MVT::i8:
1257 case MVT::i16:
1258 case MVT::i32:
1259 case MVT::i64:
1260 if (NumIntRegs < 6)
1261 ++NumIntRegs;
1262 else
1263 NumBytes += 8;
1264 break;
1265 case MVT::f32:
1266 case MVT::f64:
1267 case MVT::v16i8:
1268 case MVT::v8i16:
1269 case MVT::v4i32:
1270 case MVT::v2i64:
1271 case MVT::v4f32:
1272 case MVT::v2f64:
1273 if (NumXMMRegs < 8)
1274 NumXMMRegs++;
1275 else if (ArgVT == MVT::f32 || ArgVT == MVT::f64)
1276 NumBytes += 8;
1277 else {
1278 // XMM arguments have to be aligned on 16-byte boundary.
1279 NumBytes = ((NumBytes + 15) / 16) * 16;
1280 NumBytes += 16;
1281 }
1282 break;
1283 }
1284 }
1285
1286 Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy()));
1287
1288 // Arguments go on the stack in reverse order, as specified by the ABI.
1289 unsigned ArgOffset = 0;
1290 NumIntRegs = 0;
1291 NumXMMRegs = 0;
Chris Lattner35a08552007-02-25 07:10:00 +00001292 SmallVector<std::pair<unsigned, SDOperand>, 8> RegsToPass;
1293 SmallVector<SDOperand, 8> MemOpChains;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001294 SDOperand StackPtr = DAG.getRegister(X86StackPtr, getPointerTy());
1295 for (unsigned i = 0; i != NumOps; ++i) {
1296 SDOperand Arg = Op.getOperand(5+2*i);
1297 MVT::ValueType ArgVT = Arg.getValueType();
1298
1299 switch (ArgVT) {
1300 default: assert(0 && "Unexpected ValueType for argument!");
1301 case MVT::i8:
1302 case MVT::i16:
1303 case MVT::i32:
1304 case MVT::i64:
1305 if (NumIntRegs < 6) {
1306 unsigned Reg = 0;
1307 switch (ArgVT) {
1308 default: break;
1309 case MVT::i8: Reg = GPR8ArgRegs[NumIntRegs]; break;
1310 case MVT::i16: Reg = GPR16ArgRegs[NumIntRegs]; break;
1311 case MVT::i32: Reg = GPR32ArgRegs[NumIntRegs]; break;
1312 case MVT::i64: Reg = GPR64ArgRegs[NumIntRegs]; break;
1313 }
1314 RegsToPass.push_back(std::make_pair(Reg, Arg));
1315 ++NumIntRegs;
1316 } else {
1317 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
1318 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
Evan Chengab51cf22006-10-13 21:14:26 +00001319 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001320 ArgOffset += 8;
1321 }
1322 break;
1323 case MVT::f32:
1324 case MVT::f64:
1325 case MVT::v16i8:
1326 case MVT::v8i16:
1327 case MVT::v4i32:
1328 case MVT::v2i64:
1329 case MVT::v4f32:
1330 case MVT::v2f64:
1331 if (NumXMMRegs < 8) {
1332 RegsToPass.push_back(std::make_pair(XMMArgRegs[NumXMMRegs], Arg));
1333 NumXMMRegs++;
1334 } else {
1335 if (ArgVT != MVT::f32 && ArgVT != MVT::f64) {
1336 // XMM arguments have to be aligned on 16-byte boundary.
1337 ArgOffset = ((ArgOffset + 15) / 16) * 16;
1338 }
1339 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
1340 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
Evan Chengab51cf22006-10-13 21:14:26 +00001341 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001342 if (ArgVT == MVT::f32 || ArgVT == MVT::f64)
1343 ArgOffset += 8;
1344 else
1345 ArgOffset += 16;
1346 }
1347 }
1348 }
1349
1350 if (!MemOpChains.empty())
1351 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
1352 &MemOpChains[0], MemOpChains.size());
1353
1354 // Build a sequence of copy-to-reg nodes chained together with token chain
1355 // and flag operands which copy the outgoing args into registers.
1356 SDOperand InFlag;
1357 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
1358 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second,
1359 InFlag);
1360 InFlag = Chain.getValue(1);
1361 }
1362
1363 if (isVarArg) {
1364 // From AMD64 ABI document:
1365 // For calls that may call functions that use varargs or stdargs
1366 // (prototype-less calls or calls to functions containing ellipsis (...) in
1367 // the declaration) %al is used as hidden argument to specify the number
1368 // of SSE registers used. The contents of %al do not need to match exactly
1369 // the number of registers, but must be an ubound on the number of SSE
1370 // registers used and is in the range 0 - 8 inclusive.
1371 Chain = DAG.getCopyToReg(Chain, X86::AL,
1372 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag);
1373 InFlag = Chain.getValue(1);
1374 }
1375
1376 // If the callee is a GlobalAddress node (quite common, every direct call is)
1377 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
Anton Korobeynikov37d080b2006-11-20 10:46:14 +00001378 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
Anton Korobeynikov430e68a12006-12-22 22:29:05 +00001379 // We should use extra load for direct calls to dllimported functions in
1380 // non-JIT mode.
1381 if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(),
1382 getTargetMachine(), true))
Anton Korobeynikov37d080b2006-11-20 10:46:14 +00001383 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy());
1384 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001385 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
1386
Chris Lattnere56fef92007-02-25 06:40:16 +00001387 // Returns a chain & a flag for retval copy to use.
1388 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Chris Lattner35a08552007-02-25 07:10:00 +00001389 SmallVector<SDOperand, 8> Ops;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001390 Ops.push_back(Chain);
1391 Ops.push_back(Callee);
1392
1393 // Add argument registers to the end of the list so that they are known live
1394 // into the call.
1395 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00001396 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001397 RegsToPass[i].second.getValueType()));
1398
1399 if (InFlag.Val)
1400 Ops.push_back(InFlag);
1401
1402 // FIXME: Do not generate X86ISD::TAILCALL for now.
1403 Chain = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
1404 NodeTys, &Ops[0], Ops.size());
1405 InFlag = Chain.getValue(1);
1406
Chris Lattnerd6b853ad2007-02-25 07:18:38 +00001407 // Returns a flag for retval copy to use.
1408 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001409 Ops.clear();
1410 Ops.push_back(Chain);
1411 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
1412 Ops.push_back(DAG.getConstant(0, getPointerTy()));
1413 Ops.push_back(InFlag);
1414 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size());
1415 if (RetVT != MVT::Other)
1416 InFlag = Chain.getValue(1);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00001417
Chris Lattner35a08552007-02-25 07:10:00 +00001418 SmallVector<SDOperand, 8> ResultVals;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001419 switch (RetVT) {
1420 default: assert(0 && "Unknown value type to return!");
Chris Lattnerd6b853ad2007-02-25 07:18:38 +00001421 case MVT::Other:
1422 NodeTys = DAG.getVTList(MVT::Other);
1423 break;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001424 case MVT::i8:
1425 Chain = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag).getValue(1);
1426 ResultVals.push_back(Chain.getValue(0));
Chris Lattnere56fef92007-02-25 06:40:16 +00001427 NodeTys = DAG.getVTList(MVT::i8, MVT::Other);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001428 break;
1429 case MVT::i16:
1430 Chain = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag).getValue(1);
1431 ResultVals.push_back(Chain.getValue(0));
Chris Lattnere56fef92007-02-25 06:40:16 +00001432 NodeTys = DAG.getVTList(MVT::i16, MVT::Other);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001433 break;
1434 case MVT::i32:
1435 Chain = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag).getValue(1);
1436 ResultVals.push_back(Chain.getValue(0));
Chris Lattnere56fef92007-02-25 06:40:16 +00001437 NodeTys = DAG.getVTList(MVT::i32, MVT::Other);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001438 break;
1439 case MVT::i64:
1440 if (Op.Val->getValueType(1) == MVT::i64) {
1441 // FIXME: __int128 support?
1442 Chain = DAG.getCopyFromReg(Chain, X86::RAX, MVT::i64, InFlag).getValue(1);
1443 ResultVals.push_back(Chain.getValue(0));
1444 Chain = DAG.getCopyFromReg(Chain, X86::RDX, MVT::i64,
1445 Chain.getValue(2)).getValue(1);
1446 ResultVals.push_back(Chain.getValue(0));
Chris Lattnere56fef92007-02-25 06:40:16 +00001447 NodeTys = DAG.getVTList(MVT::i64, MVT::i64, MVT::Other);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001448 } else {
1449 Chain = DAG.getCopyFromReg(Chain, X86::RAX, MVT::i64, InFlag).getValue(1);
1450 ResultVals.push_back(Chain.getValue(0));
Chris Lattnere56fef92007-02-25 06:40:16 +00001451 NodeTys = DAG.getVTList(MVT::i64, MVT::Other);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001452 }
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001453 break;
1454 case MVT::f32:
1455 case MVT::f64:
1456 case MVT::v16i8:
1457 case MVT::v8i16:
1458 case MVT::v4i32:
1459 case MVT::v2i64:
1460 case MVT::v4f32:
1461 case MVT::v2f64:
1462 // FIXME: long double support?
1463 Chain = DAG.getCopyFromReg(Chain, X86::XMM0, RetVT, InFlag).getValue(1);
1464 ResultVals.push_back(Chain.getValue(0));
Chris Lattnere56fef92007-02-25 06:40:16 +00001465 NodeTys = DAG.getVTList(RetVT, MVT::Other);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001466 break;
1467 }
1468
Chris Lattnerd6b853ad2007-02-25 07:18:38 +00001469 // Merge everything together with a MERGE_VALUES node.
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001470 ResultVals.push_back(Chain);
1471 SDOperand Res = DAG.getNode(ISD::MERGE_VALUES, NodeTys,
1472 &ResultVals[0], ResultVals.size());
1473 return Res.getValue(Op.ResNo);
1474}
1475
Chris Lattner76ac0682005-11-15 00:40:23 +00001476//===----------------------------------------------------------------------===//
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001477// Fast & FastCall Calling Convention implementation
Chris Lattner76ac0682005-11-15 00:40:23 +00001478//===----------------------------------------------------------------------===//
1479//
1480// The X86 'fast' calling convention passes up to two integer arguments in
1481// registers (an appropriate portion of EAX/EDX), passes arguments in C order,
1482// and requires that the callee pop its arguments off the stack (allowing proper
1483// tail calls), and has the same return value conventions as C calling convs.
1484//
1485// This calling convention always arranges for the callee pop value to be 8n+4
1486// bytes, which is needed for tail recursion elimination and stack alignment
1487// reasons.
1488//
1489// Note that this can be enhanced in the future to pass fp vals in registers
1490// (when we have a global fp allocator) and do other tricks.
1491//
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001492//===----------------------------------------------------------------------===//
1493// The X86 'fastcall' calling convention passes up to two integer arguments in
1494// registers (an appropriate portion of ECX/EDX), passes arguments in C order,
1495// and requires that the callee pop its arguments off the stack (allowing proper
1496// tail calls), and has the same return value conventions as C calling convs.
1497//
1498// This calling convention always arranges for the callee pop value to be 8n+4
1499// bytes, which is needed for tail recursion elimination and stack alignment
1500// reasons.
Chris Lattner76ac0682005-11-15 00:40:23 +00001501
Evan Cheng48940d12006-04-27 01:32:22 +00001502
Evan Cheng17e734f2006-05-23 21:06:34 +00001503SDOperand
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001504X86TargetLowering::LowerFastCCArguments(SDOperand Op, SelectionDAG &DAG,
1505 bool isFastCall) {
Evan Cheng17e734f2006-05-23 21:06:34 +00001506 unsigned NumArgs = Op.Val->getNumValues()-1;
Chris Lattner76ac0682005-11-15 00:40:23 +00001507 MachineFunction &MF = DAG.getMachineFunction();
1508 MachineFrameInfo *MFI = MF.getFrameInfo();
Evan Cheng17e734f2006-05-23 21:06:34 +00001509 SDOperand Root = Op.getOperand(0);
Chris Lattner35a08552007-02-25 07:10:00 +00001510 SmallVector<SDOperand, 8> ArgValues;
Chris Lattner76ac0682005-11-15 00:40:23 +00001511
Evan Cheng48940d12006-04-27 01:32:22 +00001512 // Add DAG nodes to load the arguments... On entry to a function the stack
1513 // frame looks like this:
1514 //
1515 // [ESP] -- return address
1516 // [ESP + 4] -- first nonreg argument (leftmost lexically)
Evan Chengcbfb3d02006-05-26 18:37:16 +00001517 // [ESP + 8] -- second nonreg argument, if 1st argument is <= 4 bytes in size
Evan Cheng48940d12006-04-27 01:32:22 +00001518 // ...
Chris Lattner76ac0682005-11-15 00:40:23 +00001519 unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot
1520
1521 // Keep track of the number of integer regs passed so far. This can be either
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001522 // 0 (neither EAX/ECX or EDX used), 1 (EAX/ECX is used) or 2 (EAX/ECX and EDX
1523 // are both used).
Chris Lattner76ac0682005-11-15 00:40:23 +00001524 unsigned NumIntRegs = 0;
Evan Cheng89001ad2006-04-27 08:31:10 +00001525 unsigned NumXMMRegs = 0; // XMM regs used for parameter passing.
Evan Cheng2a330942006-05-25 00:59:30 +00001526
1527 static const unsigned XMMArgRegs[] = {
Evan Chengbfb5ea62006-05-26 19:22:06 +00001528 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3
Evan Cheng2a330942006-05-25 00:59:30 +00001529 };
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00001530
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001531 static const unsigned GPRArgRegs[][2][2] = {
1532 {{ X86::AL, X86::DL }, { X86::CL, X86::DL }},
1533 {{ X86::AX, X86::DX }, { X86::CX, X86::DX }},
1534 {{ X86::EAX, X86::EDX }, { X86::ECX, X86::EDX }}
1535 };
1536
1537 static const TargetRegisterClass* GPRClasses[3] = {
1538 X86::GR8RegisterClass, X86::GR16RegisterClass, X86::GR32RegisterClass
1539 };
1540
1541 unsigned GPRInd = (isFastCall ? 1 : 0);
Evan Chenge0bcfbe2006-04-26 01:20:17 +00001542 for (unsigned i = 0; i < NumArgs; ++i) {
Evan Cheng17e734f2006-05-23 21:06:34 +00001543 MVT::ValueType ObjectVT = Op.getValue(i).getValueType();
1544 unsigned ArgIncrement = 4;
1545 unsigned ObjSize = 0;
Evan Cheng17e734f2006-05-23 21:06:34 +00001546 unsigned ObjXMMRegs = 0;
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001547 unsigned ObjIntRegs = 0;
1548 unsigned Reg = 0;
1549 SDOperand ArgValue;
Chris Lattner76ac0682005-11-15 00:40:23 +00001550
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001551 HowToPassCallArgument(ObjectVT,
1552 true, // Use as much registers as possible
1553 NumIntRegs, NumXMMRegs,
1554 (isFastCall ? 2 : FASTCC_NUM_INT_ARGS_INREGS),
1555 ObjSize, ObjIntRegs, ObjXMMRegs,
1556 !isFastCall);
1557
Evan Chenga01e7992006-05-26 18:39:59 +00001558 if (ObjSize > 4)
Evan Cheng17e734f2006-05-23 21:06:34 +00001559 ArgIncrement = ObjSize;
Evan Cheng48940d12006-04-27 01:32:22 +00001560
Evan Cheng17e734f2006-05-23 21:06:34 +00001561 if (ObjIntRegs || ObjXMMRegs) {
1562 switch (ObjectVT) {
1563 default: assert(0 && "Unhandled argument type!");
Evan Cheng17e734f2006-05-23 21:06:34 +00001564 case MVT::i8:
Evan Cheng17e734f2006-05-23 21:06:34 +00001565 case MVT::i16:
Nick Lewycky0c497222007-01-28 15:39:16 +00001566 case MVT::i32: {
1567 unsigned RegToUse = GPRArgRegs[ObjectVT-MVT::i8][GPRInd][NumIntRegs];
1568 Reg = AddLiveIn(MF, RegToUse, GPRClasses[ObjectVT-MVT::i8]);
1569 ArgValue = DAG.getCopyFromReg(Root, Reg, ObjectVT);
1570 break;
1571 }
Evan Cheng17e734f2006-05-23 21:06:34 +00001572 case MVT::v16i8:
1573 case MVT::v8i16:
1574 case MVT::v4i32:
1575 case MVT::v2i64:
1576 case MVT::v4f32:
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001577 case MVT::v2f64: {
1578 assert(!isFastCall && "Unhandled argument type!");
Evan Cheng17e734f2006-05-23 21:06:34 +00001579 Reg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], X86::VR128RegisterClass);
1580 ArgValue = DAG.getCopyFromReg(Root, Reg, ObjectVT);
1581 break;
Evan Cheng48940d12006-04-27 01:32:22 +00001582 }
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001583 }
Evan Cheng17e734f2006-05-23 21:06:34 +00001584 NumIntRegs += ObjIntRegs;
1585 NumXMMRegs += ObjXMMRegs;
Chris Lattner76ac0682005-11-15 00:40:23 +00001586 }
Evan Cheng17e734f2006-05-23 21:06:34 +00001587 if (ObjSize) {
Evan Chengb92f4182006-05-26 20:37:47 +00001588 // XMM arguments have to be aligned on 16-byte boundary.
1589 if (ObjSize == 16)
1590 ArgOffset = ((ArgOffset + 15) / 16) * 16;
Evan Cheng17e734f2006-05-23 21:06:34 +00001591 // Create the SelectionDAG nodes corresponding to a load from this
1592 // parameter.
1593 int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
1594 SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy());
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001595 ArgValue = DAG.getLoad(Op.Val->getValueType(i), Root, FIN, NULL, 0);
1596
Evan Cheng17e734f2006-05-23 21:06:34 +00001597 ArgOffset += ArgIncrement; // Move on to the next argument.
1598 }
1599
1600 ArgValues.push_back(ArgValue);
Chris Lattner76ac0682005-11-15 00:40:23 +00001601 }
1602
Evan Cheng17e734f2006-05-23 21:06:34 +00001603 ArgValues.push_back(Root);
1604
Chris Lattner76ac0682005-11-15 00:40:23 +00001605 // Make sure the instruction takes 8n+4 bytes to make sure the start of the
1606 // arguments and the arguments after the retaddr has been pushed are aligned.
1607 if ((ArgOffset & 7) == 0)
1608 ArgOffset += 4;
1609
1610 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs.
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001611 RegSaveFrameIndex = 0xAAAAAAA; // X86-64 only.
Chris Lattner76ac0682005-11-15 00:40:23 +00001612 ReturnAddrIndex = 0; // No return address slot generated yet.
1613 BytesToPopOnReturn = ArgOffset; // Callee pops all stack arguments.
1614 BytesCallerReserves = 0;
1615
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001616 MF.getInfo<X86FunctionInfo>()->setBytesToPopOnReturn(BytesToPopOnReturn);
1617
Chris Lattner76ac0682005-11-15 00:40:23 +00001618 // Finally, inform the code generator which regs we return values in.
Evan Cheng17e734f2006-05-23 21:06:34 +00001619 switch (getValueType(MF.getFunction()->getReturnType())) {
Chris Lattner76ac0682005-11-15 00:40:23 +00001620 default: assert(0 && "Unknown type!");
1621 case MVT::isVoid: break;
Chris Lattnerf598d732006-10-03 17:18:42 +00001622 case MVT::i1:
Chris Lattner76ac0682005-11-15 00:40:23 +00001623 case MVT::i8:
1624 case MVT::i16:
1625 case MVT::i32:
1626 MF.addLiveOut(X86::EAX);
1627 break;
1628 case MVT::i64:
1629 MF.addLiveOut(X86::EAX);
1630 MF.addLiveOut(X86::EDX);
1631 break;
1632 case MVT::f32:
1633 case MVT::f64:
1634 MF.addLiveOut(X86::ST0);
1635 break;
Evan Cheng5ee96892006-05-25 18:56:34 +00001636 case MVT::v16i8:
1637 case MVT::v8i16:
1638 case MVT::v4i32:
1639 case MVT::v2i64:
1640 case MVT::v4f32:
1641 case MVT::v2f64:
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001642 assert(!isFastCall && "Unknown result type");
Evan Cheng88decde2006-04-28 21:29:37 +00001643 MF.addLiveOut(X86::XMM0);
1644 break;
1645 }
Evan Cheng88decde2006-04-28 21:29:37 +00001646
Evan Cheng17e734f2006-05-23 21:06:34 +00001647 // Return the new list of results.
Chris Lattner35a08552007-02-25 07:10:00 +00001648 return DAG.getNode(ISD::MERGE_VALUES, Op.Val->getVTList(),
1649 &ArgValues[0], ArgValues.size());
Chris Lattner76ac0682005-11-15 00:40:23 +00001650}
1651
Chris Lattner104aa5d2006-09-26 03:57:53 +00001652SDOperand X86TargetLowering::LowerFastCCCallTo(SDOperand Op, SelectionDAG &DAG,
Chris Lattner7802f3e2007-02-25 09:06:15 +00001653 unsigned CC) {
Evan Cheng2a330942006-05-25 00:59:30 +00001654 SDOperand Chain = Op.getOperand(0);
Evan Cheng2a330942006-05-25 00:59:30 +00001655 bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0;
1656 SDOperand Callee = Op.getOperand(4);
1657 MVT::ValueType RetVT= Op.Val->getValueType(0);
1658 unsigned NumOps = (Op.getNumOperands() - 5) / 2;
1659
Chris Lattner76ac0682005-11-15 00:40:23 +00001660 // Count how many bytes are to be pushed on the stack.
1661 unsigned NumBytes = 0;
1662
1663 // Keep track of the number of integer regs passed so far. This can be either
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001664 // 0 (neither EAX/ECX or EDX used), 1 (EAX/ECX is used) or 2 (EAX/ECX and EDX
1665 // are both used).
Chris Lattner76ac0682005-11-15 00:40:23 +00001666 unsigned NumIntRegs = 0;
Evan Cheng2a330942006-05-25 00:59:30 +00001667 unsigned NumXMMRegs = 0; // XMM regs used for parameter passing.
Chris Lattner76ac0682005-11-15 00:40:23 +00001668
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001669 static const unsigned GPRArgRegs[][2][2] = {
1670 {{ X86::AL, X86::DL }, { X86::CL, X86::DL }},
1671 {{ X86::AX, X86::DX }, { X86::CX, X86::DX }},
1672 {{ X86::EAX, X86::EDX }, { X86::ECX, X86::EDX }}
Evan Cheng2a330942006-05-25 00:59:30 +00001673 };
1674 static const unsigned XMMArgRegs[] = {
Evan Chengbfb5ea62006-05-26 19:22:06 +00001675 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3
Evan Cheng2a330942006-05-25 00:59:30 +00001676 };
1677
Chris Lattner7802f3e2007-02-25 09:06:15 +00001678 bool isFastCall = CC == CallingConv::X86_FastCall;
1679 unsigned GPRInd = isFastCall ? 1 : 0;
Evan Cheng2a330942006-05-25 00:59:30 +00001680 for (unsigned i = 0; i != NumOps; ++i) {
1681 SDOperand Arg = Op.getOperand(5+2*i);
1682
1683 switch (Arg.getValueType()) {
Chris Lattner76ac0682005-11-15 00:40:23 +00001684 default: assert(0 && "Unknown value type!");
Chris Lattner76ac0682005-11-15 00:40:23 +00001685 case MVT::i8:
1686 case MVT::i16:
Nick Lewyckyc68bbef2006-09-21 02:08:31 +00001687 case MVT::i32: {
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00001688 unsigned MaxNumIntRegs = (isFastCall ? 2 : FASTCC_NUM_INT_ARGS_INREGS);
1689 if (NumIntRegs < MaxNumIntRegs) {
1690 ++NumIntRegs;
1691 break;
1692 }
Nick Lewyckyc68bbef2006-09-21 02:08:31 +00001693 } // Fall through
Chris Lattner76ac0682005-11-15 00:40:23 +00001694 case MVT::f32:
1695 NumBytes += 4;
1696 break;
Chris Lattner76ac0682005-11-15 00:40:23 +00001697 case MVT::f64:
1698 NumBytes += 8;
1699 break;
Evan Cheng2a330942006-05-25 00:59:30 +00001700 case MVT::v16i8:
1701 case MVT::v8i16:
1702 case MVT::v4i32:
1703 case MVT::v2i64:
1704 case MVT::v4f32:
Evan Cheng5ee96892006-05-25 18:56:34 +00001705 case MVT::v2f64:
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001706 assert(!isFastCall && "Unknown value type!");
1707 if (NumXMMRegs < 4)
1708 NumXMMRegs++;
1709 else {
1710 // XMM arguments have to be aligned on 16-byte boundary.
1711 NumBytes = ((NumBytes + 15) / 16) * 16;
1712 NumBytes += 16;
1713 }
1714 break;
Chris Lattner76ac0682005-11-15 00:40:23 +00001715 }
Evan Cheng2a330942006-05-25 00:59:30 +00001716 }
Chris Lattner76ac0682005-11-15 00:40:23 +00001717
1718 // Make sure the instruction takes 8n+4 bytes to make sure the start of the
1719 // arguments and the arguments after the retaddr has been pushed are aligned.
1720 if ((NumBytes & 7) == 0)
1721 NumBytes += 4;
1722
Chris Lattner62c34842006-02-13 09:00:43 +00001723 Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy()));
Chris Lattner76ac0682005-11-15 00:40:23 +00001724
1725 // Arguments go on the stack in reverse order, as specified by the ABI.
1726 unsigned ArgOffset = 0;
Chris Lattner76ac0682005-11-15 00:40:23 +00001727 NumIntRegs = 0;
Chris Lattner35a08552007-02-25 07:10:00 +00001728 SmallVector<std::pair<unsigned, SDOperand>, 8> RegsToPass;
1729 SmallVector<SDOperand, 8> MemOpChains;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001730 SDOperand StackPtr = DAG.getRegister(X86StackPtr, getPointerTy());
Evan Cheng2a330942006-05-25 00:59:30 +00001731 for (unsigned i = 0; i != NumOps; ++i) {
1732 SDOperand Arg = Op.getOperand(5+2*i);
1733
1734 switch (Arg.getValueType()) {
Chris Lattner76ac0682005-11-15 00:40:23 +00001735 default: assert(0 && "Unexpected ValueType for argument!");
Chris Lattner76ac0682005-11-15 00:40:23 +00001736 case MVT::i8:
1737 case MVT::i16:
Nick Lewyckyc68bbef2006-09-21 02:08:31 +00001738 case MVT::i32: {
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00001739 unsigned MaxNumIntRegs = (isFastCall ? 2 : FASTCC_NUM_INT_ARGS_INREGS);
1740 if (NumIntRegs < MaxNumIntRegs) {
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001741 unsigned RegToUse =
1742 GPRArgRegs[Arg.getValueType()-MVT::i8][GPRInd][NumIntRegs];
1743 RegsToPass.push_back(std::make_pair(RegToUse, Arg));
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00001744 ++NumIntRegs;
1745 break;
1746 }
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001747 } // Fall through
Chris Lattner76ac0682005-11-15 00:40:23 +00001748 case MVT::f32: {
1749 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
Evan Cheng2a330942006-05-25 00:59:30 +00001750 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
Evan Chengab51cf22006-10-13 21:14:26 +00001751 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
Chris Lattner76ac0682005-11-15 00:40:23 +00001752 ArgOffset += 4;
1753 break;
1754 }
Evan Cheng2a330942006-05-25 00:59:30 +00001755 case MVT::f64: {
Chris Lattner76ac0682005-11-15 00:40:23 +00001756 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
Evan Cheng2a330942006-05-25 00:59:30 +00001757 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
Evan Chengab51cf22006-10-13 21:14:26 +00001758 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
Chris Lattner76ac0682005-11-15 00:40:23 +00001759 ArgOffset += 8;
1760 break;
1761 }
Evan Cheng2a330942006-05-25 00:59:30 +00001762 case MVT::v16i8:
1763 case MVT::v8i16:
1764 case MVT::v4i32:
1765 case MVT::v2i64:
1766 case MVT::v4f32:
Evan Cheng5ee96892006-05-25 18:56:34 +00001767 case MVT::v2f64:
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001768 assert(!isFastCall && "Unexpected ValueType for argument!");
1769 if (NumXMMRegs < 4) {
1770 RegsToPass.push_back(std::make_pair(XMMArgRegs[NumXMMRegs], Arg));
1771 NumXMMRegs++;
1772 } else {
1773 // XMM arguments have to be aligned on 16-byte boundary.
1774 ArgOffset = ((ArgOffset + 15) / 16) * 16;
1775 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
1776 PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
1777 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
1778 ArgOffset += 16;
1779 }
1780 break;
Evan Cheng2a330942006-05-25 00:59:30 +00001781 }
Chris Lattner76ac0682005-11-15 00:40:23 +00001782 }
Chris Lattner76ac0682005-11-15 00:40:23 +00001783
Evan Cheng2a330942006-05-25 00:59:30 +00001784 if (!MemOpChains.empty())
Chris Lattnerc24a1d32006-08-08 02:23:42 +00001785 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
1786 &MemOpChains[0], MemOpChains.size());
Chris Lattner76ac0682005-11-15 00:40:23 +00001787
Nate Begeman7e5496d2006-02-17 00:03:04 +00001788 // Build a sequence of copy-to-reg nodes chained together with token chain
1789 // and flag operands which copy the outgoing args into registers.
1790 SDOperand InFlag;
Evan Cheng2a330942006-05-25 00:59:30 +00001791 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
1792 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second,
1793 InFlag);
Nate Begeman7e5496d2006-02-17 00:03:04 +00001794 InFlag = Chain.getValue(1);
1795 }
1796
Evan Cheng2a330942006-05-25 00:59:30 +00001797 // If the callee is a GlobalAddress node (quite common, every direct call is)
1798 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
Anton Korobeynikov37d080b2006-11-20 10:46:14 +00001799 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
Anton Korobeynikov430e68a12006-12-22 22:29:05 +00001800 // We should use extra load for direct calls to dllimported functions in
1801 // non-JIT mode.
1802 if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(),
1803 getTargetMachine(), true))
Anton Korobeynikov37d080b2006-11-20 10:46:14 +00001804 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy());
1805 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
Evan Cheng2a330942006-05-25 00:59:30 +00001806 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
1807
Evan Cheng84a041e2007-02-21 21:18:14 +00001808 // ELF / PIC requires GOT in the EBX register before function calls via PLT
1809 // GOT pointer.
Anton Korobeynikov037c8672007-01-28 13:31:35 +00001810 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
1811 Subtarget->isPICStyleGOT()) {
1812 Chain = DAG.getCopyToReg(Chain, X86::EBX,
1813 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
1814 InFlag);
1815 InFlag = Chain.getValue(1);
1816 }
1817
Chris Lattnere56fef92007-02-25 06:40:16 +00001818 // Returns a chain & a flag for retval copy to use.
1819 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Chris Lattner35a08552007-02-25 07:10:00 +00001820 SmallVector<SDOperand, 8> Ops;
Nate Begeman7e5496d2006-02-17 00:03:04 +00001821 Ops.push_back(Chain);
1822 Ops.push_back(Callee);
Evan Chengca254862006-06-14 18:17:40 +00001823
1824 // Add argument registers to the end of the list so that they are known live
1825 // into the call.
1826 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00001827 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
Evan Chengca254862006-06-14 18:17:40 +00001828 RegsToPass[i].second.getValueType()));
1829
Evan Cheng84a041e2007-02-21 21:18:14 +00001830 // Add an implicit use GOT pointer in EBX.
1831 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
1832 Subtarget->isPICStyleGOT())
1833 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy()));
1834
Nate Begeman7e5496d2006-02-17 00:03:04 +00001835 if (InFlag.Val)
1836 Ops.push_back(InFlag);
1837
1838 // FIXME: Do not generate X86ISD::TAILCALL for now.
Chris Lattner3d826992006-05-16 06:45:34 +00001839 Chain = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
Chris Lattnerc24a1d32006-08-08 02:23:42 +00001840 NodeTys, &Ops[0], Ops.size());
Nate Begeman7e5496d2006-02-17 00:03:04 +00001841 InFlag = Chain.getValue(1);
1842
Chris Lattnerd6b853ad2007-02-25 07:18:38 +00001843 // Returns a flag for retval copy to use.
1844 NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Nate Begeman7e5496d2006-02-17 00:03:04 +00001845 Ops.clear();
1846 Ops.push_back(Chain);
Evan Cheng2a330942006-05-25 00:59:30 +00001847 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
1848 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
Nate Begeman7e5496d2006-02-17 00:03:04 +00001849 Ops.push_back(InFlag);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00001850 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size());
Evan Cheng2a330942006-05-25 00:59:30 +00001851 if (RetVT != MVT::Other)
1852 InFlag = Chain.getValue(1);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00001853
Chris Lattner35a08552007-02-25 07:10:00 +00001854 SmallVector<SDOperand, 8> ResultVals;
Evan Cheng2a330942006-05-25 00:59:30 +00001855 switch (RetVT) {
1856 default: assert(0 && "Unknown value type to return!");
Chris Lattnerd6b853ad2007-02-25 07:18:38 +00001857 case MVT::Other:
1858 NodeTys = DAG.getVTList(MVT::Other);
1859 break;
Evan Cheng2a330942006-05-25 00:59:30 +00001860 case MVT::i8:
1861 Chain = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag).getValue(1);
1862 ResultVals.push_back(Chain.getValue(0));
Chris Lattnere56fef92007-02-25 06:40:16 +00001863 NodeTys = DAG.getVTList(MVT::i8, MVT::Other);
Evan Cheng2a330942006-05-25 00:59:30 +00001864 break;
1865 case MVT::i16:
1866 Chain = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag).getValue(1);
1867 ResultVals.push_back(Chain.getValue(0));
Chris Lattnere56fef92007-02-25 06:40:16 +00001868 NodeTys = DAG.getVTList(MVT::i16, MVT::Other);
Evan Cheng2a330942006-05-25 00:59:30 +00001869 break;
1870 case MVT::i32:
1871 if (Op.Val->getValueType(1) == MVT::i32) {
1872 Chain = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag).getValue(1);
1873 ResultVals.push_back(Chain.getValue(0));
1874 Chain = DAG.getCopyFromReg(Chain, X86::EDX, MVT::i32,
1875 Chain.getValue(2)).getValue(1);
1876 ResultVals.push_back(Chain.getValue(0));
Chris Lattnere56fef92007-02-25 06:40:16 +00001877 NodeTys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
Evan Cheng2a330942006-05-25 00:59:30 +00001878 } else {
1879 Chain = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag).getValue(1);
1880 ResultVals.push_back(Chain.getValue(0));
Chris Lattnere56fef92007-02-25 06:40:16 +00001881 NodeTys = DAG.getVTList(MVT::i32, MVT::Other);
Evan Cheng172fce72006-01-06 00:43:03 +00001882 }
Evan Cheng2a330942006-05-25 00:59:30 +00001883 break;
1884 case MVT::v16i8:
1885 case MVT::v8i16:
1886 case MVT::v4i32:
1887 case MVT::v2i64:
1888 case MVT::v4f32:
1889 case MVT::v2f64:
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00001890 if (isFastCall) {
1891 assert(0 && "Unknown value type to return!");
1892 } else {
1893 Chain = DAG.getCopyFromReg(Chain, X86::XMM0, RetVT, InFlag).getValue(1);
1894 ResultVals.push_back(Chain.getValue(0));
Chris Lattnere56fef92007-02-25 06:40:16 +00001895 NodeTys = DAG.getVTList(RetVT, MVT::Other);
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00001896 }
1897 break;
Evan Cheng2a330942006-05-25 00:59:30 +00001898 case MVT::f32:
1899 case MVT::f64: {
Chris Lattner35a08552007-02-25 07:10:00 +00001900 SDVTList Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag);
1901 SmallVector<SDOperand, 8> Ops;
Evan Cheng2a330942006-05-25 00:59:30 +00001902 Ops.push_back(Chain);
1903 Ops.push_back(InFlag);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00001904 SDOperand RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys,
1905 &Ops[0], Ops.size());
Evan Cheng2a330942006-05-25 00:59:30 +00001906 Chain = RetVal.getValue(1);
1907 InFlag = RetVal.getValue(2);
1908 if (X86ScalarSSE) {
1909 // FIXME: Currently the FST is flagged to the FP_GET_RESULT. This
1910 // shouldn't be necessary except that RFP cannot be live across
1911 // multiple blocks. When stackifier is fixed, they can be uncoupled.
1912 MachineFunction &MF = DAG.getMachineFunction();
1913 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8);
1914 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
Chris Lattner35a08552007-02-25 07:10:00 +00001915 Tys = DAG.getVTList(MVT::Other);
Evan Cheng2a330942006-05-25 00:59:30 +00001916 Ops.clear();
Nate Begeman7e5496d2006-02-17 00:03:04 +00001917 Ops.push_back(Chain);
Evan Cheng2a330942006-05-25 00:59:30 +00001918 Ops.push_back(RetVal);
1919 Ops.push_back(StackSlot);
1920 Ops.push_back(DAG.getValueType(RetVT));
Nate Begeman7e5496d2006-02-17 00:03:04 +00001921 Ops.push_back(InFlag);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00001922 Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size());
Evan Chenge71fe34d2006-10-09 20:57:25 +00001923 RetVal = DAG.getLoad(RetVT, Chain, StackSlot, NULL, 0);
Evan Cheng2a330942006-05-25 00:59:30 +00001924 Chain = RetVal.getValue(1);
1925 }
Evan Cheng172fce72006-01-06 00:43:03 +00001926
Evan Cheng2a330942006-05-25 00:59:30 +00001927 if (RetVT == MVT::f32 && !X86ScalarSSE)
1928 // FIXME: we would really like to remember that this FP_ROUND
1929 // operation is okay to eliminate if we allow excess FP precision.
1930 RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal);
1931 ResultVals.push_back(RetVal);
Chris Lattnere56fef92007-02-25 06:40:16 +00001932 NodeTys = DAG.getVTList(RetVT, MVT::Other);
1933
Evan Cheng2a330942006-05-25 00:59:30 +00001934 break;
1935 }
Chris Lattner76ac0682005-11-15 00:40:23 +00001936 }
Nate Begeman7e5496d2006-02-17 00:03:04 +00001937
Chris Lattnerd6b853ad2007-02-25 07:18:38 +00001938 // Merge everything together with a MERGE_VALUES node.
Evan Cheng2a330942006-05-25 00:59:30 +00001939 ResultVals.push_back(Chain);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00001940 SDOperand Res = DAG.getNode(ISD::MERGE_VALUES, NodeTys,
1941 &ResultVals[0], ResultVals.size());
Evan Cheng2a330942006-05-25 00:59:30 +00001942 return Res.getValue(Op.ResNo);
Chris Lattner76ac0682005-11-15 00:40:23 +00001943}
1944
1945SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) {
1946 if (ReturnAddrIndex == 0) {
1947 // Set up a frame object for the return address.
1948 MachineFunction &MF = DAG.getMachineFunction();
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001949 if (Subtarget->is64Bit())
1950 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(8, -8);
1951 else
1952 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4);
Chris Lattner76ac0682005-11-15 00:40:23 +00001953 }
1954
Evan Cheng11b0a5d2006-09-08 06:48:29 +00001955 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy());
Chris Lattner76ac0682005-11-15 00:40:23 +00001956}
1957
1958
1959
Evan Cheng45df7f82006-01-30 23:41:35 +00001960/// translateX86CC - do a one to one translation of a ISD::CondCode to the X86
1961/// specific condition code. It returns a false if it cannot do a direct
Chris Lattner7a627672006-09-13 03:22:10 +00001962/// translation. X86CC is the translated CondCode. LHS/RHS are modified as
1963/// needed.
Evan Cheng78038292006-04-05 23:38:46 +00001964static bool translateX86CC(ISD::CondCode SetCCOpcode, bool isFP,
Chris Lattner7a627672006-09-13 03:22:10 +00001965 unsigned &X86CC, SDOperand &LHS, SDOperand &RHS,
1966 SelectionDAG &DAG) {
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001967 X86CC = X86::COND_INVALID;
Evan Cheng172fce72006-01-06 00:43:03 +00001968 if (!isFP) {
Chris Lattner971e3392006-09-13 17:04:54 +00001969 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
1970 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) {
1971 // X > -1 -> X == 0, jump !sign.
1972 RHS = DAG.getConstant(0, RHS.getValueType());
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001973 X86CC = X86::COND_NS;
Chris Lattner971e3392006-09-13 17:04:54 +00001974 return true;
1975 } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) {
1976 // X < 0 -> X == 0, jump on sign.
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001977 X86CC = X86::COND_S;
Chris Lattner971e3392006-09-13 17:04:54 +00001978 return true;
1979 }
Chris Lattner7a627672006-09-13 03:22:10 +00001980 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00001981
Evan Cheng172fce72006-01-06 00:43:03 +00001982 switch (SetCCOpcode) {
1983 default: break;
Chris Lattnerc0fb5672006-10-20 17:42:20 +00001984 case ISD::SETEQ: X86CC = X86::COND_E; break;
1985 case ISD::SETGT: X86CC = X86::COND_G; break;
1986 case ISD::SETGE: X86CC = X86::COND_GE; break;
1987 case ISD::SETLT: X86CC = X86::COND_L; break;
1988 case ISD::SETLE: X86CC = X86::COND_LE; break;
1989 case ISD::SETNE: X86CC = X86::COND_NE; break;
1990 case ISD::SETULT: X86CC = X86::COND_B; break;
1991 case ISD::SETUGT: X86CC = X86::COND_A; break;
1992 case ISD::SETULE: X86CC = X86::COND_BE; break;
1993 case ISD::SETUGE: X86CC = X86::COND_AE; break;
Evan Cheng172fce72006-01-06 00:43:03 +00001994 }
1995 } else {
1996 // On a floating point condition, the flags are set as follows:
1997 // ZF PF CF op
1998 // 0 | 0 | 0 | X > Y
1999 // 0 | 0 | 1 | X < Y
2000 // 1 | 0 | 0 | X == Y
2001 // 1 | 1 | 1 | unordered
Chris Lattner7a627672006-09-13 03:22:10 +00002002 bool Flip = false;
Evan Cheng172fce72006-01-06 00:43:03 +00002003 switch (SetCCOpcode) {
2004 default: break;
2005 case ISD::SETUEQ:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00002006 case ISD::SETEQ: X86CC = X86::COND_E; break;
Evan Chengb3b41c42006-04-17 07:24:10 +00002007 case ISD::SETOLT: Flip = true; // Fallthrough
Evan Cheng172fce72006-01-06 00:43:03 +00002008 case ISD::SETOGT:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00002009 case ISD::SETGT: X86CC = X86::COND_A; break;
Evan Chengb3b41c42006-04-17 07:24:10 +00002010 case ISD::SETOLE: Flip = true; // Fallthrough
Evan Cheng172fce72006-01-06 00:43:03 +00002011 case ISD::SETOGE:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00002012 case ISD::SETGE: X86CC = X86::COND_AE; break;
Evan Chengb3b41c42006-04-17 07:24:10 +00002013 case ISD::SETUGT: Flip = true; // Fallthrough
Evan Cheng172fce72006-01-06 00:43:03 +00002014 case ISD::SETULT:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00002015 case ISD::SETLT: X86CC = X86::COND_B; break;
Evan Chengb3b41c42006-04-17 07:24:10 +00002016 case ISD::SETUGE: Flip = true; // Fallthrough
Evan Cheng172fce72006-01-06 00:43:03 +00002017 case ISD::SETULE:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00002018 case ISD::SETLE: X86CC = X86::COND_BE; break;
Evan Cheng172fce72006-01-06 00:43:03 +00002019 case ISD::SETONE:
Chris Lattnerc0fb5672006-10-20 17:42:20 +00002020 case ISD::SETNE: X86CC = X86::COND_NE; break;
2021 case ISD::SETUO: X86CC = X86::COND_P; break;
2022 case ISD::SETO: X86CC = X86::COND_NP; break;
Evan Cheng172fce72006-01-06 00:43:03 +00002023 }
Chris Lattner7a627672006-09-13 03:22:10 +00002024 if (Flip)
2025 std::swap(LHS, RHS);
Evan Cheng172fce72006-01-06 00:43:03 +00002026 }
Evan Cheng45df7f82006-01-30 23:41:35 +00002027
Chris Lattnerc0fb5672006-10-20 17:42:20 +00002028 return X86CC != X86::COND_INVALID;
Evan Cheng172fce72006-01-06 00:43:03 +00002029}
2030
Evan Cheng339edad2006-01-11 00:33:36 +00002031/// hasFPCMov - is there a floating point cmov for the specific X86 condition
2032/// code. Current x86 isa includes the following FP cmov instructions:
Evan Cheng73a1ad92006-01-10 20:26:56 +00002033/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
Evan Cheng339edad2006-01-11 00:33:36 +00002034static bool hasFPCMov(unsigned X86CC) {
Evan Cheng73a1ad92006-01-10 20:26:56 +00002035 switch (X86CC) {
2036 default:
2037 return false;
Chris Lattnerc0fb5672006-10-20 17:42:20 +00002038 case X86::COND_B:
2039 case X86::COND_BE:
2040 case X86::COND_E:
2041 case X86::COND_P:
2042 case X86::COND_A:
2043 case X86::COND_AE:
2044 case X86::COND_NE:
2045 case X86::COND_NP:
Evan Cheng73a1ad92006-01-10 20:26:56 +00002046 return true;
2047 }
2048}
2049
Evan Chengc995b452006-04-06 23:23:56 +00002050/// isUndefOrInRange - Op is either an undef node or a ConstantSDNode. Return
Evan Chengac847262006-04-07 21:53:05 +00002051/// true if Op is undef or if its value falls within the specified range (L, H].
Evan Chengc995b452006-04-06 23:23:56 +00002052static bool isUndefOrInRange(SDOperand Op, unsigned Low, unsigned Hi) {
2053 if (Op.getOpcode() == ISD::UNDEF)
2054 return true;
2055
2056 unsigned Val = cast<ConstantSDNode>(Op)->getValue();
Evan Chengac847262006-04-07 21:53:05 +00002057 return (Val >= Low && Val < Hi);
2058}
2059
2060/// isUndefOrEqual - Op is either an undef node or a ConstantSDNode. Return
2061/// true if Op is undef or if its value equal to the specified value.
2062static bool isUndefOrEqual(SDOperand Op, unsigned Val) {
2063 if (Op.getOpcode() == ISD::UNDEF)
2064 return true;
2065 return cast<ConstantSDNode>(Op)->getValue() == Val;
Evan Chengc995b452006-04-06 23:23:56 +00002066}
2067
Evan Cheng68ad48b2006-03-22 18:59:22 +00002068/// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand
2069/// specifies a shuffle of elements that is suitable for input to PSHUFD.
2070bool X86::isPSHUFDMask(SDNode *N) {
2071 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2072
2073 if (N->getNumOperands() != 4)
2074 return false;
2075
2076 // Check if the value doesn't reference the second vector.
Evan Chengb7fedff2006-03-29 23:07:14 +00002077 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
Evan Cheng99d72052006-03-31 00:30:29 +00002078 SDOperand Arg = N->getOperand(i);
2079 if (Arg.getOpcode() == ISD::UNDEF) continue;
2080 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2081 if (cast<ConstantSDNode>(Arg)->getValue() >= 4)
Evan Chengb7fedff2006-03-29 23:07:14 +00002082 return false;
2083 }
2084
2085 return true;
2086}
2087
2088/// isPSHUFHWMask - Return true if the specified VECTOR_SHUFFLE operand
Evan Cheng59a63552006-04-05 01:47:37 +00002089/// specifies a shuffle of elements that is suitable for input to PSHUFHW.
Evan Chengb7fedff2006-03-29 23:07:14 +00002090bool X86::isPSHUFHWMask(SDNode *N) {
2091 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2092
2093 if (N->getNumOperands() != 8)
2094 return false;
2095
2096 // Lower quadword copied in order.
2097 for (unsigned i = 0; i != 4; ++i) {
Evan Cheng99d72052006-03-31 00:30:29 +00002098 SDOperand Arg = N->getOperand(i);
2099 if (Arg.getOpcode() == ISD::UNDEF) continue;
2100 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2101 if (cast<ConstantSDNode>(Arg)->getValue() != i)
Evan Chengb7fedff2006-03-29 23:07:14 +00002102 return false;
2103 }
2104
2105 // Upper quadword shuffled.
2106 for (unsigned i = 4; i != 8; ++i) {
Evan Cheng99d72052006-03-31 00:30:29 +00002107 SDOperand Arg = N->getOperand(i);
2108 if (Arg.getOpcode() == ISD::UNDEF) continue;
2109 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2110 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
Evan Chengb7fedff2006-03-29 23:07:14 +00002111 if (Val < 4 || Val > 7)
2112 return false;
2113 }
2114
2115 return true;
2116}
2117
2118/// isPSHUFLWMask - Return true if the specified VECTOR_SHUFFLE operand
Evan Cheng59a63552006-04-05 01:47:37 +00002119/// specifies a shuffle of elements that is suitable for input to PSHUFLW.
Evan Chengb7fedff2006-03-29 23:07:14 +00002120bool X86::isPSHUFLWMask(SDNode *N) {
2121 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2122
2123 if (N->getNumOperands() != 8)
2124 return false;
2125
2126 // Upper quadword copied in order.
Evan Chengac847262006-04-07 21:53:05 +00002127 for (unsigned i = 4; i != 8; ++i)
2128 if (!isUndefOrEqual(N->getOperand(i), i))
Evan Chengb7fedff2006-03-29 23:07:14 +00002129 return false;
Evan Chengb7fedff2006-03-29 23:07:14 +00002130
2131 // Lower quadword shuffled.
Evan Chengac847262006-04-07 21:53:05 +00002132 for (unsigned i = 0; i != 4; ++i)
2133 if (!isUndefOrInRange(N->getOperand(i), 0, 4))
Evan Chengb7fedff2006-03-29 23:07:14 +00002134 return false;
Evan Cheng68ad48b2006-03-22 18:59:22 +00002135
2136 return true;
2137}
2138
Evan Chengd27fb3e2006-03-24 01:18:28 +00002139/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand
2140/// specifies a shuffle of elements that is suitable for input to SHUFP*.
Chris Lattner35a08552007-02-25 07:10:00 +00002141static bool isSHUFPMask(const SDOperand *Elems, unsigned NumElems) {
Evan Cheng60f0b892006-04-20 08:58:49 +00002142 if (NumElems != 2 && NumElems != 4) return false;
Evan Chengd27fb3e2006-03-24 01:18:28 +00002143
Evan Cheng60f0b892006-04-20 08:58:49 +00002144 unsigned Half = NumElems / 2;
2145 for (unsigned i = 0; i < Half; ++i)
Chris Lattner35a08552007-02-25 07:10:00 +00002146 if (!isUndefOrInRange(Elems[i], 0, NumElems))
Evan Cheng60f0b892006-04-20 08:58:49 +00002147 return false;
2148 for (unsigned i = Half; i < NumElems; ++i)
Chris Lattner35a08552007-02-25 07:10:00 +00002149 if (!isUndefOrInRange(Elems[i], NumElems, NumElems*2))
Evan Cheng60f0b892006-04-20 08:58:49 +00002150 return false;
Evan Chengd27fb3e2006-03-24 01:18:28 +00002151
2152 return true;
2153}
2154
Evan Cheng60f0b892006-04-20 08:58:49 +00002155bool X86::isSHUFPMask(SDNode *N) {
2156 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00002157 return ::isSHUFPMask(N->op_begin(), N->getNumOperands());
Evan Cheng60f0b892006-04-20 08:58:49 +00002158}
2159
2160/// isCommutedSHUFP - Returns true if the shuffle mask is except
2161/// the reverse of what x86 shuffles want. x86 shuffles requires the lower
2162/// half elements to come from vector 1 (which would equal the dest.) and
2163/// the upper half to come from vector 2.
Chris Lattner35a08552007-02-25 07:10:00 +00002164static bool isCommutedSHUFP(const SDOperand *Ops, unsigned NumOps) {
2165 if (NumOps != 2 && NumOps != 4) return false;
Evan Cheng60f0b892006-04-20 08:58:49 +00002166
Chris Lattner35a08552007-02-25 07:10:00 +00002167 unsigned Half = NumOps / 2;
Evan Cheng60f0b892006-04-20 08:58:49 +00002168 for (unsigned i = 0; i < Half; ++i)
Chris Lattner35a08552007-02-25 07:10:00 +00002169 if (!isUndefOrInRange(Ops[i], NumOps, NumOps*2))
Evan Cheng60f0b892006-04-20 08:58:49 +00002170 return false;
Chris Lattner35a08552007-02-25 07:10:00 +00002171 for (unsigned i = Half; i < NumOps; ++i)
2172 if (!isUndefOrInRange(Ops[i], 0, NumOps))
Evan Cheng60f0b892006-04-20 08:58:49 +00002173 return false;
2174 return true;
2175}
2176
2177static bool isCommutedSHUFP(SDNode *N) {
2178 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00002179 return isCommutedSHUFP(N->op_begin(), N->getNumOperands());
Evan Cheng60f0b892006-04-20 08:58:49 +00002180}
2181
Evan Cheng2595a682006-03-24 02:58:06 +00002182/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand
2183/// specifies a shuffle of elements that is suitable for input to MOVHLPS.
2184bool X86::isMOVHLPSMask(SDNode *N) {
2185 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2186
Evan Cheng1a194a52006-03-28 06:50:32 +00002187 if (N->getNumOperands() != 4)
Evan Cheng2595a682006-03-24 02:58:06 +00002188 return false;
2189
Evan Cheng1a194a52006-03-28 06:50:32 +00002190 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3
Evan Chengac847262006-04-07 21:53:05 +00002191 return isUndefOrEqual(N->getOperand(0), 6) &&
2192 isUndefOrEqual(N->getOperand(1), 7) &&
2193 isUndefOrEqual(N->getOperand(2), 2) &&
2194 isUndefOrEqual(N->getOperand(3), 3);
Evan Cheng1a194a52006-03-28 06:50:32 +00002195}
2196
Evan Cheng922e1912006-11-07 22:14:24 +00002197/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form
2198/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef,
2199/// <2, 3, 2, 3>
2200bool X86::isMOVHLPS_v_undef_Mask(SDNode *N) {
2201 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2202
2203 if (N->getNumOperands() != 4)
2204 return false;
2205
2206 // Expect bit0 == 2, bit1 == 3, bit2 == 2, bit3 == 3
2207 return isUndefOrEqual(N->getOperand(0), 2) &&
2208 isUndefOrEqual(N->getOperand(1), 3) &&
2209 isUndefOrEqual(N->getOperand(2), 2) &&
2210 isUndefOrEqual(N->getOperand(3), 3);
2211}
2212
Evan Chengc995b452006-04-06 23:23:56 +00002213/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand
2214/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}.
2215bool X86::isMOVLPMask(SDNode *N) {
2216 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2217
2218 unsigned NumElems = N->getNumOperands();
2219 if (NumElems != 2 && NumElems != 4)
2220 return false;
2221
Evan Chengac847262006-04-07 21:53:05 +00002222 for (unsigned i = 0; i < NumElems/2; ++i)
2223 if (!isUndefOrEqual(N->getOperand(i), i + NumElems))
2224 return false;
Evan Chengc995b452006-04-06 23:23:56 +00002225
Evan Chengac847262006-04-07 21:53:05 +00002226 for (unsigned i = NumElems/2; i < NumElems; ++i)
2227 if (!isUndefOrEqual(N->getOperand(i), i))
2228 return false;
Evan Chengc995b452006-04-06 23:23:56 +00002229
2230 return true;
2231}
2232
2233/// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand
Evan Cheng7855e4d2006-04-19 20:35:22 +00002234/// specifies a shuffle of elements that is suitable for input to MOVHP{S|D}
2235/// and MOVLHPS.
Evan Chengc995b452006-04-06 23:23:56 +00002236bool X86::isMOVHPMask(SDNode *N) {
2237 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2238
2239 unsigned NumElems = N->getNumOperands();
2240 if (NumElems != 2 && NumElems != 4)
2241 return false;
2242
Evan Chengac847262006-04-07 21:53:05 +00002243 for (unsigned i = 0; i < NumElems/2; ++i)
2244 if (!isUndefOrEqual(N->getOperand(i), i))
2245 return false;
Evan Chengc995b452006-04-06 23:23:56 +00002246
2247 for (unsigned i = 0; i < NumElems/2; ++i) {
2248 SDOperand Arg = N->getOperand(i + NumElems/2);
Evan Chengac847262006-04-07 21:53:05 +00002249 if (!isUndefOrEqual(Arg, i + NumElems))
2250 return false;
Evan Chengc995b452006-04-06 23:23:56 +00002251 }
2252
2253 return true;
2254}
2255
Evan Cheng5df75882006-03-28 00:39:58 +00002256/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand
2257/// specifies a shuffle of elements that is suitable for input to UNPCKL.
Chris Lattner35a08552007-02-25 07:10:00 +00002258bool static isUNPCKLMask(const SDOperand *Elts, unsigned NumElts,
2259 bool V2IsSplat = false) {
2260 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
Evan Cheng5df75882006-03-28 00:39:58 +00002261 return false;
2262
Chris Lattner35a08552007-02-25 07:10:00 +00002263 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) {
2264 SDOperand BitI = Elts[i];
2265 SDOperand BitI1 = Elts[i+1];
Evan Chengac847262006-04-07 21:53:05 +00002266 if (!isUndefOrEqual(BitI, j))
2267 return false;
Evan Cheng60f0b892006-04-20 08:58:49 +00002268 if (V2IsSplat) {
Chris Lattner35a08552007-02-25 07:10:00 +00002269 if (isUndefOrEqual(BitI1, NumElts))
Evan Cheng60f0b892006-04-20 08:58:49 +00002270 return false;
2271 } else {
Chris Lattner35a08552007-02-25 07:10:00 +00002272 if (!isUndefOrEqual(BitI1, j + NumElts))
Evan Cheng60f0b892006-04-20 08:58:49 +00002273 return false;
2274 }
Evan Cheng5df75882006-03-28 00:39:58 +00002275 }
2276
2277 return true;
2278}
2279
Evan Cheng60f0b892006-04-20 08:58:49 +00002280bool X86::isUNPCKLMask(SDNode *N, bool V2IsSplat) {
2281 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00002282 return ::isUNPCKLMask(N->op_begin(), N->getNumOperands(), V2IsSplat);
Evan Cheng60f0b892006-04-20 08:58:49 +00002283}
2284
Evan Cheng2bc32802006-03-28 02:43:26 +00002285/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand
2286/// specifies a shuffle of elements that is suitable for input to UNPCKH.
Chris Lattner35a08552007-02-25 07:10:00 +00002287bool static isUNPCKHMask(const SDOperand *Elts, unsigned NumElts,
2288 bool V2IsSplat = false) {
2289 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
Evan Cheng2bc32802006-03-28 02:43:26 +00002290 return false;
2291
Chris Lattner35a08552007-02-25 07:10:00 +00002292 for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) {
2293 SDOperand BitI = Elts[i];
2294 SDOperand BitI1 = Elts[i+1];
2295 if (!isUndefOrEqual(BitI, j + NumElts/2))
Evan Chengac847262006-04-07 21:53:05 +00002296 return false;
Evan Cheng60f0b892006-04-20 08:58:49 +00002297 if (V2IsSplat) {
Chris Lattner35a08552007-02-25 07:10:00 +00002298 if (isUndefOrEqual(BitI1, NumElts))
Evan Cheng60f0b892006-04-20 08:58:49 +00002299 return false;
2300 } else {
Chris Lattner35a08552007-02-25 07:10:00 +00002301 if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts))
Evan Cheng60f0b892006-04-20 08:58:49 +00002302 return false;
2303 }
Evan Cheng2bc32802006-03-28 02:43:26 +00002304 }
2305
2306 return true;
2307}
2308
Evan Cheng60f0b892006-04-20 08:58:49 +00002309bool X86::isUNPCKHMask(SDNode *N, bool V2IsSplat) {
2310 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00002311 return ::isUNPCKHMask(N->op_begin(), N->getNumOperands(), V2IsSplat);
Evan Cheng60f0b892006-04-20 08:58:49 +00002312}
2313
Evan Chengf3b52c82006-04-05 07:20:06 +00002314/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form
2315/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef,
2316/// <0, 0, 1, 1>
2317bool X86::isUNPCKL_v_undef_Mask(SDNode *N) {
2318 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2319
2320 unsigned NumElems = N->getNumOperands();
2321 if (NumElems != 4 && NumElems != 8 && NumElems != 16)
2322 return false;
2323
2324 for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) {
2325 SDOperand BitI = N->getOperand(i);
2326 SDOperand BitI1 = N->getOperand(i+1);
2327
Evan Chengac847262006-04-07 21:53:05 +00002328 if (!isUndefOrEqual(BitI, j))
2329 return false;
2330 if (!isUndefOrEqual(BitI1, j))
2331 return false;
Evan Chengf3b52c82006-04-05 07:20:06 +00002332 }
2333
2334 return true;
2335}
2336
Evan Chenge8b51802006-04-21 01:05:10 +00002337/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand
2338/// specifies a shuffle of elements that is suitable for input to MOVSS,
2339/// MOVSD, and MOVD, i.e. setting the lowest element.
Chris Lattner35a08552007-02-25 07:10:00 +00002340static bool isMOVLMask(const SDOperand *Elts, unsigned NumElts) {
2341 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
Evan Cheng12ba3e22006-04-11 00:19:04 +00002342 return false;
2343
Chris Lattner35a08552007-02-25 07:10:00 +00002344 if (!isUndefOrEqual(Elts[0], NumElts))
Evan Cheng12ba3e22006-04-11 00:19:04 +00002345 return false;
2346
Chris Lattner35a08552007-02-25 07:10:00 +00002347 for (unsigned i = 1; i < NumElts; ++i) {
2348 if (!isUndefOrEqual(Elts[i], i))
Evan Cheng12ba3e22006-04-11 00:19:04 +00002349 return false;
2350 }
2351
2352 return true;
2353}
Evan Chengf3b52c82006-04-05 07:20:06 +00002354
Evan Chenge8b51802006-04-21 01:05:10 +00002355bool X86::isMOVLMask(SDNode *N) {
Evan Cheng60f0b892006-04-20 08:58:49 +00002356 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00002357 return ::isMOVLMask(N->op_begin(), N->getNumOperands());
Evan Cheng60f0b892006-04-20 08:58:49 +00002358}
2359
Evan Chenge8b51802006-04-21 01:05:10 +00002360/// isCommutedMOVL - Returns true if the shuffle mask is except the reverse
2361/// of what x86 movss want. X86 movs requires the lowest element to be lowest
Evan Cheng60f0b892006-04-20 08:58:49 +00002362/// element of vector 2 and the other elements to come from vector 1 in order.
Chris Lattner35a08552007-02-25 07:10:00 +00002363static bool isCommutedMOVL(const SDOperand *Ops, unsigned NumOps,
2364 bool V2IsSplat = false,
Evan Cheng89c5d042006-09-08 01:50:06 +00002365 bool V2IsUndef = false) {
Chris Lattner35a08552007-02-25 07:10:00 +00002366 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16)
Evan Cheng60f0b892006-04-20 08:58:49 +00002367 return false;
2368
2369 if (!isUndefOrEqual(Ops[0], 0))
2370 return false;
2371
Chris Lattner35a08552007-02-25 07:10:00 +00002372 for (unsigned i = 1; i < NumOps; ++i) {
Evan Cheng60f0b892006-04-20 08:58:49 +00002373 SDOperand Arg = Ops[i];
Chris Lattner35a08552007-02-25 07:10:00 +00002374 if (!(isUndefOrEqual(Arg, i+NumOps) ||
2375 (V2IsUndef && isUndefOrInRange(Arg, NumOps, NumOps*2)) ||
2376 (V2IsSplat && isUndefOrEqual(Arg, NumOps))))
Evan Cheng89c5d042006-09-08 01:50:06 +00002377 return false;
Evan Cheng60f0b892006-04-20 08:58:49 +00002378 }
2379
2380 return true;
2381}
2382
Evan Cheng89c5d042006-09-08 01:50:06 +00002383static bool isCommutedMOVL(SDNode *N, bool V2IsSplat = false,
2384 bool V2IsUndef = false) {
Evan Cheng60f0b892006-04-20 08:58:49 +00002385 assert(N->getOpcode() == ISD::BUILD_VECTOR);
Chris Lattner35a08552007-02-25 07:10:00 +00002386 return isCommutedMOVL(N->op_begin(), N->getNumOperands(),
2387 V2IsSplat, V2IsUndef);
Evan Cheng60f0b892006-04-20 08:58:49 +00002388}
2389
Evan Cheng5d247f82006-04-14 21:59:03 +00002390/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand
2391/// specifies a shuffle of elements that is suitable for input to MOVSHDUP.
2392bool X86::isMOVSHDUPMask(SDNode *N) {
2393 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2394
2395 if (N->getNumOperands() != 4)
2396 return false;
2397
2398 // Expect 1, 1, 3, 3
2399 for (unsigned i = 0; i < 2; ++i) {
2400 SDOperand Arg = N->getOperand(i);
2401 if (Arg.getOpcode() == ISD::UNDEF) continue;
2402 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2403 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2404 if (Val != 1) return false;
2405 }
Evan Cheng6222cf22006-04-15 05:37:34 +00002406
2407 bool HasHi = false;
Evan Cheng5d247f82006-04-14 21:59:03 +00002408 for (unsigned i = 2; i < 4; ++i) {
2409 SDOperand Arg = N->getOperand(i);
2410 if (Arg.getOpcode() == ISD::UNDEF) continue;
2411 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2412 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2413 if (Val != 3) return false;
Evan Cheng6222cf22006-04-15 05:37:34 +00002414 HasHi = true;
Evan Cheng5d247f82006-04-14 21:59:03 +00002415 }
Evan Cheng65bb7202006-04-15 03:13:24 +00002416
Evan Cheng6222cf22006-04-15 05:37:34 +00002417 // Don't use movshdup if it can be done with a shufps.
2418 return HasHi;
Evan Cheng5d247f82006-04-14 21:59:03 +00002419}
2420
2421/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand
2422/// specifies a shuffle of elements that is suitable for input to MOVSLDUP.
2423bool X86::isMOVSLDUPMask(SDNode *N) {
2424 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2425
2426 if (N->getNumOperands() != 4)
2427 return false;
2428
2429 // Expect 0, 0, 2, 2
2430 for (unsigned i = 0; i < 2; ++i) {
2431 SDOperand Arg = N->getOperand(i);
2432 if (Arg.getOpcode() == ISD::UNDEF) continue;
2433 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2434 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2435 if (Val != 0) return false;
2436 }
Evan Cheng6222cf22006-04-15 05:37:34 +00002437
2438 bool HasHi = false;
Evan Cheng5d247f82006-04-14 21:59:03 +00002439 for (unsigned i = 2; i < 4; ++i) {
2440 SDOperand Arg = N->getOperand(i);
2441 if (Arg.getOpcode() == ISD::UNDEF) continue;
2442 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2443 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2444 if (Val != 2) return false;
Evan Cheng6222cf22006-04-15 05:37:34 +00002445 HasHi = true;
Evan Cheng5d247f82006-04-14 21:59:03 +00002446 }
Evan Cheng65bb7202006-04-15 03:13:24 +00002447
Evan Cheng6222cf22006-04-15 05:37:34 +00002448 // Don't use movshdup if it can be done with a shufps.
2449 return HasHi;
Evan Cheng5d247f82006-04-14 21:59:03 +00002450}
2451
Evan Chengd097e672006-03-22 02:53:00 +00002452/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies
2453/// a splat of a single element.
Evan Cheng5022b342006-04-17 20:43:08 +00002454static bool isSplatMask(SDNode *N) {
Evan Chengd097e672006-03-22 02:53:00 +00002455 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2456
Evan Chengd097e672006-03-22 02:53:00 +00002457 // This is a splat operation if each element of the permute is the same, and
2458 // if the value doesn't reference the second vector.
Evan Cheng4a1b0d32006-04-19 23:28:59 +00002459 unsigned NumElems = N->getNumOperands();
2460 SDOperand ElementBase;
2461 unsigned i = 0;
2462 for (; i != NumElems; ++i) {
2463 SDOperand Elt = N->getOperand(i);
Reid Spencerde46e482006-11-02 20:25:50 +00002464 if (isa<ConstantSDNode>(Elt)) {
Evan Cheng4a1b0d32006-04-19 23:28:59 +00002465 ElementBase = Elt;
2466 break;
2467 }
2468 }
2469
2470 if (!ElementBase.Val)
2471 return false;
2472
2473 for (; i != NumElems; ++i) {
Evan Cheng99d72052006-03-31 00:30:29 +00002474 SDOperand Arg = N->getOperand(i);
2475 if (Arg.getOpcode() == ISD::UNDEF) continue;
2476 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
Evan Cheng4a1b0d32006-04-19 23:28:59 +00002477 if (Arg != ElementBase) return false;
Evan Chengd097e672006-03-22 02:53:00 +00002478 }
2479
2480 // Make sure it is a splat of the first vector operand.
Evan Cheng4a1b0d32006-04-19 23:28:59 +00002481 return cast<ConstantSDNode>(ElementBase)->getValue() < NumElems;
Evan Chengd097e672006-03-22 02:53:00 +00002482}
2483
Evan Cheng5022b342006-04-17 20:43:08 +00002484/// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies
2485/// a splat of a single element and it's a 2 or 4 element mask.
2486bool X86::isSplatMask(SDNode *N) {
2487 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2488
Evan Cheng4a1b0d32006-04-19 23:28:59 +00002489 // We can only splat 64-bit, and 32-bit quantities with a single instruction.
Evan Cheng5022b342006-04-17 20:43:08 +00002490 if (N->getNumOperands() != 4 && N->getNumOperands() != 2)
2491 return false;
2492 return ::isSplatMask(N);
2493}
2494
Evan Chenge056dd52006-10-27 21:08:32 +00002495/// isSplatLoMask - Return true if the specified VECTOR_SHUFFLE operand
2496/// specifies a splat of zero element.
2497bool X86::isSplatLoMask(SDNode *N) {
2498 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2499
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00002500 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
Evan Chenge056dd52006-10-27 21:08:32 +00002501 if (!isUndefOrEqual(N->getOperand(i), 0))
2502 return false;
2503 return true;
2504}
2505
Evan Cheng8fdbdf22006-03-22 08:01:21 +00002506/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle
2507/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP*
2508/// instructions.
2509unsigned X86::getShuffleSHUFImmediate(SDNode *N) {
Evan Chengd097e672006-03-22 02:53:00 +00002510 unsigned NumOperands = N->getNumOperands();
2511 unsigned Shift = (NumOperands == 4) ? 2 : 1;
2512 unsigned Mask = 0;
Evan Cheng8160fd32006-03-28 23:41:33 +00002513 for (unsigned i = 0; i < NumOperands; ++i) {
Evan Cheng99d72052006-03-31 00:30:29 +00002514 unsigned Val = 0;
2515 SDOperand Arg = N->getOperand(NumOperands-i-1);
2516 if (Arg.getOpcode() != ISD::UNDEF)
2517 Val = cast<ConstantSDNode>(Arg)->getValue();
Evan Chengd27fb3e2006-03-24 01:18:28 +00002518 if (Val >= NumOperands) Val -= NumOperands;
Evan Cheng8fdbdf22006-03-22 08:01:21 +00002519 Mask |= Val;
Evan Cheng8160fd32006-03-28 23:41:33 +00002520 if (i != NumOperands - 1)
2521 Mask <<= Shift;
2522 }
Evan Cheng8fdbdf22006-03-22 08:01:21 +00002523
2524 return Mask;
2525}
2526
Evan Chengb7fedff2006-03-29 23:07:14 +00002527/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle
2528/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW
2529/// instructions.
2530unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) {
2531 unsigned Mask = 0;
2532 // 8 nodes, but we only care about the last 4.
2533 for (unsigned i = 7; i >= 4; --i) {
Evan Cheng99d72052006-03-31 00:30:29 +00002534 unsigned Val = 0;
2535 SDOperand Arg = N->getOperand(i);
2536 if (Arg.getOpcode() != ISD::UNDEF)
2537 Val = cast<ConstantSDNode>(Arg)->getValue();
Evan Chengb7fedff2006-03-29 23:07:14 +00002538 Mask |= (Val - 4);
2539 if (i != 4)
2540 Mask <<= 2;
2541 }
2542
2543 return Mask;
2544}
2545
2546/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle
2547/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW
2548/// instructions.
2549unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) {
2550 unsigned Mask = 0;
2551 // 8 nodes, but we only care about the first 4.
2552 for (int i = 3; i >= 0; --i) {
Evan Cheng99d72052006-03-31 00:30:29 +00002553 unsigned Val = 0;
2554 SDOperand Arg = N->getOperand(i);
2555 if (Arg.getOpcode() != ISD::UNDEF)
2556 Val = cast<ConstantSDNode>(Arg)->getValue();
Evan Chengb7fedff2006-03-29 23:07:14 +00002557 Mask |= Val;
2558 if (i != 0)
2559 Mask <<= 2;
2560 }
2561
2562 return Mask;
2563}
2564
Evan Cheng59a63552006-04-05 01:47:37 +00002565/// isPSHUFHW_PSHUFLWMask - true if the specified VECTOR_SHUFFLE operand
2566/// specifies a 8 element shuffle that can be broken into a pair of
2567/// PSHUFHW and PSHUFLW.
2568static bool isPSHUFHW_PSHUFLWMask(SDNode *N) {
2569 assert(N->getOpcode() == ISD::BUILD_VECTOR);
2570
2571 if (N->getNumOperands() != 8)
2572 return false;
2573
2574 // Lower quadword shuffled.
2575 for (unsigned i = 0; i != 4; ++i) {
2576 SDOperand Arg = N->getOperand(i);
2577 if (Arg.getOpcode() == ISD::UNDEF) continue;
2578 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2579 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2580 if (Val > 4)
2581 return false;
2582 }
2583
2584 // Upper quadword shuffled.
2585 for (unsigned i = 4; i != 8; ++i) {
2586 SDOperand Arg = N->getOperand(i);
2587 if (Arg.getOpcode() == ISD::UNDEF) continue;
2588 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2589 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2590 if (Val < 4 || Val > 7)
2591 return false;
2592 }
2593
2594 return true;
2595}
2596
Evan Chengc995b452006-04-06 23:23:56 +00002597/// CommuteVectorShuffle - Swap vector_shuffle operandsas well as
2598/// values in ther permute mask.
Evan Chengc415c5b2006-10-25 21:49:50 +00002599static SDOperand CommuteVectorShuffle(SDOperand Op, SDOperand &V1,
2600 SDOperand &V2, SDOperand &Mask,
2601 SelectionDAG &DAG) {
Evan Chengc995b452006-04-06 23:23:56 +00002602 MVT::ValueType VT = Op.getValueType();
2603 MVT::ValueType MaskVT = Mask.getValueType();
2604 MVT::ValueType EltVT = MVT::getVectorBaseType(MaskVT);
2605 unsigned NumElems = Mask.getNumOperands();
Chris Lattner35a08552007-02-25 07:10:00 +00002606 SmallVector<SDOperand, 8> MaskVec;
Evan Chengc995b452006-04-06 23:23:56 +00002607
2608 for (unsigned i = 0; i != NumElems; ++i) {
2609 SDOperand Arg = Mask.getOperand(i);
Evan Chenga3caaee2006-04-19 22:48:17 +00002610 if (Arg.getOpcode() == ISD::UNDEF) {
2611 MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT));
2612 continue;
2613 }
Evan Chengc995b452006-04-06 23:23:56 +00002614 assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
2615 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2616 if (Val < NumElems)
2617 MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT));
2618 else
2619 MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT));
2620 }
2621
Evan Chengc415c5b2006-10-25 21:49:50 +00002622 std::swap(V1, V2);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002623 Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size());
Evan Chengc415c5b2006-10-25 21:49:50 +00002624 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask);
Evan Chengc995b452006-04-06 23:23:56 +00002625}
2626
Evan Cheng7855e4d2006-04-19 20:35:22 +00002627/// ShouldXformToMOVHLPS - Return true if the node should be transformed to
2628/// match movhlps. The lower half elements should come from upper half of
2629/// V1 (and in order), and the upper half elements should come from the upper
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00002630/// half of V2 (and in order).
Evan Cheng7855e4d2006-04-19 20:35:22 +00002631static bool ShouldXformToMOVHLPS(SDNode *Mask) {
2632 unsigned NumElems = Mask->getNumOperands();
2633 if (NumElems != 4)
2634 return false;
2635 for (unsigned i = 0, e = 2; i != e; ++i)
2636 if (!isUndefOrEqual(Mask->getOperand(i), i+2))
2637 return false;
2638 for (unsigned i = 2; i != 4; ++i)
2639 if (!isUndefOrEqual(Mask->getOperand(i), i+4))
2640 return false;
2641 return true;
2642}
2643
Evan Chengc995b452006-04-06 23:23:56 +00002644/// isScalarLoadToVector - Returns true if the node is a scalar load that
2645/// is promoted to a vector.
Evan Cheng7855e4d2006-04-19 20:35:22 +00002646static inline bool isScalarLoadToVector(SDNode *N) {
2647 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2648 N = N->getOperand(0).Val;
Evan Chenge71fe34d2006-10-09 20:57:25 +00002649 return ISD::isNON_EXTLoad(N);
Evan Chengc995b452006-04-06 23:23:56 +00002650 }
2651 return false;
2652}
2653
Evan Cheng7855e4d2006-04-19 20:35:22 +00002654/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to
2655/// match movlp{s|d}. The lower half elements should come from lower half of
2656/// V1 (and in order), and the upper half elements should come from the upper
2657/// half of V2 (and in order). And since V1 will become the source of the
2658/// MOVLP, it must be either a vector load or a scalar load to vector.
Evan Chenge646abb2006-10-09 21:39:25 +00002659static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, SDNode *Mask) {
Evan Chenge71fe34d2006-10-09 20:57:25 +00002660 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1))
Evan Cheng7855e4d2006-04-19 20:35:22 +00002661 return false;
Evan Chenge646abb2006-10-09 21:39:25 +00002662 // Is V2 is a vector load, don't do this transformation. We will try to use
2663 // load folding shufps op.
2664 if (ISD::isNON_EXTLoad(V2))
2665 return false;
Evan Chengc995b452006-04-06 23:23:56 +00002666
Evan Cheng7855e4d2006-04-19 20:35:22 +00002667 unsigned NumElems = Mask->getNumOperands();
2668 if (NumElems != 2 && NumElems != 4)
2669 return false;
2670 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
2671 if (!isUndefOrEqual(Mask->getOperand(i), i))
2672 return false;
2673 for (unsigned i = NumElems/2; i != NumElems; ++i)
2674 if (!isUndefOrEqual(Mask->getOperand(i), i+NumElems))
2675 return false;
2676 return true;
Evan Chengc995b452006-04-06 23:23:56 +00002677}
2678
Evan Cheng60f0b892006-04-20 08:58:49 +00002679/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are
2680/// all the same.
2681static bool isSplatVector(SDNode *N) {
2682 if (N->getOpcode() != ISD::BUILD_VECTOR)
2683 return false;
Evan Chengc995b452006-04-06 23:23:56 +00002684
Evan Cheng60f0b892006-04-20 08:58:49 +00002685 SDOperand SplatValue = N->getOperand(0);
2686 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
2687 if (N->getOperand(i) != SplatValue)
Evan Chengc995b452006-04-06 23:23:56 +00002688 return false;
2689 return true;
2690}
2691
Evan Cheng89c5d042006-09-08 01:50:06 +00002692/// isUndefShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved
2693/// to an undef.
2694static bool isUndefShuffle(SDNode *N) {
2695 if (N->getOpcode() != ISD::BUILD_VECTOR)
2696 return false;
2697
2698 SDOperand V1 = N->getOperand(0);
2699 SDOperand V2 = N->getOperand(1);
2700 SDOperand Mask = N->getOperand(2);
2701 unsigned NumElems = Mask.getNumOperands();
2702 for (unsigned i = 0; i != NumElems; ++i) {
2703 SDOperand Arg = Mask.getOperand(i);
2704 if (Arg.getOpcode() != ISD::UNDEF) {
2705 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2706 if (Val < NumElems && V1.getOpcode() != ISD::UNDEF)
2707 return false;
2708 else if (Val >= NumElems && V2.getOpcode() != ISD::UNDEF)
2709 return false;
2710 }
2711 }
2712 return true;
2713}
2714
Evan Cheng60f0b892006-04-20 08:58:49 +00002715/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements
2716/// that point to V2 points to its first element.
2717static SDOperand NormalizeMask(SDOperand Mask, SelectionDAG &DAG) {
2718 assert(Mask.getOpcode() == ISD::BUILD_VECTOR);
2719
2720 bool Changed = false;
Chris Lattner35a08552007-02-25 07:10:00 +00002721 SmallVector<SDOperand, 8> MaskVec;
Evan Cheng60f0b892006-04-20 08:58:49 +00002722 unsigned NumElems = Mask.getNumOperands();
2723 for (unsigned i = 0; i != NumElems; ++i) {
2724 SDOperand Arg = Mask.getOperand(i);
2725 if (Arg.getOpcode() != ISD::UNDEF) {
2726 unsigned Val = cast<ConstantSDNode>(Arg)->getValue();
2727 if (Val > NumElems) {
2728 Arg = DAG.getConstant(NumElems, Arg.getValueType());
2729 Changed = true;
2730 }
2731 }
2732 MaskVec.push_back(Arg);
2733 }
2734
2735 if (Changed)
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002736 Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(),
2737 &MaskVec[0], MaskVec.size());
Evan Cheng60f0b892006-04-20 08:58:49 +00002738 return Mask;
2739}
2740
Evan Chenge8b51802006-04-21 01:05:10 +00002741/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd
2742/// operation of specified width.
2743static SDOperand getMOVLMask(unsigned NumElems, SelectionDAG &DAG) {
Evan Cheng60f0b892006-04-20 08:58:49 +00002744 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2745 MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT);
2746
Chris Lattner35a08552007-02-25 07:10:00 +00002747 SmallVector<SDOperand, 8> MaskVec;
Evan Cheng60f0b892006-04-20 08:58:49 +00002748 MaskVec.push_back(DAG.getConstant(NumElems, BaseVT));
2749 for (unsigned i = 1; i != NumElems; ++i)
2750 MaskVec.push_back(DAG.getConstant(i, BaseVT));
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002751 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size());
Evan Cheng60f0b892006-04-20 08:58:49 +00002752}
2753
Evan Cheng5022b342006-04-17 20:43:08 +00002754/// getUnpacklMask - Returns a vector_shuffle mask for an unpackl operation
2755/// of specified width.
2756static SDOperand getUnpacklMask(unsigned NumElems, SelectionDAG &DAG) {
2757 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2758 MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002759 SmallVector<SDOperand, 8> MaskVec;
Evan Cheng5022b342006-04-17 20:43:08 +00002760 for (unsigned i = 0, e = NumElems/2; i != e; ++i) {
2761 MaskVec.push_back(DAG.getConstant(i, BaseVT));
2762 MaskVec.push_back(DAG.getConstant(i + NumElems, BaseVT));
2763 }
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002764 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size());
Evan Cheng5022b342006-04-17 20:43:08 +00002765}
2766
Evan Cheng60f0b892006-04-20 08:58:49 +00002767/// getUnpackhMask - Returns a vector_shuffle mask for an unpackh operation
2768/// of specified width.
2769static SDOperand getUnpackhMask(unsigned NumElems, SelectionDAG &DAG) {
2770 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2771 MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT);
2772 unsigned Half = NumElems/2;
Chris Lattner35a08552007-02-25 07:10:00 +00002773 SmallVector<SDOperand, 8> MaskVec;
Evan Cheng60f0b892006-04-20 08:58:49 +00002774 for (unsigned i = 0; i != Half; ++i) {
2775 MaskVec.push_back(DAG.getConstant(i + Half, BaseVT));
2776 MaskVec.push_back(DAG.getConstant(i + NumElems + Half, BaseVT));
2777 }
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002778 return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size());
Evan Cheng60f0b892006-04-20 08:58:49 +00002779}
2780
Evan Chenge8b51802006-04-21 01:05:10 +00002781/// getZeroVector - Returns a vector of specified type with all zero elements.
2782///
2783static SDOperand getZeroVector(MVT::ValueType VT, SelectionDAG &DAG) {
2784 assert(MVT::isVector(VT) && "Expected a vector type");
2785 unsigned NumElems = getVectorNumElements(VT);
2786 MVT::ValueType EVT = MVT::getVectorBaseType(VT);
2787 bool isFP = MVT::isFloatingPoint(EVT);
2788 SDOperand Zero = isFP ? DAG.getConstantFP(0.0, EVT) : DAG.getConstant(0, EVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002789 SmallVector<SDOperand, 8> ZeroVec(NumElems, Zero);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002790 return DAG.getNode(ISD::BUILD_VECTOR, VT, &ZeroVec[0], ZeroVec.size());
Evan Chenge8b51802006-04-21 01:05:10 +00002791}
2792
Evan Cheng5022b342006-04-17 20:43:08 +00002793/// PromoteSplat - Promote a splat of v8i16 or v16i8 to v4i32.
2794///
2795static SDOperand PromoteSplat(SDOperand Op, SelectionDAG &DAG) {
2796 SDOperand V1 = Op.getOperand(0);
Evan Chenge8b51802006-04-21 01:05:10 +00002797 SDOperand Mask = Op.getOperand(2);
Evan Cheng5022b342006-04-17 20:43:08 +00002798 MVT::ValueType VT = Op.getValueType();
Evan Chenge8b51802006-04-21 01:05:10 +00002799 unsigned NumElems = Mask.getNumOperands();
2800 Mask = getUnpacklMask(NumElems, DAG);
Evan Cheng5022b342006-04-17 20:43:08 +00002801 while (NumElems != 4) {
Evan Chenge8b51802006-04-21 01:05:10 +00002802 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask);
Evan Cheng5022b342006-04-17 20:43:08 +00002803 NumElems >>= 1;
2804 }
2805 V1 = DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, V1);
2806
2807 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4);
Evan Chenge8b51802006-04-21 01:05:10 +00002808 Mask = getZeroVector(MaskVT, DAG);
Evan Cheng5022b342006-04-17 20:43:08 +00002809 SDOperand Shuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v4i32, V1,
Evan Chenge8b51802006-04-21 01:05:10 +00002810 DAG.getNode(ISD::UNDEF, MVT::v4i32), Mask);
Evan Cheng5022b342006-04-17 20:43:08 +00002811 return DAG.getNode(ISD::BIT_CONVERT, VT, Shuffle);
2812}
2813
Evan Chenge8b51802006-04-21 01:05:10 +00002814/// isZeroNode - Returns true if Elt is a constant zero or a floating point
2815/// constant +0.0.
2816static inline bool isZeroNode(SDOperand Elt) {
2817 return ((isa<ConstantSDNode>(Elt) &&
2818 cast<ConstantSDNode>(Elt)->getValue() == 0) ||
2819 (isa<ConstantFPSDNode>(Elt) &&
2820 cast<ConstantFPSDNode>(Elt)->isExactlyValue(0.0)));
2821}
2822
Evan Cheng14215c32006-04-21 23:03:30 +00002823/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified
2824/// vector and zero or undef vector.
2825static SDOperand getShuffleVectorZeroOrUndef(SDOperand V2, MVT::ValueType VT,
Evan Chenge8b51802006-04-21 01:05:10 +00002826 unsigned NumElems, unsigned Idx,
Evan Cheng14215c32006-04-21 23:03:30 +00002827 bool isZero, SelectionDAG &DAG) {
2828 SDOperand V1 = isZero ? getZeroVector(VT, DAG) : DAG.getNode(ISD::UNDEF, VT);
Evan Chenge8b51802006-04-21 01:05:10 +00002829 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2830 MVT::ValueType EVT = MVT::getVectorBaseType(MaskVT);
2831 SDOperand Zero = DAG.getConstant(0, EVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002832 SmallVector<SDOperand, 8> MaskVec(NumElems, Zero);
Evan Chenge8b51802006-04-21 01:05:10 +00002833 MaskVec[Idx] = DAG.getConstant(NumElems, EVT);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002834 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2835 &MaskVec[0], MaskVec.size());
Evan Cheng14215c32006-04-21 23:03:30 +00002836 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask);
Evan Chenge8b51802006-04-21 01:05:10 +00002837}
2838
Evan Chengb0461082006-04-24 18:01:45 +00002839/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8.
2840///
2841static SDOperand LowerBuildVectorv16i8(SDOperand Op, unsigned NonZeros,
2842 unsigned NumNonZero, unsigned NumZero,
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002843 SelectionDAG &DAG, TargetLowering &TLI) {
Evan Chengb0461082006-04-24 18:01:45 +00002844 if (NumNonZero > 8)
2845 return SDOperand();
2846
2847 SDOperand V(0, 0);
2848 bool First = true;
2849 for (unsigned i = 0; i < 16; ++i) {
2850 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0;
2851 if (ThisIsNonZero && First) {
2852 if (NumZero)
2853 V = getZeroVector(MVT::v8i16, DAG);
2854 else
2855 V = DAG.getNode(ISD::UNDEF, MVT::v8i16);
2856 First = false;
2857 }
2858
2859 if ((i & 1) != 0) {
2860 SDOperand ThisElt(0, 0), LastElt(0, 0);
2861 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0;
2862 if (LastIsNonZero) {
2863 LastElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i-1));
2864 }
2865 if (ThisIsNonZero) {
2866 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i));
2867 ThisElt = DAG.getNode(ISD::SHL, MVT::i16,
2868 ThisElt, DAG.getConstant(8, MVT::i8));
2869 if (LastIsNonZero)
2870 ThisElt = DAG.getNode(ISD::OR, MVT::i16, ThisElt, LastElt);
2871 } else
2872 ThisElt = LastElt;
2873
2874 if (ThisElt.Val)
2875 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, ThisElt,
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002876 DAG.getConstant(i/2, TLI.getPointerTy()));
Evan Chengb0461082006-04-24 18:01:45 +00002877 }
2878 }
2879
2880 return DAG.getNode(ISD::BIT_CONVERT, MVT::v16i8, V);
2881}
2882
2883/// LowerBuildVectorv16i8 - Custom lower build_vector of v8i16.
2884///
2885static SDOperand LowerBuildVectorv8i16(SDOperand Op, unsigned NonZeros,
2886 unsigned NumNonZero, unsigned NumZero,
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002887 SelectionDAG &DAG, TargetLowering &TLI) {
Evan Chengb0461082006-04-24 18:01:45 +00002888 if (NumNonZero > 4)
2889 return SDOperand();
2890
2891 SDOperand V(0, 0);
2892 bool First = true;
2893 for (unsigned i = 0; i < 8; ++i) {
2894 bool isNonZero = (NonZeros & (1 << i)) != 0;
2895 if (isNonZero) {
2896 if (First) {
2897 if (NumZero)
2898 V = getZeroVector(MVT::v8i16, DAG);
2899 else
2900 V = DAG.getNode(ISD::UNDEF, MVT::v8i16);
2901 First = false;
2902 }
2903 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, Op.getOperand(i),
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002904 DAG.getConstant(i, TLI.getPointerTy()));
Evan Chengb0461082006-04-24 18:01:45 +00002905 }
2906 }
2907
2908 return V;
2909}
2910
Evan Chenga9467aa2006-04-25 20:13:52 +00002911SDOperand
2912X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
2913 // All zero's are handled with pxor.
2914 if (ISD::isBuildVectorAllZeros(Op.Val))
2915 return Op;
2916
2917 // All one's are handled with pcmpeqd.
2918 if (ISD::isBuildVectorAllOnes(Op.Val))
2919 return Op;
2920
2921 MVT::ValueType VT = Op.getValueType();
2922 MVT::ValueType EVT = MVT::getVectorBaseType(VT);
2923 unsigned EVTBits = MVT::getSizeInBits(EVT);
2924
2925 unsigned NumElems = Op.getNumOperands();
2926 unsigned NumZero = 0;
2927 unsigned NumNonZero = 0;
2928 unsigned NonZeros = 0;
2929 std::set<SDOperand> Values;
2930 for (unsigned i = 0; i < NumElems; ++i) {
2931 SDOperand Elt = Op.getOperand(i);
2932 if (Elt.getOpcode() != ISD::UNDEF) {
2933 Values.insert(Elt);
2934 if (isZeroNode(Elt))
2935 NumZero++;
2936 else {
2937 NonZeros |= (1 << i);
2938 NumNonZero++;
2939 }
2940 }
2941 }
2942
2943 if (NumNonZero == 0)
2944 // Must be a mix of zero and undef. Return a zero vector.
2945 return getZeroVector(VT, DAG);
2946
2947 // Splat is obviously ok. Let legalizer expand it to a shuffle.
2948 if (Values.size() == 1)
2949 return SDOperand();
2950
2951 // Special case for single non-zero element.
Evan Cheng798b3062006-10-25 20:48:19 +00002952 if (NumNonZero == 1) {
Evan Chenga9467aa2006-04-25 20:13:52 +00002953 unsigned Idx = CountTrailingZeros_32(NonZeros);
2954 SDOperand Item = Op.getOperand(Idx);
2955 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item);
2956 if (Idx == 0)
2957 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector.
2958 return getShuffleVectorZeroOrUndef(Item, VT, NumElems, Idx,
2959 NumZero > 0, DAG);
2960
2961 if (EVTBits == 32) {
2962 // Turn it into a shuffle of zero and zero-extended scalar to vector.
2963 Item = getShuffleVectorZeroOrUndef(Item, VT, NumElems, 0, NumZero > 0,
2964 DAG);
2965 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
2966 MVT::ValueType MaskEVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00002967 SmallVector<SDOperand, 8> MaskVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00002968 for (unsigned i = 0; i < NumElems; i++)
2969 MaskVec.push_back(DAG.getConstant((i == Idx) ? 0 : 1, MaskEVT));
Chris Lattnerc24a1d32006-08-08 02:23:42 +00002970 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
2971 &MaskVec[0], MaskVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00002972 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Item,
2973 DAG.getNode(ISD::UNDEF, VT), Mask);
2974 }
2975 }
2976
Evan Cheng8c5766e2006-10-04 18:33:38 +00002977 // Let legalizer expand 2-wide build_vector's.
Evan Chenga9467aa2006-04-25 20:13:52 +00002978 if (EVTBits == 64)
2979 return SDOperand();
2980
2981 // If element VT is < 32 bits, convert it to inserts into a zero vector.
2982 if (EVTBits == 8) {
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002983 SDOperand V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG,
2984 *this);
Evan Chenga9467aa2006-04-25 20:13:52 +00002985 if (V.Val) return V;
2986 }
2987
2988 if (EVTBits == 16) {
Evan Cheng11b0a5d2006-09-08 06:48:29 +00002989 SDOperand V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG,
2990 *this);
Evan Chenga9467aa2006-04-25 20:13:52 +00002991 if (V.Val) return V;
2992 }
2993
2994 // If element VT is == 32 bits, turn it into a number of shuffles.
Chris Lattner35a08552007-02-25 07:10:00 +00002995 SmallVector<SDOperand, 8> V;
2996 V.resize(NumElems);
Evan Chenga9467aa2006-04-25 20:13:52 +00002997 if (NumElems == 4 && NumZero > 0) {
2998 for (unsigned i = 0; i < 4; ++i) {
2999 bool isZero = !(NonZeros & (1 << i));
3000 if (isZero)
3001 V[i] = getZeroVector(VT, DAG);
3002 else
3003 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i));
3004 }
3005
3006 for (unsigned i = 0; i < 2; ++i) {
3007 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) {
3008 default: break;
3009 case 0:
3010 V[i] = V[i*2]; // Must be a zero vector.
3011 break;
3012 case 1:
3013 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2+1], V[i*2],
3014 getMOVLMask(NumElems, DAG));
3015 break;
3016 case 2:
3017 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1],
3018 getMOVLMask(NumElems, DAG));
3019 break;
3020 case 3:
3021 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1],
3022 getUnpacklMask(NumElems, DAG));
3023 break;
3024 }
3025 }
3026
Evan Cheng9fee4422006-05-16 07:21:53 +00003027 // Take advantage of the fact GR32 to VR128 scalar_to_vector (i.e. movd)
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003028 // clears the upper bits.
Evan Chenga9467aa2006-04-25 20:13:52 +00003029 // FIXME: we can do the same for v4f32 case when we know both parts of
3030 // the lower half come from scalar_to_vector (loadf32). We should do
3031 // that in post legalizer dag combiner with target specific hooks.
Evan Cheng798b3062006-10-25 20:48:19 +00003032 if (MVT::isInteger(EVT) && (NonZeros & (0x3 << 2)) == 0)
Evan Chenga9467aa2006-04-25 20:13:52 +00003033 return V[0];
3034 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
3035 MVT::ValueType EVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00003036 SmallVector<SDOperand, 8> MaskVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00003037 bool Reverse = (NonZeros & 0x3) == 2;
3038 for (unsigned i = 0; i < 2; ++i)
3039 if (Reverse)
3040 MaskVec.push_back(DAG.getConstant(1-i, EVT));
3041 else
3042 MaskVec.push_back(DAG.getConstant(i, EVT));
3043 Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2;
3044 for (unsigned i = 0; i < 2; ++i)
3045 if (Reverse)
3046 MaskVec.push_back(DAG.getConstant(1-i+NumElems, EVT));
3047 else
3048 MaskVec.push_back(DAG.getConstant(i+NumElems, EVT));
Chris Lattnered728e82006-08-11 17:38:39 +00003049 SDOperand ShufMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
3050 &MaskVec[0], MaskVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003051 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[0], V[1], ShufMask);
3052 }
3053
3054 if (Values.size() > 2) {
3055 // Expand into a number of unpckl*.
3056 // e.g. for v4f32
3057 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0>
3058 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1>
3059 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0>
3060 SDOperand UnpckMask = getUnpacklMask(NumElems, DAG);
3061 for (unsigned i = 0; i < NumElems; ++i)
3062 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i));
3063 NumElems >>= 1;
3064 while (NumElems != 0) {
3065 for (unsigned i = 0; i < NumElems; ++i)
3066 V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i], V[i + NumElems],
3067 UnpckMask);
3068 NumElems >>= 1;
3069 }
3070 return V[0];
3071 }
3072
3073 return SDOperand();
3074}
3075
3076SDOperand
3077X86TargetLowering::LowerVECTOR_SHUFFLE(SDOperand Op, SelectionDAG &DAG) {
3078 SDOperand V1 = Op.getOperand(0);
3079 SDOperand V2 = Op.getOperand(1);
3080 SDOperand PermMask = Op.getOperand(2);
3081 MVT::ValueType VT = Op.getValueType();
3082 unsigned NumElems = PermMask.getNumOperands();
3083 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF;
3084 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
Evan Cheng949bcc92006-10-16 06:36:00 +00003085 bool V1IsSplat = false;
3086 bool V2IsSplat = false;
Evan Chenga9467aa2006-04-25 20:13:52 +00003087
Evan Cheng89c5d042006-09-08 01:50:06 +00003088 if (isUndefShuffle(Op.Val))
3089 return DAG.getNode(ISD::UNDEF, VT);
3090
Evan Chenga9467aa2006-04-25 20:13:52 +00003091 if (isSplatMask(PermMask.Val)) {
3092 if (NumElems <= 4) return Op;
3093 // Promote it to a v4i32 splat.
Evan Cheng798b3062006-10-25 20:48:19 +00003094 return PromoteSplat(Op, DAG);
Evan Chenga9467aa2006-04-25 20:13:52 +00003095 }
3096
Evan Cheng798b3062006-10-25 20:48:19 +00003097 if (X86::isMOVLMask(PermMask.Val))
3098 return (V1IsUndef) ? V2 : Op;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003099
Evan Cheng798b3062006-10-25 20:48:19 +00003100 if (X86::isMOVSHDUPMask(PermMask.Val) ||
3101 X86::isMOVSLDUPMask(PermMask.Val) ||
3102 X86::isMOVHLPSMask(PermMask.Val) ||
3103 X86::isMOVHPMask(PermMask.Val) ||
3104 X86::isMOVLPMask(PermMask.Val))
3105 return Op;
Evan Chenga9467aa2006-04-25 20:13:52 +00003106
Evan Cheng798b3062006-10-25 20:48:19 +00003107 if (ShouldXformToMOVHLPS(PermMask.Val) ||
3108 ShouldXformToMOVLP(V1.Val, V2.Val, PermMask.Val))
Evan Chengc415c5b2006-10-25 21:49:50 +00003109 return CommuteVectorShuffle(Op, V1, V2, PermMask, DAG);
Evan Chenga9467aa2006-04-25 20:13:52 +00003110
Evan Chengc415c5b2006-10-25 21:49:50 +00003111 bool Commuted = false;
Evan Cheng798b3062006-10-25 20:48:19 +00003112 V1IsSplat = isSplatVector(V1.Val);
3113 V2IsSplat = isSplatVector(V2.Val);
3114 if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) {
Evan Chengc415c5b2006-10-25 21:49:50 +00003115 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG);
Evan Cheng798b3062006-10-25 20:48:19 +00003116 std::swap(V1IsSplat, V2IsSplat);
3117 std::swap(V1IsUndef, V2IsUndef);
Evan Chengc415c5b2006-10-25 21:49:50 +00003118 Commuted = true;
Evan Cheng798b3062006-10-25 20:48:19 +00003119 }
3120
3121 if (isCommutedMOVL(PermMask.Val, V2IsSplat, V2IsUndef)) {
3122 if (V2IsUndef) return V1;
Evan Chengc415c5b2006-10-25 21:49:50 +00003123 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG);
Evan Cheng798b3062006-10-25 20:48:19 +00003124 if (V2IsSplat) {
3125 // V2 is a splat, so the mask may be malformed. That is, it may point
3126 // to any V2 element. The instruction selectior won't like this. Get
3127 // a corrected mask and commute to form a proper MOVS{S|D}.
3128 SDOperand NewMask = getMOVLMask(NumElems, DAG);
3129 if (NewMask.Val != PermMask.Val)
3130 Op = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask);
Evan Chenga9467aa2006-04-25 20:13:52 +00003131 }
Evan Cheng798b3062006-10-25 20:48:19 +00003132 return Op;
Evan Cheng949bcc92006-10-16 06:36:00 +00003133 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003134
Evan Cheng949bcc92006-10-16 06:36:00 +00003135 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) ||
3136 X86::isUNPCKLMask(PermMask.Val) ||
3137 X86::isUNPCKHMask(PermMask.Val))
3138 return Op;
Evan Cheng8c5766e2006-10-04 18:33:38 +00003139
Evan Cheng798b3062006-10-25 20:48:19 +00003140 if (V2IsSplat) {
3141 // Normalize mask so all entries that point to V2 points to its first
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003142 // element then try to match unpck{h|l} again. If match, return a
Evan Cheng798b3062006-10-25 20:48:19 +00003143 // new vector_shuffle with the corrected mask.
3144 SDOperand NewMask = NormalizeMask(PermMask, DAG);
3145 if (NewMask.Val != PermMask.Val) {
3146 if (X86::isUNPCKLMask(PermMask.Val, true)) {
3147 SDOperand NewMask = getUnpacklMask(NumElems, DAG);
3148 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask);
3149 } else if (X86::isUNPCKHMask(PermMask.Val, true)) {
3150 SDOperand NewMask = getUnpackhMask(NumElems, DAG);
3151 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask);
Evan Chenga9467aa2006-04-25 20:13:52 +00003152 }
3153 }
3154 }
3155
3156 // Normalize the node to match x86 shuffle ops if needed
Evan Chengc415c5b2006-10-25 21:49:50 +00003157 if (V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(PermMask.Val))
3158 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG);
3159
3160 if (Commuted) {
3161 // Commute is back and try unpck* again.
3162 Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG);
3163 if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) ||
3164 X86::isUNPCKLMask(PermMask.Val) ||
3165 X86::isUNPCKHMask(PermMask.Val))
3166 return Op;
3167 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003168
3169 // If VT is integer, try PSHUF* first, then SHUFP*.
3170 if (MVT::isInteger(VT)) {
3171 if (X86::isPSHUFDMask(PermMask.Val) ||
3172 X86::isPSHUFHWMask(PermMask.Val) ||
3173 X86::isPSHUFLWMask(PermMask.Val)) {
3174 if (V2.getOpcode() != ISD::UNDEF)
3175 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1,
3176 DAG.getNode(ISD::UNDEF, V1.getValueType()),PermMask);
3177 return Op;
3178 }
3179
3180 if (X86::isSHUFPMask(PermMask.Val))
3181 return Op;
3182
3183 // Handle v8i16 shuffle high / low shuffle node pair.
3184 if (VT == MVT::v8i16 && isPSHUFHW_PSHUFLWMask(PermMask.Val)) {
3185 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
3186 MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00003187 SmallVector<SDOperand, 8> MaskVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00003188 for (unsigned i = 0; i != 4; ++i)
3189 MaskVec.push_back(PermMask.getOperand(i));
3190 for (unsigned i = 4; i != 8; ++i)
3191 MaskVec.push_back(DAG.getConstant(i, BaseVT));
Chris Lattnered728e82006-08-11 17:38:39 +00003192 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
3193 &MaskVec[0], MaskVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003194 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask);
3195 MaskVec.clear();
3196 for (unsigned i = 0; i != 4; ++i)
3197 MaskVec.push_back(DAG.getConstant(i, BaseVT));
3198 for (unsigned i = 4; i != 8; ++i)
3199 MaskVec.push_back(PermMask.getOperand(i));
Chris Lattnered728e82006-08-11 17:38:39 +00003200 Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0],MaskVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003201 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask);
3202 }
3203 } else {
3204 // Floating point cases in the other order.
3205 if (X86::isSHUFPMask(PermMask.Val))
3206 return Op;
3207 if (X86::isPSHUFDMask(PermMask.Val) ||
3208 X86::isPSHUFHWMask(PermMask.Val) ||
3209 X86::isPSHUFLWMask(PermMask.Val)) {
3210 if (V2.getOpcode() != ISD::UNDEF)
3211 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1,
3212 DAG.getNode(ISD::UNDEF, V1.getValueType()),PermMask);
3213 return Op;
3214 }
3215 }
3216
3217 if (NumElems == 4) {
Evan Chenga9467aa2006-04-25 20:13:52 +00003218 MVT::ValueType MaskVT = PermMask.getValueType();
3219 MVT::ValueType MaskEVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00003220 SmallVector<std::pair<int, int>, 8> Locs;
Evan Cheng3cd43622006-04-28 07:03:38 +00003221 Locs.reserve(NumElems);
Chris Lattner35a08552007-02-25 07:10:00 +00003222 SmallVector<SDOperand, 8> Mask1(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT));
3223 SmallVector<SDOperand, 8> Mask2(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT));
Evan Cheng3cd43622006-04-28 07:03:38 +00003224 unsigned NumHi = 0;
3225 unsigned NumLo = 0;
3226 // If no more than two elements come from either vector. This can be
3227 // implemented with two shuffles. First shuffle gather the elements.
3228 // The second shuffle, which takes the first shuffle as both of its
3229 // vector operands, put the elements into the right order.
3230 for (unsigned i = 0; i != NumElems; ++i) {
3231 SDOperand Elt = PermMask.getOperand(i);
3232 if (Elt.getOpcode() == ISD::UNDEF) {
3233 Locs[i] = std::make_pair(-1, -1);
3234 } else {
3235 unsigned Val = cast<ConstantSDNode>(Elt)->getValue();
3236 if (Val < NumElems) {
3237 Locs[i] = std::make_pair(0, NumLo);
3238 Mask1[NumLo] = Elt;
3239 NumLo++;
3240 } else {
3241 Locs[i] = std::make_pair(1, NumHi);
3242 if (2+NumHi < NumElems)
3243 Mask1[2+NumHi] = Elt;
3244 NumHi++;
3245 }
3246 }
3247 }
3248 if (NumLo <= 2 && NumHi <= 2) {
3249 V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2,
Chris Lattnered728e82006-08-11 17:38:39 +00003250 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
3251 &Mask1[0], Mask1.size()));
Evan Cheng3cd43622006-04-28 07:03:38 +00003252 for (unsigned i = 0; i != NumElems; ++i) {
3253 if (Locs[i].first == -1)
3254 continue;
3255 else {
3256 unsigned Idx = (i < NumElems/2) ? 0 : NumElems;
3257 Idx += Locs[i].first * (NumElems/2) + Locs[i].second;
3258 Mask2[i] = DAG.getConstant(Idx, MaskEVT);
3259 }
3260 }
3261
3262 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1,
Chris Lattnered728e82006-08-11 17:38:39 +00003263 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
3264 &Mask2[0], Mask2.size()));
Evan Cheng3cd43622006-04-28 07:03:38 +00003265 }
3266
3267 // Break it into (shuffle shuffle_hi, shuffle_lo).
3268 Locs.clear();
Chris Lattner35a08552007-02-25 07:10:00 +00003269 SmallVector<SDOperand,8> LoMask(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT));
3270 SmallVector<SDOperand,8> HiMask(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT));
3271 SmallVector<SDOperand,8> *MaskPtr = &LoMask;
Evan Chenga9467aa2006-04-25 20:13:52 +00003272 unsigned MaskIdx = 0;
3273 unsigned LoIdx = 0;
3274 unsigned HiIdx = NumElems/2;
3275 for (unsigned i = 0; i != NumElems; ++i) {
3276 if (i == NumElems/2) {
3277 MaskPtr = &HiMask;
3278 MaskIdx = 1;
3279 LoIdx = 0;
3280 HiIdx = NumElems/2;
3281 }
3282 SDOperand Elt = PermMask.getOperand(i);
3283 if (Elt.getOpcode() == ISD::UNDEF) {
3284 Locs[i] = std::make_pair(-1, -1);
3285 } else if (cast<ConstantSDNode>(Elt)->getValue() < NumElems) {
3286 Locs[i] = std::make_pair(MaskIdx, LoIdx);
3287 (*MaskPtr)[LoIdx] = Elt;
3288 LoIdx++;
3289 } else {
3290 Locs[i] = std::make_pair(MaskIdx, HiIdx);
3291 (*MaskPtr)[HiIdx] = Elt;
3292 HiIdx++;
3293 }
3294 }
3295
Chris Lattner3d826992006-05-16 06:45:34 +00003296 SDOperand LoShuffle =
3297 DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2,
Chris Lattnered728e82006-08-11 17:38:39 +00003298 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
3299 &LoMask[0], LoMask.size()));
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003300 SDOperand HiShuffle =
Chris Lattner3d826992006-05-16 06:45:34 +00003301 DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2,
Chris Lattnered728e82006-08-11 17:38:39 +00003302 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
3303 &HiMask[0], HiMask.size()));
Chris Lattner35a08552007-02-25 07:10:00 +00003304 SmallVector<SDOperand, 8> MaskOps;
Evan Chenga9467aa2006-04-25 20:13:52 +00003305 for (unsigned i = 0; i != NumElems; ++i) {
3306 if (Locs[i].first == -1) {
3307 MaskOps.push_back(DAG.getNode(ISD::UNDEF, MaskEVT));
3308 } else {
3309 unsigned Idx = Locs[i].first * NumElems + Locs[i].second;
3310 MaskOps.push_back(DAG.getConstant(Idx, MaskEVT));
3311 }
3312 }
3313 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, LoShuffle, HiShuffle,
Chris Lattnered728e82006-08-11 17:38:39 +00003314 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
3315 &MaskOps[0], MaskOps.size()));
Evan Chenga9467aa2006-04-25 20:13:52 +00003316 }
3317
3318 return SDOperand();
3319}
3320
3321SDOperand
3322X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) {
3323 if (!isa<ConstantSDNode>(Op.getOperand(1)))
3324 return SDOperand();
3325
3326 MVT::ValueType VT = Op.getValueType();
3327 // TODO: handle v16i8.
3328 if (MVT::getSizeInBits(VT) == 16) {
3329 // Transform it so it match pextrw which produces a 32-bit result.
3330 MVT::ValueType EVT = (MVT::ValueType)(VT+1);
3331 SDOperand Extract = DAG.getNode(X86ISD::PEXTRW, EVT,
3332 Op.getOperand(0), Op.getOperand(1));
3333 SDOperand Assert = DAG.getNode(ISD::AssertZext, EVT, Extract,
3334 DAG.getValueType(VT));
3335 return DAG.getNode(ISD::TRUNCATE, VT, Assert);
3336 } else if (MVT::getSizeInBits(VT) == 32) {
3337 SDOperand Vec = Op.getOperand(0);
3338 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue();
3339 if (Idx == 0)
3340 return Op;
Evan Chenga9467aa2006-04-25 20:13:52 +00003341 // SHUFPS the element to the lowest double word, then movss.
3342 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4);
Chris Lattner35a08552007-02-25 07:10:00 +00003343 SmallVector<SDOperand, 8> IdxVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00003344 IdxVec.push_back(DAG.getConstant(Idx, MVT::getVectorBaseType(MaskVT)));
3345 IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT)));
3346 IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT)));
3347 IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT)));
Chris Lattnered728e82006-08-11 17:38:39 +00003348 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
3349 &IdxVec[0], IdxVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003350 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(),
Evan Cheng922e1912006-11-07 22:14:24 +00003351 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask);
Evan Chenga9467aa2006-04-25 20:13:52 +00003352 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec,
Evan Chengde7156f2006-06-15 08:14:54 +00003353 DAG.getConstant(0, getPointerTy()));
Evan Chenga9467aa2006-04-25 20:13:52 +00003354 } else if (MVT::getSizeInBits(VT) == 64) {
3355 SDOperand Vec = Op.getOperand(0);
3356 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue();
3357 if (Idx == 0)
3358 return Op;
3359
3360 // UNPCKHPD the element to the lowest double word, then movsd.
3361 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored
3362 // to a f64mem, the whole operation is folded into a single MOVHPDmr.
3363 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4);
Chris Lattner35a08552007-02-25 07:10:00 +00003364 SmallVector<SDOperand, 8> IdxVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00003365 IdxVec.push_back(DAG.getConstant(1, MVT::getVectorBaseType(MaskVT)));
3366 IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT)));
Chris Lattnered728e82006-08-11 17:38:39 +00003367 SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
3368 &IdxVec[0], IdxVec.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003369 Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(),
3370 Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask);
3371 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec,
Evan Chengde7156f2006-06-15 08:14:54 +00003372 DAG.getConstant(0, getPointerTy()));
Evan Chenga9467aa2006-04-25 20:13:52 +00003373 }
3374
3375 return SDOperand();
3376}
3377
3378SDOperand
3379X86TargetLowering::LowerINSERT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) {
Evan Cheng9fee4422006-05-16 07:21:53 +00003380 // Transform it so it match pinsrw which expects a 16-bit value in a GR32
Evan Chenga9467aa2006-04-25 20:13:52 +00003381 // as its second argument.
3382 MVT::ValueType VT = Op.getValueType();
3383 MVT::ValueType BaseVT = MVT::getVectorBaseType(VT);
3384 SDOperand N0 = Op.getOperand(0);
3385 SDOperand N1 = Op.getOperand(1);
3386 SDOperand N2 = Op.getOperand(2);
3387 if (MVT::getSizeInBits(BaseVT) == 16) {
3388 if (N1.getValueType() != MVT::i32)
3389 N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1);
3390 if (N2.getValueType() != MVT::i32)
3391 N2 = DAG.getConstant(cast<ConstantSDNode>(N2)->getValue(), MVT::i32);
3392 return DAG.getNode(X86ISD::PINSRW, VT, N0, N1, N2);
3393 } else if (MVT::getSizeInBits(BaseVT) == 32) {
3394 unsigned Idx = cast<ConstantSDNode>(N2)->getValue();
3395 if (Idx == 0) {
3396 // Use a movss.
3397 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, N1);
3398 MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4);
3399 MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT);
Chris Lattner35a08552007-02-25 07:10:00 +00003400 SmallVector<SDOperand, 8> MaskVec;
Evan Chenga9467aa2006-04-25 20:13:52 +00003401 MaskVec.push_back(DAG.getConstant(4, BaseVT));
3402 for (unsigned i = 1; i <= 3; ++i)
3403 MaskVec.push_back(DAG.getConstant(i, BaseVT));
3404 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, N0, N1,
Chris Lattnered728e82006-08-11 17:38:39 +00003405 DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
3406 &MaskVec[0], MaskVec.size()));
Evan Chenga9467aa2006-04-25 20:13:52 +00003407 } else {
3408 // Use two pinsrw instructions to insert a 32 bit value.
3409 Idx <<= 1;
3410 if (MVT::isFloatingPoint(N1.getValueType())) {
Evan Chenge71fe34d2006-10-09 20:57:25 +00003411 if (ISD::isNON_EXTLoad(N1.Val)) {
Evan Cheng9fee4422006-05-16 07:21:53 +00003412 // Just load directly from f32mem to GR32.
Evan Chenge71fe34d2006-10-09 20:57:25 +00003413 LoadSDNode *LD = cast<LoadSDNode>(N1);
3414 N1 = DAG.getLoad(MVT::i32, LD->getChain(), LD->getBasePtr(),
3415 LD->getSrcValue(), LD->getSrcValueOffset());
Evan Chenga9467aa2006-04-25 20:13:52 +00003416 } else {
3417 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v4f32, N1);
3418 N1 = DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, N1);
3419 N1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, N1,
Evan Chengde7156f2006-06-15 08:14:54 +00003420 DAG.getConstant(0, getPointerTy()));
Evan Chenga9467aa2006-04-25 20:13:52 +00003421 }
3422 }
3423 N0 = DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, N0);
3424 N0 = DAG.getNode(X86ISD::PINSRW, MVT::v8i16, N0, N1,
Evan Chengde7156f2006-06-15 08:14:54 +00003425 DAG.getConstant(Idx, getPointerTy()));
Evan Chenga9467aa2006-04-25 20:13:52 +00003426 N1 = DAG.getNode(ISD::SRL, MVT::i32, N1, DAG.getConstant(16, MVT::i8));
3427 N0 = DAG.getNode(X86ISD::PINSRW, MVT::v8i16, N0, N1,
Evan Chengde7156f2006-06-15 08:14:54 +00003428 DAG.getConstant(Idx+1, getPointerTy()));
Evan Chenga9467aa2006-04-25 20:13:52 +00003429 return DAG.getNode(ISD::BIT_CONVERT, VT, N0);
3430 }
3431 }
3432
3433 return SDOperand();
3434}
3435
3436SDOperand
3437X86TargetLowering::LowerSCALAR_TO_VECTOR(SDOperand Op, SelectionDAG &DAG) {
3438 SDOperand AnyExt = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, Op.getOperand(0));
3439 return DAG.getNode(X86ISD::S2VEC, Op.getValueType(), AnyExt);
3440}
3441
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003442// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
Evan Chenga9467aa2006-04-25 20:13:52 +00003443// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is
3444// one of the above mentioned nodes. It has to be wrapped because otherwise
3445// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
3446// be used to form addressing mode. These wrapped nodes will be selected
3447// into MOV32ri.
3448SDOperand
3449X86TargetLowering::LowerConstantPool(SDOperand Op, SelectionDAG &DAG) {
3450 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
Evan Cheng0b169222006-11-29 23:19:46 +00003451 SDOperand Result = DAG.getTargetConstantPool(CP->getConstVal(),
3452 getPointerTy(),
3453 CP->getAlignment());
Evan Cheng62cdc3f2006-12-05 04:01:03 +00003454 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result);
Anton Korobeynikova0554d92007-01-12 19:20:47 +00003455 // With PIC, the address is actually $g + Offset.
3456 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
3457 !Subtarget->isPICStyleRIPRel()) {
3458 Result = DAG.getNode(ISD::ADD, getPointerTy(),
3459 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
3460 Result);
Evan Chenga9467aa2006-04-25 20:13:52 +00003461 }
3462
3463 return Result;
3464}
3465
3466SDOperand
3467X86TargetLowering::LowerGlobalAddress(SDOperand Op, SelectionDAG &DAG) {
3468 GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
Evan Cheng0b169222006-11-29 23:19:46 +00003469 SDOperand Result = DAG.getTargetGlobalAddress(GV, getPointerTy());
Evan Cheng62cdc3f2006-12-05 04:01:03 +00003470 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result);
Anton Korobeynikova0554d92007-01-12 19:20:47 +00003471 // With PIC, the address is actually $g + Offset.
3472 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
3473 !Subtarget->isPICStyleRIPRel()) {
3474 Result = DAG.getNode(ISD::ADD, getPointerTy(),
3475 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
3476 Result);
Evan Chenga9467aa2006-04-25 20:13:52 +00003477 }
Anton Korobeynikov430e68a12006-12-22 22:29:05 +00003478
3479 // For Darwin & Mingw32, external and weak symbols are indirect, so we want to
3480 // load the value at address GV, not the value of GV itself. This means that
3481 // the GlobalAddress must be in the base or index register of the address, not
3482 // the GV offset field. Platform check is inside GVRequiresExtraLoad() call
Anton Korobeynikova0554d92007-01-12 19:20:47 +00003483 // The same applies for external symbols during PIC codegen
Anton Korobeynikov430e68a12006-12-22 22:29:05 +00003484 if (Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false))
3485 Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), Result, NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003486
3487 return Result;
3488}
3489
3490SDOperand
3491X86TargetLowering::LowerExternalSymbol(SDOperand Op, SelectionDAG &DAG) {
3492 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol();
Evan Cheng0b169222006-11-29 23:19:46 +00003493 SDOperand Result = DAG.getTargetExternalSymbol(Sym, getPointerTy());
Evan Cheng62cdc3f2006-12-05 04:01:03 +00003494 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result);
Anton Korobeynikova0554d92007-01-12 19:20:47 +00003495 // With PIC, the address is actually $g + Offset.
3496 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
3497 !Subtarget->isPICStyleRIPRel()) {
3498 Result = DAG.getNode(ISD::ADD, getPointerTy(),
3499 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
3500 Result);
3501 }
3502
3503 return Result;
3504}
3505
3506SDOperand X86TargetLowering::LowerJumpTable(SDOperand Op, SelectionDAG &DAG) {
3507 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
3508 SDOperand Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy());
3509 Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result);
3510 // With PIC, the address is actually $g + Offset.
3511 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
3512 !Subtarget->isPICStyleRIPRel()) {
3513 Result = DAG.getNode(ISD::ADD, getPointerTy(),
3514 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
3515 Result);
Evan Chenga9467aa2006-04-25 20:13:52 +00003516 }
3517
3518 return Result;
3519}
3520
3521SDOperand X86TargetLowering::LowerShift(SDOperand Op, SelectionDAG &DAG) {
Evan Cheng9c249c32006-01-09 18:33:28 +00003522 assert(Op.getNumOperands() == 3 && Op.getValueType() == MVT::i32 &&
3523 "Not an i64 shift!");
3524 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
3525 SDOperand ShOpLo = Op.getOperand(0);
3526 SDOperand ShOpHi = Op.getOperand(1);
3527 SDOperand ShAmt = Op.getOperand(2);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003528 SDOperand Tmp1 = isSRA ?
3529 DAG.getNode(ISD::SRA, MVT::i32, ShOpHi, DAG.getConstant(31, MVT::i8)) :
3530 DAG.getConstant(0, MVT::i32);
Evan Cheng9c249c32006-01-09 18:33:28 +00003531
3532 SDOperand Tmp2, Tmp3;
3533 if (Op.getOpcode() == ISD::SHL_PARTS) {
3534 Tmp2 = DAG.getNode(X86ISD::SHLD, MVT::i32, ShOpHi, ShOpLo, ShAmt);
3535 Tmp3 = DAG.getNode(ISD::SHL, MVT::i32, ShOpLo, ShAmt);
3536 } else {
3537 Tmp2 = DAG.getNode(X86ISD::SHRD, MVT::i32, ShOpLo, ShOpHi, ShAmt);
Evan Cheng267ba592006-01-19 01:46:14 +00003538 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, MVT::i32, ShOpHi, ShAmt);
Evan Cheng9c249c32006-01-09 18:33:28 +00003539 }
3540
Evan Cheng4259a0f2006-09-11 02:19:56 +00003541 const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag);
3542 SDOperand AndNode = DAG.getNode(ISD::AND, MVT::i8, ShAmt,
3543 DAG.getConstant(32, MVT::i8));
3544 SDOperand COps[]={DAG.getEntryNode(), AndNode, DAG.getConstant(0, MVT::i8)};
3545 SDOperand InFlag = DAG.getNode(X86ISD::CMP, VTs, 2, COps, 3).getValue(1);
Evan Cheng9c249c32006-01-09 18:33:28 +00003546
3547 SDOperand Hi, Lo;
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003548 SDOperand CC = DAG.getConstant(X86::COND_NE, MVT::i8);
Evan Cheng9c249c32006-01-09 18:33:28 +00003549
Evan Cheng4259a0f2006-09-11 02:19:56 +00003550 VTs = DAG.getNodeValueTypes(MVT::i32, MVT::Flag);
3551 SmallVector<SDOperand, 4> Ops;
Evan Cheng9c249c32006-01-09 18:33:28 +00003552 if (Op.getOpcode() == ISD::SHL_PARTS) {
3553 Ops.push_back(Tmp2);
3554 Ops.push_back(Tmp3);
3555 Ops.push_back(CC);
3556 Ops.push_back(InFlag);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003557 Hi = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size());
Evan Cheng9c249c32006-01-09 18:33:28 +00003558 InFlag = Hi.getValue(1);
3559
3560 Ops.clear();
3561 Ops.push_back(Tmp3);
3562 Ops.push_back(Tmp1);
3563 Ops.push_back(CC);
3564 Ops.push_back(InFlag);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003565 Lo = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size());
Evan Cheng9c249c32006-01-09 18:33:28 +00003566 } else {
3567 Ops.push_back(Tmp2);
3568 Ops.push_back(Tmp3);
3569 Ops.push_back(CC);
Evan Cheng12181af2006-01-09 22:29:54 +00003570 Ops.push_back(InFlag);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003571 Lo = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size());
Evan Cheng9c249c32006-01-09 18:33:28 +00003572 InFlag = Lo.getValue(1);
3573
3574 Ops.clear();
3575 Ops.push_back(Tmp3);
3576 Ops.push_back(Tmp1);
3577 Ops.push_back(CC);
3578 Ops.push_back(InFlag);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003579 Hi = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size());
Evan Cheng9c249c32006-01-09 18:33:28 +00003580 }
3581
Evan Cheng4259a0f2006-09-11 02:19:56 +00003582 VTs = DAG.getNodeValueTypes(MVT::i32, MVT::i32);
Evan Cheng9c249c32006-01-09 18:33:28 +00003583 Ops.clear();
3584 Ops.push_back(Lo);
3585 Ops.push_back(Hi);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003586 return DAG.getNode(ISD::MERGE_VALUES, VTs, 2, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003587}
Evan Cheng6305e502006-01-12 22:54:21 +00003588
Evan Chenga9467aa2006-04-25 20:13:52 +00003589SDOperand X86TargetLowering::LowerSINT_TO_FP(SDOperand Op, SelectionDAG &DAG) {
3590 assert(Op.getOperand(0).getValueType() <= MVT::i64 &&
3591 Op.getOperand(0).getValueType() >= MVT::i16 &&
3592 "Unknown SINT_TO_FP to lower!");
3593
3594 SDOperand Result;
3595 MVT::ValueType SrcVT = Op.getOperand(0).getValueType();
3596 unsigned Size = MVT::getSizeInBits(SrcVT)/8;
3597 MachineFunction &MF = DAG.getMachineFunction();
3598 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
3599 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
Evan Chengdf9ac472006-10-05 23:01:46 +00003600 SDOperand Chain = DAG.getStore(DAG.getEntryNode(), Op.getOperand(0),
Evan Chengab51cf22006-10-13 21:14:26 +00003601 StackSlot, NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003602
3603 // Build the FILD
Chris Lattner35a08552007-02-25 07:10:00 +00003604 SDVTList Tys;
3605 if (X86ScalarSSE)
3606 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag);
3607 else
3608 Tys = DAG.getVTList(MVT::f64, MVT::Other);
3609 SmallVector<SDOperand, 8> Ops;
Evan Chenga9467aa2006-04-25 20:13:52 +00003610 Ops.push_back(Chain);
3611 Ops.push_back(StackSlot);
3612 Ops.push_back(DAG.getValueType(SrcVT));
3613 Result = DAG.getNode(X86ScalarSSE ? X86ISD::FILD_FLAG :X86ISD::FILD,
Chris Lattnerc24a1d32006-08-08 02:23:42 +00003614 Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003615
3616 if (X86ScalarSSE) {
3617 Chain = Result.getValue(1);
3618 SDOperand InFlag = Result.getValue(2);
3619
3620 // FIXME: Currently the FST is flagged to the FILD_FLAG. This
3621 // shouldn't be necessary except that RFP cannot be live across
3622 // multiple blocks. When stackifier is fixed, they can be uncoupled.
Chris Lattner76ac0682005-11-15 00:40:23 +00003623 MachineFunction &MF = DAG.getMachineFunction();
Evan Chenga9467aa2006-04-25 20:13:52 +00003624 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8);
Chris Lattner76ac0682005-11-15 00:40:23 +00003625 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
Chris Lattner35a08552007-02-25 07:10:00 +00003626 Tys = DAG.getVTList(MVT::Other);
3627 SmallVector<SDOperand, 8> Ops;
Evan Cheng6305e502006-01-12 22:54:21 +00003628 Ops.push_back(Chain);
Evan Chenga9467aa2006-04-25 20:13:52 +00003629 Ops.push_back(Result);
Chris Lattner76ac0682005-11-15 00:40:23 +00003630 Ops.push_back(StackSlot);
Evan Chenga9467aa2006-04-25 20:13:52 +00003631 Ops.push_back(DAG.getValueType(Op.getValueType()));
3632 Ops.push_back(InFlag);
Chris Lattnerc24a1d32006-08-08 02:23:42 +00003633 Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size());
Evan Chenge71fe34d2006-10-09 20:57:25 +00003634 Result = DAG.getLoad(Op.getValueType(), Chain, StackSlot, NULL, 0);
Chris Lattner76ac0682005-11-15 00:40:23 +00003635 }
Chris Lattner76ac0682005-11-15 00:40:23 +00003636
Evan Chenga9467aa2006-04-25 20:13:52 +00003637 return Result;
3638}
3639
3640SDOperand X86TargetLowering::LowerFP_TO_SINT(SDOperand Op, SelectionDAG &DAG) {
3641 assert(Op.getValueType() <= MVT::i64 && Op.getValueType() >= MVT::i16 &&
3642 "Unknown FP_TO_SINT to lower!");
3643 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary
3644 // stack slot.
3645 MachineFunction &MF = DAG.getMachineFunction();
3646 unsigned MemSize = MVT::getSizeInBits(Op.getValueType())/8;
3647 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize);
3648 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
3649
3650 unsigned Opc;
3651 switch (Op.getValueType()) {
Chris Lattner76ac0682005-11-15 00:40:23 +00003652 default: assert(0 && "Invalid FP_TO_SINT to lower!");
3653 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
3654 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
3655 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
Evan Chenga9467aa2006-04-25 20:13:52 +00003656 }
Chris Lattner76ac0682005-11-15 00:40:23 +00003657
Evan Chenga9467aa2006-04-25 20:13:52 +00003658 SDOperand Chain = DAG.getEntryNode();
3659 SDOperand Value = Op.getOperand(0);
3660 if (X86ScalarSSE) {
3661 assert(Op.getValueType() == MVT::i64 && "Invalid FP_TO_SINT to lower!");
Evan Chengab51cf22006-10-13 21:14:26 +00003662 Chain = DAG.getStore(Chain, Value, StackSlot, NULL, 0);
Chris Lattner35a08552007-02-25 07:10:00 +00003663 SDVTList Tys = DAG.getVTList(MVT::f64, MVT::Other);
3664 SDOperand Ops[] = {
3665 Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType())
3666 };
3667 Value = DAG.getNode(X86ISD::FLD, Tys, Ops, 3);
Evan Chenga9467aa2006-04-25 20:13:52 +00003668 Chain = Value.getValue(1);
3669 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize);
3670 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
3671 }
Chris Lattner76ac0682005-11-15 00:40:23 +00003672
Evan Chenga9467aa2006-04-25 20:13:52 +00003673 // Build the FP_TO_INT*_IN_MEM
Chris Lattner35a08552007-02-25 07:10:00 +00003674 SDOperand Ops[] = { Chain, Value, StackSlot };
3675 SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops, 3);
Evan Cheng172fce72006-01-06 00:43:03 +00003676
Evan Chenga9467aa2006-04-25 20:13:52 +00003677 // Load the result.
Evan Chenge71fe34d2006-10-09 20:57:25 +00003678 return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003679}
3680
3681SDOperand X86TargetLowering::LowerFABS(SDOperand Op, SelectionDAG &DAG) {
3682 MVT::ValueType VT = Op.getValueType();
3683 const Type *OpNTy = MVT::getTypeForValueType(VT);
3684 std::vector<Constant*> CV;
3685 if (VT == MVT::f64) {
3686 CV.push_back(ConstantFP::get(OpNTy, BitsToDouble(~(1ULL << 63))));
3687 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3688 } else {
3689 CV.push_back(ConstantFP::get(OpNTy, BitsToFloat(~(1U << 31))));
3690 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3691 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3692 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3693 }
3694 Constant *CS = ConstantStruct::get(CV);
3695 SDOperand CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4);
Chris Lattner35a08552007-02-25 07:10:00 +00003696 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
Evan Chengbd1c5a82006-08-11 09:08:15 +00003697 SmallVector<SDOperand, 3> Ops;
3698 Ops.push_back(DAG.getEntryNode());
3699 Ops.push_back(CPIdx);
3700 Ops.push_back(DAG.getSrcValue(NULL));
3701 SDOperand Mask = DAG.getNode(X86ISD::LOAD_PACK, Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003702 return DAG.getNode(X86ISD::FAND, VT, Op.getOperand(0), Mask);
3703}
3704
3705SDOperand X86TargetLowering::LowerFNEG(SDOperand Op, SelectionDAG &DAG) {
3706 MVT::ValueType VT = Op.getValueType();
3707 const Type *OpNTy = MVT::getTypeForValueType(VT);
3708 std::vector<Constant*> CV;
3709 if (VT == MVT::f64) {
3710 CV.push_back(ConstantFP::get(OpNTy, BitsToDouble(1ULL << 63)));
3711 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3712 } else {
3713 CV.push_back(ConstantFP::get(OpNTy, BitsToFloat(1U << 31)));
3714 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3715 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3716 CV.push_back(ConstantFP::get(OpNTy, 0.0));
3717 }
3718 Constant *CS = ConstantStruct::get(CV);
3719 SDOperand CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4);
Chris Lattner35a08552007-02-25 07:10:00 +00003720 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
Evan Chengbd1c5a82006-08-11 09:08:15 +00003721 SmallVector<SDOperand, 3> Ops;
3722 Ops.push_back(DAG.getEntryNode());
3723 Ops.push_back(CPIdx);
3724 Ops.push_back(DAG.getSrcValue(NULL));
3725 SDOperand Mask = DAG.getNode(X86ISD::LOAD_PACK, Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003726 return DAG.getNode(X86ISD::FXOR, VT, Op.getOperand(0), Mask);
3727}
3728
Evan Cheng4363e882007-01-05 07:55:56 +00003729SDOperand X86TargetLowering::LowerFCOPYSIGN(SDOperand Op, SelectionDAG &DAG) {
Evan Cheng82241c82007-01-05 21:37:56 +00003730 SDOperand Op0 = Op.getOperand(0);
3731 SDOperand Op1 = Op.getOperand(1);
Evan Cheng4363e882007-01-05 07:55:56 +00003732 MVT::ValueType VT = Op.getValueType();
Evan Cheng82241c82007-01-05 21:37:56 +00003733 MVT::ValueType SrcVT = Op1.getValueType();
Evan Cheng4363e882007-01-05 07:55:56 +00003734 const Type *SrcTy = MVT::getTypeForValueType(SrcVT);
Evan Cheng82241c82007-01-05 21:37:56 +00003735
3736 // If second operand is smaller, extend it first.
3737 if (MVT::getSizeInBits(SrcVT) < MVT::getSizeInBits(VT)) {
3738 Op1 = DAG.getNode(ISD::FP_EXTEND, VT, Op1);
3739 SrcVT = VT;
3740 }
3741
Evan Cheng4363e882007-01-05 07:55:56 +00003742 // First get the sign bit of second operand.
3743 std::vector<Constant*> CV;
3744 if (SrcVT == MVT::f64) {
3745 CV.push_back(ConstantFP::get(SrcTy, BitsToDouble(1ULL << 63)));
3746 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3747 } else {
3748 CV.push_back(ConstantFP::get(SrcTy, BitsToFloat(1U << 31)));
3749 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3750 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3751 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3752 }
3753 Constant *CS = ConstantStruct::get(CV);
3754 SDOperand CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4);
Chris Lattnere56fef92007-02-25 06:40:16 +00003755 SDVTList Tys = DAG.getVTList(SrcVT, MVT::Other);
Evan Cheng4363e882007-01-05 07:55:56 +00003756 SmallVector<SDOperand, 3> Ops;
3757 Ops.push_back(DAG.getEntryNode());
3758 Ops.push_back(CPIdx);
3759 Ops.push_back(DAG.getSrcValue(NULL));
Evan Cheng82241c82007-01-05 21:37:56 +00003760 SDOperand Mask1 = DAG.getNode(X86ISD::LOAD_PACK, Tys, &Ops[0], Ops.size());
3761 SDOperand SignBit = DAG.getNode(X86ISD::FAND, SrcVT, Op1, Mask1);
Evan Cheng4363e882007-01-05 07:55:56 +00003762
3763 // Shift sign bit right or left if the two operands have different types.
3764 if (MVT::getSizeInBits(SrcVT) > MVT::getSizeInBits(VT)) {
3765 // Op0 is MVT::f32, Op1 is MVT::f64.
3766 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2f64, SignBit);
3767 SignBit = DAG.getNode(X86ISD::FSRL, MVT::v2f64, SignBit,
3768 DAG.getConstant(32, MVT::i32));
3769 SignBit = DAG.getNode(ISD::BIT_CONVERT, MVT::v4f32, SignBit);
3770 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::f32, SignBit,
3771 DAG.getConstant(0, getPointerTy()));
Evan Cheng4363e882007-01-05 07:55:56 +00003772 }
3773
Evan Cheng82241c82007-01-05 21:37:56 +00003774 // Clear first operand sign bit.
3775 CV.clear();
3776 if (VT == MVT::f64) {
3777 CV.push_back(ConstantFP::get(SrcTy, BitsToDouble(~(1ULL << 63))));
3778 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3779 } else {
3780 CV.push_back(ConstantFP::get(SrcTy, BitsToFloat(~(1U << 31))));
3781 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3782 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3783 CV.push_back(ConstantFP::get(SrcTy, 0.0));
3784 }
3785 CS = ConstantStruct::get(CV);
3786 CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4);
Chris Lattnere56fef92007-02-25 06:40:16 +00003787 Tys = DAG.getVTList(VT, MVT::Other);
Evan Cheng82241c82007-01-05 21:37:56 +00003788 Ops.clear();
3789 Ops.push_back(DAG.getEntryNode());
3790 Ops.push_back(CPIdx);
3791 Ops.push_back(DAG.getSrcValue(NULL));
3792 SDOperand Mask2 = DAG.getNode(X86ISD::LOAD_PACK, Tys, &Ops[0], Ops.size());
3793 SDOperand Val = DAG.getNode(X86ISD::FAND, VT, Op0, Mask2);
3794
3795 // Or the value with the sign bit.
3796 return DAG.getNode(X86ISD::FOR, VT, Val, SignBit);
Evan Cheng4363e882007-01-05 07:55:56 +00003797}
3798
Evan Cheng4259a0f2006-09-11 02:19:56 +00003799SDOperand X86TargetLowering::LowerSETCC(SDOperand Op, SelectionDAG &DAG,
3800 SDOperand Chain) {
Evan Chenga9467aa2006-04-25 20:13:52 +00003801 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer");
3802 SDOperand Cond;
Evan Cheng4259a0f2006-09-11 02:19:56 +00003803 SDOperand Op0 = Op.getOperand(0);
3804 SDOperand Op1 = Op.getOperand(1);
Evan Chenga9467aa2006-04-25 20:13:52 +00003805 SDOperand CC = Op.getOperand(2);
3806 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
Evan Cheng694810c2006-10-12 19:12:56 +00003807 const MVT::ValueType *VTs1 = DAG.getNodeValueTypes(MVT::Other, MVT::Flag);
3808 const MVT::ValueType *VTs2 = DAG.getNodeValueTypes(MVT::i8, MVT::Flag);
Evan Chenga9467aa2006-04-25 20:13:52 +00003809 bool isFP = MVT::isFloatingPoint(Op.getOperand(1).getValueType());
Evan Chenga9467aa2006-04-25 20:13:52 +00003810 unsigned X86CC;
Evan Chenga9467aa2006-04-25 20:13:52 +00003811
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003812 if (translateX86CC(cast<CondCodeSDNode>(CC)->get(), isFP, X86CC,
Chris Lattner7a627672006-09-13 03:22:10 +00003813 Op0, Op1, DAG)) {
Evan Cheng4259a0f2006-09-11 02:19:56 +00003814 SDOperand Ops1[] = { Chain, Op0, Op1 };
Evan Cheng694810c2006-10-12 19:12:56 +00003815 Cond = DAG.getNode(X86ISD::CMP, VTs1, 2, Ops1, 3).getValue(1);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003816 SDOperand Ops2[] = { DAG.getConstant(X86CC, MVT::i8), Cond };
Evan Cheng694810c2006-10-12 19:12:56 +00003817 return DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops2, 2);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003818 }
3819
3820 assert(isFP && "Illegal integer SetCC!");
3821
3822 SDOperand COps[] = { Chain, Op0, Op1 };
Evan Cheng694810c2006-10-12 19:12:56 +00003823 Cond = DAG.getNode(X86ISD::CMP, VTs1, 2, COps, 3).getValue(1);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003824
3825 switch (SetCCOpcode) {
3826 default: assert(false && "Illegal floating point SetCC!");
3827 case ISD::SETOEQ: { // !PF & ZF
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003828 SDOperand Ops1[] = { DAG.getConstant(X86::COND_NP, MVT::i8), Cond };
Evan Cheng694810c2006-10-12 19:12:56 +00003829 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops1, 2);
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003830 SDOperand Ops2[] = { DAG.getConstant(X86::COND_E, MVT::i8),
Evan Cheng4259a0f2006-09-11 02:19:56 +00003831 Tmp1.getValue(1) };
Evan Cheng694810c2006-10-12 19:12:56 +00003832 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops2, 2);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003833 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
3834 }
3835 case ISD::SETUNE: { // PF | !ZF
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003836 SDOperand Ops1[] = { DAG.getConstant(X86::COND_P, MVT::i8), Cond };
Evan Cheng694810c2006-10-12 19:12:56 +00003837 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops1, 2);
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003838 SDOperand Ops2[] = { DAG.getConstant(X86::COND_NE, MVT::i8),
Evan Cheng4259a0f2006-09-11 02:19:56 +00003839 Tmp1.getValue(1) };
Evan Cheng694810c2006-10-12 19:12:56 +00003840 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops2, 2);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003841 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
3842 }
Evan Chengc1583db2005-12-21 20:21:51 +00003843 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003844}
Evan Cheng45df7f82006-01-30 23:41:35 +00003845
Evan Chenga9467aa2006-04-25 20:13:52 +00003846SDOperand X86TargetLowering::LowerSELECT(SDOperand Op, SelectionDAG &DAG) {
Evan Cheng4259a0f2006-09-11 02:19:56 +00003847 bool addTest = true;
3848 SDOperand Chain = DAG.getEntryNode();
3849 SDOperand Cond = Op.getOperand(0);
3850 SDOperand CC;
3851 const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag);
Evan Cheng944d1e92006-01-26 02:13:10 +00003852
Evan Cheng4259a0f2006-09-11 02:19:56 +00003853 if (Cond.getOpcode() == ISD::SETCC)
3854 Cond = LowerSETCC(Cond, DAG, Chain);
3855
3856 if (Cond.getOpcode() == X86ISD::SETCC) {
3857 CC = Cond.getOperand(0);
3858
Evan Chenga9467aa2006-04-25 20:13:52 +00003859 // If condition flag is set by a X86ISD::CMP, then make a copy of it
Evan Cheng4259a0f2006-09-11 02:19:56 +00003860 // (since flag operand cannot be shared). Use it as the condition setting
3861 // operand in place of the X86ISD::SETCC.
3862 // If the X86ISD::SETCC has more than one use, then perhaps it's better
Evan Chenga9467aa2006-04-25 20:13:52 +00003863 // to use a test instead of duplicating the X86ISD::CMP (for register
Evan Cheng4259a0f2006-09-11 02:19:56 +00003864 // pressure reason)?
3865 SDOperand Cmp = Cond.getOperand(1);
3866 unsigned Opc = Cmp.getOpcode();
3867 bool IllegalFPCMov = !X86ScalarSSE &&
3868 MVT::isFloatingPoint(Op.getValueType()) &&
3869 !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended());
3870 if ((Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI) &&
3871 !IllegalFPCMov) {
3872 SDOperand Ops[] = { Chain, Cmp.getOperand(1), Cmp.getOperand(2) };
3873 Cond = DAG.getNode(Opc, VTs, 2, Ops, 3);
3874 addTest = false;
3875 }
3876 }
Evan Cheng73a1ad92006-01-10 20:26:56 +00003877
Evan Chenga9467aa2006-04-25 20:13:52 +00003878 if (addTest) {
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003879 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003880 SDOperand Ops[] = { Chain, Cond, DAG.getConstant(0, MVT::i8) };
3881 Cond = DAG.getNode(X86ISD::CMP, VTs, 2, Ops, 3);
Evan Cheng225a4d02005-12-17 01:21:05 +00003882 }
Evan Cheng45df7f82006-01-30 23:41:35 +00003883
Evan Cheng4259a0f2006-09-11 02:19:56 +00003884 VTs = DAG.getNodeValueTypes(Op.getValueType(), MVT::Flag);
3885 SmallVector<SDOperand, 4> Ops;
Evan Chenga9467aa2006-04-25 20:13:52 +00003886 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
3887 // condition is true.
3888 Ops.push_back(Op.getOperand(2));
3889 Ops.push_back(Op.getOperand(1));
3890 Ops.push_back(CC);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003891 Ops.push_back(Cond.getValue(1));
3892 return DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00003893}
Evan Cheng944d1e92006-01-26 02:13:10 +00003894
Evan Chenga9467aa2006-04-25 20:13:52 +00003895SDOperand X86TargetLowering::LowerBRCOND(SDOperand Op, SelectionDAG &DAG) {
Evan Cheng4259a0f2006-09-11 02:19:56 +00003896 bool addTest = true;
3897 SDOperand Chain = Op.getOperand(0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003898 SDOperand Cond = Op.getOperand(1);
3899 SDOperand Dest = Op.getOperand(2);
3900 SDOperand CC;
Evan Cheng4259a0f2006-09-11 02:19:56 +00003901 const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag);
3902
Evan Chenga9467aa2006-04-25 20:13:52 +00003903 if (Cond.getOpcode() == ISD::SETCC)
Evan Cheng4259a0f2006-09-11 02:19:56 +00003904 Cond = LowerSETCC(Cond, DAG, Chain);
Evan Chenga9467aa2006-04-25 20:13:52 +00003905
3906 if (Cond.getOpcode() == X86ISD::SETCC) {
Evan Cheng4259a0f2006-09-11 02:19:56 +00003907 CC = Cond.getOperand(0);
Evan Chenga9467aa2006-04-25 20:13:52 +00003908
Evan Cheng4259a0f2006-09-11 02:19:56 +00003909 // If condition flag is set by a X86ISD::CMP, then make a copy of it
3910 // (since flag operand cannot be shared). Use it as the condition setting
3911 // operand in place of the X86ISD::SETCC.
3912 // If the X86ISD::SETCC has more than one use, then perhaps it's better
3913 // to use a test instead of duplicating the X86ISD::CMP (for register
3914 // pressure reason)?
3915 SDOperand Cmp = Cond.getOperand(1);
3916 unsigned Opc = Cmp.getOpcode();
3917 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI) {
3918 SDOperand Ops[] = { Chain, Cmp.getOperand(1), Cmp.getOperand(2) };
3919 Cond = DAG.getNode(Opc, VTs, 2, Ops, 3);
3920 addTest = false;
3921 }
3922 }
Evan Chengfb22e862006-01-13 01:03:02 +00003923
Evan Chenga9467aa2006-04-25 20:13:52 +00003924 if (addTest) {
Chris Lattnerc0fb5672006-10-20 17:42:20 +00003925 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
Evan Cheng4259a0f2006-09-11 02:19:56 +00003926 SDOperand Ops[] = { Chain, Cond, DAG.getConstant(0, MVT::i8) };
3927 Cond = DAG.getNode(X86ISD::CMP, VTs, 2, Ops, 3);
Evan Cheng6fc31042005-12-19 23:12:38 +00003928 }
Evan Chenga9467aa2006-04-25 20:13:52 +00003929 return DAG.getNode(X86ISD::BRCOND, Op.getValueType(),
Evan Cheng4259a0f2006-09-11 02:19:56 +00003930 Cond, Op.getOperand(2), CC, Cond.getValue(1));
Evan Chenga9467aa2006-04-25 20:13:52 +00003931}
Evan Chengae986f12006-01-11 22:15:48 +00003932
Evan Cheng2a330942006-05-25 00:59:30 +00003933SDOperand X86TargetLowering::LowerCALL(SDOperand Op, SelectionDAG &DAG) {
3934 unsigned CallingConv= cast<ConstantSDNode>(Op.getOperand(1))->getValue();
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003935
Evan Cheng11b0a5d2006-09-08 06:48:29 +00003936 if (Subtarget->is64Bit())
Chris Lattner7802f3e2007-02-25 09:06:15 +00003937 return LowerX86_64CCCCallTo(Op, DAG, CallingConv);
Evan Cheng2a330942006-05-25 00:59:30 +00003938 else
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003939 switch (CallingConv) {
Chris Lattnerfc360392006-09-27 18:29:38 +00003940 default:
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003941 assert(0 && "Unsupported calling convention");
Chris Lattnerfc360392006-09-27 18:29:38 +00003942 case CallingConv::Fast:
Chris Lattner0cd99602007-02-25 08:59:22 +00003943 if (EnableFastCC)
Chris Lattner7802f3e2007-02-25 09:06:15 +00003944 return LowerFastCCCallTo(Op, DAG, CallingConv);
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003945 // Falls through
Chris Lattnerfc360392006-09-27 18:29:38 +00003946 case CallingConv::C:
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00003947 case CallingConv::X86_StdCall:
Chris Lattner7802f3e2007-02-25 09:06:15 +00003948 return LowerCCCCallTo(Op, DAG, CallingConv);
Chris Lattnerfc360392006-09-27 18:29:38 +00003949 case CallingConv::X86_FastCall:
Chris Lattner7802f3e2007-02-25 09:06:15 +00003950 return LowerFastCCCallTo(Op, DAG, CallingConv);
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00003951 }
Evan Cheng2a330942006-05-25 00:59:30 +00003952}
3953
Chris Lattnerdfda38f2007-02-25 08:15:11 +00003954SDOperand X86TargetLowering::LowerRET(SDOperand Op, SelectionDAG &DAG) {
3955 assert((Op.getNumOperands() & 1) == 1 && "ISD::RET should have odd # args");
3956
3957 // Support up returning up to two registers.
3958 MVT::ValueType VTs[2];
3959 unsigned DestRegs[2];
3960 unsigned NumRegs = Op.getNumOperands() / 2;
3961 assert(NumRegs <= 2 && "Can only return up to two regs!");
3962
3963 for (unsigned i = 0; i != NumRegs; ++i)
3964 VTs[i] = Op.getOperand(i*2+1).getValueType();
3965
3966 // Determine which register each value should be copied into.
Chris Lattner3c763092007-02-25 08:29:00 +00003967 GetRetValueLocs(VTs, NumRegs, DestRegs, Subtarget,
3968 DAG.getMachineFunction().getFunction()->getCallingConv());
Chris Lattnerdfda38f2007-02-25 08:15:11 +00003969
3970 // If this is the first return lowered for this function, add the regs to the
3971 // liveout set for the function.
3972 if (DAG.getMachineFunction().liveout_empty()) {
3973 for (unsigned i = 0; i != NumRegs; ++i)
3974 DAG.getMachineFunction().addLiveOut(DestRegs[i]);
3975 }
3976
3977 SDOperand Chain = Op.getOperand(0);
3978 SDOperand Flag;
3979
3980 // Copy the result values into the output registers.
3981 if (NumRegs != 1 || DestRegs[0] != X86::ST0) {
3982 for (unsigned i = 0; i != NumRegs; ++i) {
3983 Chain = DAG.getCopyToReg(Chain, DestRegs[i], Op.getOperand(i*2+1), Flag);
3984 Flag = Chain.getValue(1);
3985 }
3986 } else {
3987 // We need to handle a destination of ST0 specially, because it isn't really
3988 // a register.
3989 SDOperand Value = Op.getOperand(1);
3990
3991 // If this is an FP return with ScalarSSE, we need to move the value from
3992 // an XMM register onto the fp-stack.
3993 if (X86ScalarSSE) {
3994 SDOperand MemLoc;
3995
3996 // If this is a load into a scalarsse value, don't store the loaded value
3997 // back to the stack, only to reload it: just replace the scalar-sse load.
3998 if (ISD::isNON_EXTLoad(Value.Val) &&
3999 (Chain == Value.getValue(1) || Chain == Value.getOperand(0))) {
4000 Chain = Value.getOperand(0);
4001 MemLoc = Value.getOperand(1);
4002 } else {
4003 // Spill the value to memory and reload it into top of stack.
4004 unsigned Size = MVT::getSizeInBits(VTs[0])/8;
4005 MachineFunction &MF = DAG.getMachineFunction();
4006 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
4007 MemLoc = DAG.getFrameIndex(SSFI, getPointerTy());
4008 Chain = DAG.getStore(Op.getOperand(0), Value, MemLoc, NULL, 0);
4009 }
4010 SDVTList Tys = DAG.getVTList(MVT::f64, MVT::Other);
4011 SDOperand Ops[] = { Chain, MemLoc, DAG.getValueType(VTs[0]) };
4012 Value = DAG.getNode(X86ISD::FLD, Tys, Ops, 3);
4013 Chain = Value.getValue(1);
4014 }
4015
4016 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
4017 SDOperand Ops[] = { Chain, Value };
4018 Chain = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops, 2);
4019 Flag = Chain.getValue(1);
4020 }
4021
4022 SDOperand BytesToPop = DAG.getConstant(getBytesToPopOnReturn(), MVT::i16);
4023 if (Flag.Val)
4024 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, Chain, BytesToPop, Flag);
4025 else
4026 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, Chain, BytesToPop);
Evan Chenga9467aa2006-04-25 20:13:52 +00004027}
4028
Evan Chenge0bcfbe2006-04-26 01:20:17 +00004029SDOperand
4030X86TargetLowering::LowerFORMAL_ARGUMENTS(SDOperand Op, SelectionDAG &DAG) {
Evan Chengdc614c12006-06-06 23:30:24 +00004031 MachineFunction &MF = DAG.getMachineFunction();
4032 const Function* Fn = MF.getFunction();
4033 if (Fn->hasExternalLinkage() &&
Anton Korobeynikov4efbbc92007-01-03 11:43:14 +00004034 Subtarget->isTargetCygMing() &&
Evan Cheng0e14a562006-06-09 06:24:42 +00004035 Fn->getName() == "main")
Evan Chengdc614c12006-06-06 23:30:24 +00004036 MF.getInfo<X86FunctionInfo>()->setForceFramePointer(true);
4037
Evan Cheng17e734f2006-05-23 21:06:34 +00004038 unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue();
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004039 if (Subtarget->is64Bit())
4040 return LowerX86_64CCCArguments(Op, DAG);
Evan Cheng17e734f2006-05-23 21:06:34 +00004041 else
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00004042 switch(CC) {
Chris Lattnerfc360392006-09-27 18:29:38 +00004043 default:
4044 assert(0 && "Unsupported calling convention");
4045 case CallingConv::Fast:
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00004046 if (EnableFastCC) {
4047 return LowerFastCCArguments(Op, DAG);
4048 }
4049 // Falls through
Chris Lattnerfc360392006-09-27 18:29:38 +00004050 case CallingConv::C:
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00004051 return LowerCCCArguments(Op, DAG);
Chris Lattnerfc360392006-09-27 18:29:38 +00004052 case CallingConv::X86_StdCall:
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00004053 MF.getInfo<X86FunctionInfo>()->setDecorationStyle(StdCall);
Anton Korobeynikov037c8672007-01-28 13:31:35 +00004054 return LowerCCCArguments(Op, DAG, true);
Chris Lattnerfc360392006-09-27 18:29:38 +00004055 case CallingConv::X86_FastCall:
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00004056 MF.getInfo<X86FunctionInfo>()->setDecorationStyle(FastCall);
Anton Korobeynikov037c8672007-01-28 13:31:35 +00004057 return LowerFastCCArguments(Op, DAG, true);
Anton Korobeynikov3c5b3df2006-09-20 22:03:51 +00004058 }
Evan Chenge0bcfbe2006-04-26 01:20:17 +00004059}
4060
Evan Chenga9467aa2006-04-25 20:13:52 +00004061SDOperand X86TargetLowering::LowerMEMSET(SDOperand Op, SelectionDAG &DAG) {
4062 SDOperand InFlag(0, 0);
4063 SDOperand Chain = Op.getOperand(0);
4064 unsigned Align =
4065 (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
4066 if (Align == 0) Align = 1;
4067
4068 ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3));
4069 // If not DWORD aligned, call memset if size is less than the threshold.
4070 // It knows how to align to the right boundary first.
4071 if ((Align & 3) != 0 ||
4072 (I && I->getValue() < Subtarget->getMinRepStrSizeThreshold())) {
4073 MVT::ValueType IntPtr = getPointerTy();
Owen Anderson20a631f2006-05-03 01:29:57 +00004074 const Type *IntPtrTy = getTargetData()->getIntPtrType();
Reid Spencere63b6512006-12-31 05:55:36 +00004075 TargetLowering::ArgListTy Args;
4076 TargetLowering::ArgListEntry Entry;
4077 Entry.Node = Op.getOperand(1);
4078 Entry.Ty = IntPtrTy;
4079 Entry.isSigned = false;
Anton Korobeynikov037c8672007-01-28 13:31:35 +00004080 Entry.isInReg = false;
4081 Entry.isSRet = false;
Reid Spencere63b6512006-12-31 05:55:36 +00004082 Args.push_back(Entry);
Reid Spencere87b5e92007-01-03 17:24:59 +00004083 // Extend the unsigned i8 argument to be an int value for the call.
Reid Spencere63b6512006-12-31 05:55:36 +00004084 Entry.Node = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Op.getOperand(2));
4085 Entry.Ty = IntPtrTy;
4086 Entry.isSigned = false;
Anton Korobeynikov037c8672007-01-28 13:31:35 +00004087 Entry.isInReg = false;
4088 Entry.isSRet = false;
Reid Spencere63b6512006-12-31 05:55:36 +00004089 Args.push_back(Entry);
4090 Entry.Node = Op.getOperand(3);
4091 Args.push_back(Entry);
Evan Chenga9467aa2006-04-25 20:13:52 +00004092 std::pair<SDOperand,SDOperand> CallResult =
Reid Spencere63b6512006-12-31 05:55:36 +00004093 LowerCallTo(Chain, Type::VoidTy, false, false, CallingConv::C, false,
Evan Chenga9467aa2006-04-25 20:13:52 +00004094 DAG.getExternalSymbol("memset", IntPtr), Args, DAG);
4095 return CallResult.second;
Evan Chengd5e905d2006-03-21 23:01:21 +00004096 }
Evan Chengd097e672006-03-22 02:53:00 +00004097
Evan Chenga9467aa2006-04-25 20:13:52 +00004098 MVT::ValueType AVT;
4099 SDOperand Count;
4100 ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Op.getOperand(2));
4101 unsigned BytesLeft = 0;
4102 bool TwoRepStos = false;
4103 if (ValC) {
4104 unsigned ValReg;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004105 uint64_t Val = ValC->getValue() & 255;
Evan Chengc995b452006-04-06 23:23:56 +00004106
Evan Chenga9467aa2006-04-25 20:13:52 +00004107 // If the value is a constant, then we can potentially use larger sets.
4108 switch (Align & 3) {
4109 case 2: // WORD aligned
4110 AVT = MVT::i16;
Evan Chenga9467aa2006-04-25 20:13:52 +00004111 ValReg = X86::AX;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004112 Val = (Val << 8) | Val;
Evan Chenga9467aa2006-04-25 20:13:52 +00004113 break;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004114 case 0: // DWORD aligned
Evan Chenga9467aa2006-04-25 20:13:52 +00004115 AVT = MVT::i32;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004116 ValReg = X86::EAX;
Evan Chenga9467aa2006-04-25 20:13:52 +00004117 Val = (Val << 8) | Val;
4118 Val = (Val << 16) | Val;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004119 if (Subtarget->is64Bit() && ((Align & 0xF) == 0)) { // QWORD aligned
4120 AVT = MVT::i64;
4121 ValReg = X86::RAX;
4122 Val = (Val << 32) | Val;
4123 }
Evan Chenga9467aa2006-04-25 20:13:52 +00004124 break;
4125 default: // Byte aligned
4126 AVT = MVT::i8;
Evan Chenga9467aa2006-04-25 20:13:52 +00004127 ValReg = X86::AL;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004128 Count = Op.getOperand(3);
Evan Chenga9467aa2006-04-25 20:13:52 +00004129 break;
Evan Chenga3caaee2006-04-19 22:48:17 +00004130 }
4131
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004132 if (AVT > MVT::i8) {
4133 if (I) {
4134 unsigned UBytes = MVT::getSizeInBits(AVT) / 8;
4135 Count = DAG.getConstant(I->getValue() / UBytes, getPointerTy());
4136 BytesLeft = I->getValue() % UBytes;
4137 } else {
4138 assert(AVT >= MVT::i32 &&
4139 "Do not use rep;stos if not at least DWORD aligned");
4140 Count = DAG.getNode(ISD::SRL, Op.getOperand(3).getValueType(),
4141 Op.getOperand(3), DAG.getConstant(2, MVT::i8));
4142 TwoRepStos = true;
4143 }
4144 }
4145
Evan Chenga9467aa2006-04-25 20:13:52 +00004146 Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT),
4147 InFlag);
4148 InFlag = Chain.getValue(1);
4149 } else {
4150 AVT = MVT::i8;
4151 Count = Op.getOperand(3);
4152 Chain = DAG.getCopyToReg(Chain, X86::AL, Op.getOperand(2), InFlag);
4153 InFlag = Chain.getValue(1);
Evan Chengd097e672006-03-22 02:53:00 +00004154 }
Evan Chengb0461082006-04-24 18:01:45 +00004155
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004156 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX,
4157 Count, InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00004158 InFlag = Chain.getValue(1);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004159 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI,
4160 Op.getOperand(1), InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00004161 InFlag = Chain.getValue(1);
Evan Cheng9b9cc4f2006-03-27 07:00:16 +00004162
Chris Lattnere56fef92007-02-25 06:40:16 +00004163 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Chris Lattner35a08552007-02-25 07:10:00 +00004164 SmallVector<SDOperand, 8> Ops;
Evan Chenga9467aa2006-04-25 20:13:52 +00004165 Ops.push_back(Chain);
4166 Ops.push_back(DAG.getValueType(AVT));
4167 Ops.push_back(InFlag);
Evan Cheng5c68bba2006-08-11 07:35:45 +00004168 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size());
Evan Chengb0461082006-04-24 18:01:45 +00004169
Evan Chenga9467aa2006-04-25 20:13:52 +00004170 if (TwoRepStos) {
4171 InFlag = Chain.getValue(1);
4172 Count = Op.getOperand(3);
4173 MVT::ValueType CVT = Count.getValueType();
4174 SDOperand Left = DAG.getNode(ISD::AND, CVT, Count,
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004175 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
4176 Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX,
4177 Left, InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00004178 InFlag = Chain.getValue(1);
Chris Lattnere56fef92007-02-25 06:40:16 +00004179 Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Evan Chenga9467aa2006-04-25 20:13:52 +00004180 Ops.clear();
4181 Ops.push_back(Chain);
4182 Ops.push_back(DAG.getValueType(MVT::i8));
4183 Ops.push_back(InFlag);
Evan Cheng5c68bba2006-08-11 07:35:45 +00004184 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00004185 } else if (BytesLeft) {
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004186 // Issue stores for the last 1 - 7 bytes.
Evan Chenga9467aa2006-04-25 20:13:52 +00004187 SDOperand Value;
4188 unsigned Val = ValC->getValue() & 255;
4189 unsigned Offset = I->getValue() - BytesLeft;
4190 SDOperand DstAddr = Op.getOperand(1);
4191 MVT::ValueType AddrVT = DstAddr.getValueType();
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004192 if (BytesLeft >= 4) {
4193 Val = (Val << 8) | Val;
4194 Val = (Val << 16) | Val;
4195 Value = DAG.getConstant(Val, MVT::i32);
Evan Chengdf9ac472006-10-05 23:01:46 +00004196 Chain = DAG.getStore(Chain, Value,
4197 DAG.getNode(ISD::ADD, AddrVT, DstAddr,
4198 DAG.getConstant(Offset, AddrVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00004199 NULL, 0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004200 BytesLeft -= 4;
4201 Offset += 4;
4202 }
Evan Chenga9467aa2006-04-25 20:13:52 +00004203 if (BytesLeft >= 2) {
4204 Value = DAG.getConstant((Val << 8) | Val, MVT::i16);
Evan Chengdf9ac472006-10-05 23:01:46 +00004205 Chain = DAG.getStore(Chain, Value,
4206 DAG.getNode(ISD::ADD, AddrVT, DstAddr,
4207 DAG.getConstant(Offset, AddrVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00004208 NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00004209 BytesLeft -= 2;
4210 Offset += 2;
Evan Cheng082c8782006-03-24 07:29:27 +00004211 }
Evan Chenga9467aa2006-04-25 20:13:52 +00004212 if (BytesLeft == 1) {
4213 Value = DAG.getConstant(Val, MVT::i8);
Evan Chengdf9ac472006-10-05 23:01:46 +00004214 Chain = DAG.getStore(Chain, Value,
4215 DAG.getNode(ISD::ADD, AddrVT, DstAddr,
4216 DAG.getConstant(Offset, AddrVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00004217 NULL, 0);
Evan Cheng14215c32006-04-21 23:03:30 +00004218 }
Evan Cheng082c8782006-03-24 07:29:27 +00004219 }
Evan Chengebf10062006-04-03 20:53:28 +00004220
Evan Chenga9467aa2006-04-25 20:13:52 +00004221 return Chain;
4222}
Evan Chengebf10062006-04-03 20:53:28 +00004223
Evan Chenga9467aa2006-04-25 20:13:52 +00004224SDOperand X86TargetLowering::LowerMEMCPY(SDOperand Op, SelectionDAG &DAG) {
4225 SDOperand Chain = Op.getOperand(0);
4226 unsigned Align =
4227 (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
4228 if (Align == 0) Align = 1;
Evan Chengebf10062006-04-03 20:53:28 +00004229
Evan Chenga9467aa2006-04-25 20:13:52 +00004230 ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3));
4231 // If not DWORD aligned, call memcpy if size is less than the threshold.
4232 // It knows how to align to the right boundary first.
4233 if ((Align & 3) != 0 ||
4234 (I && I->getValue() < Subtarget->getMinRepStrSizeThreshold())) {
4235 MVT::ValueType IntPtr = getPointerTy();
Reid Spencere63b6512006-12-31 05:55:36 +00004236 TargetLowering::ArgListTy Args;
4237 TargetLowering::ArgListEntry Entry;
Anton Korobeynikov037c8672007-01-28 13:31:35 +00004238 Entry.Ty = getTargetData()->getIntPtrType();
4239 Entry.isSigned = false;
4240 Entry.isInReg = false;
4241 Entry.isSRet = false;
Reid Spencere63b6512006-12-31 05:55:36 +00004242 Entry.Node = Op.getOperand(1); Args.push_back(Entry);
4243 Entry.Node = Op.getOperand(2); Args.push_back(Entry);
4244 Entry.Node = Op.getOperand(3); Args.push_back(Entry);
Evan Chenga9467aa2006-04-25 20:13:52 +00004245 std::pair<SDOperand,SDOperand> CallResult =
Reid Spencere63b6512006-12-31 05:55:36 +00004246 LowerCallTo(Chain, Type::VoidTy, false, false, CallingConv::C, false,
Evan Chenga9467aa2006-04-25 20:13:52 +00004247 DAG.getExternalSymbol("memcpy", IntPtr), Args, DAG);
4248 return CallResult.second;
Evan Chengcbffa462006-03-31 19:22:53 +00004249 }
Evan Chenga9467aa2006-04-25 20:13:52 +00004250
4251 MVT::ValueType AVT;
4252 SDOperand Count;
4253 unsigned BytesLeft = 0;
4254 bool TwoRepMovs = false;
4255 switch (Align & 3) {
4256 case 2: // WORD aligned
4257 AVT = MVT::i16;
Evan Chenga9467aa2006-04-25 20:13:52 +00004258 break;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004259 case 0: // DWORD aligned
Evan Chenga9467aa2006-04-25 20:13:52 +00004260 AVT = MVT::i32;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004261 if (Subtarget->is64Bit() && ((Align & 0xF) == 0)) // QWORD aligned
4262 AVT = MVT::i64;
Evan Chenga9467aa2006-04-25 20:13:52 +00004263 break;
4264 default: // Byte aligned
4265 AVT = MVT::i8;
4266 Count = Op.getOperand(3);
4267 break;
4268 }
4269
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004270 if (AVT > MVT::i8) {
4271 if (I) {
4272 unsigned UBytes = MVT::getSizeInBits(AVT) / 8;
4273 Count = DAG.getConstant(I->getValue() / UBytes, getPointerTy());
4274 BytesLeft = I->getValue() % UBytes;
4275 } else {
4276 assert(AVT >= MVT::i32 &&
4277 "Do not use rep;movs if not at least DWORD aligned");
4278 Count = DAG.getNode(ISD::SRL, Op.getOperand(3).getValueType(),
4279 Op.getOperand(3), DAG.getConstant(2, MVT::i8));
4280 TwoRepMovs = true;
4281 }
4282 }
4283
Evan Chenga9467aa2006-04-25 20:13:52 +00004284 SDOperand InFlag(0, 0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004285 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX,
4286 Count, InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00004287 InFlag = Chain.getValue(1);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004288 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI,
4289 Op.getOperand(1), InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00004290 InFlag = Chain.getValue(1);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004291 Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RSI : X86::ESI,
4292 Op.getOperand(2), InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00004293 InFlag = Chain.getValue(1);
4294
Chris Lattnere56fef92007-02-25 06:40:16 +00004295 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Chris Lattner35a08552007-02-25 07:10:00 +00004296 SmallVector<SDOperand, 8> Ops;
Evan Chenga9467aa2006-04-25 20:13:52 +00004297 Ops.push_back(Chain);
4298 Ops.push_back(DAG.getValueType(AVT));
4299 Ops.push_back(InFlag);
Evan Cheng5c68bba2006-08-11 07:35:45 +00004300 Chain = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00004301
4302 if (TwoRepMovs) {
4303 InFlag = Chain.getValue(1);
4304 Count = Op.getOperand(3);
4305 MVT::ValueType CVT = Count.getValueType();
4306 SDOperand Left = DAG.getNode(ISD::AND, CVT, Count,
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004307 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
4308 Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX,
4309 Left, InFlag);
Evan Chenga9467aa2006-04-25 20:13:52 +00004310 InFlag = Chain.getValue(1);
Chris Lattnere56fef92007-02-25 06:40:16 +00004311 Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Evan Chenga9467aa2006-04-25 20:13:52 +00004312 Ops.clear();
4313 Ops.push_back(Chain);
4314 Ops.push_back(DAG.getValueType(MVT::i8));
4315 Ops.push_back(InFlag);
Evan Cheng5c68bba2006-08-11 07:35:45 +00004316 Chain = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00004317 } else if (BytesLeft) {
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004318 // Issue loads and stores for the last 1 - 7 bytes.
Evan Chenga9467aa2006-04-25 20:13:52 +00004319 unsigned Offset = I->getValue() - BytesLeft;
4320 SDOperand DstAddr = Op.getOperand(1);
4321 MVT::ValueType DstVT = DstAddr.getValueType();
4322 SDOperand SrcAddr = Op.getOperand(2);
4323 MVT::ValueType SrcVT = SrcAddr.getValueType();
4324 SDOperand Value;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004325 if (BytesLeft >= 4) {
4326 Value = DAG.getLoad(MVT::i32, Chain,
4327 DAG.getNode(ISD::ADD, SrcVT, SrcAddr,
4328 DAG.getConstant(Offset, SrcVT)),
Evan Chenge71fe34d2006-10-09 20:57:25 +00004329 NULL, 0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004330 Chain = Value.getValue(1);
Evan Chengdf9ac472006-10-05 23:01:46 +00004331 Chain = DAG.getStore(Chain, Value,
4332 DAG.getNode(ISD::ADD, DstVT, DstAddr,
4333 DAG.getConstant(Offset, DstVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00004334 NULL, 0);
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004335 BytesLeft -= 4;
4336 Offset += 4;
4337 }
Evan Chenga9467aa2006-04-25 20:13:52 +00004338 if (BytesLeft >= 2) {
4339 Value = DAG.getLoad(MVT::i16, Chain,
4340 DAG.getNode(ISD::ADD, SrcVT, SrcAddr,
4341 DAG.getConstant(Offset, SrcVT)),
Evan Chenge71fe34d2006-10-09 20:57:25 +00004342 NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00004343 Chain = Value.getValue(1);
Evan Chengdf9ac472006-10-05 23:01:46 +00004344 Chain = DAG.getStore(Chain, Value,
4345 DAG.getNode(ISD::ADD, DstVT, DstAddr,
4346 DAG.getConstant(Offset, DstVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00004347 NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00004348 BytesLeft -= 2;
4349 Offset += 2;
Evan Chengcbffa462006-03-31 19:22:53 +00004350 }
4351
Evan Chenga9467aa2006-04-25 20:13:52 +00004352 if (BytesLeft == 1) {
4353 Value = DAG.getLoad(MVT::i8, Chain,
4354 DAG.getNode(ISD::ADD, SrcVT, SrcAddr,
4355 DAG.getConstant(Offset, SrcVT)),
Evan Chenge71fe34d2006-10-09 20:57:25 +00004356 NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00004357 Chain = Value.getValue(1);
Evan Chengdf9ac472006-10-05 23:01:46 +00004358 Chain = DAG.getStore(Chain, Value,
4359 DAG.getNode(ISD::ADD, DstVT, DstAddr,
4360 DAG.getConstant(Offset, DstVT)),
Evan Chengab51cf22006-10-13 21:14:26 +00004361 NULL, 0);
Evan Chenga9467aa2006-04-25 20:13:52 +00004362 }
Evan Chengcbffa462006-03-31 19:22:53 +00004363 }
Evan Chenga9467aa2006-04-25 20:13:52 +00004364
4365 return Chain;
4366}
4367
4368SDOperand
4369X86TargetLowering::LowerREADCYCLCECOUNTER(SDOperand Op, SelectionDAG &DAG) {
Chris Lattnere56fef92007-02-25 06:40:16 +00004370 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Chris Lattner35a08552007-02-25 07:10:00 +00004371 SDOperand TheOp = Op.getOperand(0);
4372 SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheOp, 1);
Evan Cheng28a9e9b2006-11-29 08:28:13 +00004373 if (Subtarget->is64Bit()) {
4374 SDOperand Copy1 = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1));
4375 SDOperand Copy2 = DAG.getCopyFromReg(Copy1.getValue(1), X86::RDX,
4376 MVT::i64, Copy1.getValue(2));
4377 SDOperand Tmp = DAG.getNode(ISD::SHL, MVT::i64, Copy2,
4378 DAG.getConstant(32, MVT::i8));
Chris Lattner35a08552007-02-25 07:10:00 +00004379 SDOperand Ops[] = {
4380 DAG.getNode(ISD::OR, MVT::i64, Copy1, Tmp), Copy2.getValue(1)
4381 };
Chris Lattnere56fef92007-02-25 06:40:16 +00004382
4383 Tys = DAG.getVTList(MVT::i64, MVT::Other);
Chris Lattner35a08552007-02-25 07:10:00 +00004384 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops, 2);
Evan Cheng28a9e9b2006-11-29 08:28:13 +00004385 }
Chris Lattner35a08552007-02-25 07:10:00 +00004386
4387 SDOperand Copy1 = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1));
4388 SDOperand Copy2 = DAG.getCopyFromReg(Copy1.getValue(1), X86::EDX,
4389 MVT::i32, Copy1.getValue(2));
4390 SDOperand Ops[] = { Copy1, Copy2, Copy2.getValue(1) };
4391 Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
4392 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops, 3);
Evan Chenga9467aa2006-04-25 20:13:52 +00004393}
4394
4395SDOperand X86TargetLowering::LowerVASTART(SDOperand Op, SelectionDAG &DAG) {
Evan Chengab51cf22006-10-13 21:14:26 +00004396 SrcValueSDNode *SV = cast<SrcValueSDNode>(Op.getOperand(2));
4397
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004398 if (!Subtarget->is64Bit()) {
4399 // vastart just stores the address of the VarArgsFrameIndex slot into the
4400 // memory location argument.
4401 SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy());
Evan Chengab51cf22006-10-13 21:14:26 +00004402 return DAG.getStore(Op.getOperand(0), FR,Op.getOperand(1), SV->getValue(),
4403 SV->getOffset());
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004404 }
4405
4406 // __va_list_tag:
4407 // gp_offset (0 - 6 * 8)
4408 // fp_offset (48 - 48 + 8 * 16)
4409 // overflow_arg_area (point to parameters coming in memory).
4410 // reg_save_area
Chris Lattner35a08552007-02-25 07:10:00 +00004411 SmallVector<SDOperand, 8> MemOps;
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004412 SDOperand FIN = Op.getOperand(1);
4413 // Store gp_offset
Evan Chengdf9ac472006-10-05 23:01:46 +00004414 SDOperand Store = DAG.getStore(Op.getOperand(0),
4415 DAG.getConstant(VarArgsGPOffset, MVT::i32),
Evan Chengab51cf22006-10-13 21:14:26 +00004416 FIN, SV->getValue(), SV->getOffset());
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004417 MemOps.push_back(Store);
4418
4419 // Store fp_offset
4420 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
4421 DAG.getConstant(4, getPointerTy()));
Evan Chengdf9ac472006-10-05 23:01:46 +00004422 Store = DAG.getStore(Op.getOperand(0),
4423 DAG.getConstant(VarArgsFPOffset, MVT::i32),
Evan Chengab51cf22006-10-13 21:14:26 +00004424 FIN, SV->getValue(), SV->getOffset());
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004425 MemOps.push_back(Store);
4426
4427 // Store ptr to overflow_arg_area
4428 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
4429 DAG.getConstant(4, getPointerTy()));
4430 SDOperand OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy());
Evan Chengab51cf22006-10-13 21:14:26 +00004431 Store = DAG.getStore(Op.getOperand(0), OVFIN, FIN, SV->getValue(),
4432 SV->getOffset());
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004433 MemOps.push_back(Store);
4434
4435 // Store ptr to reg_save_area.
4436 FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN,
4437 DAG.getConstant(8, getPointerTy()));
4438 SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy());
Evan Chengab51cf22006-10-13 21:14:26 +00004439 Store = DAG.getStore(Op.getOperand(0), RSFIN, FIN, SV->getValue(),
4440 SV->getOffset());
Evan Cheng11b0a5d2006-09-08 06:48:29 +00004441 MemOps.push_back(Store);
4442 return DAG.getNode(ISD::TokenFactor, MVT::Other, &MemOps[0], MemOps.size());
Evan Chenga9467aa2006-04-25 20:13:52 +00004443}
4444
4445SDOperand
4446X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDOperand Op, SelectionDAG &DAG) {
4447 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getValue();
4448 switch (IntNo) {
4449 default: return SDOperand(); // Don't custom lower most intrinsics.
Evan Cheng78038292006-04-05 23:38:46 +00004450 // Comparison intrinsics.
Evan Chenga9467aa2006-04-25 20:13:52 +00004451 case Intrinsic::x86_sse_comieq_ss:
4452 case Intrinsic::x86_sse_comilt_ss:
4453 case Intrinsic::x86_sse_comile_ss:
4454 case Intrinsic::x86_sse_comigt_ss:
4455 case Intrinsic::x86_sse_comige_ss:
4456 case Intrinsic::x86_sse_comineq_ss:
4457 case Intrinsic::x86_sse_ucomieq_ss:
4458 case Intrinsic::x86_sse_ucomilt_ss:
4459 case Intrinsic::x86_sse_ucomile_ss:
4460 case Intrinsic::x86_sse_ucomigt_ss:
4461 case Intrinsic::x86_sse_ucomige_ss:
4462 case Intrinsic::x86_sse_ucomineq_ss:
4463 case Intrinsic::x86_sse2_comieq_sd:
4464 case Intrinsic::x86_sse2_comilt_sd:
4465 case Intrinsic::x86_sse2_comile_sd:
4466 case Intrinsic::x86_sse2_comigt_sd:
4467 case Intrinsic::x86_sse2_comige_sd:
4468 case Intrinsic::x86_sse2_comineq_sd:
4469 case Intrinsic::x86_sse2_ucomieq_sd:
4470 case Intrinsic::x86_sse2_ucomilt_sd:
4471 case Intrinsic::x86_sse2_ucomile_sd:
4472 case Intrinsic::x86_sse2_ucomigt_sd:
4473 case Intrinsic::x86_sse2_ucomige_sd:
4474 case Intrinsic::x86_sse2_ucomineq_sd: {
4475 unsigned Opc = 0;
4476 ISD::CondCode CC = ISD::SETCC_INVALID;
4477 switch (IntNo) {
4478 default: break;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004479 case Intrinsic::x86_sse_comieq_ss:
4480 case Intrinsic::x86_sse2_comieq_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00004481 Opc = X86ISD::COMI;
4482 CC = ISD::SETEQ;
4483 break;
Evan Cheng78038292006-04-05 23:38:46 +00004484 case Intrinsic::x86_sse_comilt_ss:
Evan Cheng78038292006-04-05 23:38:46 +00004485 case Intrinsic::x86_sse2_comilt_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00004486 Opc = X86ISD::COMI;
4487 CC = ISD::SETLT;
4488 break;
4489 case Intrinsic::x86_sse_comile_ss:
Evan Cheng78038292006-04-05 23:38:46 +00004490 case Intrinsic::x86_sse2_comile_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00004491 Opc = X86ISD::COMI;
4492 CC = ISD::SETLE;
4493 break;
4494 case Intrinsic::x86_sse_comigt_ss:
Evan Cheng78038292006-04-05 23:38:46 +00004495 case Intrinsic::x86_sse2_comigt_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00004496 Opc = X86ISD::COMI;
4497 CC = ISD::SETGT;
4498 break;
4499 case Intrinsic::x86_sse_comige_ss:
Evan Cheng78038292006-04-05 23:38:46 +00004500 case Intrinsic::x86_sse2_comige_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00004501 Opc = X86ISD::COMI;
4502 CC = ISD::SETGE;
4503 break;
4504 case Intrinsic::x86_sse_comineq_ss:
Evan Cheng78038292006-04-05 23:38:46 +00004505 case Intrinsic::x86_sse2_comineq_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00004506 Opc = X86ISD::COMI;
4507 CC = ISD::SETNE;
4508 break;
4509 case Intrinsic::x86_sse_ucomieq_ss:
Evan Cheng78038292006-04-05 23:38:46 +00004510 case Intrinsic::x86_sse2_ucomieq_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00004511 Opc = X86ISD::UCOMI;
4512 CC = ISD::SETEQ;
4513 break;
4514 case Intrinsic::x86_sse_ucomilt_ss:
Evan Cheng78038292006-04-05 23:38:46 +00004515 case Intrinsic::x86_sse2_ucomilt_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00004516 Opc = X86ISD::UCOMI;
4517 CC = ISD::SETLT;
4518 break;
4519 case Intrinsic::x86_sse_ucomile_ss:
Evan Cheng78038292006-04-05 23:38:46 +00004520 case Intrinsic::x86_sse2_ucomile_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00004521 Opc = X86ISD::UCOMI;
4522 CC = ISD::SETLE;
4523 break;
4524 case Intrinsic::x86_sse_ucomigt_ss:
Evan Cheng78038292006-04-05 23:38:46 +00004525 case Intrinsic::x86_sse2_ucomigt_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00004526 Opc = X86ISD::UCOMI;
4527 CC = ISD::SETGT;
4528 break;
4529 case Intrinsic::x86_sse_ucomige_ss:
Evan Cheng78038292006-04-05 23:38:46 +00004530 case Intrinsic::x86_sse2_ucomige_sd:
Evan Chenga9467aa2006-04-25 20:13:52 +00004531 Opc = X86ISD::UCOMI;
4532 CC = ISD::SETGE;
4533 break;
4534 case Intrinsic::x86_sse_ucomineq_ss:
4535 case Intrinsic::x86_sse2_ucomineq_sd:
4536 Opc = X86ISD::UCOMI;
4537 CC = ISD::SETNE;
4538 break;
Evan Cheng78038292006-04-05 23:38:46 +00004539 }
Evan Cheng4259a0f2006-09-11 02:19:56 +00004540
Evan Chenga9467aa2006-04-25 20:13:52 +00004541 unsigned X86CC;
Chris Lattner7a627672006-09-13 03:22:10 +00004542 SDOperand LHS = Op.getOperand(1);
4543 SDOperand RHS = Op.getOperand(2);
4544 translateX86CC(CC, true, X86CC, LHS, RHS, DAG);
Evan Cheng4259a0f2006-09-11 02:19:56 +00004545
4546 const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag);
Chris Lattner7a627672006-09-13 03:22:10 +00004547 SDOperand Ops1[] = { DAG.getEntryNode(), LHS, RHS };
Evan Cheng4259a0f2006-09-11 02:19:56 +00004548 SDOperand Cond = DAG.getNode(Opc, VTs, 2, Ops1, 3);
4549 VTs = DAG.getNodeValueTypes(MVT::i8, MVT::Flag);
4550 SDOperand Ops2[] = { DAG.getConstant(X86CC, MVT::i8), Cond };
4551 SDOperand SetCC = DAG.getNode(X86ISD::SETCC, VTs, 2, Ops2, 2);
Evan Chenga9467aa2006-04-25 20:13:52 +00004552 return DAG.getNode(ISD::ANY_EXTEND, MVT::i32, SetCC);
Evan Cheng78038292006-04-05 23:38:46 +00004553 }
Evan Cheng5c59d492005-12-23 07:31:11 +00004554 }
Chris Lattner76ac0682005-11-15 00:40:23 +00004555}
Evan Cheng6af02632005-12-20 06:22:03 +00004556
Nate Begemaneda59972007-01-29 22:58:52 +00004557SDOperand X86TargetLowering::LowerRETURNADDR(SDOperand Op, SelectionDAG &DAG) {
4558 // Depths > 0 not supported yet!
4559 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0)
4560 return SDOperand();
4561
4562 // Just load the return address
4563 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG);
4564 return DAG.getLoad(getPointerTy(), DAG.getEntryNode(), RetAddrFI, NULL, 0);
4565}
4566
4567SDOperand X86TargetLowering::LowerFRAMEADDR(SDOperand Op, SelectionDAG &DAG) {
4568 // Depths > 0 not supported yet!
4569 if (cast<ConstantSDNode>(Op.getOperand(0))->getValue() > 0)
4570 return SDOperand();
4571
4572 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG);
4573 return DAG.getNode(ISD::SUB, getPointerTy(), RetAddrFI,
4574 DAG.getConstant(4, getPointerTy()));
4575}
4576
Evan Chenga9467aa2006-04-25 20:13:52 +00004577/// LowerOperation - Provide custom lowering hooks for some operations.
4578///
4579SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
4580 switch (Op.getOpcode()) {
4581 default: assert(0 && "Should not custom lower this!");
4582 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
4583 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
4584 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
4585 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
4586 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
4587 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
4588 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
4589 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG);
4590 case ISD::SHL_PARTS:
4591 case ISD::SRA_PARTS:
4592 case ISD::SRL_PARTS: return LowerShift(Op, DAG);
4593 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
4594 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG);
4595 case ISD::FABS: return LowerFABS(Op, DAG);
4596 case ISD::FNEG: return LowerFNEG(Op, DAG);
Evan Cheng4363e882007-01-05 07:55:56 +00004597 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
Evan Cheng4259a0f2006-09-11 02:19:56 +00004598 case ISD::SETCC: return LowerSETCC(Op, DAG, DAG.getEntryNode());
Evan Chenga9467aa2006-04-25 20:13:52 +00004599 case ISD::SELECT: return LowerSELECT(Op, DAG);
4600 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
4601 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
Evan Cheng2a330942006-05-25 00:59:30 +00004602 case ISD::CALL: return LowerCALL(Op, DAG);
Evan Chenga9467aa2006-04-25 20:13:52 +00004603 case ISD::RET: return LowerRET(Op, DAG);
Evan Chenge0bcfbe2006-04-26 01:20:17 +00004604 case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG);
Evan Chenga9467aa2006-04-25 20:13:52 +00004605 case ISD::MEMSET: return LowerMEMSET(Op, DAG);
4606 case ISD::MEMCPY: return LowerMEMCPY(Op, DAG);
4607 case ISD::READCYCLECOUNTER: return LowerREADCYCLCECOUNTER(Op, DAG);
4608 case ISD::VASTART: return LowerVASTART(Op, DAG);
4609 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
Nate Begemaneda59972007-01-29 22:58:52 +00004610 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
4611 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
Evan Chenga9467aa2006-04-25 20:13:52 +00004612 }
Jim Laskey3796abe2007-02-21 22:54:50 +00004613 return SDOperand();
Evan Chenga9467aa2006-04-25 20:13:52 +00004614}
4615
Evan Cheng6af02632005-12-20 06:22:03 +00004616const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
4617 switch (Opcode) {
4618 default: return NULL;
Evan Cheng9c249c32006-01-09 18:33:28 +00004619 case X86ISD::SHLD: return "X86ISD::SHLD";
4620 case X86ISD::SHRD: return "X86ISD::SHRD";
Evan Cheng2dd217b2006-01-31 03:14:29 +00004621 case X86ISD::FAND: return "X86ISD::FAND";
Evan Cheng4363e882007-01-05 07:55:56 +00004622 case X86ISD::FOR: return "X86ISD::FOR";
Evan Cheng72d5c252006-01-31 22:28:30 +00004623 case X86ISD::FXOR: return "X86ISD::FXOR";
Evan Cheng4363e882007-01-05 07:55:56 +00004624 case X86ISD::FSRL: return "X86ISD::FSRL";
Evan Cheng6305e502006-01-12 22:54:21 +00004625 case X86ISD::FILD: return "X86ISD::FILD";
Evan Cheng11613a52006-02-04 02:20:30 +00004626 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
Evan Cheng6af02632005-12-20 06:22:03 +00004627 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
4628 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
4629 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
Evan Chenga74ce622005-12-21 02:39:21 +00004630 case X86ISD::FLD: return "X86ISD::FLD";
Evan Cheng45e190982006-01-05 00:27:02 +00004631 case X86ISD::FST: return "X86ISD::FST";
4632 case X86ISD::FP_GET_RESULT: return "X86ISD::FP_GET_RESULT";
Evan Chenga74ce622005-12-21 02:39:21 +00004633 case X86ISD::FP_SET_RESULT: return "X86ISD::FP_SET_RESULT";
Evan Cheng6af02632005-12-20 06:22:03 +00004634 case X86ISD::CALL: return "X86ISD::CALL";
4635 case X86ISD::TAILCALL: return "X86ISD::TAILCALL";
4636 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG";
4637 case X86ISD::CMP: return "X86ISD::CMP";
Evan Cheng78038292006-04-05 23:38:46 +00004638 case X86ISD::COMI: return "X86ISD::COMI";
4639 case X86ISD::UCOMI: return "X86ISD::UCOMI";
Evan Chengc1583db2005-12-21 20:21:51 +00004640 case X86ISD::SETCC: return "X86ISD::SETCC";
Evan Cheng6af02632005-12-20 06:22:03 +00004641 case X86ISD::CMOV: return "X86ISD::CMOV";
4642 case X86ISD::BRCOND: return "X86ISD::BRCOND";
Evan Chenga74ce622005-12-21 02:39:21 +00004643 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
Evan Cheng084a1022006-03-04 01:12:00 +00004644 case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
4645 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
Evan Cheng72d5c252006-01-31 22:28:30 +00004646 case X86ISD::LOAD_PACK: return "X86ISD::LOAD_PACK";
Evan Cheng5987cfb2006-07-07 08:33:52 +00004647 case X86ISD::LOAD_UA: return "X86ISD::LOAD_UA";
Evan Cheng5588de92006-02-18 00:15:05 +00004648 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
Evan Chenge0ed6ec2006-02-23 20:41:18 +00004649 case X86ISD::Wrapper: return "X86ISD::Wrapper";
Evan Chenge7ee6a52006-03-24 23:15:12 +00004650 case X86ISD::S2VEC: return "X86ISD::S2VEC";
Evan Chengcbffa462006-03-31 19:22:53 +00004651 case X86ISD::PEXTRW: return "X86ISD::PEXTRW";
Evan Cheng5fd7c692006-03-31 21:55:24 +00004652 case X86ISD::PINSRW: return "X86ISD::PINSRW";
Evan Cheng49683ba2006-11-10 21:43:37 +00004653 case X86ISD::FMAX: return "X86ISD::FMAX";
4654 case X86ISD::FMIN: return "X86ISD::FMIN";
Evan Cheng6af02632005-12-20 06:22:03 +00004655 }
4656}
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004657
Evan Cheng02612422006-07-05 22:17:51 +00004658/// isLegalAddressImmediate - Return true if the integer value or
4659/// GlobalValue can be used as the offset of the target addressing mode.
4660bool X86TargetLowering::isLegalAddressImmediate(int64_t V) const {
4661 // X86 allows a sign-extended 32-bit immediate field.
4662 return (V > -(1LL << 32) && V < (1LL << 32)-1);
4663}
4664
4665bool X86TargetLowering::isLegalAddressImmediate(GlobalValue *GV) const {
Evan Cheng7a9238c2006-11-29 23:48:14 +00004666 // In 64-bit mode, GV is 64-bit so it won't fit in the 32-bit displacement
4667 // field unless we are in small code model.
4668 if (Subtarget->is64Bit() &&
4669 getTargetMachine().getCodeModel() != CodeModel::Small)
Evan Cheng02612422006-07-05 22:17:51 +00004670 return false;
Anton Korobeynikov430e68a12006-12-22 22:29:05 +00004671
4672 return (!Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false));
Evan Cheng02612422006-07-05 22:17:51 +00004673}
4674
4675/// isShuffleMaskLegal - Targets can use this to indicate that they only
4676/// support *some* VECTOR_SHUFFLE operations, those with specific masks.
4677/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
4678/// are assumed to be legal.
4679bool
4680X86TargetLowering::isShuffleMaskLegal(SDOperand Mask, MVT::ValueType VT) const {
4681 // Only do shuffles on 128-bit vector types for now.
4682 if (MVT::getSizeInBits(VT) == 64) return false;
4683 return (Mask.Val->getNumOperands() <= 4 ||
4684 isSplatMask(Mask.Val) ||
4685 isPSHUFHW_PSHUFLWMask(Mask.Val) ||
4686 X86::isUNPCKLMask(Mask.Val) ||
4687 X86::isUNPCKL_v_undef_Mask(Mask.Val) ||
4688 X86::isUNPCKHMask(Mask.Val));
4689}
4690
4691bool X86TargetLowering::isVectorClearMaskLegal(std::vector<SDOperand> &BVOps,
4692 MVT::ValueType EVT,
4693 SelectionDAG &DAG) const {
4694 unsigned NumElts = BVOps.size();
4695 // Only do shuffles on 128-bit vector types for now.
4696 if (MVT::getSizeInBits(EVT) * NumElts == 64) return false;
4697 if (NumElts == 2) return true;
4698 if (NumElts == 4) {
Chris Lattner35a08552007-02-25 07:10:00 +00004699 return (isMOVLMask(&BVOps[0], 4) ||
4700 isCommutedMOVL(&BVOps[0], 4, true) ||
4701 isSHUFPMask(&BVOps[0], 4) ||
4702 isCommutedSHUFP(&BVOps[0], 4));
Evan Cheng02612422006-07-05 22:17:51 +00004703 }
4704 return false;
4705}
4706
4707//===----------------------------------------------------------------------===//
4708// X86 Scheduler Hooks
4709//===----------------------------------------------------------------------===//
4710
4711MachineBasicBlock *
4712X86TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI,
4713 MachineBasicBlock *BB) {
Evan Cheng20350c42006-11-27 23:37:22 +00004714 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
Evan Cheng02612422006-07-05 22:17:51 +00004715 switch (MI->getOpcode()) {
4716 default: assert(false && "Unexpected instr type to insert");
4717 case X86::CMOV_FR32:
4718 case X86::CMOV_FR64:
4719 case X86::CMOV_V4F32:
4720 case X86::CMOV_V2F64:
4721 case X86::CMOV_V2I64: {
4722 // To "insert" a SELECT_CC instruction, we actually have to insert the
4723 // diamond control-flow pattern. The incoming instruction knows the
4724 // destination vreg to set, the condition code register to branch on, the
4725 // true/false values to select between, and a branch opcode to use.
4726 const BasicBlock *LLVM_BB = BB->getBasicBlock();
4727 ilist<MachineBasicBlock>::iterator It = BB;
4728 ++It;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004729
Evan Cheng02612422006-07-05 22:17:51 +00004730 // thisMBB:
4731 // ...
4732 // TrueVal = ...
4733 // cmpTY ccX, r1, r2
4734 // bCC copy1MBB
4735 // fallthrough --> copy0MBB
4736 MachineBasicBlock *thisMBB = BB;
4737 MachineBasicBlock *copy0MBB = new MachineBasicBlock(LLVM_BB);
4738 MachineBasicBlock *sinkMBB = new MachineBasicBlock(LLVM_BB);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004739 unsigned Opc =
Chris Lattnerc0fb5672006-10-20 17:42:20 +00004740 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm());
Evan Cheng20350c42006-11-27 23:37:22 +00004741 BuildMI(BB, TII->get(Opc)).addMBB(sinkMBB);
Evan Cheng02612422006-07-05 22:17:51 +00004742 MachineFunction *F = BB->getParent();
4743 F->getBasicBlockList().insert(It, copy0MBB);
4744 F->getBasicBlockList().insert(It, sinkMBB);
4745 // Update machine-CFG edges by first adding all successors of the current
4746 // block to the new block which will contain the Phi node for the select.
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004747 for(MachineBasicBlock::succ_iterator i = BB->succ_begin(),
Evan Cheng02612422006-07-05 22:17:51 +00004748 e = BB->succ_end(); i != e; ++i)
4749 sinkMBB->addSuccessor(*i);
4750 // Next, remove all successors of the current block, and add the true
4751 // and fallthrough blocks as its successors.
4752 while(!BB->succ_empty())
4753 BB->removeSuccessor(BB->succ_begin());
4754 BB->addSuccessor(copy0MBB);
4755 BB->addSuccessor(sinkMBB);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004756
Evan Cheng02612422006-07-05 22:17:51 +00004757 // copy0MBB:
4758 // %FalseValue = ...
4759 // # fallthrough to sinkMBB
4760 BB = copy0MBB;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004761
Evan Cheng02612422006-07-05 22:17:51 +00004762 // Update machine-CFG edges
4763 BB->addSuccessor(sinkMBB);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004764
Evan Cheng02612422006-07-05 22:17:51 +00004765 // sinkMBB:
4766 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
4767 // ...
4768 BB = sinkMBB;
Evan Cheng20350c42006-11-27 23:37:22 +00004769 BuildMI(BB, TII->get(X86::PHI), MI->getOperand(0).getReg())
Evan Cheng02612422006-07-05 22:17:51 +00004770 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
4771 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
4772
4773 delete MI; // The pseudo instruction is gone now.
4774 return BB;
4775 }
4776
4777 case X86::FP_TO_INT16_IN_MEM:
4778 case X86::FP_TO_INT32_IN_MEM:
4779 case X86::FP_TO_INT64_IN_MEM: {
4780 // Change the floating point control register to use "round towards zero"
4781 // mode when truncating to an integer value.
4782 MachineFunction *F = BB->getParent();
4783 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2);
Evan Cheng20350c42006-11-27 23:37:22 +00004784 addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx);
Evan Cheng02612422006-07-05 22:17:51 +00004785
4786 // Load the old value of the high byte of the control word...
4787 unsigned OldCW =
4788 F->getSSARegMap()->createVirtualRegister(X86::GR16RegisterClass);
Evan Cheng20350c42006-11-27 23:37:22 +00004789 addFrameReference(BuildMI(BB, TII->get(X86::MOV16rm), OldCW), CWFrameIdx);
Evan Cheng02612422006-07-05 22:17:51 +00004790
4791 // Set the high part to be round to zero...
Evan Cheng20350c42006-11-27 23:37:22 +00004792 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mi)), CWFrameIdx)
4793 .addImm(0xC7F);
Evan Cheng02612422006-07-05 22:17:51 +00004794
4795 // Reload the modified control word now...
Evan Cheng20350c42006-11-27 23:37:22 +00004796 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx);
Evan Cheng02612422006-07-05 22:17:51 +00004797
4798 // Restore the memory image of control word to original value
Evan Cheng20350c42006-11-27 23:37:22 +00004799 addFrameReference(BuildMI(BB, TII->get(X86::MOV16mr)), CWFrameIdx)
4800 .addReg(OldCW);
Evan Cheng02612422006-07-05 22:17:51 +00004801
4802 // Get the X86 opcode to use.
4803 unsigned Opc;
4804 switch (MI->getOpcode()) {
4805 default: assert(0 && "illegal opcode!");
4806 case X86::FP_TO_INT16_IN_MEM: Opc = X86::FpIST16m; break;
4807 case X86::FP_TO_INT32_IN_MEM: Opc = X86::FpIST32m; break;
4808 case X86::FP_TO_INT64_IN_MEM: Opc = X86::FpIST64m; break;
4809 }
4810
4811 X86AddressMode AM;
4812 MachineOperand &Op = MI->getOperand(0);
4813 if (Op.isRegister()) {
4814 AM.BaseType = X86AddressMode::RegBase;
4815 AM.Base.Reg = Op.getReg();
4816 } else {
4817 AM.BaseType = X86AddressMode::FrameIndexBase;
4818 AM.Base.FrameIndex = Op.getFrameIndex();
4819 }
4820 Op = MI->getOperand(1);
4821 if (Op.isImmediate())
Chris Lattnerc0fb5672006-10-20 17:42:20 +00004822 AM.Scale = Op.getImm();
Evan Cheng02612422006-07-05 22:17:51 +00004823 Op = MI->getOperand(2);
4824 if (Op.isImmediate())
Chris Lattnerc0fb5672006-10-20 17:42:20 +00004825 AM.IndexReg = Op.getImm();
Evan Cheng02612422006-07-05 22:17:51 +00004826 Op = MI->getOperand(3);
4827 if (Op.isGlobalAddress()) {
4828 AM.GV = Op.getGlobal();
4829 } else {
Chris Lattnerc0fb5672006-10-20 17:42:20 +00004830 AM.Disp = Op.getImm();
Evan Cheng02612422006-07-05 22:17:51 +00004831 }
Evan Cheng20350c42006-11-27 23:37:22 +00004832 addFullAddress(BuildMI(BB, TII->get(Opc)), AM)
4833 .addReg(MI->getOperand(4).getReg());
Evan Cheng02612422006-07-05 22:17:51 +00004834
4835 // Reload the original control word now.
Evan Cheng20350c42006-11-27 23:37:22 +00004836 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx);
Evan Cheng02612422006-07-05 22:17:51 +00004837
4838 delete MI; // The pseudo instruction is gone now.
4839 return BB;
4840 }
4841 }
4842}
4843
4844//===----------------------------------------------------------------------===//
4845// X86 Optimization Hooks
4846//===----------------------------------------------------------------------===//
4847
Nate Begeman8a77efe2006-02-16 21:11:51 +00004848void X86TargetLowering::computeMaskedBitsForTargetNode(const SDOperand Op,
4849 uint64_t Mask,
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004850 uint64_t &KnownZero,
Nate Begeman8a77efe2006-02-16 21:11:51 +00004851 uint64_t &KnownOne,
4852 unsigned Depth) const {
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004853 unsigned Opc = Op.getOpcode();
Evan Cheng6d196db2006-04-05 06:11:20 +00004854 assert((Opc >= ISD::BUILTIN_OP_END ||
4855 Opc == ISD::INTRINSIC_WO_CHAIN ||
4856 Opc == ISD::INTRINSIC_W_CHAIN ||
4857 Opc == ISD::INTRINSIC_VOID) &&
4858 "Should use MaskedValueIsZero if you don't know whether Op"
4859 " is a target node!");
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004860
Evan Cheng6d196db2006-04-05 06:11:20 +00004861 KnownZero = KnownOne = 0; // Don't know anything.
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004862 switch (Opc) {
Evan Cheng6d196db2006-04-05 06:11:20 +00004863 default: break;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00004864 case X86ISD::SETCC:
Nate Begeman8a77efe2006-02-16 21:11:51 +00004865 KnownZero |= (MVT::getIntVTBitMask(Op.getValueType()) ^ 1ULL);
4866 break;
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004867 }
Evan Cheng9cdc16c2005-12-21 23:05:39 +00004868}
Chris Lattnerc642aa52006-01-31 19:43:35 +00004869
Evan Cheng5987cfb2006-07-07 08:33:52 +00004870/// getShuffleScalarElt - Returns the scalar element that will make up the ith
4871/// element of the result of the vector shuffle.
4872static SDOperand getShuffleScalarElt(SDNode *N, unsigned i, SelectionDAG &DAG) {
4873 MVT::ValueType VT = N->getValueType(0);
4874 SDOperand PermMask = N->getOperand(2);
4875 unsigned NumElems = PermMask.getNumOperands();
4876 SDOperand V = (i < NumElems) ? N->getOperand(0) : N->getOperand(1);
4877 i %= NumElems;
4878 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR) {
4879 return (i == 0)
4880 ? V.getOperand(0) : DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(VT));
4881 } else if (V.getOpcode() == ISD::VECTOR_SHUFFLE) {
4882 SDOperand Idx = PermMask.getOperand(i);
4883 if (Idx.getOpcode() == ISD::UNDEF)
4884 return DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(VT));
4885 return getShuffleScalarElt(V.Val,cast<ConstantSDNode>(Idx)->getValue(),DAG);
4886 }
4887 return SDOperand();
4888}
4889
4890/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the
4891/// node is a GlobalAddress + an offset.
4892static bool isGAPlusOffset(SDNode *N, GlobalValue* &GA, int64_t &Offset) {
Evan Chengae1cd752006-11-30 21:55:46 +00004893 unsigned Opc = N->getOpcode();
Evan Cheng62cdc3f2006-12-05 04:01:03 +00004894 if (Opc == X86ISD::Wrapper) {
Evan Cheng5987cfb2006-07-07 08:33:52 +00004895 if (dyn_cast<GlobalAddressSDNode>(N->getOperand(0))) {
4896 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal();
4897 return true;
4898 }
Evan Chengae1cd752006-11-30 21:55:46 +00004899 } else if (Opc == ISD::ADD) {
Evan Cheng5987cfb2006-07-07 08:33:52 +00004900 SDOperand N1 = N->getOperand(0);
4901 SDOperand N2 = N->getOperand(1);
4902 if (isGAPlusOffset(N1.Val, GA, Offset)) {
4903 ConstantSDNode *V = dyn_cast<ConstantSDNode>(N2);
4904 if (V) {
4905 Offset += V->getSignExtended();
4906 return true;
4907 }
4908 } else if (isGAPlusOffset(N2.Val, GA, Offset)) {
4909 ConstantSDNode *V = dyn_cast<ConstantSDNode>(N1);
4910 if (V) {
4911 Offset += V->getSignExtended();
4912 return true;
4913 }
4914 }
4915 }
4916 return false;
4917}
4918
4919/// isConsecutiveLoad - Returns true if N is loading from an address of Base
4920/// + Dist * Size.
4921static bool isConsecutiveLoad(SDNode *N, SDNode *Base, int Dist, int Size,
4922 MachineFrameInfo *MFI) {
4923 if (N->getOperand(0).Val != Base->getOperand(0).Val)
4924 return false;
4925
4926 SDOperand Loc = N->getOperand(1);
4927 SDOperand BaseLoc = Base->getOperand(1);
4928 if (Loc.getOpcode() == ISD::FrameIndex) {
4929 if (BaseLoc.getOpcode() != ISD::FrameIndex)
4930 return false;
4931 int FI = dyn_cast<FrameIndexSDNode>(Loc)->getIndex();
4932 int BFI = dyn_cast<FrameIndexSDNode>(BaseLoc)->getIndex();
4933 int FS = MFI->getObjectSize(FI);
4934 int BFS = MFI->getObjectSize(BFI);
4935 if (FS != BFS || FS != Size) return false;
4936 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Size);
4937 } else {
4938 GlobalValue *GV1 = NULL;
4939 GlobalValue *GV2 = NULL;
4940 int64_t Offset1 = 0;
4941 int64_t Offset2 = 0;
4942 bool isGA1 = isGAPlusOffset(Loc.Val, GV1, Offset1);
4943 bool isGA2 = isGAPlusOffset(BaseLoc.Val, GV2, Offset2);
4944 if (isGA1 && isGA2 && GV1 == GV2)
4945 return Offset1 == (Offset2 + Dist*Size);
4946 }
4947
4948 return false;
4949}
4950
Evan Cheng79cf9a52006-07-10 21:37:44 +00004951static bool isBaseAlignment16(SDNode *Base, MachineFrameInfo *MFI,
4952 const X86Subtarget *Subtarget) {
Evan Cheng5987cfb2006-07-07 08:33:52 +00004953 GlobalValue *GV;
4954 int64_t Offset;
4955 if (isGAPlusOffset(Base, GV, Offset))
4956 return (GV->getAlignment() >= 16 && (Offset % 16) == 0);
4957 else {
4958 assert(Base->getOpcode() == ISD::FrameIndex && "Unexpected base node!");
4959 int BFI = dyn_cast<FrameIndexSDNode>(Base)->getIndex();
Evan Cheng79cf9a52006-07-10 21:37:44 +00004960 if (BFI < 0)
4961 // Fixed objects do not specify alignment, however the offsets are known.
4962 return ((Subtarget->getStackAlignment() % 16) == 0 &&
4963 (MFI->getObjectOffset(BFI) % 16) == 0);
4964 else
4965 return MFI->getObjectAlignment(BFI) >= 16;
Evan Cheng5987cfb2006-07-07 08:33:52 +00004966 }
4967 return false;
4968}
4969
4970
4971/// PerformShuffleCombine - Combine a vector_shuffle that is equal to
4972/// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load
4973/// if the load addresses are consecutive, non-overlapping, and in the right
4974/// order.
Evan Cheng79cf9a52006-07-10 21:37:44 +00004975static SDOperand PerformShuffleCombine(SDNode *N, SelectionDAG &DAG,
4976 const X86Subtarget *Subtarget) {
Evan Cheng5987cfb2006-07-07 08:33:52 +00004977 MachineFunction &MF = DAG.getMachineFunction();
4978 MachineFrameInfo *MFI = MF.getFrameInfo();
4979 MVT::ValueType VT = N->getValueType(0);
4980 MVT::ValueType EVT = MVT::getVectorBaseType(VT);
4981 SDOperand PermMask = N->getOperand(2);
4982 int NumElems = (int)PermMask.getNumOperands();
4983 SDNode *Base = NULL;
4984 for (int i = 0; i < NumElems; ++i) {
4985 SDOperand Idx = PermMask.getOperand(i);
4986 if (Idx.getOpcode() == ISD::UNDEF) {
4987 if (!Base) return SDOperand();
4988 } else {
4989 SDOperand Arg =
4990 getShuffleScalarElt(N, cast<ConstantSDNode>(Idx)->getValue(), DAG);
Evan Chenge71fe34d2006-10-09 20:57:25 +00004991 if (!Arg.Val || !ISD::isNON_EXTLoad(Arg.Val))
Evan Cheng5987cfb2006-07-07 08:33:52 +00004992 return SDOperand();
4993 if (!Base)
4994 Base = Arg.Val;
4995 else if (!isConsecutiveLoad(Arg.Val, Base,
4996 i, MVT::getSizeInBits(EVT)/8,MFI))
4997 return SDOperand();
4998 }
4999 }
5000
Evan Cheng79cf9a52006-07-10 21:37:44 +00005001 bool isAlign16 = isBaseAlignment16(Base->getOperand(1).Val, MFI, Subtarget);
Evan Chenge71fe34d2006-10-09 20:57:25 +00005002 if (isAlign16) {
5003 LoadSDNode *LD = cast<LoadSDNode>(Base);
5004 return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(),
5005 LD->getSrcValueOffset());
5006 } else {
Evan Cheng5987cfb2006-07-07 08:33:52 +00005007 // Just use movups, it's shorter.
Chris Lattnere56fef92007-02-25 06:40:16 +00005008 SDVTList Tys = DAG.getVTList(MVT::v4f32, MVT::Other);
Evan Chengbd1c5a82006-08-11 09:08:15 +00005009 SmallVector<SDOperand, 3> Ops;
5010 Ops.push_back(Base->getOperand(0));
5011 Ops.push_back(Base->getOperand(1));
5012 Ops.push_back(Base->getOperand(2));
Evan Cheng5987cfb2006-07-07 08:33:52 +00005013 return DAG.getNode(ISD::BIT_CONVERT, VT,
Evan Chengbd1c5a82006-08-11 09:08:15 +00005014 DAG.getNode(X86ISD::LOAD_UA, Tys, &Ops[0], Ops.size()));
Evan Cheng5c68bba2006-08-11 07:35:45 +00005015 }
Evan Cheng5987cfb2006-07-07 08:33:52 +00005016}
5017
Chris Lattner9259b1e2006-10-04 06:57:07 +00005018/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes.
5019static SDOperand PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
5020 const X86Subtarget *Subtarget) {
5021 SDOperand Cond = N->getOperand(0);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005022
Chris Lattner9259b1e2006-10-04 06:57:07 +00005023 // If we have SSE[12] support, try to form min/max nodes.
5024 if (Subtarget->hasSSE2() &&
5025 (N->getValueType(0) == MVT::f32 || N->getValueType(0) == MVT::f64)) {
5026 if (Cond.getOpcode() == ISD::SETCC) {
5027 // Get the LHS/RHS of the select.
5028 SDOperand LHS = N->getOperand(1);
5029 SDOperand RHS = N->getOperand(2);
5030 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005031
Evan Cheng49683ba2006-11-10 21:43:37 +00005032 unsigned Opcode = 0;
Chris Lattner9259b1e2006-10-04 06:57:07 +00005033 if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) {
Chris Lattnerf2ef2432006-10-05 04:11:26 +00005034 switch (CC) {
5035 default: break;
5036 case ISD::SETOLE: // (X <= Y) ? X : Y -> min
5037 case ISD::SETULE:
5038 case ISD::SETLE:
5039 if (!UnsafeFPMath) break;
5040 // FALL THROUGH.
5041 case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min
5042 case ISD::SETLT:
Evan Cheng49683ba2006-11-10 21:43:37 +00005043 Opcode = X86ISD::FMIN;
Chris Lattnerf2ef2432006-10-05 04:11:26 +00005044 break;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005045
Chris Lattnerf2ef2432006-10-05 04:11:26 +00005046 case ISD::SETOGT: // (X > Y) ? X : Y -> max
5047 case ISD::SETUGT:
5048 case ISD::SETGT:
5049 if (!UnsafeFPMath) break;
5050 // FALL THROUGH.
5051 case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max
5052 case ISD::SETGE:
Evan Cheng49683ba2006-11-10 21:43:37 +00005053 Opcode = X86ISD::FMAX;
Chris Lattnerf2ef2432006-10-05 04:11:26 +00005054 break;
5055 }
Chris Lattner9259b1e2006-10-04 06:57:07 +00005056 } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) {
Chris Lattnerf2ef2432006-10-05 04:11:26 +00005057 switch (CC) {
5058 default: break;
5059 case ISD::SETOGT: // (X > Y) ? Y : X -> min
5060 case ISD::SETUGT:
5061 case ISD::SETGT:
5062 if (!UnsafeFPMath) break;
5063 // FALL THROUGH.
5064 case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min
5065 case ISD::SETGE:
Evan Cheng49683ba2006-11-10 21:43:37 +00005066 Opcode = X86ISD::FMIN;
Chris Lattnerf2ef2432006-10-05 04:11:26 +00005067 break;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005068
Chris Lattnerf2ef2432006-10-05 04:11:26 +00005069 case ISD::SETOLE: // (X <= Y) ? Y : X -> max
5070 case ISD::SETULE:
5071 case ISD::SETLE:
5072 if (!UnsafeFPMath) break;
5073 // FALL THROUGH.
5074 case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max
5075 case ISD::SETLT:
Evan Cheng49683ba2006-11-10 21:43:37 +00005076 Opcode = X86ISD::FMAX;
Chris Lattnerf2ef2432006-10-05 04:11:26 +00005077 break;
5078 }
Chris Lattner9259b1e2006-10-04 06:57:07 +00005079 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005080
Evan Cheng49683ba2006-11-10 21:43:37 +00005081 if (Opcode)
5082 return DAG.getNode(Opcode, N->getValueType(0), LHS, RHS);
Chris Lattner9259b1e2006-10-04 06:57:07 +00005083 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005084
Chris Lattner9259b1e2006-10-04 06:57:07 +00005085 }
5086
5087 return SDOperand();
5088}
5089
5090
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005091SDOperand X86TargetLowering::PerformDAGCombine(SDNode *N,
Evan Cheng5987cfb2006-07-07 08:33:52 +00005092 DAGCombinerInfo &DCI) const {
Evan Cheng5987cfb2006-07-07 08:33:52 +00005093 SelectionDAG &DAG = DCI.DAG;
5094 switch (N->getOpcode()) {
5095 default: break;
5096 case ISD::VECTOR_SHUFFLE:
Evan Cheng79cf9a52006-07-10 21:37:44 +00005097 return PerformShuffleCombine(N, DAG, Subtarget);
Chris Lattner9259b1e2006-10-04 06:57:07 +00005098 case ISD::SELECT:
5099 return PerformSELECTCombine(N, DAG, Subtarget);
Evan Cheng5987cfb2006-07-07 08:33:52 +00005100 }
5101
5102 return SDOperand();
5103}
5104
Evan Cheng02612422006-07-05 22:17:51 +00005105//===----------------------------------------------------------------------===//
5106// X86 Inline Assembly Support
5107//===----------------------------------------------------------------------===//
5108
Chris Lattner298ef372006-07-11 02:54:03 +00005109/// getConstraintType - Given a constraint letter, return the type of
5110/// constraint it is for this target.
5111X86TargetLowering::ConstraintType
5112X86TargetLowering::getConstraintType(char ConstraintLetter) const {
5113 switch (ConstraintLetter) {
Chris Lattnerc8db1072006-07-12 16:59:49 +00005114 case 'A':
5115 case 'r':
5116 case 'R':
5117 case 'l':
5118 case 'q':
5119 case 'Q':
5120 case 'x':
5121 case 'Y':
5122 return C_RegisterClass;
Chris Lattner298ef372006-07-11 02:54:03 +00005123 default: return TargetLowering::getConstraintType(ConstraintLetter);
5124 }
5125}
5126
Chris Lattner44daa502006-10-31 20:13:11 +00005127/// isOperandValidForConstraint - Return the specified operand (possibly
5128/// modified) if the specified SDOperand is valid for the specified target
5129/// constraint letter, otherwise return null.
5130SDOperand X86TargetLowering::
5131isOperandValidForConstraint(SDOperand Op, char Constraint, SelectionDAG &DAG) {
5132 switch (Constraint) {
5133 default: break;
5134 case 'i':
5135 // Literal immediates are always ok.
5136 if (isa<ConstantSDNode>(Op)) return Op;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005137
Chris Lattner44daa502006-10-31 20:13:11 +00005138 // If we are in non-pic codegen mode, we allow the address of a global to
5139 // be used with 'i'.
5140 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
5141 if (getTargetMachine().getRelocationModel() == Reloc::PIC_)
5142 return SDOperand(0, 0);
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005143
Chris Lattner44daa502006-10-31 20:13:11 +00005144 if (GA->getOpcode() != ISD::TargetGlobalAddress)
5145 Op = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0),
5146 GA->getOffset());
5147 return Op;
5148 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005149
Chris Lattner44daa502006-10-31 20:13:11 +00005150 // Otherwise, not valid for this mode.
5151 return SDOperand(0, 0);
5152 }
5153 return TargetLowering::isOperandValidForConstraint(Op, Constraint, DAG);
5154}
5155
5156
Chris Lattnerc642aa52006-01-31 19:43:35 +00005157std::vector<unsigned> X86TargetLowering::
Chris Lattner7ad77df2006-02-22 00:56:39 +00005158getRegClassForInlineAsmConstraint(const std::string &Constraint,
5159 MVT::ValueType VT) const {
Chris Lattnerc642aa52006-01-31 19:43:35 +00005160 if (Constraint.size() == 1) {
5161 // FIXME: not handling fp-stack yet!
5162 // FIXME: not handling MMX registers yet ('y' constraint).
5163 switch (Constraint[0]) { // GCC X86 Constraint Letters
Chris Lattner298ef372006-07-11 02:54:03 +00005164 default: break; // Unknown constraint letter
5165 case 'A': // EAX/EDX
5166 if (VT == MVT::i32 || VT == MVT::i64)
5167 return make_vector<unsigned>(X86::EAX, X86::EDX, 0);
5168 break;
Chris Lattnerc642aa52006-01-31 19:43:35 +00005169 case 'r': // GENERAL_REGS
5170 case 'R': // LEGACY_REGS
Chris Lattnerd139ddd2006-12-04 22:38:21 +00005171 if (VT == MVT::i64 && Subtarget->is64Bit())
5172 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX,
5173 X86::RSI, X86::RDI, X86::RBP, X86::RSP,
5174 X86::R8, X86::R9, X86::R10, X86::R11,
5175 X86::R12, X86::R13, X86::R14, X86::R15, 0);
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00005176 if (VT == MVT::i32)
5177 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX,
5178 X86::ESI, X86::EDI, X86::EBP, X86::ESP, 0);
5179 else if (VT == MVT::i16)
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005180 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX,
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00005181 X86::SI, X86::DI, X86::BP, X86::SP, 0);
5182 else if (VT == MVT::i8)
Chris Lattnera16201c2006-12-05 17:29:40 +00005183 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0);
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00005184 break;
Chris Lattnerc642aa52006-01-31 19:43:35 +00005185 case 'l': // INDEX_REGS
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00005186 if (VT == MVT::i32)
5187 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX,
5188 X86::ESI, X86::EDI, X86::EBP, 0);
5189 else if (VT == MVT::i16)
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005190 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX,
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00005191 X86::SI, X86::DI, X86::BP, 0);
5192 else if (VT == MVT::i8)
5193 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::DL, 0);
5194 break;
Chris Lattnerc642aa52006-01-31 19:43:35 +00005195 case 'q': // Q_REGS (GENERAL_REGS in 64-bit mode)
5196 case 'Q': // Q_REGS
Chris Lattner6d4a2dc2006-05-06 00:29:37 +00005197 if (VT == MVT::i32)
5198 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0);
5199 else if (VT == MVT::i16)
5200 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0);
5201 else if (VT == MVT::i8)
5202 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::DL, 0);
5203 break;
Chris Lattnerc642aa52006-01-31 19:43:35 +00005204 case 'x': // SSE_REGS if SSE1 allowed
5205 if (Subtarget->hasSSE1())
5206 return make_vector<unsigned>(X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
5207 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7,
5208 0);
5209 return std::vector<unsigned>();
5210 case 'Y': // SSE_REGS if SSE2 allowed
5211 if (Subtarget->hasSSE2())
5212 return make_vector<unsigned>(X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
5213 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7,
5214 0);
5215 return std::vector<unsigned>();
5216 }
5217 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005218
Chris Lattner7ad77df2006-02-22 00:56:39 +00005219 return std::vector<unsigned>();
Chris Lattnerc642aa52006-01-31 19:43:35 +00005220}
Chris Lattner524129d2006-07-31 23:26:50 +00005221
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005222std::pair<unsigned, const TargetRegisterClass*>
Chris Lattner524129d2006-07-31 23:26:50 +00005223X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
5224 MVT::ValueType VT) const {
5225 // Use the default implementation in TargetLowering to convert the register
5226 // constraint into a member of a register class.
5227 std::pair<unsigned, const TargetRegisterClass*> Res;
5228 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
Chris Lattnerf6a69662006-10-31 19:42:44 +00005229
5230 // Not found as a standard register?
5231 if (Res.second == 0) {
5232 // GCC calls "st(0)" just plain "st".
5233 if (StringsEqualNoCase("{st}", Constraint)) {
5234 Res.first = X86::ST0;
5235 Res.second = X86::RSTRegisterClass;
5236 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005237
Chris Lattnerf6a69662006-10-31 19:42:44 +00005238 return Res;
5239 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005240
Chris Lattner524129d2006-07-31 23:26:50 +00005241 // Otherwise, check to see if this is a register class of the wrong value
5242 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to
5243 // turn into {ax},{dx}.
5244 if (Res.second->hasType(VT))
5245 return Res; // Correct type already, nothing to do.
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005246
Chris Lattner524129d2006-07-31 23:26:50 +00005247 // All of the single-register GCC register classes map their values onto
5248 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we
5249 // really want an 8-bit or 32-bit register, map to the appropriate register
5250 // class and return the appropriate register.
5251 if (Res.second != X86::GR16RegisterClass)
5252 return Res;
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005253
Chris Lattner524129d2006-07-31 23:26:50 +00005254 if (VT == MVT::i8) {
5255 unsigned DestReg = 0;
5256 switch (Res.first) {
5257 default: break;
5258 case X86::AX: DestReg = X86::AL; break;
5259 case X86::DX: DestReg = X86::DL; break;
5260 case X86::CX: DestReg = X86::CL; break;
5261 case X86::BX: DestReg = X86::BL; break;
5262 }
5263 if (DestReg) {
5264 Res.first = DestReg;
5265 Res.second = Res.second = X86::GR8RegisterClass;
5266 }
5267 } else if (VT == MVT::i32) {
5268 unsigned DestReg = 0;
5269 switch (Res.first) {
5270 default: break;
5271 case X86::AX: DestReg = X86::EAX; break;
5272 case X86::DX: DestReg = X86::EDX; break;
5273 case X86::CX: DestReg = X86::ECX; break;
5274 case X86::BX: DestReg = X86::EBX; break;
5275 case X86::SI: DestReg = X86::ESI; break;
5276 case X86::DI: DestReg = X86::EDI; break;
5277 case X86::BP: DestReg = X86::EBP; break;
5278 case X86::SP: DestReg = X86::ESP; break;
5279 }
5280 if (DestReg) {
5281 Res.first = DestReg;
5282 Res.second = Res.second = X86::GR32RegisterClass;
5283 }
Evan Cheng11b0a5d2006-09-08 06:48:29 +00005284 } else if (VT == MVT::i64) {
5285 unsigned DestReg = 0;
5286 switch (Res.first) {
5287 default: break;
5288 case X86::AX: DestReg = X86::RAX; break;
5289 case X86::DX: DestReg = X86::RDX; break;
5290 case X86::CX: DestReg = X86::RCX; break;
5291 case X86::BX: DestReg = X86::RBX; break;
5292 case X86::SI: DestReg = X86::RSI; break;
5293 case X86::DI: DestReg = X86::RDI; break;
5294 case X86::BP: DestReg = X86::RBP; break;
5295 case X86::SP: DestReg = X86::RSP; break;
5296 }
5297 if (DestReg) {
5298 Res.first = DestReg;
5299 Res.second = Res.second = X86::GR64RegisterClass;
5300 }
Chris Lattner524129d2006-07-31 23:26:50 +00005301 }
Anton Korobeynikov5b96cde2006-11-21 00:01:06 +00005302
Chris Lattner524129d2006-07-31 23:26:50 +00005303 return Res;
5304}