blob: ca6ff929a77d2c67b6cf119f1a0843b76bf51181 [file] [log] [blame]
Arnold Schwaighofer92226dd2007-10-12 21:53:12 +00001//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00002//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner4ee451d2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00007//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the interfaces that X86 uses to lower LLVM code into a
11// selection DAG.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
Evan Cheng0cc39452006-01-16 21:21:29 +000016#include "X86InstrBuilder.h"
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +000017#include "X86ISelLowering.h"
18#include "X86TargetMachine.h"
19#include "llvm/CallingConv.h"
Evan Cheng223547a2006-01-31 22:28:30 +000020#include "llvm/Constants.h"
Evan Cheng347d5f72006-04-28 21:29:37 +000021#include "llvm/DerivedTypes.h"
Chris Lattnerb903bed2009-06-26 21:20:29 +000022#include "llvm/GlobalAlias.h"
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +000023#include "llvm/GlobalVariable.h"
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +000024#include "llvm/Function.h"
Chris Lattnerb8105652009-07-20 17:51:36 +000025#include "llvm/Instructions.h"
Evan Cheng6be2c582006-04-05 23:38:46 +000026#include "llvm/Intrinsics.h"
Owen Andersona90b3dc2009-07-15 21:51:10 +000027#include "llvm/LLVMContext.h"
Evan Cheng14b32e12007-12-11 01:46:18 +000028#include "llvm/ADT/BitVector.h"
Evan Cheng30b37b52006-03-13 23:18:16 +000029#include "llvm/ADT/VectorExtras.h"
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +000030#include "llvm/CodeGen/MachineFrameInfo.h"
Evan Cheng4a460802006-01-11 00:33:36 +000031#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineInstrBuilder.h"
Evan Chenga844bde2008-02-02 04:07:54 +000033#include "llvm/CodeGen/MachineModuleInfo.h"
Chris Lattner84bc5422007-12-31 04:13:23 +000034#include "llvm/CodeGen/MachineRegisterInfo.h"
Dan Gohman69de1932008-02-06 22:27:42 +000035#include "llvm/CodeGen/PseudoSourceValue.h"
Evan Chengef6ffb12006-01-31 03:14:29 +000036#include "llvm/Support/MathExtras.h"
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +000037#include "llvm/Support/Debug.h"
Torok Edwinab7c09b2009-07-08 18:01:40 +000038#include "llvm/Support/ErrorHandling.h"
Chris Lattnerf0144122009-07-28 03:13:23 +000039#include "llvm/Target/TargetLoweringObjectFile.h"
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +000040#include "llvm/Target/TargetOptions.h"
Evan Cheng14b32e12007-12-11 01:46:18 +000041#include "llvm/ADT/SmallSet.h"
Chris Lattner1a60aa72006-10-31 19:42:44 +000042#include "llvm/ADT/StringExtras.h"
Mon P Wang3c81d352008-11-23 04:37:22 +000043#include "llvm/Support/CommandLine.h"
Torok Edwindac237e2009-07-08 20:53:28 +000044#include "llvm/Support/raw_ostream.h"
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +000045using namespace llvm;
46
Mon P Wang3c81d352008-11-23 04:37:22 +000047static cl::opt<bool>
Mon P Wang9f22a4a2008-11-24 02:10:43 +000048DisableMMX("disable-mmx", cl::Hidden, cl::desc("Disable use of MMX"));
Mon P Wang3c81d352008-11-23 04:37:22 +000049
Evan Cheng10e86422008-04-25 19:11:04 +000050// Forward declarations.
Owen Andersone50ed302009-08-10 22:56:29 +000051static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
Nate Begeman9008ca62009-04-27 18:41:29 +000052 SDValue V2);
Evan Cheng10e86422008-04-25 19:11:04 +000053
Chris Lattnerf0144122009-07-28 03:13:23 +000054static TargetLoweringObjectFile *createTLOF(X86TargetMachine &TM) {
55 switch (TM.getSubtarget<X86Subtarget>().TargetType) {
56 default: llvm_unreachable("unknown subtarget type");
57 case X86Subtarget::isDarwin:
Chris Lattnerf26e03b2009-07-31 17:42:42 +000058 return new TargetLoweringObjectFileMachO();
Chris Lattnerf0144122009-07-28 03:13:23 +000059 case X86Subtarget::isELF:
60 return new TargetLoweringObjectFileELF();
61 case X86Subtarget::isMingw:
62 case X86Subtarget::isCygwin:
63 case X86Subtarget::isWindows:
64 return new TargetLoweringObjectFileCOFF();
65 }
66
67}
68
Dan Gohmanc9f5f3f2008-05-14 01:58:56 +000069X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
Chris Lattnerf0144122009-07-28 03:13:23 +000070 : TargetLowering(TM, createTLOF(TM)) {
Evan Cheng559806f2006-01-27 08:10:46 +000071 Subtarget = &TM.getSubtarget<X86Subtarget>();
Dale Johannesenf1fc3a82007-09-23 14:52:20 +000072 X86ScalarSSEf64 = Subtarget->hasSSE2();
73 X86ScalarSSEf32 = Subtarget->hasSSE1();
Evan Cheng25ab6902006-09-08 06:48:29 +000074 X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP;
Anton Korobeynikovbff66b02008-09-09 18:22:57 +000075
Anton Korobeynikov2365f512007-07-14 14:06:15 +000076 RegInfo = TM.getRegisterInfo();
Anton Korobeynikovbff66b02008-09-09 18:22:57 +000077 TD = getTargetData();
Anton Korobeynikov2365f512007-07-14 14:06:15 +000078
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +000079 // Set up the TargetLowering object.
80
81 // X86 is weird, it always uses i8 for shift amounts and setcc results.
Owen Anderson825b72b2009-08-11 20:47:22 +000082 setShiftAmountType(MVT::i8);
Duncan Sands03228082008-11-23 15:47:28 +000083 setBooleanContents(ZeroOrOneBooleanContent);
Evan Cheng0b2afbd2006-01-25 09:15:17 +000084 setSchedulingPreference(SchedulingForRegPressure);
Evan Cheng25ab6902006-09-08 06:48:29 +000085 setStackPointerRegisterToSaveRestore(X86StackPtr);
Evan Cheng714554d2006-03-16 21:47:42 +000086
Anton Korobeynikovd27a2582006-12-10 23:12:42 +000087 if (Subtarget->isTargetDarwin()) {
Evan Chengdf57fa02006-03-17 20:31:41 +000088 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp.
Anton Korobeynikovd27a2582006-12-10 23:12:42 +000089 setUseUnderscoreSetJmp(false);
90 setUseUnderscoreLongJmp(false);
Anton Korobeynikov317848f2007-01-03 11:43:14 +000091 } else if (Subtarget->isTargetMingw()) {
Anton Korobeynikovd27a2582006-12-10 23:12:42 +000092 // MS runtime is weird: it exports _setjmp, but longjmp!
93 setUseUnderscoreSetJmp(true);
94 setUseUnderscoreLongJmp(false);
95 } else {
96 setUseUnderscoreSetJmp(true);
97 setUseUnderscoreLongJmp(true);
98 }
Scott Michelfdc40a02009-02-17 22:15:04 +000099
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000100 // Set up the register classes.
Owen Anderson825b72b2009-08-11 20:47:22 +0000101 addRegisterClass(MVT::i8, X86::GR8RegisterClass);
102 addRegisterClass(MVT::i16, X86::GR16RegisterClass);
103 addRegisterClass(MVT::i32, X86::GR32RegisterClass);
Evan Cheng25ab6902006-09-08 06:48:29 +0000104 if (Subtarget->is64Bit())
Owen Anderson825b72b2009-08-11 20:47:22 +0000105 addRegisterClass(MVT::i64, X86::GR64RegisterClass);
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000106
Owen Anderson825b72b2009-08-11 20:47:22 +0000107 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote);
Evan Chengc5484282006-10-04 00:56:09 +0000108
Scott Michelfdc40a02009-02-17 22:15:04 +0000109 // We don't accept any truncstore of integer registers.
Owen Anderson825b72b2009-08-11 20:47:22 +0000110 setTruncStoreAction(MVT::i64, MVT::i32, Expand);
111 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
112 setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
113 setTruncStoreAction(MVT::i32, MVT::i16, Expand);
114 setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
115 setTruncStoreAction(MVT::i16, MVT::i8, Expand);
Evan Cheng7f042682008-10-15 02:05:31 +0000116
117 // SETOEQ and SETUNE require checking two conditions.
Owen Anderson825b72b2009-08-11 20:47:22 +0000118 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand);
119 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand);
120 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand);
121 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand);
122 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand);
123 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand);
Chris Lattnerddf89562008-01-17 19:59:44 +0000124
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000125 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
126 // operation.
Owen Anderson825b72b2009-08-11 20:47:22 +0000127 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote);
128 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote);
129 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote);
Evan Cheng6892f282006-01-17 02:32:49 +0000130
Evan Cheng25ab6902006-09-08 06:48:29 +0000131 if (Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000132 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote);
133 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand);
Eli Friedman948e95a2009-05-23 09:59:16 +0000134 } else if (!UseSoftFloat) {
135 if (X86ScalarSSEf64) {
Dale Johannesen1c15bf52008-10-21 20:50:01 +0000136 // We have an impenetrably clever algorithm for ui64->double only.
Owen Anderson825b72b2009-08-11 20:47:22 +0000137 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom);
Bill Wendling105be5a2009-03-13 08:41:47 +0000138 }
Eli Friedman948e95a2009-05-23 09:59:16 +0000139 // We have an algorithm for SSE2, and we turn this into a 64-bit
140 // FILD for other targets.
Owen Anderson825b72b2009-08-11 20:47:22 +0000141 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom);
Evan Cheng25ab6902006-09-08 06:48:29 +0000142 }
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000143
144 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
145 // this operation.
Owen Anderson825b72b2009-08-11 20:47:22 +0000146 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote);
147 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote);
Bill Wendling105be5a2009-03-13 08:41:47 +0000148
Devang Patel6a784892009-06-05 18:48:29 +0000149 if (!UseSoftFloat) {
Bill Wendling105be5a2009-03-13 08:41:47 +0000150 // SSE has no i16 to fp conversion, only i32
151 if (X86ScalarSSEf32) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000152 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote);
Bill Wendling105be5a2009-03-13 08:41:47 +0000153 // f32 and f64 cases are Legal, f80 case is not
Owen Anderson825b72b2009-08-11 20:47:22 +0000154 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom);
Bill Wendling105be5a2009-03-13 08:41:47 +0000155 } else {
Owen Anderson825b72b2009-08-11 20:47:22 +0000156 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom);
157 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom);
Bill Wendling105be5a2009-03-13 08:41:47 +0000158 }
Dale Johannesen9e3d3ab2007-09-14 22:26:36 +0000159 } else {
Owen Anderson825b72b2009-08-11 20:47:22 +0000160 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote);
161 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Promote);
Evan Cheng5298bcc2006-02-17 07:01:52 +0000162 }
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000163
Dale Johannesen73328d12007-09-19 23:55:34 +0000164 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
165 // are Legal, f80 is custom lowered.
Owen Anderson825b72b2009-08-11 20:47:22 +0000166 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom);
167 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom);
Evan Cheng6dab0532006-01-30 08:02:57 +0000168
Evan Cheng02568ff2006-01-30 22:13:22 +0000169 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
170 // this operation.
Owen Anderson825b72b2009-08-11 20:47:22 +0000171 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote);
172 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote);
Evan Cheng02568ff2006-01-30 22:13:22 +0000173
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000174 if (X86ScalarSSEf32) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000175 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote);
Dale Johannesen9e3d3ab2007-09-14 22:26:36 +0000176 // f32 and f64 cases are Legal, f80 case is not
Owen Anderson825b72b2009-08-11 20:47:22 +0000177 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom);
Evan Cheng02568ff2006-01-30 22:13:22 +0000178 } else {
Owen Anderson825b72b2009-08-11 20:47:22 +0000179 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom);
180 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom);
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000181 }
182
183 // Handle FP_TO_UINT by promoting the destination to a larger signed
184 // conversion.
Owen Anderson825b72b2009-08-11 20:47:22 +0000185 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote);
186 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote);
187 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote);
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000188
Evan Cheng25ab6902006-09-08 06:48:29 +0000189 if (Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000190 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand);
191 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote);
Eli Friedman948e95a2009-05-23 09:59:16 +0000192 } else if (!UseSoftFloat) {
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000193 if (X86ScalarSSEf32 && !Subtarget->hasSSE3())
Evan Cheng25ab6902006-09-08 06:48:29 +0000194 // Expand FP_TO_UINT into a select.
195 // FIXME: We would like to use a Custom expander here eventually to do
196 // the optimal thing for SSE vs. the default expansion in the legalizer.
Owen Anderson825b72b2009-08-11 20:47:22 +0000197 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand);
Evan Cheng25ab6902006-09-08 06:48:29 +0000198 else
Eli Friedman948e95a2009-05-23 09:59:16 +0000199 // With SSE3 we can use fisttpll to convert to a signed i64; without
200 // SSE, we're stuck with a fistpll.
Owen Anderson825b72b2009-08-11 20:47:22 +0000201 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom);
Evan Cheng25ab6902006-09-08 06:48:29 +0000202 }
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000203
Chris Lattner399610a2006-12-05 18:22:22 +0000204 // TODO: when we have SSE, these could be more efficient, by using movd/movq.
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000205 if (!X86ScalarSSEf64) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000206 setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand);
207 setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand);
Chris Lattnerf3597a12006-12-05 18:45:06 +0000208 }
Chris Lattner21f66852005-12-23 05:15:23 +0000209
Dan Gohmanb00ee212008-02-18 19:34:53 +0000210 // Scalar integer divide and remainder are lowered to use operations that
211 // produce two results, to match the available instructions. This exposes
212 // the two-result form to trivial CSE, which is able to combine x/y and x%y
213 // into a single instruction.
214 //
215 // Scalar integer multiply-high is also lowered to use two-result
216 // operations, to match the available instructions. However, plain multiply
217 // (low) operations are left as Legal, as there are single-result
218 // instructions for this in x86. Using the two-result multiply instructions
219 // when both high and low results are needed must be arranged by dagcombine.
Owen Anderson825b72b2009-08-11 20:47:22 +0000220 setOperationAction(ISD::MULHS , MVT::i8 , Expand);
221 setOperationAction(ISD::MULHU , MVT::i8 , Expand);
222 setOperationAction(ISD::SDIV , MVT::i8 , Expand);
223 setOperationAction(ISD::UDIV , MVT::i8 , Expand);
224 setOperationAction(ISD::SREM , MVT::i8 , Expand);
225 setOperationAction(ISD::UREM , MVT::i8 , Expand);
226 setOperationAction(ISD::MULHS , MVT::i16 , Expand);
227 setOperationAction(ISD::MULHU , MVT::i16 , Expand);
228 setOperationAction(ISD::SDIV , MVT::i16 , Expand);
229 setOperationAction(ISD::UDIV , MVT::i16 , Expand);
230 setOperationAction(ISD::SREM , MVT::i16 , Expand);
231 setOperationAction(ISD::UREM , MVT::i16 , Expand);
232 setOperationAction(ISD::MULHS , MVT::i32 , Expand);
233 setOperationAction(ISD::MULHU , MVT::i32 , Expand);
234 setOperationAction(ISD::SDIV , MVT::i32 , Expand);
235 setOperationAction(ISD::UDIV , MVT::i32 , Expand);
236 setOperationAction(ISD::SREM , MVT::i32 , Expand);
237 setOperationAction(ISD::UREM , MVT::i32 , Expand);
238 setOperationAction(ISD::MULHS , MVT::i64 , Expand);
239 setOperationAction(ISD::MULHU , MVT::i64 , Expand);
240 setOperationAction(ISD::SDIV , MVT::i64 , Expand);
241 setOperationAction(ISD::UDIV , MVT::i64 , Expand);
242 setOperationAction(ISD::SREM , MVT::i64 , Expand);
243 setOperationAction(ISD::UREM , MVT::i64 , Expand);
Dan Gohmana37c9f72007-09-25 18:23:27 +0000244
Owen Anderson825b72b2009-08-11 20:47:22 +0000245 setOperationAction(ISD::BR_JT , MVT::Other, Expand);
246 setOperationAction(ISD::BRCOND , MVT::Other, Custom);
247 setOperationAction(ISD::BR_CC , MVT::Other, Expand);
248 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand);
Evan Cheng25ab6902006-09-08 06:48:29 +0000249 if (Subtarget->is64Bit())
Owen Anderson825b72b2009-08-11 20:47:22 +0000250 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
251 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal);
252 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
253 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
254 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand);
255 setOperationAction(ISD::FREM , MVT::f32 , Expand);
256 setOperationAction(ISD::FREM , MVT::f64 , Expand);
257 setOperationAction(ISD::FREM , MVT::f80 , Expand);
258 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom);
Scott Michelfdc40a02009-02-17 22:15:04 +0000259
Owen Anderson825b72b2009-08-11 20:47:22 +0000260 setOperationAction(ISD::CTPOP , MVT::i8 , Expand);
261 setOperationAction(ISD::CTTZ , MVT::i8 , Custom);
262 setOperationAction(ISD::CTLZ , MVT::i8 , Custom);
263 setOperationAction(ISD::CTPOP , MVT::i16 , Expand);
264 setOperationAction(ISD::CTTZ , MVT::i16 , Custom);
265 setOperationAction(ISD::CTLZ , MVT::i16 , Custom);
266 setOperationAction(ISD::CTPOP , MVT::i32 , Expand);
267 setOperationAction(ISD::CTTZ , MVT::i32 , Custom);
268 setOperationAction(ISD::CTLZ , MVT::i32 , Custom);
Evan Cheng25ab6902006-09-08 06:48:29 +0000269 if (Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000270 setOperationAction(ISD::CTPOP , MVT::i64 , Expand);
271 setOperationAction(ISD::CTTZ , MVT::i64 , Custom);
272 setOperationAction(ISD::CTLZ , MVT::i64 , Custom);
Evan Cheng25ab6902006-09-08 06:48:29 +0000273 }
274
Owen Anderson825b72b2009-08-11 20:47:22 +0000275 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
276 setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
Nate Begeman35ef9132006-01-11 21:21:00 +0000277
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000278 // These should be promoted to a larger select which is supported.
Owen Anderson825b72b2009-08-11 20:47:22 +0000279 setOperationAction(ISD::SELECT , MVT::i1 , Promote);
280 setOperationAction(ISD::SELECT , MVT::i8 , Promote);
Nate Begeman4c5dcf52006-02-17 00:03:04 +0000281 // X86 wants to expand cmov itself.
Owen Anderson825b72b2009-08-11 20:47:22 +0000282 setOperationAction(ISD::SELECT , MVT::i16 , Custom);
283 setOperationAction(ISD::SELECT , MVT::i32 , Custom);
284 setOperationAction(ISD::SELECT , MVT::f32 , Custom);
285 setOperationAction(ISD::SELECT , MVT::f64 , Custom);
286 setOperationAction(ISD::SELECT , MVT::f80 , Custom);
287 setOperationAction(ISD::SETCC , MVT::i8 , Custom);
288 setOperationAction(ISD::SETCC , MVT::i16 , Custom);
289 setOperationAction(ISD::SETCC , MVT::i32 , Custom);
290 setOperationAction(ISD::SETCC , MVT::f32 , Custom);
291 setOperationAction(ISD::SETCC , MVT::f64 , Custom);
292 setOperationAction(ISD::SETCC , MVT::f80 , Custom);
Evan Cheng25ab6902006-09-08 06:48:29 +0000293 if (Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000294 setOperationAction(ISD::SELECT , MVT::i64 , Custom);
295 setOperationAction(ISD::SETCC , MVT::i64 , Custom);
Evan Cheng25ab6902006-09-08 06:48:29 +0000296 }
Owen Anderson825b72b2009-08-11 20:47:22 +0000297 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
Anton Korobeynikov2365f512007-07-14 14:06:15 +0000298
Nate Begeman4c5dcf52006-02-17 00:03:04 +0000299 // Darwin ABI issue.
Owen Anderson825b72b2009-08-11 20:47:22 +0000300 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom);
301 setOperationAction(ISD::JumpTable , MVT::i32 , Custom);
302 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom);
303 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom);
Anton Korobeynikov6625eff2008-05-04 21:36:32 +0000304 if (Subtarget->is64Bit())
Owen Anderson825b72b2009-08-11 20:47:22 +0000305 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
306 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom);
Evan Cheng25ab6902006-09-08 06:48:29 +0000307 if (Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000308 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom);
309 setOperationAction(ISD::JumpTable , MVT::i64 , Custom);
310 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom);
311 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom);
Evan Cheng25ab6902006-09-08 06:48:29 +0000312 }
Nate Begeman4c5dcf52006-02-17 00:03:04 +0000313 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86)
Owen Anderson825b72b2009-08-11 20:47:22 +0000314 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom);
315 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom);
316 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom);
Dan Gohman4c1fa612008-03-03 22:22:09 +0000317 if (Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000318 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom);
319 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom);
320 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom);
Dan Gohman4c1fa612008-03-03 22:22:09 +0000321 }
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000322
Evan Chengd2cde682008-03-10 19:38:10 +0000323 if (Subtarget->hasSSE1())
Owen Anderson825b72b2009-08-11 20:47:22 +0000324 setOperationAction(ISD::PREFETCH , MVT::Other, Legal);
Evan Cheng27b7db52008-03-08 00:58:38 +0000325
Andrew Lenharthd497d9f2008-02-16 14:46:26 +0000326 if (!Subtarget->hasSSE2())
Owen Anderson825b72b2009-08-11 20:47:22 +0000327 setOperationAction(ISD::MEMBARRIER , MVT::Other, Expand);
Andrew Lenharthd497d9f2008-02-16 14:46:26 +0000328
Mon P Wang63307c32008-05-05 19:05:59 +0000329 // Expand certain atomics
Owen Anderson825b72b2009-08-11 20:47:22 +0000330 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, Custom);
331 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, Custom);
332 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom);
333 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom);
Bill Wendling5bf1b4e2008-08-20 00:28:16 +0000334
Owen Anderson825b72b2009-08-11 20:47:22 +0000335 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i8, Custom);
336 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i16, Custom);
337 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
338 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
Andrew Lenharthd497d9f2008-02-16 14:46:26 +0000339
Dale Johannesen48c1bc22008-10-02 18:53:47 +0000340 if (!Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000341 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, Custom);
342 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
343 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
344 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, Custom);
345 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, Custom);
346 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i64, Custom);
347 setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, Custom);
Dale Johannesen48c1bc22008-10-02 18:53:47 +0000348 }
349
Dan Gohman7f460202008-06-30 20:59:49 +0000350 // Use the default ISD::DBG_STOPPOINT, ISD::DECLARE expansion.
Owen Anderson825b72b2009-08-11 20:47:22 +0000351 setOperationAction(ISD::DBG_STOPPOINT, MVT::Other, Expand);
Evan Cheng3c992d22006-03-07 02:02:57 +0000352 // FIXME - use subtarget debug flags
Anton Korobeynikovab4022f2006-10-31 08:31:24 +0000353 if (!Subtarget->isTargetDarwin() &&
354 !Subtarget->isTargetELF() &&
Dan Gohman44066042008-07-01 00:05:16 +0000355 !Subtarget->isTargetCygMing()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000356 setOperationAction(ISD::DBG_LABEL, MVT::Other, Expand);
357 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
Dan Gohman44066042008-07-01 00:05:16 +0000358 }
Chris Lattnerf73bae12005-11-29 06:16:21 +0000359
Owen Anderson825b72b2009-08-11 20:47:22 +0000360 setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand);
361 setOperationAction(ISD::EHSELECTION, MVT::i64, Expand);
362 setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand);
363 setOperationAction(ISD::EHSELECTION, MVT::i32, Expand);
Anton Korobeynikovce3b4652007-05-02 19:53:33 +0000364 if (Subtarget->is64Bit()) {
Anton Korobeynikovce3b4652007-05-02 19:53:33 +0000365 setExceptionPointerRegister(X86::RAX);
366 setExceptionSelectorRegister(X86::RDX);
367 } else {
368 setExceptionPointerRegister(X86::EAX);
369 setExceptionSelectorRegister(X86::EDX);
370 }
Owen Anderson825b72b2009-08-11 20:47:22 +0000371 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
372 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
Anton Korobeynikov260a6b82008-09-08 21:12:11 +0000373
Owen Anderson825b72b2009-08-11 20:47:22 +0000374 setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom);
Duncan Sandsb116fac2007-07-27 20:02:49 +0000375
Owen Anderson825b72b2009-08-11 20:47:22 +0000376 setOperationAction(ISD::TRAP, MVT::Other, Legal);
Anton Korobeynikov66fac792008-01-15 07:02:33 +0000377
Nate Begemanacc398c2006-01-25 18:21:52 +0000378 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
Owen Anderson825b72b2009-08-11 20:47:22 +0000379 setOperationAction(ISD::VASTART , MVT::Other, Custom);
380 setOperationAction(ISD::VAEND , MVT::Other, Expand);
Dan Gohman9018e832008-05-10 01:26:14 +0000381 if (Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000382 setOperationAction(ISD::VAARG , MVT::Other, Custom);
383 setOperationAction(ISD::VACOPY , MVT::Other, Custom);
Dan Gohman9018e832008-05-10 01:26:14 +0000384 } else {
Owen Anderson825b72b2009-08-11 20:47:22 +0000385 setOperationAction(ISD::VAARG , MVT::Other, Expand);
386 setOperationAction(ISD::VACOPY , MVT::Other, Expand);
Dan Gohman9018e832008-05-10 01:26:14 +0000387 }
Evan Chengae642192007-03-02 23:16:35 +0000388
Owen Anderson825b72b2009-08-11 20:47:22 +0000389 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
390 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
Evan Cheng25ab6902006-09-08 06:48:29 +0000391 if (Subtarget->is64Bit())
Owen Anderson825b72b2009-08-11 20:47:22 +0000392 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
Anton Korobeynikov57fc00d2007-04-17 09:20:00 +0000393 if (Subtarget->isTargetCygMing())
Owen Anderson825b72b2009-08-11 20:47:22 +0000394 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
Anton Korobeynikov57fc00d2007-04-17 09:20:00 +0000395 else
Owen Anderson825b72b2009-08-11 20:47:22 +0000396 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
Chris Lattnerb99329e2006-01-13 02:42:53 +0000397
Evan Chengc7ce29b2009-02-13 22:36:38 +0000398 if (!UseSoftFloat && X86ScalarSSEf64) {
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000399 // f32 and f64 use SSE.
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000400 // Set up the FP register classes.
Owen Anderson825b72b2009-08-11 20:47:22 +0000401 addRegisterClass(MVT::f32, X86::FR32RegisterClass);
402 addRegisterClass(MVT::f64, X86::FR64RegisterClass);
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000403
Evan Cheng223547a2006-01-31 22:28:30 +0000404 // Use ANDPD to simulate FABS.
Owen Anderson825b72b2009-08-11 20:47:22 +0000405 setOperationAction(ISD::FABS , MVT::f64, Custom);
406 setOperationAction(ISD::FABS , MVT::f32, Custom);
Evan Cheng223547a2006-01-31 22:28:30 +0000407
408 // Use XORP to simulate FNEG.
Owen Anderson825b72b2009-08-11 20:47:22 +0000409 setOperationAction(ISD::FNEG , MVT::f64, Custom);
410 setOperationAction(ISD::FNEG , MVT::f32, Custom);
Evan Cheng223547a2006-01-31 22:28:30 +0000411
Evan Cheng68c47cb2007-01-05 07:55:56 +0000412 // Use ANDPD and ORPD to simulate FCOPYSIGN.
Owen Anderson825b72b2009-08-11 20:47:22 +0000413 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
414 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
Evan Cheng68c47cb2007-01-05 07:55:56 +0000415
Evan Chengd25e9e82006-02-02 00:28:23 +0000416 // We don't support sin/cos/fmod
Owen Anderson825b72b2009-08-11 20:47:22 +0000417 setOperationAction(ISD::FSIN , MVT::f64, Expand);
418 setOperationAction(ISD::FCOS , MVT::f64, Expand);
419 setOperationAction(ISD::FSIN , MVT::f32, Expand);
420 setOperationAction(ISD::FCOS , MVT::f32, Expand);
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000421
Chris Lattnera54aa942006-01-29 06:26:08 +0000422 // Expand FP immediates into loads from the stack, except for the special
423 // cases we handle.
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000424 addLegalFPImmediate(APFloat(+0.0)); // xorpd
425 addLegalFPImmediate(APFloat(+0.0f)); // xorps
Evan Chengc7ce29b2009-02-13 22:36:38 +0000426 } else if (!UseSoftFloat && X86ScalarSSEf32) {
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000427 // Use SSE for f32, x87 for f64.
428 // Set up the FP register classes.
Owen Anderson825b72b2009-08-11 20:47:22 +0000429 addRegisterClass(MVT::f32, X86::FR32RegisterClass);
430 addRegisterClass(MVT::f64, X86::RFP64RegisterClass);
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000431
432 // Use ANDPS to simulate FABS.
Owen Anderson825b72b2009-08-11 20:47:22 +0000433 setOperationAction(ISD::FABS , MVT::f32, Custom);
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000434
435 // Use XORP to simulate FNEG.
Owen Anderson825b72b2009-08-11 20:47:22 +0000436 setOperationAction(ISD::FNEG , MVT::f32, Custom);
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000437
Owen Anderson825b72b2009-08-11 20:47:22 +0000438 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000439
440 // Use ANDPS and ORPS to simulate FCOPYSIGN.
Owen Anderson825b72b2009-08-11 20:47:22 +0000441 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
442 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000443
444 // We don't support sin/cos/fmod
Owen Anderson825b72b2009-08-11 20:47:22 +0000445 setOperationAction(ISD::FSIN , MVT::f32, Expand);
446 setOperationAction(ISD::FCOS , MVT::f32, Expand);
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000447
Nate Begemane1795842008-02-14 08:57:00 +0000448 // Special cases we handle for FP constants.
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000449 addLegalFPImmediate(APFloat(+0.0f)); // xorps
450 addLegalFPImmediate(APFloat(+0.0)); // FLD0
451 addLegalFPImmediate(APFloat(+1.0)); // FLD1
452 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
453 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
454
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000455 if (!UnsafeFPMath) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000456 setOperationAction(ISD::FSIN , MVT::f64 , Expand);
457 setOperationAction(ISD::FCOS , MVT::f64 , Expand);
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000458 }
Evan Chengc7ce29b2009-02-13 22:36:38 +0000459 } else if (!UseSoftFloat) {
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000460 // f32 and f64 in x87.
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000461 // Set up the FP register classes.
Owen Anderson825b72b2009-08-11 20:47:22 +0000462 addRegisterClass(MVT::f64, X86::RFP64RegisterClass);
463 addRegisterClass(MVT::f32, X86::RFP32RegisterClass);
Anton Korobeynikov12c49af2006-11-21 00:01:06 +0000464
Owen Anderson825b72b2009-08-11 20:47:22 +0000465 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
466 setOperationAction(ISD::UNDEF, MVT::f32, Expand);
467 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
468 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
Dale Johannesen5411a392007-08-09 01:04:01 +0000469
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000470 if (!UnsafeFPMath) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000471 setOperationAction(ISD::FSIN , MVT::f64 , Expand);
472 setOperationAction(ISD::FCOS , MVT::f64 , Expand);
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000473 }
Dale Johannesenf04afdb2007-08-30 00:23:21 +0000474 addLegalFPImmediate(APFloat(+0.0)); // FLD0
475 addLegalFPImmediate(APFloat(+1.0)); // FLD1
476 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
477 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
Dale Johannesenf1fc3a82007-09-23 14:52:20 +0000478 addLegalFPImmediate(APFloat(+0.0f)); // FLD0
479 addLegalFPImmediate(APFloat(+1.0f)); // FLD1
480 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
481 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000482 }
Evan Cheng470a6ad2006-02-22 02:26:30 +0000483
Dale Johannesen59a58732007-08-05 18:49:15 +0000484 // Long double always uses X87.
Evan Cheng92722532009-03-26 23:06:32 +0000485 if (!UseSoftFloat) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000486 addRegisterClass(MVT::f80, X86::RFP80RegisterClass);
487 setOperationAction(ISD::UNDEF, MVT::f80, Expand);
488 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
Evan Chengc7ce29b2009-02-13 22:36:38 +0000489 {
490 bool ignored;
491 APFloat TmpFlt(+0.0);
492 TmpFlt.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven,
493 &ignored);
494 addLegalFPImmediate(TmpFlt); // FLD0
495 TmpFlt.changeSign();
496 addLegalFPImmediate(TmpFlt); // FLD0/FCHS
497 APFloat TmpFlt2(+1.0);
498 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven,
499 &ignored);
500 addLegalFPImmediate(TmpFlt2); // FLD1
501 TmpFlt2.changeSign();
502 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
503 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000504
Evan Chengc7ce29b2009-02-13 22:36:38 +0000505 if (!UnsafeFPMath) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000506 setOperationAction(ISD::FSIN , MVT::f80 , Expand);
507 setOperationAction(ISD::FCOS , MVT::f80 , Expand);
Evan Chengc7ce29b2009-02-13 22:36:38 +0000508 }
Dale Johannesen2f429012007-09-26 21:10:55 +0000509 }
Dale Johannesen59a58732007-08-05 18:49:15 +0000510
Dan Gohmanf96e4de2007-10-11 23:21:31 +0000511 // Always use a library call for pow.
Owen Anderson825b72b2009-08-11 20:47:22 +0000512 setOperationAction(ISD::FPOW , MVT::f32 , Expand);
513 setOperationAction(ISD::FPOW , MVT::f64 , Expand);
514 setOperationAction(ISD::FPOW , MVT::f80 , Expand);
Dan Gohmanf96e4de2007-10-11 23:21:31 +0000515
Owen Anderson825b72b2009-08-11 20:47:22 +0000516 setOperationAction(ISD::FLOG, MVT::f80, Expand);
517 setOperationAction(ISD::FLOG2, MVT::f80, Expand);
518 setOperationAction(ISD::FLOG10, MVT::f80, Expand);
519 setOperationAction(ISD::FEXP, MVT::f80, Expand);
520 setOperationAction(ISD::FEXP2, MVT::f80, Expand);
Dale Johannesen7794f2a2008-09-04 00:47:13 +0000521
Mon P Wangf007a8b2008-11-06 05:31:54 +0000522 // First set operation action for all vector types to either promote
Mon P Wang0c397192008-10-30 08:01:45 +0000523 // (for widening) or expand (for scalarization). Then we will selectively
524 // turn on ones that can be effectively codegen'd.
Owen Anderson825b72b2009-08-11 20:47:22 +0000525 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
526 VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) {
527 setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand);
528 setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand);
529 setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand);
530 setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand);
531 setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand);
532 setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand);
533 setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand);
534 setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand);
535 setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand);
536 setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand);
537 setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand);
538 setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand);
539 setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand);
540 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand);
541 setOperationAction(ISD::EXTRACT_VECTOR_ELT,(MVT::SimpleValueType)VT,Expand);
542 setOperationAction(ISD::EXTRACT_SUBVECTOR,(MVT::SimpleValueType)VT,Expand);
543 setOperationAction(ISD::INSERT_VECTOR_ELT,(MVT::SimpleValueType)VT, Expand);
544 setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand);
545 setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand);
546 setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand);
547 setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand);
548 setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand);
549 setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand);
550 setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand);
551 setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand);
552 setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand);
553 setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand);
554 setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand);
555 setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand);
556 setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand);
557 setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand);
558 setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand);
559 setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand);
560 setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand);
561 setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand);
562 setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand);
563 setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand);
564 setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand);
565 setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand);
566 setOperationAction(ISD::FLOG, (MVT::SimpleValueType)VT, Expand);
567 setOperationAction(ISD::FLOG2, (MVT::SimpleValueType)VT, Expand);
568 setOperationAction(ISD::FLOG10, (MVT::SimpleValueType)VT, Expand);
569 setOperationAction(ISD::FEXP, (MVT::SimpleValueType)VT, Expand);
570 setOperationAction(ISD::FEXP2, (MVT::SimpleValueType)VT, Expand);
571 setOperationAction(ISD::FP_TO_UINT, (MVT::SimpleValueType)VT, Expand);
572 setOperationAction(ISD::FP_TO_SINT, (MVT::SimpleValueType)VT, Expand);
573 setOperationAction(ISD::UINT_TO_FP, (MVT::SimpleValueType)VT, Expand);
574 setOperationAction(ISD::SINT_TO_FP, (MVT::SimpleValueType)VT, Expand);
Evan Chengd30bf012006-03-01 01:11:20 +0000575 }
576
Evan Chengc7ce29b2009-02-13 22:36:38 +0000577 // FIXME: In order to prevent SSE instructions being expanded to MMX ones
578 // with -msoft-float, disable use of MMX as well.
Evan Cheng92722532009-03-26 23:06:32 +0000579 if (!UseSoftFloat && !DisableMMX && Subtarget->hasMMX()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000580 addRegisterClass(MVT::v8i8, X86::VR64RegisterClass);
581 addRegisterClass(MVT::v4i16, X86::VR64RegisterClass);
582 addRegisterClass(MVT::v2i32, X86::VR64RegisterClass);
583 addRegisterClass(MVT::v2f32, X86::VR64RegisterClass);
584 addRegisterClass(MVT::v1i64, X86::VR64RegisterClass);
Evan Cheng470a6ad2006-02-22 02:26:30 +0000585
Owen Anderson825b72b2009-08-11 20:47:22 +0000586 setOperationAction(ISD::ADD, MVT::v8i8, Legal);
587 setOperationAction(ISD::ADD, MVT::v4i16, Legal);
588 setOperationAction(ISD::ADD, MVT::v2i32, Legal);
589 setOperationAction(ISD::ADD, MVT::v1i64, Legal);
Bill Wendling2f88dcd2007-03-08 22:09:11 +0000590
Owen Anderson825b72b2009-08-11 20:47:22 +0000591 setOperationAction(ISD::SUB, MVT::v8i8, Legal);
592 setOperationAction(ISD::SUB, MVT::v4i16, Legal);
593 setOperationAction(ISD::SUB, MVT::v2i32, Legal);
594 setOperationAction(ISD::SUB, MVT::v1i64, Legal);
Bill Wendlingc1fb0472007-03-10 09:57:05 +0000595
Owen Anderson825b72b2009-08-11 20:47:22 +0000596 setOperationAction(ISD::MULHS, MVT::v4i16, Legal);
597 setOperationAction(ISD::MUL, MVT::v4i16, Legal);
Bill Wendling74027e92007-03-15 21:24:36 +0000598
Owen Anderson825b72b2009-08-11 20:47:22 +0000599 setOperationAction(ISD::AND, MVT::v8i8, Promote);
600 AddPromotedToType (ISD::AND, MVT::v8i8, MVT::v1i64);
601 setOperationAction(ISD::AND, MVT::v4i16, Promote);
602 AddPromotedToType (ISD::AND, MVT::v4i16, MVT::v1i64);
603 setOperationAction(ISD::AND, MVT::v2i32, Promote);
604 AddPromotedToType (ISD::AND, MVT::v2i32, MVT::v1i64);
605 setOperationAction(ISD::AND, MVT::v1i64, Legal);
Bill Wendling1b7a81d2007-03-16 09:44:46 +0000606
Owen Anderson825b72b2009-08-11 20:47:22 +0000607 setOperationAction(ISD::OR, MVT::v8i8, Promote);
608 AddPromotedToType (ISD::OR, MVT::v8i8, MVT::v1i64);
609 setOperationAction(ISD::OR, MVT::v4i16, Promote);
610 AddPromotedToType (ISD::OR, MVT::v4i16, MVT::v1i64);
611 setOperationAction(ISD::OR, MVT::v2i32, Promote);
612 AddPromotedToType (ISD::OR, MVT::v2i32, MVT::v1i64);
613 setOperationAction(ISD::OR, MVT::v1i64, Legal);
Bill Wendling1b7a81d2007-03-16 09:44:46 +0000614
Owen Anderson825b72b2009-08-11 20:47:22 +0000615 setOperationAction(ISD::XOR, MVT::v8i8, Promote);
616 AddPromotedToType (ISD::XOR, MVT::v8i8, MVT::v1i64);
617 setOperationAction(ISD::XOR, MVT::v4i16, Promote);
618 AddPromotedToType (ISD::XOR, MVT::v4i16, MVT::v1i64);
619 setOperationAction(ISD::XOR, MVT::v2i32, Promote);
620 AddPromotedToType (ISD::XOR, MVT::v2i32, MVT::v1i64);
621 setOperationAction(ISD::XOR, MVT::v1i64, Legal);
Bill Wendling1b7a81d2007-03-16 09:44:46 +0000622
Owen Anderson825b72b2009-08-11 20:47:22 +0000623 setOperationAction(ISD::LOAD, MVT::v8i8, Promote);
624 AddPromotedToType (ISD::LOAD, MVT::v8i8, MVT::v1i64);
625 setOperationAction(ISD::LOAD, MVT::v4i16, Promote);
626 AddPromotedToType (ISD::LOAD, MVT::v4i16, MVT::v1i64);
627 setOperationAction(ISD::LOAD, MVT::v2i32, Promote);
628 AddPromotedToType (ISD::LOAD, MVT::v2i32, MVT::v1i64);
629 setOperationAction(ISD::LOAD, MVT::v2f32, Promote);
630 AddPromotedToType (ISD::LOAD, MVT::v2f32, MVT::v1i64);
631 setOperationAction(ISD::LOAD, MVT::v1i64, Legal);
Bill Wendling2f88dcd2007-03-08 22:09:11 +0000632
Owen Anderson825b72b2009-08-11 20:47:22 +0000633 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Custom);
634 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom);
635 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Custom);
636 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f32, Custom);
637 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i64, Custom);
Bill Wendlinga348c562007-03-22 18:42:45 +0000638
Owen Anderson825b72b2009-08-11 20:47:22 +0000639 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom);
640 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom);
641 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i32, Custom);
642 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v1i64, Custom);
Bill Wendling826f36f2007-03-28 00:57:11 +0000643
Owen Anderson825b72b2009-08-11 20:47:22 +0000644 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f32, Custom);
645 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Custom);
646 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Custom);
647 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Custom);
Bill Wendling3180e202008-07-20 02:32:23 +0000648
Owen Anderson825b72b2009-08-11 20:47:22 +0000649 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom);
Mon P Wang9e5ecb82008-12-12 01:25:51 +0000650
Owen Anderson825b72b2009-08-11 20:47:22 +0000651 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand);
652 setOperationAction(ISD::TRUNCATE, MVT::v8i8, Expand);
653 setOperationAction(ISD::SELECT, MVT::v8i8, Promote);
654 setOperationAction(ISD::SELECT, MVT::v4i16, Promote);
655 setOperationAction(ISD::SELECT, MVT::v2i32, Promote);
656 setOperationAction(ISD::SELECT, MVT::v1i64, Custom);
657 setOperationAction(ISD::VSETCC, MVT::v8i8, Custom);
658 setOperationAction(ISD::VSETCC, MVT::v4i16, Custom);
659 setOperationAction(ISD::VSETCC, MVT::v2i32, Custom);
Evan Cheng470a6ad2006-02-22 02:26:30 +0000660 }
661
Evan Cheng92722532009-03-26 23:06:32 +0000662 if (!UseSoftFloat && Subtarget->hasSSE1()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000663 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass);
Evan Cheng470a6ad2006-02-22 02:26:30 +0000664
Owen Anderson825b72b2009-08-11 20:47:22 +0000665 setOperationAction(ISD::FADD, MVT::v4f32, Legal);
666 setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
667 setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
668 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
669 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
670 setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
671 setOperationAction(ISD::LOAD, MVT::v4f32, Legal);
672 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
673 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
674 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
675 setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
676 setOperationAction(ISD::VSETCC, MVT::v4f32, Custom);
Evan Cheng470a6ad2006-02-22 02:26:30 +0000677 }
678
Evan Cheng92722532009-03-26 23:06:32 +0000679 if (!UseSoftFloat && Subtarget->hasSSE2()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000680 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass);
Evan Chengc7ce29b2009-02-13 22:36:38 +0000681
Bill Wendlingf9abd7e2009-03-11 22:30:01 +0000682 // FIXME: Unfortunately -soft-float and -no-implicit-float means XMM
683 // registers cannot be used even for integer operations.
Owen Anderson825b72b2009-08-11 20:47:22 +0000684 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass);
685 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass);
686 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass);
687 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass);
Evan Cheng470a6ad2006-02-22 02:26:30 +0000688
Owen Anderson825b72b2009-08-11 20:47:22 +0000689 setOperationAction(ISD::ADD, MVT::v16i8, Legal);
690 setOperationAction(ISD::ADD, MVT::v8i16, Legal);
691 setOperationAction(ISD::ADD, MVT::v4i32, Legal);
692 setOperationAction(ISD::ADD, MVT::v2i64, Legal);
693 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
694 setOperationAction(ISD::SUB, MVT::v16i8, Legal);
695 setOperationAction(ISD::SUB, MVT::v8i16, Legal);
696 setOperationAction(ISD::SUB, MVT::v4i32, Legal);
697 setOperationAction(ISD::SUB, MVT::v2i64, Legal);
698 setOperationAction(ISD::MUL, MVT::v8i16, Legal);
699 setOperationAction(ISD::FADD, MVT::v2f64, Legal);
700 setOperationAction(ISD::FSUB, MVT::v2f64, Legal);
701 setOperationAction(ISD::FMUL, MVT::v2f64, Legal);
702 setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
703 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal);
704 setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
Evan Cheng2c3ae372006-04-12 21:21:57 +0000705
Owen Anderson825b72b2009-08-11 20:47:22 +0000706 setOperationAction(ISD::VSETCC, MVT::v2f64, Custom);
707 setOperationAction(ISD::VSETCC, MVT::v16i8, Custom);
708 setOperationAction(ISD::VSETCC, MVT::v8i16, Custom);
709 setOperationAction(ISD::VSETCC, MVT::v4i32, Custom);
Nate Begemanc2616e42008-05-12 20:34:32 +0000710
Owen Anderson825b72b2009-08-11 20:47:22 +0000711 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom);
712 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom);
713 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
714 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
715 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
Evan Chengf7c378e2006-04-10 07:23:14 +0000716
Evan Cheng2c3ae372006-04-12 21:21:57 +0000717 // Custom lower build_vector, vector_shuffle, and extract_vector_elt.
Owen Anderson825b72b2009-08-11 20:47:22 +0000718 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; ++i) {
719 EVT VT = (MVT::SimpleValueType)i;
Nate Begeman844e0f92007-12-11 01:41:33 +0000720 // Do not attempt to custom lower non-power-of-2 vectors
Duncan Sands83ec4b62008-06-06 12:08:01 +0000721 if (!isPowerOf2_32(VT.getVectorNumElements()))
Nate Begeman844e0f92007-12-11 01:41:33 +0000722 continue;
David Greene9b9838d2009-06-29 16:47:10 +0000723 // Do not attempt to custom lower non-128-bit vectors
724 if (!VT.is128BitVector())
725 continue;
Owen Anderson825b72b2009-08-11 20:47:22 +0000726 setOperationAction(ISD::BUILD_VECTOR,
727 VT.getSimpleVT().SimpleTy, Custom);
728 setOperationAction(ISD::VECTOR_SHUFFLE,
729 VT.getSimpleVT().SimpleTy, Custom);
730 setOperationAction(ISD::EXTRACT_VECTOR_ELT,
731 VT.getSimpleVT().SimpleTy, Custom);
Evan Cheng2c3ae372006-04-12 21:21:57 +0000732 }
Bill Wendlingf9abd7e2009-03-11 22:30:01 +0000733
Owen Anderson825b72b2009-08-11 20:47:22 +0000734 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom);
735 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom);
736 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom);
737 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom);
738 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom);
739 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom);
Bill Wendlingf9abd7e2009-03-11 22:30:01 +0000740
Nate Begemancdd1eec2008-02-12 22:51:28 +0000741 if (Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000742 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom);
743 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom);
Nate Begemancdd1eec2008-02-12 22:51:28 +0000744 }
Evan Cheng2c3ae372006-04-12 21:21:57 +0000745
Anton Korobeynikov12c49af2006-11-21 00:01:06 +0000746 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64.
Owen Anderson825b72b2009-08-11 20:47:22 +0000747 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; i++) {
748 MVT::SimpleValueType SVT = (MVT::SimpleValueType)i;
Owen Andersone50ed302009-08-10 22:56:29 +0000749 EVT VT = SVT;
David Greene9b9838d2009-06-29 16:47:10 +0000750
751 // Do not attempt to promote non-128-bit vectors
752 if (!VT.is128BitVector()) {
753 continue;
754 }
Owen Andersond6662ad2009-08-10 20:46:15 +0000755 setOperationAction(ISD::AND, SVT, Promote);
Owen Anderson825b72b2009-08-11 20:47:22 +0000756 AddPromotedToType (ISD::AND, SVT, MVT::v2i64);
Owen Andersond6662ad2009-08-10 20:46:15 +0000757 setOperationAction(ISD::OR, SVT, Promote);
Owen Anderson825b72b2009-08-11 20:47:22 +0000758 AddPromotedToType (ISD::OR, SVT, MVT::v2i64);
Owen Andersond6662ad2009-08-10 20:46:15 +0000759 setOperationAction(ISD::XOR, SVT, Promote);
Owen Anderson825b72b2009-08-11 20:47:22 +0000760 AddPromotedToType (ISD::XOR, SVT, MVT::v2i64);
Owen Andersond6662ad2009-08-10 20:46:15 +0000761 setOperationAction(ISD::LOAD, SVT, Promote);
Owen Anderson825b72b2009-08-11 20:47:22 +0000762 AddPromotedToType (ISD::LOAD, SVT, MVT::v2i64);
Owen Andersond6662ad2009-08-10 20:46:15 +0000763 setOperationAction(ISD::SELECT, SVT, Promote);
Owen Anderson825b72b2009-08-11 20:47:22 +0000764 AddPromotedToType (ISD::SELECT, SVT, MVT::v2i64);
Evan Chengf7c378e2006-04-10 07:23:14 +0000765 }
Evan Cheng2c3ae372006-04-12 21:21:57 +0000766
Owen Anderson825b72b2009-08-11 20:47:22 +0000767 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
Chris Lattnerd43d00c2008-01-24 08:07:48 +0000768
Evan Cheng2c3ae372006-04-12 21:21:57 +0000769 // Custom lower v2i64 and v2f64 selects.
Owen Anderson825b72b2009-08-11 20:47:22 +0000770 setOperationAction(ISD::LOAD, MVT::v2f64, Legal);
771 setOperationAction(ISD::LOAD, MVT::v2i64, Legal);
772 setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
773 setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
Scott Michelfdc40a02009-02-17 22:15:04 +0000774
Owen Anderson825b72b2009-08-11 20:47:22 +0000775 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal);
776 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal);
Eli Friedman23ef1052009-06-06 03:57:58 +0000777 if (!DisableMMX && Subtarget->hasMMX()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000778 setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
779 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
Eli Friedman23ef1052009-06-06 03:57:58 +0000780 }
Evan Cheng470a6ad2006-02-22 02:26:30 +0000781 }
Evan Chengc7ce29b2009-02-13 22:36:38 +0000782
Nate Begeman14d12ca2008-02-11 04:19:36 +0000783 if (Subtarget->hasSSE41()) {
784 // FIXME: Do we need to handle scalar-to-vector here?
Owen Anderson825b72b2009-08-11 20:47:22 +0000785 setOperationAction(ISD::MUL, MVT::v4i32, Legal);
Nate Begeman14d12ca2008-02-11 04:19:36 +0000786
787 // i8 and i16 vectors are custom , because the source register and source
788 // source memory operand types are not the same width. f32 vectors are
789 // custom since the immediate controlling the insert encodes additional
790 // information.
Owen Anderson825b72b2009-08-11 20:47:22 +0000791 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
792 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
793 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
794 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
Nate Begeman14d12ca2008-02-11 04:19:36 +0000795
Owen Anderson825b72b2009-08-11 20:47:22 +0000796 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom);
797 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom);
798 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom);
799 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
Nate Begeman14d12ca2008-02-11 04:19:36 +0000800
801 if (Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000802 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal);
803 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal);
Nate Begeman14d12ca2008-02-11 04:19:36 +0000804 }
805 }
Evan Cheng470a6ad2006-02-22 02:26:30 +0000806
Nate Begeman30a0de92008-07-17 16:51:19 +0000807 if (Subtarget->hasSSE42()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000808 setOperationAction(ISD::VSETCC, MVT::v2i64, Custom);
Nate Begeman30a0de92008-07-17 16:51:19 +0000809 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000810
David Greene9b9838d2009-06-29 16:47:10 +0000811 if (!UseSoftFloat && Subtarget->hasAVX()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000812 addRegisterClass(MVT::v8f32, X86::VR256RegisterClass);
813 addRegisterClass(MVT::v4f64, X86::VR256RegisterClass);
814 addRegisterClass(MVT::v8i32, X86::VR256RegisterClass);
815 addRegisterClass(MVT::v4i64, X86::VR256RegisterClass);
David Greened94c1012009-06-29 22:50:51 +0000816
Owen Anderson825b72b2009-08-11 20:47:22 +0000817 setOperationAction(ISD::LOAD, MVT::v8f32, Legal);
818 setOperationAction(ISD::LOAD, MVT::v8i32, Legal);
819 setOperationAction(ISD::LOAD, MVT::v4f64, Legal);
820 setOperationAction(ISD::LOAD, MVT::v4i64, Legal);
821 setOperationAction(ISD::FADD, MVT::v8f32, Legal);
822 setOperationAction(ISD::FSUB, MVT::v8f32, Legal);
823 setOperationAction(ISD::FMUL, MVT::v8f32, Legal);
824 setOperationAction(ISD::FDIV, MVT::v8f32, Legal);
825 setOperationAction(ISD::FSQRT, MVT::v8f32, Legal);
826 setOperationAction(ISD::FNEG, MVT::v8f32, Custom);
827 //setOperationAction(ISD::BUILD_VECTOR, MVT::v8f32, Custom);
828 //setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8f32, Custom);
829 //setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8f32, Custom);
830 //setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
831 //setOperationAction(ISD::VSETCC, MVT::v8f32, Custom);
David Greene9b9838d2009-06-29 16:47:10 +0000832
833 // Operations to consider commented out -v16i16 v32i8
Owen Anderson825b72b2009-08-11 20:47:22 +0000834 //setOperationAction(ISD::ADD, MVT::v16i16, Legal);
835 setOperationAction(ISD::ADD, MVT::v8i32, Custom);
836 setOperationAction(ISD::ADD, MVT::v4i64, Custom);
837 //setOperationAction(ISD::SUB, MVT::v32i8, Legal);
838 //setOperationAction(ISD::SUB, MVT::v16i16, Legal);
839 setOperationAction(ISD::SUB, MVT::v8i32, Custom);
840 setOperationAction(ISD::SUB, MVT::v4i64, Custom);
841 //setOperationAction(ISD::MUL, MVT::v16i16, Legal);
842 setOperationAction(ISD::FADD, MVT::v4f64, Legal);
843 setOperationAction(ISD::FSUB, MVT::v4f64, Legal);
844 setOperationAction(ISD::FMUL, MVT::v4f64, Legal);
845 setOperationAction(ISD::FDIV, MVT::v4f64, Legal);
846 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal);
847 setOperationAction(ISD::FNEG, MVT::v4f64, Custom);
David Greene9b9838d2009-06-29 16:47:10 +0000848
Owen Anderson825b72b2009-08-11 20:47:22 +0000849 setOperationAction(ISD::VSETCC, MVT::v4f64, Custom);
850 // setOperationAction(ISD::VSETCC, MVT::v32i8, Custom);
851 // setOperationAction(ISD::VSETCC, MVT::v16i16, Custom);
852 setOperationAction(ISD::VSETCC, MVT::v8i32, Custom);
David Greene9b9838d2009-06-29 16:47:10 +0000853
Owen Anderson825b72b2009-08-11 20:47:22 +0000854 // setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32i8, Custom);
855 // setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i16, Custom);
856 // setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i16, Custom);
857 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i32, Custom);
858 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8f32, Custom);
David Greene9b9838d2009-06-29 16:47:10 +0000859
Owen Anderson825b72b2009-08-11 20:47:22 +0000860 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom);
861 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i64, Custom);
862 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f64, Custom);
863 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i64, Custom);
864 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f64, Custom);
865 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f64, Custom);
David Greene9b9838d2009-06-29 16:47:10 +0000866
867#if 0
868 // Not sure we want to do this since there are no 256-bit integer
869 // operations in AVX
870
871 // Custom lower build_vector, vector_shuffle, and extract_vector_elt.
872 // This includes 256-bit vectors
Owen Anderson825b72b2009-08-11 20:47:22 +0000873 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v4i64; ++i) {
874 EVT VT = (MVT::SimpleValueType)i;
David Greene9b9838d2009-06-29 16:47:10 +0000875
876 // Do not attempt to custom lower non-power-of-2 vectors
877 if (!isPowerOf2_32(VT.getVectorNumElements()))
878 continue;
879
880 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
881 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
882 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
883 }
884
885 if (Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +0000886 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i64, Custom);
887 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i64, Custom);
David Greene9b9838d2009-06-29 16:47:10 +0000888 }
889#endif
890
891#if 0
892 // Not sure we want to do this since there are no 256-bit integer
893 // operations in AVX
894
895 // Promote v32i8, v16i16, v8i32 load, select, and, or, xor to v4i64.
896 // Including 256-bit vectors
Owen Anderson825b72b2009-08-11 20:47:22 +0000897 for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v4i64; i++) {
898 EVT VT = (MVT::SimpleValueType)i;
David Greene9b9838d2009-06-29 16:47:10 +0000899
900 if (!VT.is256BitVector()) {
901 continue;
902 }
903 setOperationAction(ISD::AND, VT, Promote);
Owen Anderson825b72b2009-08-11 20:47:22 +0000904 AddPromotedToType (ISD::AND, VT, MVT::v4i64);
David Greene9b9838d2009-06-29 16:47:10 +0000905 setOperationAction(ISD::OR, VT, Promote);
Owen Anderson825b72b2009-08-11 20:47:22 +0000906 AddPromotedToType (ISD::OR, VT, MVT::v4i64);
David Greene9b9838d2009-06-29 16:47:10 +0000907 setOperationAction(ISD::XOR, VT, Promote);
Owen Anderson825b72b2009-08-11 20:47:22 +0000908 AddPromotedToType (ISD::XOR, VT, MVT::v4i64);
David Greene9b9838d2009-06-29 16:47:10 +0000909 setOperationAction(ISD::LOAD, VT, Promote);
Owen Anderson825b72b2009-08-11 20:47:22 +0000910 AddPromotedToType (ISD::LOAD, VT, MVT::v4i64);
David Greene9b9838d2009-06-29 16:47:10 +0000911 setOperationAction(ISD::SELECT, VT, Promote);
Owen Anderson825b72b2009-08-11 20:47:22 +0000912 AddPromotedToType (ISD::SELECT, VT, MVT::v4i64);
David Greene9b9838d2009-06-29 16:47:10 +0000913 }
914
Owen Anderson825b72b2009-08-11 20:47:22 +0000915 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
David Greene9b9838d2009-06-29 16:47:10 +0000916#endif
917 }
918
Evan Cheng6be2c582006-04-05 23:38:46 +0000919 // We want to custom lower some of our intrinsics.
Owen Anderson825b72b2009-08-11 20:47:22 +0000920 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
Evan Cheng6be2c582006-04-05 23:38:46 +0000921
Bill Wendling74c37652008-12-09 22:08:41 +0000922 // Add/Sub/Mul with overflow operations are custom lowered.
Owen Anderson825b72b2009-08-11 20:47:22 +0000923 setOperationAction(ISD::SADDO, MVT::i32, Custom);
924 setOperationAction(ISD::SADDO, MVT::i64, Custom);
925 setOperationAction(ISD::UADDO, MVT::i32, Custom);
926 setOperationAction(ISD::UADDO, MVT::i64, Custom);
927 setOperationAction(ISD::SSUBO, MVT::i32, Custom);
928 setOperationAction(ISD::SSUBO, MVT::i64, Custom);
929 setOperationAction(ISD::USUBO, MVT::i32, Custom);
930 setOperationAction(ISD::USUBO, MVT::i64, Custom);
931 setOperationAction(ISD::SMULO, MVT::i32, Custom);
932 setOperationAction(ISD::SMULO, MVT::i64, Custom);
Bill Wendling41ea7e72008-11-24 19:21:46 +0000933
Evan Chengd54f2d52009-03-31 19:38:51 +0000934 if (!Subtarget->is64Bit()) {
935 // These libcalls are not available in 32-bit.
936 setLibcallName(RTLIB::SHL_I128, 0);
937 setLibcallName(RTLIB::SRL_I128, 0);
938 setLibcallName(RTLIB::SRA_I128, 0);
939 }
940
Evan Cheng206ee9d2006-07-07 08:33:52 +0000941 // We have target-specific dag combine patterns for the following nodes:
942 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
Evan Chengd880b972008-05-09 21:53:03 +0000943 setTargetDAGCombine(ISD::BUILD_VECTOR);
Chris Lattner83e6c992006-10-04 06:57:07 +0000944 setTargetDAGCombine(ISD::SELECT);
Nate Begeman740ab032009-01-26 00:52:55 +0000945 setTargetDAGCombine(ISD::SHL);
946 setTargetDAGCombine(ISD::SRA);
947 setTargetDAGCombine(ISD::SRL);
Chris Lattner149a4e52008-02-22 02:09:43 +0000948 setTargetDAGCombine(ISD::STORE);
Owen Anderson99177002009-06-29 18:04:45 +0000949 setTargetDAGCombine(ISD::MEMBARRIER);
Evan Cheng0b0cd912009-03-28 05:57:29 +0000950 if (Subtarget->is64Bit())
951 setTargetDAGCombine(ISD::MUL);
Evan Cheng206ee9d2006-07-07 08:33:52 +0000952
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000953 computeRegisterProperties();
954
Evan Cheng87ed7162006-02-14 08:25:08 +0000955 // FIXME: These should be based on subtarget info. Plus, the values should
956 // be smaller when we are in optimizing for size mode.
Dan Gohman87060f52008-06-30 21:00:56 +0000957 maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
958 maxStoresPerMemcpy = 16; // For @llvm.memcpy -> sequence of stores
959 maxStoresPerMemmove = 3; // For @llvm.memmove -> sequence of stores
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000960 allowUnalignedMemoryAccesses = true; // x86 supports it!
Evan Chengfb8075d2008-02-28 00:43:03 +0000961 setPrefLoopAlignment(16);
Evan Cheng6ebf7bc2009-05-13 21:42:09 +0000962 benefitFromCodePlacementOpt = true;
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +0000963}
964
Scott Michel5b8f82e2008-03-10 15:42:14 +0000965
Owen Anderson825b72b2009-08-11 20:47:22 +0000966MVT::SimpleValueType X86TargetLowering::getSetCCResultType(EVT VT) const {
967 return MVT::i8;
Scott Michel5b8f82e2008-03-10 15:42:14 +0000968}
969
970
Evan Cheng29286502008-01-23 23:17:41 +0000971/// getMaxByValAlign - Helper for getByValTypeAlignment to determine
972/// the desired ByVal argument alignment.
973static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) {
974 if (MaxAlign == 16)
975 return;
976 if (const VectorType *VTy = dyn_cast<VectorType>(Ty)) {
977 if (VTy->getBitWidth() == 128)
978 MaxAlign = 16;
Evan Cheng29286502008-01-23 23:17:41 +0000979 } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
980 unsigned EltAlign = 0;
981 getMaxByValAlign(ATy->getElementType(), EltAlign);
982 if (EltAlign > MaxAlign)
983 MaxAlign = EltAlign;
984 } else if (const StructType *STy = dyn_cast<StructType>(Ty)) {
985 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
986 unsigned EltAlign = 0;
987 getMaxByValAlign(STy->getElementType(i), EltAlign);
988 if (EltAlign > MaxAlign)
989 MaxAlign = EltAlign;
990 if (MaxAlign == 16)
991 break;
992 }
993 }
994 return;
995}
996
997/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
998/// function arguments in the caller parameter area. For X86, aggregates
Dale Johannesen0c191872008-02-08 19:48:20 +0000999/// that contain SSE vectors are placed at 16-byte boundaries while the rest
1000/// are at 4-byte boundaries.
Evan Cheng29286502008-01-23 23:17:41 +00001001unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const {
Evan Cheng1887c1c2008-08-21 21:00:15 +00001002 if (Subtarget->is64Bit()) {
1003 // Max of 8 and alignment of type.
Anton Korobeynikovbff66b02008-09-09 18:22:57 +00001004 unsigned TyAlign = TD->getABITypeAlignment(Ty);
Evan Cheng1887c1c2008-08-21 21:00:15 +00001005 if (TyAlign > 8)
1006 return TyAlign;
1007 return 8;
1008 }
1009
Evan Cheng29286502008-01-23 23:17:41 +00001010 unsigned Align = 4;
Dale Johannesen0c191872008-02-08 19:48:20 +00001011 if (Subtarget->hasSSE1())
1012 getMaxByValAlign(Ty, Align);
Evan Cheng29286502008-01-23 23:17:41 +00001013 return Align;
1014}
Chris Lattner2b02a442007-02-25 08:29:00 +00001015
Evan Chengf0df0312008-05-15 08:39:06 +00001016/// getOptimalMemOpType - Returns the target specific optimal type for load
Evan Cheng0ef8de32008-05-15 22:13:02 +00001017/// and store operations as a result of memset, memcpy, and memmove
Owen Anderson825b72b2009-08-11 20:47:22 +00001018/// lowering. It returns MVT::iAny if SelectionDAG should be responsible for
Evan Chengf0df0312008-05-15 08:39:06 +00001019/// determining it.
Owen Andersone50ed302009-08-10 22:56:29 +00001020EVT
Evan Chengf0df0312008-05-15 08:39:06 +00001021X86TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned Align,
Devang Patel578efa92009-06-05 21:57:13 +00001022 bool isSrcConst, bool isSrcStr,
1023 SelectionDAG &DAG) const {
Chris Lattner4002a1b2008-10-28 05:49:35 +00001024 // FIXME: This turns off use of xmm stores for memset/memcpy on targets like
1025 // linux. This is because the stack realignment code can't handle certain
1026 // cases like PR2962. This should be removed when PR2962 is fixed.
Devang Patel578efa92009-06-05 21:57:13 +00001027 const Function *F = DAG.getMachineFunction().getFunction();
1028 bool NoImplicitFloatOps = F->hasFnAttr(Attribute::NoImplicitFloat);
1029 if (!NoImplicitFloatOps && Subtarget->getStackAlignment() >= 16) {
Chris Lattner4002a1b2008-10-28 05:49:35 +00001030 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE2() && Size >= 16)
Owen Anderson825b72b2009-08-11 20:47:22 +00001031 return MVT::v4i32;
Chris Lattner4002a1b2008-10-28 05:49:35 +00001032 if ((isSrcConst || isSrcStr) && Subtarget->hasSSE1() && Size >= 16)
Owen Anderson825b72b2009-08-11 20:47:22 +00001033 return MVT::v4f32;
Chris Lattner4002a1b2008-10-28 05:49:35 +00001034 }
Evan Chengf0df0312008-05-15 08:39:06 +00001035 if (Subtarget->is64Bit() && Size >= 8)
Owen Anderson825b72b2009-08-11 20:47:22 +00001036 return MVT::i64;
1037 return MVT::i32;
Evan Chengf0df0312008-05-15 08:39:06 +00001038}
1039
Evan Chengcc415862007-11-09 01:32:10 +00001040/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC
1041/// jumptable.
Dan Gohman475871a2008-07-27 21:46:04 +00001042SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
Evan Chengcc415862007-11-09 01:32:10 +00001043 SelectionDAG &DAG) const {
1044 if (usesGlobalOffsetTable())
Dale Johannesenb300d2a2009-02-07 00:55:49 +00001045 return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy());
Chris Lattnere4df7562009-07-09 03:15:51 +00001046 if (!Subtarget->is64Bit())
Dale Johannesenb300d2a2009-02-07 00:55:49 +00001047 // This doesn't have DebugLoc associated with it, but is not really the
1048 // same as a Register.
1049 return DAG.getNode(X86ISD::GlobalBaseReg, DebugLoc::getUnknownLoc(),
1050 getPointerTy());
Evan Chengcc415862007-11-09 01:32:10 +00001051 return Table;
1052}
1053
Bill Wendlingb4202b82009-07-01 18:50:55 +00001054/// getFunctionAlignment - Return the Log2 alignment of this function.
Bill Wendling20c568f2009-06-30 22:38:32 +00001055unsigned X86TargetLowering::getFunctionAlignment(const Function *F) const {
1056 return F->hasFnAttr(Attribute::OptimizeForSize) ? 1 : 4;
1057}
1058
Chris Lattner2b02a442007-02-25 08:29:00 +00001059//===----------------------------------------------------------------------===//
1060// Return Value Calling Convention Implementation
1061//===----------------------------------------------------------------------===//
1062
Chris Lattner59ed56b2007-02-28 04:55:35 +00001063#include "X86GenCallingConv.inc"
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00001064
Dan Gohman98ca4f22009-08-05 01:29:28 +00001065SDValue
1066X86TargetLowering::LowerReturn(SDValue Chain,
1067 unsigned CallConv, bool isVarArg,
1068 const SmallVectorImpl<ISD::OutputArg> &Outs,
1069 DebugLoc dl, SelectionDAG &DAG) {
Scott Michelfdc40a02009-02-17 22:15:04 +00001070
Chris Lattner9774c912007-02-27 05:28:59 +00001071 SmallVector<CCValAssign, 16> RVLocs;
Dan Gohman98ca4f22009-08-05 01:29:28 +00001072 CCState CCInfo(CallConv, isVarArg, getTargetMachine(),
1073 RVLocs, *DAG.getContext());
1074 CCInfo.AnalyzeReturn(Outs, RetCC_X86);
Scott Michelfdc40a02009-02-17 22:15:04 +00001075
Chris Lattner2a9bdd72007-02-25 09:12:39 +00001076 // If this is the first return lowered for this function, add the regs to the
1077 // liveout set for the function.
Chris Lattner84bc5422007-12-31 04:13:23 +00001078 if (DAG.getMachineFunction().getRegInfo().liveout_empty()) {
Chris Lattner9774c912007-02-27 05:28:59 +00001079 for (unsigned i = 0; i != RVLocs.size(); ++i)
1080 if (RVLocs[i].isRegLoc())
Chris Lattner84bc5422007-12-31 04:13:23 +00001081 DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg());
Chris Lattner2a9bdd72007-02-25 09:12:39 +00001082 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001083
Dan Gohman475871a2008-07-27 21:46:04 +00001084 SDValue Flag;
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00001085
Dan Gohman475871a2008-07-27 21:46:04 +00001086 SmallVector<SDValue, 6> RetOps;
Chris Lattner447ff682008-03-11 03:23:40 +00001087 RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
1088 // Operand #1 = Bytes To Pop
Owen Anderson825b72b2009-08-11 20:47:22 +00001089 RetOps.push_back(DAG.getConstant(getBytesToPopOnReturn(), MVT::i16));
Scott Michelfdc40a02009-02-17 22:15:04 +00001090
Chris Lattner2a9bdd72007-02-25 09:12:39 +00001091 // Copy the result values into the output registers.
Chris Lattner8e6da152008-03-10 21:08:41 +00001092 for (unsigned i = 0; i != RVLocs.size(); ++i) {
1093 CCValAssign &VA = RVLocs[i];
1094 assert(VA.isRegLoc() && "Can only return in registers!");
Dan Gohman98ca4f22009-08-05 01:29:28 +00001095 SDValue ValToCopy = Outs[i].Val;
Scott Michelfdc40a02009-02-17 22:15:04 +00001096
Chris Lattner447ff682008-03-11 03:23:40 +00001097 // Returns in ST0/ST1 are handled specially: these are pushed as operands to
1098 // the RET instruction and handled by the FP Stackifier.
Dan Gohman37eed792009-02-04 17:28:58 +00001099 if (VA.getLocReg() == X86::ST0 ||
1100 VA.getLocReg() == X86::ST1) {
Chris Lattner447ff682008-03-11 03:23:40 +00001101 // If this is a copy from an xmm register to ST(0), use an FPExtend to
1102 // change the value to the FP stack register class.
Dan Gohman37eed792009-02-04 17:28:58 +00001103 if (isScalarFPTypeInSSEReg(VA.getValVT()))
Owen Anderson825b72b2009-08-11 20:47:22 +00001104 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy);
Chris Lattner447ff682008-03-11 03:23:40 +00001105 RetOps.push_back(ValToCopy);
1106 // Don't emit a copytoreg.
1107 continue;
1108 }
Dale Johannesena68f9012008-06-24 22:01:44 +00001109
Evan Cheng242b38b2009-02-23 09:03:22 +00001110 // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64
1111 // which is returned in RAX / RDX.
Evan Cheng6140a8b2009-02-22 08:05:12 +00001112 if (Subtarget->is64Bit()) {
Owen Andersone50ed302009-08-10 22:56:29 +00001113 EVT ValVT = ValToCopy.getValueType();
Evan Cheng242b38b2009-02-23 09:03:22 +00001114 if (ValVT.isVector() && ValVT.getSizeInBits() == 64) {
Owen Anderson825b72b2009-08-11 20:47:22 +00001115 ValToCopy = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i64, ValToCopy);
Evan Cheng242b38b2009-02-23 09:03:22 +00001116 if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1)
Owen Anderson825b72b2009-08-11 20:47:22 +00001117 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, ValToCopy);
Evan Cheng242b38b2009-02-23 09:03:22 +00001118 }
Evan Cheng6140a8b2009-02-22 08:05:12 +00001119 }
1120
Dale Johannesendd64c412009-02-04 00:33:20 +00001121 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), ValToCopy, Flag);
Chris Lattner2a9bdd72007-02-25 09:12:39 +00001122 Flag = Chain.getValue(1);
1123 }
Dan Gohman61a92132008-04-21 23:59:07 +00001124
1125 // The x86-64 ABI for returning structs by value requires that we copy
1126 // the sret argument into %rax for the return. We saved the argument into
1127 // a virtual register in the entry block, so now we copy the value out
1128 // and into %rax.
1129 if (Subtarget->is64Bit() &&
1130 DAG.getMachineFunction().getFunction()->hasStructRetAttr()) {
1131 MachineFunction &MF = DAG.getMachineFunction();
1132 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
1133 unsigned Reg = FuncInfo->getSRetReturnReg();
1134 if (!Reg) {
Owen Anderson825b72b2009-08-11 20:47:22 +00001135 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64));
Dan Gohman61a92132008-04-21 23:59:07 +00001136 FuncInfo->setSRetReturnReg(Reg);
1137 }
Dale Johannesendd64c412009-02-04 00:33:20 +00001138 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg, getPointerTy());
Dan Gohman61a92132008-04-21 23:59:07 +00001139
Dale Johannesendd64c412009-02-04 00:33:20 +00001140 Chain = DAG.getCopyToReg(Chain, dl, X86::RAX, Val, Flag);
Dan Gohman61a92132008-04-21 23:59:07 +00001141 Flag = Chain.getValue(1);
1142 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001143
Chris Lattner447ff682008-03-11 03:23:40 +00001144 RetOps[0] = Chain; // Update chain.
1145
1146 // Add the flag if we have it.
Gabor Greifba36cb52008-08-28 21:40:38 +00001147 if (Flag.getNode())
Chris Lattner447ff682008-03-11 03:23:40 +00001148 RetOps.push_back(Flag);
Scott Michelfdc40a02009-02-17 22:15:04 +00001149
1150 return DAG.getNode(X86ISD::RET_FLAG, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00001151 MVT::Other, &RetOps[0], RetOps.size());
Chris Lattner2a9bdd72007-02-25 09:12:39 +00001152}
1153
Dan Gohman98ca4f22009-08-05 01:29:28 +00001154/// LowerCallResult - Lower the result values of a call into the
1155/// appropriate copies out of appropriate physical registers.
1156///
1157SDValue
1158X86TargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
1159 unsigned CallConv, bool isVarArg,
1160 const SmallVectorImpl<ISD::InputArg> &Ins,
1161 DebugLoc dl, SelectionDAG &DAG,
1162 SmallVectorImpl<SDValue> &InVals) {
Chris Lattner2a9bdd72007-02-25 09:12:39 +00001163
Chris Lattnere32bbf62007-02-28 07:09:55 +00001164 // Assign locations to each value returned by this call.
Chris Lattner9774c912007-02-27 05:28:59 +00001165 SmallVector<CCValAssign, 16> RVLocs;
Torok Edwin3f142c32009-02-01 18:15:56 +00001166 bool Is64Bit = Subtarget->is64Bit();
Dan Gohman98ca4f22009-08-05 01:29:28 +00001167 CCState CCInfo(CallConv, isVarArg, getTargetMachine(),
Owen Andersone922c022009-07-22 00:24:57 +00001168 RVLocs, *DAG.getContext());
Dan Gohman98ca4f22009-08-05 01:29:28 +00001169 CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
Scott Michelfdc40a02009-02-17 22:15:04 +00001170
Chris Lattner3085e152007-02-25 08:59:22 +00001171 // Copy all of the result registers out of their specified physreg.
Chris Lattner8e6da152008-03-10 21:08:41 +00001172 for (unsigned i = 0; i != RVLocs.size(); ++i) {
Dan Gohman37eed792009-02-04 17:28:58 +00001173 CCValAssign &VA = RVLocs[i];
Owen Andersone50ed302009-08-10 22:56:29 +00001174 EVT CopyVT = VA.getValVT();
Scott Michelfdc40a02009-02-17 22:15:04 +00001175
Torok Edwin3f142c32009-02-01 18:15:56 +00001176 // If this is x86-64, and we disabled SSE, we can't return FP values
Owen Anderson825b72b2009-08-11 20:47:22 +00001177 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&
Dan Gohman98ca4f22009-08-05 01:29:28 +00001178 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) {
Torok Edwin804e0fe2009-07-08 19:04:27 +00001179 llvm_report_error("SSE register return with SSE disabled");
Torok Edwin3f142c32009-02-01 18:15:56 +00001180 }
1181
Chris Lattner8e6da152008-03-10 21:08:41 +00001182 // If this is a call to a function that returns an fp value on the floating
1183 // point stack, but where we prefer to use the value in xmm registers, copy
1184 // it out as F80 and use a truncate to move it from fp stack reg to xmm reg.
Dan Gohman37eed792009-02-04 17:28:58 +00001185 if ((VA.getLocReg() == X86::ST0 ||
1186 VA.getLocReg() == X86::ST1) &&
1187 isScalarFPTypeInSSEReg(VA.getValVT())) {
Owen Anderson825b72b2009-08-11 20:47:22 +00001188 CopyVT = MVT::f80;
Chris Lattner3085e152007-02-25 08:59:22 +00001189 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001190
Evan Cheng79fb3b42009-02-20 20:43:02 +00001191 SDValue Val;
1192 if (Is64Bit && CopyVT.isVector() && CopyVT.getSizeInBits() == 64) {
Evan Cheng242b38b2009-02-23 09:03:22 +00001193 // For x86-64, MMX values are returned in XMM0 / XMM1 except for v1i64.
1194 if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) {
1195 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(),
Owen Anderson825b72b2009-08-11 20:47:22 +00001196 MVT::v2i64, InFlag).getValue(1);
Evan Cheng242b38b2009-02-23 09:03:22 +00001197 Val = Chain.getValue(0);
Owen Anderson825b72b2009-08-11 20:47:22 +00001198 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,
1199 Val, DAG.getConstant(0, MVT::i64));
Evan Cheng242b38b2009-02-23 09:03:22 +00001200 } else {
1201 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(),
Owen Anderson825b72b2009-08-11 20:47:22 +00001202 MVT::i64, InFlag).getValue(1);
Evan Cheng242b38b2009-02-23 09:03:22 +00001203 Val = Chain.getValue(0);
1204 }
Evan Cheng79fb3b42009-02-20 20:43:02 +00001205 Val = DAG.getNode(ISD::BIT_CONVERT, dl, CopyVT, Val);
1206 } else {
1207 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(),
1208 CopyVT, InFlag).getValue(1);
1209 Val = Chain.getValue(0);
1210 }
Chris Lattner8e6da152008-03-10 21:08:41 +00001211 InFlag = Chain.getValue(2);
Chris Lattner112dedc2007-12-29 06:41:28 +00001212
Dan Gohman37eed792009-02-04 17:28:58 +00001213 if (CopyVT != VA.getValVT()) {
Chris Lattner8e6da152008-03-10 21:08:41 +00001214 // Round the F80 the right size, which also moves to the appropriate xmm
1215 // register.
Dan Gohman37eed792009-02-04 17:28:58 +00001216 Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val,
Chris Lattner8e6da152008-03-10 21:08:41 +00001217 // This truncation won't change the value.
1218 DAG.getIntPtrConstant(1));
1219 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001220
Dan Gohman98ca4f22009-08-05 01:29:28 +00001221 InVals.push_back(Val);
Chris Lattner3085e152007-02-25 08:59:22 +00001222 }
Duncan Sands4bdcb612008-07-02 17:40:58 +00001223
Dan Gohman98ca4f22009-08-05 01:29:28 +00001224 return Chain;
Chris Lattner2b02a442007-02-25 08:29:00 +00001225}
1226
1227
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00001228//===----------------------------------------------------------------------===//
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00001229// C & StdCall & Fast Calling Convention implementation
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00001230//===----------------------------------------------------------------------===//
Anton Korobeynikovb10308e2007-01-28 13:31:35 +00001231// StdCall calling convention seems to be standard for many Windows' API
1232// routines and around. It differs from C calling convention just a little:
1233// callee should clean up the stack, not caller. Symbols should be also
1234// decorated in some fancy way :) It doesn't support any vector arguments.
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00001235// For info on fast calling convention see Fast Calling Convention (tail call)
1236// implementation LowerX86_32FastCCCallTo.
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00001237
Dan Gohman98ca4f22009-08-05 01:29:28 +00001238/// CallIsStructReturn - Determines whether a call uses struct return
Arnold Schwaighofer16a3e522008-02-26 17:50:59 +00001239/// semantics.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001240static bool CallIsStructReturn(const SmallVectorImpl<ISD::OutputArg> &Outs) {
1241 if (Outs.empty())
Gordon Henriksen86737662008-01-05 16:56:59 +00001242 return false;
Duncan Sands276dcbd2008-03-21 09:14:45 +00001243
Dan Gohman98ca4f22009-08-05 01:29:28 +00001244 return Outs[0].Flags.isSRet();
Gordon Henriksen86737662008-01-05 16:56:59 +00001245}
1246
Dan Gohman7e77b0f2009-08-01 19:14:37 +00001247/// ArgsAreStructReturn - Determines whether a function uses struct
Arnold Schwaighofer16a3e522008-02-26 17:50:59 +00001248/// return semantics.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001249static bool
1250ArgsAreStructReturn(const SmallVectorImpl<ISD::InputArg> &Ins) {
1251 if (Ins.empty())
Gordon Henriksen86737662008-01-05 16:56:59 +00001252 return false;
Duncan Sands276dcbd2008-03-21 09:14:45 +00001253
Dan Gohman98ca4f22009-08-05 01:29:28 +00001254 return Ins[0].Flags.isSRet();
Gordon Henriksen86737662008-01-05 16:56:59 +00001255}
1256
Dan Gohman7e77b0f2009-08-01 19:14:37 +00001257/// IsCalleePop - Determines whether the callee is required to pop its
1258/// own arguments. Callee pop is necessary to support tail calls.
Dan Gohman095cc292008-09-13 01:54:27 +00001259bool X86TargetLowering::IsCalleePop(bool IsVarArg, unsigned CallingConv) {
Gordon Henriksen86737662008-01-05 16:56:59 +00001260 if (IsVarArg)
1261 return false;
1262
Dan Gohman095cc292008-09-13 01:54:27 +00001263 switch (CallingConv) {
Gordon Henriksen86737662008-01-05 16:56:59 +00001264 default:
1265 return false;
1266 case CallingConv::X86_StdCall:
1267 return !Subtarget->is64Bit();
1268 case CallingConv::X86_FastCall:
1269 return !Subtarget->is64Bit();
1270 case CallingConv::Fast:
1271 return PerformTailCallOpt;
1272 }
1273}
1274
Dan Gohman095cc292008-09-13 01:54:27 +00001275/// CCAssignFnForNode - Selects the correct CCAssignFn for a the
1276/// given CallingConvention value.
1277CCAssignFn *X86TargetLowering::CCAssignFnForNode(unsigned CC) const {
Anton Korobeynikov7c1c2612008-02-20 11:22:39 +00001278 if (Subtarget->is64Bit()) {
Anton Korobeynikov1a979d92008-03-22 20:57:27 +00001279 if (Subtarget->isTargetWin64())
Anton Korobeynikov8f88cb02008-03-22 20:37:30 +00001280 return CC_X86_Win64_C;
Evan Chenge9ac9e62008-09-07 09:07:23 +00001281 else
1282 return CC_X86_64_C;
Anton Korobeynikov7c1c2612008-02-20 11:22:39 +00001283 }
1284
Gordon Henriksen86737662008-01-05 16:56:59 +00001285 if (CC == CallingConv::X86_FastCall)
1286 return CC_X86_32_FastCall;
Evan Chengb188dd92008-09-10 18:25:29 +00001287 else if (CC == CallingConv::Fast)
1288 return CC_X86_32_FastCC;
Gordon Henriksen86737662008-01-05 16:56:59 +00001289 else
1290 return CC_X86_32_C;
1291}
1292
Dan Gohman98ca4f22009-08-05 01:29:28 +00001293/// NameDecorationForCallConv - Selects the appropriate decoration to
1294/// apply to a MachineFunction containing a given calling convention.
Gordon Henriksen86737662008-01-05 16:56:59 +00001295NameDecorationStyle
Dan Gohman98ca4f22009-08-05 01:29:28 +00001296X86TargetLowering::NameDecorationForCallConv(unsigned CallConv) {
1297 if (CallConv == CallingConv::X86_FastCall)
Gordon Henriksen86737662008-01-05 16:56:59 +00001298 return FastCall;
Dan Gohman98ca4f22009-08-05 01:29:28 +00001299 else if (CallConv == CallingConv::X86_StdCall)
Gordon Henriksen86737662008-01-05 16:56:59 +00001300 return StdCall;
1301 return None;
1302}
1303
Arnold Schwaighoferc8ab8cd2008-01-11 16:49:42 +00001304
Arnold Schwaighofer16a3e522008-02-26 17:50:59 +00001305/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
1306/// by "Src" to address "Dst" with size and alignment information specified by
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001307/// the specific parameter attribute. The copy will be passed as a byval
1308/// function parameter.
Scott Michelfdc40a02009-02-17 22:15:04 +00001309static SDValue
Dan Gohman475871a2008-07-27 21:46:04 +00001310CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain,
Dale Johannesendd64c412009-02-04 00:33:20 +00001311 ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
1312 DebugLoc dl) {
Owen Anderson825b72b2009-08-11 20:47:22 +00001313 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
Dale Johannesendd64c412009-02-04 00:33:20 +00001314 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
Arnold Schwaighofer30e62c02008-04-30 09:16:33 +00001315 /*AlwaysInline=*/true, NULL, 0, NULL, 0);
Arnold Schwaighoferc8ab8cd2008-01-11 16:49:42 +00001316}
1317
Dan Gohman98ca4f22009-08-05 01:29:28 +00001318SDValue
1319X86TargetLowering::LowerMemArgument(SDValue Chain,
1320 unsigned CallConv,
1321 const SmallVectorImpl<ISD::InputArg> &Ins,
1322 DebugLoc dl, SelectionDAG &DAG,
1323 const CCValAssign &VA,
1324 MachineFrameInfo *MFI,
1325 unsigned i) {
1326
Rafael Espindola7effac52007-09-14 15:48:13 +00001327 // Create the nodes corresponding to a load from this parameter slot.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001328 ISD::ArgFlagsTy Flags = Ins[i].Flags;
1329 bool AlwaysUseMutable = (CallConv==CallingConv::Fast) && PerformTailCallOpt;
Duncan Sands276dcbd2008-03-21 09:14:45 +00001330 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal();
Evan Chenge70bb592008-01-10 02:24:25 +00001331
Arnold Schwaighofer865c6812008-02-26 09:19:59 +00001332 // FIXME: For now, all byval parameter objects are marked mutable. This can be
Scott Michelfdc40a02009-02-17 22:15:04 +00001333 // changed with more analysis.
Arnold Schwaighofer865c6812008-02-26 09:19:59 +00001334 // In case of tail call optimization mark all arguments mutable. Since they
1335 // could be overwritten by lowering of arguments in case of a tail call.
Duncan Sands83ec4b62008-06-06 12:08:01 +00001336 int FI = MFI->CreateFixedObject(VA.getValVT().getSizeInBits()/8,
Arnold Schwaighofer865c6812008-02-26 09:19:59 +00001337 VA.getLocMemOffset(), isImmutable);
Dan Gohman475871a2008-07-27 21:46:04 +00001338 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
Duncan Sands276dcbd2008-03-21 09:14:45 +00001339 if (Flags.isByVal())
Rafael Espindola7effac52007-09-14 15:48:13 +00001340 return FIN;
Dan Gohman98ca4f22009-08-05 01:29:28 +00001341 return DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
Dan Gohmana54cf172008-07-11 22:44:52 +00001342 PseudoSourceValue::getFixedStack(FI), 0);
Rafael Espindola7effac52007-09-14 15:48:13 +00001343}
1344
Dan Gohman475871a2008-07-27 21:46:04 +00001345SDValue
Dan Gohman98ca4f22009-08-05 01:29:28 +00001346X86TargetLowering::LowerFormalArguments(SDValue Chain,
1347 unsigned CallConv,
1348 bool isVarArg,
1349 const SmallVectorImpl<ISD::InputArg> &Ins,
1350 DebugLoc dl,
1351 SelectionDAG &DAG,
1352 SmallVectorImpl<SDValue> &InVals) {
1353
Evan Cheng1bc78042006-04-26 01:20:17 +00001354 MachineFunction &MF = DAG.getMachineFunction();
Gordon Henriksen86737662008-01-05 16:56:59 +00001355 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
Scott Michelfdc40a02009-02-17 22:15:04 +00001356
Gordon Henriksen86737662008-01-05 16:56:59 +00001357 const Function* Fn = MF.getFunction();
1358 if (Fn->hasExternalLinkage() &&
1359 Subtarget->isTargetCygMing() &&
1360 Fn->getName() == "main")
1361 FuncInfo->setForceFramePointer(true);
1362
1363 // Decorate the function name.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001364 FuncInfo->setDecorationStyle(NameDecorationForCallConv(CallConv));
Scott Michelfdc40a02009-02-17 22:15:04 +00001365
Evan Cheng1bc78042006-04-26 01:20:17 +00001366 MachineFrameInfo *MFI = MF.getFrameInfo();
Gordon Henriksen86737662008-01-05 16:56:59 +00001367 bool Is64Bit = Subtarget->is64Bit();
Anton Korobeynikov998a5bc2008-04-27 23:15:03 +00001368 bool IsWin64 = Subtarget->isTargetWin64();
Gordon Henriksenae636f82008-01-03 16:47:34 +00001369
Dan Gohman98ca4f22009-08-05 01:29:28 +00001370 assert(!(isVarArg && CallConv == CallingConv::Fast) &&
Gordon Henriksenae636f82008-01-03 16:47:34 +00001371 "Var args not supported with calling convention fastcc");
1372
Chris Lattner638402b2007-02-28 07:00:42 +00001373 // Assign locations to all of the incoming arguments.
Chris Lattnerf39f7712007-02-28 05:46:49 +00001374 SmallVector<CCValAssign, 16> ArgLocs;
Dan Gohman98ca4f22009-08-05 01:29:28 +00001375 CCState CCInfo(CallConv, isVarArg, getTargetMachine(),
1376 ArgLocs, *DAG.getContext());
1377 CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForNode(CallConv));
Scott Michelfdc40a02009-02-17 22:15:04 +00001378
Chris Lattnerf39f7712007-02-28 05:46:49 +00001379 unsigned LastVal = ~0U;
Anton Korobeynikov4ab15532009-08-03 08:13:56 +00001380 SDValue ArgValue;
Chris Lattnerf39f7712007-02-28 05:46:49 +00001381 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1382 CCValAssign &VA = ArgLocs[i];
1383 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later
1384 // places.
1385 assert(VA.getValNo() != LastVal &&
1386 "Don't support value assigned to multiple locs yet");
1387 LastVal = VA.getValNo();
Scott Michelfdc40a02009-02-17 22:15:04 +00001388
Chris Lattnerf39f7712007-02-28 05:46:49 +00001389 if (VA.isRegLoc()) {
Owen Andersone50ed302009-08-10 22:56:29 +00001390 EVT RegVT = VA.getLocVT();
Devang Patel8a84e442009-01-05 17:31:22 +00001391 TargetRegisterClass *RC = NULL;
Owen Anderson825b72b2009-08-11 20:47:22 +00001392 if (RegVT == MVT::i32)
Chris Lattnerf39f7712007-02-28 05:46:49 +00001393 RC = X86::GR32RegisterClass;
Owen Anderson825b72b2009-08-11 20:47:22 +00001394 else if (Is64Bit && RegVT == MVT::i64)
Gordon Henriksen86737662008-01-05 16:56:59 +00001395 RC = X86::GR64RegisterClass;
Owen Anderson825b72b2009-08-11 20:47:22 +00001396 else if (RegVT == MVT::f32)
Gordon Henriksen86737662008-01-05 16:56:59 +00001397 RC = X86::FR32RegisterClass;
Owen Anderson825b72b2009-08-11 20:47:22 +00001398 else if (RegVT == MVT::f64)
Gordon Henriksen86737662008-01-05 16:56:59 +00001399 RC = X86::FR64RegisterClass;
Duncan Sands83ec4b62008-06-06 12:08:01 +00001400 else if (RegVT.isVector() && RegVT.getSizeInBits() == 128)
Evan Chengee472b12008-04-25 07:56:45 +00001401 RC = X86::VR128RegisterClass;
Anton Korobeynikov80cb8aa2009-08-03 08:13:24 +00001402 else if (RegVT.isVector() && RegVT.getSizeInBits() == 64)
1403 RC = X86::VR64RegisterClass;
1404 else
Torok Edwinc23197a2009-07-14 16:55:14 +00001405 llvm_unreachable("Unknown argument type!");
Gordon Henriksenae636f82008-01-03 16:47:34 +00001406
Dan Gohman7e77b0f2009-08-01 19:14:37 +00001407 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
Dan Gohman98ca4f22009-08-05 01:29:28 +00001408 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
Scott Michelfdc40a02009-02-17 22:15:04 +00001409
Chris Lattnerf39f7712007-02-28 05:46:49 +00001410 // If this is an 8 or 16-bit value, it is really passed promoted to 32
1411 // bits. Insert an assert[sz]ext to capture this, then truncate to the
1412 // right size.
1413 if (VA.getLocInfo() == CCValAssign::SExt)
Dale Johannesenace16102009-02-03 19:33:06 +00001414 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
Chris Lattnerf39f7712007-02-28 05:46:49 +00001415 DAG.getValueType(VA.getValVT()));
1416 else if (VA.getLocInfo() == CCValAssign::ZExt)
Dale Johannesenace16102009-02-03 19:33:06 +00001417 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
Chris Lattnerf39f7712007-02-28 05:46:49 +00001418 DAG.getValueType(VA.getValVT()));
Anton Korobeynikov80cb8aa2009-08-03 08:13:24 +00001419 else if (VA.getLocInfo() == CCValAssign::BCvt)
Anton Korobeynikov6dde14b2009-08-03 08:14:14 +00001420 ArgValue = DAG.getNode(ISD::BIT_CONVERT, dl, VA.getValVT(), ArgValue);
Scott Michelfdc40a02009-02-17 22:15:04 +00001421
Anton Korobeynikov4ab15532009-08-03 08:13:56 +00001422 if (VA.isExtInLoc()) {
Anton Korobeynikov80cb8aa2009-08-03 08:13:24 +00001423 // Handle MMX values passed in XMM regs.
1424 if (RegVT.isVector()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00001425 ArgValue = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,
1426 ArgValue, DAG.getConstant(0, MVT::i64));
Anton Korobeynikov80cb8aa2009-08-03 08:13:24 +00001427 ArgValue = DAG.getNode(ISD::BIT_CONVERT, dl, VA.getValVT(), ArgValue);
1428 } else
1429 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
Evan Cheng44c0fd12008-04-25 20:13:28 +00001430 }
Chris Lattnerf39f7712007-02-28 05:46:49 +00001431 } else {
1432 assert(VA.isMemLoc());
Dan Gohman98ca4f22009-08-05 01:29:28 +00001433 ArgValue = LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, i);
Evan Cheng1bc78042006-04-26 01:20:17 +00001434 }
Anton Korobeynikov4ab15532009-08-03 08:13:56 +00001435
1436 // If value is passed via pointer - do a load.
1437 if (VA.getLocInfo() == CCValAssign::Indirect)
Dan Gohman98ca4f22009-08-05 01:29:28 +00001438 ArgValue = DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue, NULL, 0);
Anton Korobeynikov4ab15532009-08-03 08:13:56 +00001439
Dan Gohman98ca4f22009-08-05 01:29:28 +00001440 InVals.push_back(ArgValue);
Evan Cheng1bc78042006-04-26 01:20:17 +00001441 }
Gordon Henriksenae636f82008-01-03 16:47:34 +00001442
Dan Gohman61a92132008-04-21 23:59:07 +00001443 // The x86-64 ABI for returning structs by value requires that we copy
1444 // the sret argument into %rax for the return. Save the argument into
1445 // a virtual register so that we can access it from the return points.
Dan Gohman7e77b0f2009-08-01 19:14:37 +00001446 if (Is64Bit && MF.getFunction()->hasStructRetAttr()) {
Dan Gohman61a92132008-04-21 23:59:07 +00001447 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
1448 unsigned Reg = FuncInfo->getSRetReturnReg();
1449 if (!Reg) {
Owen Anderson825b72b2009-08-11 20:47:22 +00001450 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64));
Dan Gohman61a92132008-04-21 23:59:07 +00001451 FuncInfo->setSRetReturnReg(Reg);
1452 }
Dan Gohman98ca4f22009-08-05 01:29:28 +00001453 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[0]);
Owen Anderson825b72b2009-08-11 20:47:22 +00001454 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);
Dan Gohman61a92132008-04-21 23:59:07 +00001455 }
1456
Chris Lattnerf39f7712007-02-28 05:46:49 +00001457 unsigned StackSize = CCInfo.getNextStackOffset();
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00001458 // align stack specially for tail calls
Dan Gohman98ca4f22009-08-05 01:29:28 +00001459 if (PerformTailCallOpt && CallConv == CallingConv::Fast)
Gordon Henriksenae636f82008-01-03 16:47:34 +00001460 StackSize = GetAlignedArgumentStackSize(StackSize, DAG);
Evan Cheng25caf632006-05-23 21:06:34 +00001461
Evan Cheng1bc78042006-04-26 01:20:17 +00001462 // If the function takes variable number of arguments, make a frame index for
1463 // the start of the first vararg value... for expansion of llvm.va_start.
Gordon Henriksenae636f82008-01-03 16:47:34 +00001464 if (isVarArg) {
Dan Gohman98ca4f22009-08-05 01:29:28 +00001465 if (Is64Bit || CallConv != CallingConv::X86_FastCall) {
Gordon Henriksen86737662008-01-05 16:56:59 +00001466 VarArgsFrameIndex = MFI->CreateFixedObject(1, StackSize);
1467 }
1468 if (Is64Bit) {
Anton Korobeynikov998a5bc2008-04-27 23:15:03 +00001469 unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0;
1470
1471 // FIXME: We should really autogenerate these arrays
1472 static const unsigned GPR64ArgRegsWin64[] = {
1473 X86::RCX, X86::RDX, X86::R8, X86::R9
Gordon Henriksen86737662008-01-05 16:56:59 +00001474 };
Anton Korobeynikov998a5bc2008-04-27 23:15:03 +00001475 static const unsigned XMMArgRegsWin64[] = {
1476 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3
1477 };
1478 static const unsigned GPR64ArgRegs64Bit[] = {
1479 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
1480 };
1481 static const unsigned XMMArgRegs64Bit[] = {
Gordon Henriksen86737662008-01-05 16:56:59 +00001482 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
1483 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
1484 };
Anton Korobeynikov998a5bc2008-04-27 23:15:03 +00001485 const unsigned *GPR64ArgRegs, *XMMArgRegs;
1486
1487 if (IsWin64) {
1488 TotalNumIntRegs = 4; TotalNumXMMRegs = 4;
1489 GPR64ArgRegs = GPR64ArgRegsWin64;
1490 XMMArgRegs = XMMArgRegsWin64;
1491 } else {
1492 TotalNumIntRegs = 6; TotalNumXMMRegs = 8;
1493 GPR64ArgRegs = GPR64ArgRegs64Bit;
1494 XMMArgRegs = XMMArgRegs64Bit;
1495 }
1496 unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs,
1497 TotalNumIntRegs);
1498 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs,
1499 TotalNumXMMRegs);
1500
Devang Patel578efa92009-06-05 21:57:13 +00001501 bool NoImplicitFloatOps = Fn->hasFnAttr(Attribute::NoImplicitFloat);
Evan Chengc7ce29b2009-02-13 22:36:38 +00001502 assert(!(NumXMMRegs && !Subtarget->hasSSE1()) &&
Torok Edwin3f142c32009-02-01 18:15:56 +00001503 "SSE register cannot be used when SSE is disabled!");
Devang Patel578efa92009-06-05 21:57:13 +00001504 assert(!(NumXMMRegs && UseSoftFloat && NoImplicitFloatOps) &&
Evan Chengc7ce29b2009-02-13 22:36:38 +00001505 "SSE register cannot be used when SSE is disabled!");
Devang Patel578efa92009-06-05 21:57:13 +00001506 if (UseSoftFloat || NoImplicitFloatOps || !Subtarget->hasSSE1())
Torok Edwin3f142c32009-02-01 18:15:56 +00001507 // Kernel mode asks for SSE to be disabled, so don't push them
1508 // on the stack.
1509 TotalNumXMMRegs = 0;
Bill Wendlingf9abd7e2009-03-11 22:30:01 +00001510
Gordon Henriksen86737662008-01-05 16:56:59 +00001511 // For X86-64, if there are vararg parameters that are passed via
1512 // registers, then we must store them to their spots on the stack so they
1513 // may be loaded by deferencing the result of va_next.
1514 VarArgsGPOffset = NumIntRegs * 8;
Anton Korobeynikov998a5bc2008-04-27 23:15:03 +00001515 VarArgsFPOffset = TotalNumIntRegs * 8 + NumXMMRegs * 16;
1516 RegSaveFrameIndex = MFI->CreateStackObject(TotalNumIntRegs * 8 +
1517 TotalNumXMMRegs * 16, 16);
1518
Gordon Henriksen86737662008-01-05 16:56:59 +00001519 // Store the integer parameter registers.
Dan Gohman475871a2008-07-27 21:46:04 +00001520 SmallVector<SDValue, 8> MemOps;
1521 SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy());
Dale Johannesenace16102009-02-03 19:33:06 +00001522 SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), RSFIN,
Chris Lattner0bd48932008-01-17 07:00:52 +00001523 DAG.getIntPtrConstant(VarArgsGPOffset));
Anton Korobeynikov998a5bc2008-04-27 23:15:03 +00001524 for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) {
Bob Wilson998e1252009-04-20 18:36:57 +00001525 unsigned VReg = MF.addLiveIn(GPR64ArgRegs[NumIntRegs],
1526 X86::GR64RegisterClass);
Owen Anderson825b72b2009-08-11 20:47:22 +00001527 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
Dan Gohman475871a2008-07-27 21:46:04 +00001528 SDValue Store =
Dale Johannesenace16102009-02-03 19:33:06 +00001529 DAG.getStore(Val.getValue(1), dl, Val, FIN,
Dan Gohmana54cf172008-07-11 22:44:52 +00001530 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0);
Gordon Henriksen86737662008-01-05 16:56:59 +00001531 MemOps.push_back(Store);
Dale Johannesenace16102009-02-03 19:33:06 +00001532 FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), FIN,
Chris Lattner0bd48932008-01-17 07:00:52 +00001533 DAG.getIntPtrConstant(8));
Gordon Henriksen86737662008-01-05 16:56:59 +00001534 }
Anton Korobeynikov998a5bc2008-04-27 23:15:03 +00001535
Gordon Henriksen86737662008-01-05 16:56:59 +00001536 // Now store the XMM (fp + vector) parameter registers.
Dale Johannesenace16102009-02-03 19:33:06 +00001537 FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), RSFIN,
Chris Lattner0bd48932008-01-17 07:00:52 +00001538 DAG.getIntPtrConstant(VarArgsFPOffset));
Anton Korobeynikov998a5bc2008-04-27 23:15:03 +00001539 for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) {
Bob Wilson998e1252009-04-20 18:36:57 +00001540 unsigned VReg = MF.addLiveIn(XMMArgRegs[NumXMMRegs],
1541 X86::VR128RegisterClass);
Owen Anderson825b72b2009-08-11 20:47:22 +00001542 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::v4f32);
Dan Gohman475871a2008-07-27 21:46:04 +00001543 SDValue Store =
Dale Johannesenace16102009-02-03 19:33:06 +00001544 DAG.getStore(Val.getValue(1), dl, Val, FIN,
Dan Gohmana54cf172008-07-11 22:44:52 +00001545 PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0);
Gordon Henriksen86737662008-01-05 16:56:59 +00001546 MemOps.push_back(Store);
Dale Johannesenace16102009-02-03 19:33:06 +00001547 FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), FIN,
Chris Lattner0bd48932008-01-17 07:00:52 +00001548 DAG.getIntPtrConstant(16));
Gordon Henriksen86737662008-01-05 16:56:59 +00001549 }
1550 if (!MemOps.empty())
Owen Anderson825b72b2009-08-11 20:47:22 +00001551 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
Gordon Henriksen86737662008-01-05 16:56:59 +00001552 &MemOps[0], MemOps.size());
1553 }
Gordon Henriksenae636f82008-01-03 16:47:34 +00001554 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001555
Gordon Henriksen86737662008-01-05 16:56:59 +00001556 // Some CCs need callee pop.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001557 if (IsCalleePop(isVarArg, CallConv)) {
Gordon Henriksen86737662008-01-05 16:56:59 +00001558 BytesToPopOnReturn = StackSize; // Callee pops everything.
Anton Korobeynikovb10308e2007-01-28 13:31:35 +00001559 BytesCallerReserves = 0;
1560 } else {
Anton Korobeynikov1d9bacc2007-03-06 08:12:33 +00001561 BytesToPopOnReturn = 0; // Callee pops nothing.
Chris Lattnerf39f7712007-02-28 05:46:49 +00001562 // If this is an sret function, the return should pop the hidden pointer.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001563 if (!Is64Bit && CallConv != CallingConv::Fast && ArgsAreStructReturn(Ins))
Scott Michelfdc40a02009-02-17 22:15:04 +00001564 BytesToPopOnReturn = 4;
Chris Lattnerf39f7712007-02-28 05:46:49 +00001565 BytesCallerReserves = StackSize;
Anton Korobeynikovb10308e2007-01-28 13:31:35 +00001566 }
Gordon Henriksenae636f82008-01-03 16:47:34 +00001567
Gordon Henriksen86737662008-01-05 16:56:59 +00001568 if (!Is64Bit) {
1569 RegSaveFrameIndex = 0xAAAAAAA; // RegSaveFrameIndex is X86-64 only.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001570 if (CallConv == CallingConv::X86_FastCall)
Gordon Henriksen86737662008-01-05 16:56:59 +00001571 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs.
1572 }
Evan Cheng25caf632006-05-23 21:06:34 +00001573
Anton Korobeynikova2780e12007-08-15 17:12:32 +00001574 FuncInfo->setBytesToPopOnReturn(BytesToPopOnReturn);
Evan Cheng1bc78042006-04-26 01:20:17 +00001575
Dan Gohman98ca4f22009-08-05 01:29:28 +00001576 return Chain;
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00001577}
1578
Dan Gohman475871a2008-07-27 21:46:04 +00001579SDValue
Dan Gohman98ca4f22009-08-05 01:29:28 +00001580X86TargetLowering::LowerMemOpCallTo(SDValue Chain,
1581 SDValue StackPtr, SDValue Arg,
1582 DebugLoc dl, SelectionDAG &DAG,
Evan Chengdffbd832008-01-10 00:09:10 +00001583 const CCValAssign &VA,
Dan Gohman98ca4f22009-08-05 01:29:28 +00001584 ISD::ArgFlagsTy Flags) {
Anton Korobeynikovcf6b7392009-08-03 08:12:53 +00001585 const unsigned FirstStackArgOffset = (Subtarget->isTargetWin64() ? 32 : 0);
Anton Korobeynikovcf6b7392009-08-03 08:12:53 +00001586 unsigned LocMemOffset = FirstStackArgOffset + VA.getLocMemOffset();
Dan Gohman475871a2008-07-27 21:46:04 +00001587 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
Dale Johannesenace16102009-02-03 19:33:06 +00001588 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
Duncan Sands276dcbd2008-03-21 09:14:45 +00001589 if (Flags.isByVal()) {
Dale Johannesendd64c412009-02-04 00:33:20 +00001590 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl);
Evan Chengdffbd832008-01-10 00:09:10 +00001591 }
Dale Johannesenace16102009-02-03 19:33:06 +00001592 return DAG.getStore(Chain, dl, Arg, PtrOff,
Dan Gohman3069b872008-02-07 18:41:25 +00001593 PseudoSourceValue::getStack(), LocMemOffset);
Evan Chengdffbd832008-01-10 00:09:10 +00001594}
1595
Bill Wendling64e87322009-01-16 19:25:27 +00001596/// EmitTailCallLoadRetAddr - Emit a load of return address if tail call
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001597/// optimization is performed and it is required.
Scott Michelfdc40a02009-02-17 22:15:04 +00001598SDValue
1599X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG,
Dan Gohman475871a2008-07-27 21:46:04 +00001600 SDValue &OutRetAddr,
Scott Michelfdc40a02009-02-17 22:15:04 +00001601 SDValue Chain,
1602 bool IsTailCall,
1603 bool Is64Bit,
Dale Johannesenace16102009-02-03 19:33:06 +00001604 int FPDiff,
1605 DebugLoc dl) {
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001606 if (!IsTailCall || FPDiff==0) return Chain;
1607
1608 // Adjust the Return address stack slot.
Owen Andersone50ed302009-08-10 22:56:29 +00001609 EVT VT = getPointerTy();
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001610 OutRetAddr = getReturnAddressFrameIndex(DAG);
Bill Wendling64e87322009-01-16 19:25:27 +00001611
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001612 // Load the "old" Return address.
Dale Johannesenace16102009-02-03 19:33:06 +00001613 OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, NULL, 0);
Gabor Greifba36cb52008-08-28 21:40:38 +00001614 return SDValue(OutRetAddr.getNode(), 1);
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001615}
1616
1617/// EmitTailCallStoreRetAddr - Emit a store of the return adress if tail call
1618/// optimization is performed and it is required (FPDiff!=0).
Scott Michelfdc40a02009-02-17 22:15:04 +00001619static SDValue
1620EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF,
Dan Gohman475871a2008-07-27 21:46:04 +00001621 SDValue Chain, SDValue RetAddrFrIdx,
Dale Johannesenace16102009-02-03 19:33:06 +00001622 bool Is64Bit, int FPDiff, DebugLoc dl) {
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001623 // Store the return address to the appropriate stack slot.
1624 if (!FPDiff) return Chain;
1625 // Calculate the new stack slot for the return address.
1626 int SlotSize = Is64Bit ? 8 : 4;
Scott Michelfdc40a02009-02-17 22:15:04 +00001627 int NewReturnAddrFI =
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001628 MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize);
Owen Anderson825b72b2009-08-11 20:47:22 +00001629 EVT VT = Is64Bit ? MVT::i64 : MVT::i32;
Dan Gohman475871a2008-07-27 21:46:04 +00001630 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00001631 Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx,
Dan Gohmana54cf172008-07-11 22:44:52 +00001632 PseudoSourceValue::getFixedStack(NewReturnAddrFI), 0);
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001633 return Chain;
1634}
1635
Dan Gohman98ca4f22009-08-05 01:29:28 +00001636SDValue
1637X86TargetLowering::LowerCall(SDValue Chain, SDValue Callee,
1638 unsigned CallConv, bool isVarArg, bool isTailCall,
1639 const SmallVectorImpl<ISD::OutputArg> &Outs,
1640 const SmallVectorImpl<ISD::InputArg> &Ins,
1641 DebugLoc dl, SelectionDAG &DAG,
1642 SmallVectorImpl<SDValue> &InVals) {
Gordon Henriksenae636f82008-01-03 16:47:34 +00001643
Dan Gohman98ca4f22009-08-05 01:29:28 +00001644 MachineFunction &MF = DAG.getMachineFunction();
1645 bool Is64Bit = Subtarget->is64Bit();
1646 bool IsStructRet = CallIsStructReturn(Outs);
1647
1648 assert((!isTailCall ||
1649 (CallConv == CallingConv::Fast && PerformTailCallOpt)) &&
1650 "IsEligibleForTailCallOptimization missed a case!");
1651 assert(!(isVarArg && CallConv == CallingConv::Fast) &&
Gordon Henriksenae636f82008-01-03 16:47:34 +00001652 "Var args not supported with calling convention fastcc");
1653
Chris Lattner638402b2007-02-28 07:00:42 +00001654 // Analyze operands of the call, assigning locations to each operand.
Chris Lattner423c5f42007-02-28 05:31:48 +00001655 SmallVector<CCValAssign, 16> ArgLocs;
Dan Gohman98ca4f22009-08-05 01:29:28 +00001656 CCState CCInfo(CallConv, isVarArg, getTargetMachine(),
1657 ArgLocs, *DAG.getContext());
1658 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForNode(CallConv));
Scott Michelfdc40a02009-02-17 22:15:04 +00001659
Chris Lattner423c5f42007-02-28 05:31:48 +00001660 // Get a count of how many bytes are to be pushed on the stack.
1661 unsigned NumBytes = CCInfo.getNextStackOffset();
Dan Gohman98ca4f22009-08-05 01:29:28 +00001662 if (PerformTailCallOpt && CallConv == CallingConv::Fast)
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00001663 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG);
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00001664
Gordon Henriksen86737662008-01-05 16:56:59 +00001665 int FPDiff = 0;
Dan Gohman98ca4f22009-08-05 01:29:28 +00001666 if (isTailCall) {
Gordon Henriksen86737662008-01-05 16:56:59 +00001667 // Lower arguments at fp - stackoffset + fpdiff.
Scott Michelfdc40a02009-02-17 22:15:04 +00001668 unsigned NumBytesCallerPushed =
Gordon Henriksen86737662008-01-05 16:56:59 +00001669 MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn();
1670 FPDiff = NumBytesCallerPushed - NumBytes;
1671
1672 // Set the delta of movement of the returnaddr stackslot.
1673 // But only set if delta is greater than previous delta.
1674 if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta()))
1675 MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff);
1676 }
1677
Chris Lattnere563bbc2008-10-11 22:08:30 +00001678 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true));
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00001679
Dan Gohman475871a2008-07-27 21:46:04 +00001680 SDValue RetAddrFrIdx;
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001681 // Load return adress for tail calls.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001682 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall, Is64Bit,
Dale Johannesenace16102009-02-03 19:33:06 +00001683 FPDiff, dl);
Gordon Henriksen86737662008-01-05 16:56:59 +00001684
Dan Gohman475871a2008-07-27 21:46:04 +00001685 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
1686 SmallVector<SDValue, 8> MemOpChains;
1687 SDValue StackPtr;
Chris Lattner423c5f42007-02-28 05:31:48 +00001688
Arnold Schwaighofer30e62c02008-04-30 09:16:33 +00001689 // Walk the register/memloc assignments, inserting copies/loads. In the case
1690 // of tail call optimization arguments are handle later.
Chris Lattner423c5f42007-02-28 05:31:48 +00001691 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1692 CCValAssign &VA = ArgLocs[i];
Owen Andersone50ed302009-08-10 22:56:29 +00001693 EVT RegVT = VA.getLocVT();
Dan Gohman98ca4f22009-08-05 01:29:28 +00001694 SDValue Arg = Outs[i].Val;
1695 ISD::ArgFlagsTy Flags = Outs[i].Flags;
Dan Gohman095cc292008-09-13 01:54:27 +00001696 bool isByVal = Flags.isByVal();
Scott Michelfdc40a02009-02-17 22:15:04 +00001697
Chris Lattner423c5f42007-02-28 05:31:48 +00001698 // Promote the value if needed.
1699 switch (VA.getLocInfo()) {
Torok Edwinc23197a2009-07-14 16:55:14 +00001700 default: llvm_unreachable("Unknown loc info!");
Chris Lattner423c5f42007-02-28 05:31:48 +00001701 case CCValAssign::Full: break;
1702 case CCValAssign::SExt:
Anton Korobeynikov80cb8aa2009-08-03 08:13:24 +00001703 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg);
Chris Lattner423c5f42007-02-28 05:31:48 +00001704 break;
1705 case CCValAssign::ZExt:
Anton Korobeynikov80cb8aa2009-08-03 08:13:24 +00001706 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg);
Chris Lattner423c5f42007-02-28 05:31:48 +00001707 break;
1708 case CCValAssign::AExt:
Anton Korobeynikov80cb8aa2009-08-03 08:13:24 +00001709 if (RegVT.isVector() && RegVT.getSizeInBits() == 128) {
1710 // Special case: passing MMX values in XMM registers.
Owen Anderson825b72b2009-08-11 20:47:22 +00001711 Arg = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i64, Arg);
1712 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg);
1713 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg);
Anton Korobeynikov80cb8aa2009-08-03 08:13:24 +00001714 } else
1715 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg);
1716 break;
1717 case CCValAssign::BCvt:
1718 Arg = DAG.getNode(ISD::BIT_CONVERT, dl, RegVT, Arg);
Chris Lattner423c5f42007-02-28 05:31:48 +00001719 break;
Anton Korobeynikov4ab15532009-08-03 08:13:56 +00001720 case CCValAssign::Indirect: {
1721 // Store the argument.
1722 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT());
1723 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
1724 Chain = DAG.getStore(Chain, dl, Arg, SpillSlot,
1725 PseudoSourceValue::getFixedStack(FI), 0);
1726 Arg = SpillSlot;
1727 break;
1728 }
Evan Cheng6b5783d2006-05-25 18:56:34 +00001729 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001730
Chris Lattner423c5f42007-02-28 05:31:48 +00001731 if (VA.isRegLoc()) {
1732 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
1733 } else {
Dan Gohman98ca4f22009-08-05 01:29:28 +00001734 if (!isTailCall || (isTailCall && isByVal)) {
Arnold Schwaighoferc8ab8cd2008-01-11 16:49:42 +00001735 assert(VA.isMemLoc());
Gabor Greifba36cb52008-08-28 21:40:38 +00001736 if (StackPtr.getNode() == 0)
Dale Johannesendd64c412009-02-04 00:33:20 +00001737 StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, getPointerTy());
Scott Michelfdc40a02009-02-17 22:15:04 +00001738
Dan Gohman98ca4f22009-08-05 01:29:28 +00001739 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
1740 dl, DAG, VA, Flags));
Arnold Schwaighoferc8ab8cd2008-01-11 16:49:42 +00001741 }
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00001742 }
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00001743 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001744
Evan Cheng32fe1032006-05-25 00:59:30 +00001745 if (!MemOpChains.empty())
Owen Anderson825b72b2009-08-11 20:47:22 +00001746 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
Chris Lattnerbd564bf2006-08-08 02:23:42 +00001747 &MemOpChains[0], MemOpChains.size());
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00001748
Evan Cheng347d5f72006-04-28 21:29:37 +00001749 // Build a sequence of copy-to-reg nodes chained together with token chain
1750 // and flag operands which copy the outgoing args into registers.
Dan Gohman475871a2008-07-27 21:46:04 +00001751 SDValue InFlag;
Arnold Schwaighofer30e62c02008-04-30 09:16:33 +00001752 // Tail call byval lowering might overwrite argument registers so in case of
1753 // tail call optimization the copies to registers are lowered later.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001754 if (!isTailCall)
Arnold Schwaighofer30e62c02008-04-30 09:16:33 +00001755 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
Scott Michelfdc40a02009-02-17 22:15:04 +00001756 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
Dale Johannesendd64c412009-02-04 00:33:20 +00001757 RegsToPass[i].second, InFlag);
Arnold Schwaighofer30e62c02008-04-30 09:16:33 +00001758 InFlag = Chain.getValue(1);
1759 }
Gordon Henriksen86737662008-01-05 16:56:59 +00001760
Chris Lattner951bf7d2009-07-09 02:44:11 +00001761
Chris Lattner88e1fd52009-07-09 04:24:46 +00001762 if (Subtarget->isPICStyleGOT()) {
Chris Lattnerb133a0a2009-07-09 02:55:47 +00001763 // ELF / PIC requires GOT in the EBX register before function calls via PLT
1764 // GOT pointer.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001765 if (!isTailCall) {
Chris Lattnerb133a0a2009-07-09 02:55:47 +00001766 Chain = DAG.getCopyToReg(Chain, dl, X86::EBX,
1767 DAG.getNode(X86ISD::GlobalBaseReg,
1768 DebugLoc::getUnknownLoc(),
1769 getPointerTy()),
1770 InFlag);
1771 InFlag = Chain.getValue(1);
1772 } else {
1773 // If we are tail calling and generating PIC/GOT style code load the
1774 // address of the callee into ECX. The value in ecx is used as target of
1775 // the tail jump. This is done to circumvent the ebx/callee-saved problem
1776 // for tail calls on PIC/GOT architectures. Normally we would just put the
1777 // address of GOT into ebx and then call target@PLT. But for tail calls
1778 // ebx would be restored (since ebx is callee saved) before jumping to the
1779 // target@PLT.
1780
1781 // Note: The actual moving to ECX is done further down.
1782 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
1783 if (G && !G->getGlobal()->hasHiddenVisibility() &&
1784 !G->getGlobal()->hasProtectedVisibility())
1785 Callee = LowerGlobalAddress(Callee, DAG);
1786 else if (isa<ExternalSymbolSDNode>(Callee))
Chris Lattner15a380a2009-07-09 04:39:06 +00001787 Callee = LowerExternalSymbol(Callee, DAG);
Chris Lattnerb133a0a2009-07-09 02:55:47 +00001788 }
Anton Korobeynikov7f705592007-01-12 19:20:47 +00001789 }
Gordon Henriksenae636f82008-01-03 16:47:34 +00001790
Gordon Henriksen86737662008-01-05 16:56:59 +00001791 if (Is64Bit && isVarArg) {
1792 // From AMD64 ABI document:
1793 // For calls that may call functions that use varargs or stdargs
1794 // (prototype-less calls or calls to functions containing ellipsis (...) in
1795 // the declaration) %al is used as hidden argument to specify the number
1796 // of SSE registers used. The contents of %al do not need to match exactly
1797 // the number of registers, but must be an ubound on the number of SSE
1798 // registers used and is in the range 0 - 8 inclusive.
Anton Korobeynikov998a5bc2008-04-27 23:15:03 +00001799
1800 // FIXME: Verify this on Win64
Gordon Henriksen86737662008-01-05 16:56:59 +00001801 // Count the number of XMM registers allocated.
1802 static const unsigned XMMArgRegs[] = {
1803 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
1804 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
1805 };
1806 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8);
Scott Michelfdc40a02009-02-17 22:15:04 +00001807 assert((Subtarget->hasSSE1() || !NumXMMRegs)
Torok Edwin3f142c32009-02-01 18:15:56 +00001808 && "SSE registers cannot be used when SSE is disabled");
Scott Michelfdc40a02009-02-17 22:15:04 +00001809
Dale Johannesendd64c412009-02-04 00:33:20 +00001810 Chain = DAG.getCopyToReg(Chain, dl, X86::AL,
Owen Anderson825b72b2009-08-11 20:47:22 +00001811 DAG.getConstant(NumXMMRegs, MVT::i8), InFlag);
Gordon Henriksen86737662008-01-05 16:56:59 +00001812 InFlag = Chain.getValue(1);
1813 }
1814
Arnold Schwaighofer865c6812008-02-26 09:19:59 +00001815
Arnold Schwaighoferc8ab8cd2008-01-11 16:49:42 +00001816 // For tail calls lower the arguments to the 'real' stack slot.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001817 if (isTailCall) {
1818 // Force all the incoming stack arguments to be loaded from the stack
1819 // before any new outgoing arguments are stored to the stack, because the
1820 // outgoing stack slots may alias the incoming argument stack slots, and
1821 // the alias isn't otherwise explicit. This is slightly more conservative
1822 // than necessary, because it means that each store effectively depends
1823 // on every argument instead of just those arguments it would clobber.
1824 SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain);
1825
Dan Gohman475871a2008-07-27 21:46:04 +00001826 SmallVector<SDValue, 8> MemOpChains2;
1827 SDValue FIN;
Gordon Henriksen86737662008-01-05 16:56:59 +00001828 int FI = 0;
Arnold Schwaighofer865c6812008-02-26 09:19:59 +00001829 // Do not flag preceeding copytoreg stuff together with the following stuff.
Dan Gohman475871a2008-07-27 21:46:04 +00001830 InFlag = SDValue();
Gordon Henriksen86737662008-01-05 16:56:59 +00001831 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1832 CCValAssign &VA = ArgLocs[i];
1833 if (!VA.isRegLoc()) {
Arnold Schwaighoferc8ab8cd2008-01-11 16:49:42 +00001834 assert(VA.isMemLoc());
Dan Gohman98ca4f22009-08-05 01:29:28 +00001835 SDValue Arg = Outs[i].Val;
1836 ISD::ArgFlagsTy Flags = Outs[i].Flags;
Gordon Henriksen86737662008-01-05 16:56:59 +00001837 // Create frame index.
1838 int32_t Offset = VA.getLocMemOffset()+FPDiff;
Duncan Sands83ec4b62008-06-06 12:08:01 +00001839 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8;
Gordon Henriksen86737662008-01-05 16:56:59 +00001840 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset);
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001841 FIN = DAG.getFrameIndex(FI, getPointerTy());
Arnold Schwaighoferc8ab8cd2008-01-11 16:49:42 +00001842
Duncan Sands276dcbd2008-03-21 09:14:45 +00001843 if (Flags.isByVal()) {
Evan Cheng8e5712b2008-01-12 01:08:07 +00001844 // Copy relative to framepointer.
Dan Gohman475871a2008-07-27 21:46:04 +00001845 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset());
Gabor Greifba36cb52008-08-28 21:40:38 +00001846 if (StackPtr.getNode() == 0)
Scott Michelfdc40a02009-02-17 22:15:04 +00001847 StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr,
Dale Johannesendd64c412009-02-04 00:33:20 +00001848 getPointerTy());
Dale Johannesenace16102009-02-03 19:33:06 +00001849 Source = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, Source);
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001850
Dan Gohman98ca4f22009-08-05 01:29:28 +00001851 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN,
1852 ArgChain,
Dale Johannesendd64c412009-02-04 00:33:20 +00001853 Flags, DAG, dl));
Gordon Henriksen86737662008-01-05 16:56:59 +00001854 } else {
Evan Cheng8e5712b2008-01-12 01:08:07 +00001855 // Store relative to framepointer.
Dan Gohman69de1932008-02-06 22:27:42 +00001856 MemOpChains2.push_back(
Dan Gohman98ca4f22009-08-05 01:29:28 +00001857 DAG.getStore(ArgChain, dl, Arg, FIN,
Dan Gohmana54cf172008-07-11 22:44:52 +00001858 PseudoSourceValue::getFixedStack(FI), 0));
Scott Michelfdc40a02009-02-17 22:15:04 +00001859 }
Gordon Henriksen86737662008-01-05 16:56:59 +00001860 }
1861 }
1862
1863 if (!MemOpChains2.empty())
Owen Anderson825b72b2009-08-11 20:47:22 +00001864 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
Arnold Schwaighofer719eb022008-01-11 14:34:56 +00001865 &MemOpChains2[0], MemOpChains2.size());
Gordon Henriksen86737662008-01-05 16:56:59 +00001866
Arnold Schwaighofer30e62c02008-04-30 09:16:33 +00001867 // Copy arguments to their registers.
1868 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
Scott Michelfdc40a02009-02-17 22:15:04 +00001869 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
Dale Johannesendd64c412009-02-04 00:33:20 +00001870 RegsToPass[i].second, InFlag);
Arnold Schwaighofer30e62c02008-04-30 09:16:33 +00001871 InFlag = Chain.getValue(1);
1872 }
Dan Gohman475871a2008-07-27 21:46:04 +00001873 InFlag =SDValue();
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001874
Gordon Henriksen86737662008-01-05 16:56:59 +00001875 // Store the return address to the appropriate stack slot.
Arnold Schwaighofer4b5324a2008-04-12 18:11:06 +00001876 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit,
Dale Johannesenace16102009-02-03 19:33:06 +00001877 FPDiff, dl);
Gordon Henriksen86737662008-01-05 16:56:59 +00001878 }
1879
Evan Cheng32fe1032006-05-25 00:59:30 +00001880 // If the callee is a GlobalAddress node (quite common, every direct call is)
1881 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
Anton Korobeynikova5986852006-11-20 10:46:14 +00001882 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
Anton Korobeynikov2b2bc682006-12-22 22:29:05 +00001883 // We should use extra load for direct calls to dllimported functions in
1884 // non-JIT mode.
Chris Lattner74e726e2009-07-09 05:27:35 +00001885 GlobalValue *GV = G->getGlobal();
Chris Lattner754b7652009-07-10 05:48:03 +00001886 if (!GV->hasDLLImportLinkage()) {
Chris Lattner48a7d022009-07-09 05:02:21 +00001887 unsigned char OpFlags = 0;
1888
1889 // On ELF targets, in both X86-64 and X86-32 mode, direct calls to
1890 // external symbols most go through the PLT in PIC mode. If the symbol
1891 // has hidden or protected visibility, or if it is static or local, then
1892 // we don't need to use the PLT - we can directly call it.
1893 if (Subtarget->isTargetELF() &&
1894 getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
Chris Lattner74e726e2009-07-09 05:27:35 +00001895 GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) {
Chris Lattner48a7d022009-07-09 05:02:21 +00001896 OpFlags = X86II::MO_PLT;
Chris Lattner3b67e9b2009-07-10 20:47:30 +00001897 } else if (Subtarget->isPICStyleStubAny() &&
Chris Lattner74e726e2009-07-09 05:27:35 +00001898 (GV->isDeclaration() || GV->isWeakForLinker()) &&
1899 Subtarget->getDarwinVers() < 9) {
1900 // PC-relative references to external symbols should go through $stub,
1901 // unless we're building with the leopard linker or later, which
1902 // automatically synthesizes these stubs.
1903 OpFlags = X86II::MO_DARWIN_STUB;
1904 }
Chris Lattner48a7d022009-07-09 05:02:21 +00001905
Chris Lattner74e726e2009-07-09 05:27:35 +00001906 Callee = DAG.getTargetGlobalAddress(GV, getPointerTy(),
Chris Lattner48a7d022009-07-09 05:02:21 +00001907 G->getOffset(), OpFlags);
1908 }
Bill Wendling056292f2008-09-16 21:48:12 +00001909 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
Chris Lattner48a7d022009-07-09 05:02:21 +00001910 unsigned char OpFlags = 0;
1911
1912 // On ELF targets, in either X86-64 or X86-32 mode, direct calls to external
1913 // symbols should go through the PLT.
1914 if (Subtarget->isTargetELF() &&
Chris Lattner74e726e2009-07-09 05:27:35 +00001915 getTargetMachine().getRelocationModel() == Reloc::PIC_) {
Chris Lattner48a7d022009-07-09 05:02:21 +00001916 OpFlags = X86II::MO_PLT;
Chris Lattner3b67e9b2009-07-10 20:47:30 +00001917 } else if (Subtarget->isPICStyleStubAny() &&
Chris Lattner74e726e2009-07-09 05:27:35 +00001918 Subtarget->getDarwinVers() < 9) {
1919 // PC-relative references to external symbols should go through $stub,
1920 // unless we're building with the leopard linker or later, which
1921 // automatically synthesizes these stubs.
1922 OpFlags = X86II::MO_DARWIN_STUB;
1923 }
1924
Chris Lattner48a7d022009-07-09 05:02:21 +00001925 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy(),
1926 OpFlags);
Dan Gohman98ca4f22009-08-05 01:29:28 +00001927 } else if (isTailCall) {
Arnold Schwaighoferbbd8c332009-06-12 16:26:57 +00001928 unsigned Opc = Is64Bit ? X86::R11 : X86::EAX;
Gordon Henriksen86737662008-01-05 16:56:59 +00001929
Dale Johannesendd64c412009-02-04 00:33:20 +00001930 Chain = DAG.getCopyToReg(Chain, dl,
Scott Michelfdc40a02009-02-17 22:15:04 +00001931 DAG.getRegister(Opc, getPointerTy()),
Gordon Henriksen86737662008-01-05 16:56:59 +00001932 Callee,InFlag);
1933 Callee = DAG.getRegister(Opc, getPointerTy());
1934 // Add register as live out.
Dan Gohman7e77b0f2009-08-01 19:14:37 +00001935 MF.getRegInfo().addLiveOut(Opc);
Gordon Henriksenae636f82008-01-03 16:47:34 +00001936 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001937
Chris Lattnerd96d0722007-02-25 06:40:16 +00001938 // Returns a chain & a flag for retval copy to use.
Owen Anderson825b72b2009-08-11 20:47:22 +00001939 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Dan Gohman475871a2008-07-27 21:46:04 +00001940 SmallVector<SDValue, 8> Ops;
Gordon Henriksen86737662008-01-05 16:56:59 +00001941
Dan Gohman98ca4f22009-08-05 01:29:28 +00001942 if (isTailCall) {
Dale Johannesene8d72302009-02-06 23:05:02 +00001943 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
1944 DAG.getIntPtrConstant(0, true), InFlag);
Gordon Henriksen86737662008-01-05 16:56:59 +00001945 InFlag = Chain.getValue(1);
Gordon Henriksen86737662008-01-05 16:56:59 +00001946 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001947
Nate Begeman4c5dcf52006-02-17 00:03:04 +00001948 Ops.push_back(Chain);
1949 Ops.push_back(Callee);
Evan Chengb69d1132006-06-14 18:17:40 +00001950
Dan Gohman98ca4f22009-08-05 01:29:28 +00001951 if (isTailCall)
Owen Anderson825b72b2009-08-11 20:47:22 +00001952 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32));
Evan Chengf4684712007-02-21 21:18:14 +00001953
Gordon Henriksen86737662008-01-05 16:56:59 +00001954 // Add argument registers to the end of the list so that they are known live
1955 // into the call.
Evan Cheng9b449442008-01-07 23:08:23 +00001956 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
1957 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
1958 RegsToPass[i].second.getValueType()));
Scott Michelfdc40a02009-02-17 22:15:04 +00001959
Evan Cheng586ccac2008-03-18 23:36:35 +00001960 // Add an implicit use GOT pointer in EBX.
Dan Gohman98ca4f22009-08-05 01:29:28 +00001961 if (!isTailCall && Subtarget->isPICStyleGOT())
Evan Cheng586ccac2008-03-18 23:36:35 +00001962 Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy()));
1963
1964 // Add an implicit use of AL for x86 vararg functions.
1965 if (Is64Bit && isVarArg)
Owen Anderson825b72b2009-08-11 20:47:22 +00001966 Ops.push_back(DAG.getRegister(X86::AL, MVT::i8));
Evan Cheng586ccac2008-03-18 23:36:35 +00001967
Gabor Greifba36cb52008-08-28 21:40:38 +00001968 if (InFlag.getNode())
Evan Cheng347d5f72006-04-28 21:29:37 +00001969 Ops.push_back(InFlag);
Gordon Henriksenae636f82008-01-03 16:47:34 +00001970
Dan Gohman98ca4f22009-08-05 01:29:28 +00001971 if (isTailCall) {
1972 // If this is the first return lowered for this function, add the regs
1973 // to the liveout set for the function.
1974 if (MF.getRegInfo().liveout_empty()) {
1975 SmallVector<CCValAssign, 16> RVLocs;
1976 CCState CCInfo(CallConv, isVarArg, getTargetMachine(), RVLocs,
1977 *DAG.getContext());
1978 CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
1979 for (unsigned i = 0; i != RVLocs.size(); ++i)
1980 if (RVLocs[i].isRegLoc())
1981 MF.getRegInfo().addLiveOut(RVLocs[i].getLocReg());
1982 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001983
Dan Gohman98ca4f22009-08-05 01:29:28 +00001984 assert(((Callee.getOpcode() == ISD::Register &&
1985 (cast<RegisterSDNode>(Callee)->getReg() == X86::EAX ||
1986 cast<RegisterSDNode>(Callee)->getReg() == X86::R9)) ||
1987 Callee.getOpcode() == ISD::TargetExternalSymbol ||
1988 Callee.getOpcode() == ISD::TargetGlobalAddress) &&
1989 "Expecting an global address, external symbol, or register");
1990
1991 return DAG.getNode(X86ISD::TC_RETURN, dl,
1992 NodeTys, &Ops[0], Ops.size());
Gordon Henriksen86737662008-01-05 16:56:59 +00001993 }
1994
Dale Johannesenace16102009-02-03 19:33:06 +00001995 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, &Ops[0], Ops.size());
Evan Cheng347d5f72006-04-28 21:29:37 +00001996 InFlag = Chain.getValue(1);
Evan Chengd90eb7f2006-01-05 00:27:02 +00001997
Chris Lattner2d297092006-05-23 18:50:38 +00001998 // Create the CALLSEQ_END node.
Gordon Henriksen86737662008-01-05 16:56:59 +00001999 unsigned NumBytesForCalleeToPush;
Dan Gohman98ca4f22009-08-05 01:29:28 +00002000 if (IsCalleePop(isVarArg, CallConv))
Gordon Henriksen86737662008-01-05 16:56:59 +00002001 NumBytesForCalleeToPush = NumBytes; // Callee pops everything
Dan Gohman98ca4f22009-08-05 01:29:28 +00002002 else if (!Is64Bit && CallConv != CallingConv::Fast && IsStructRet)
Anton Korobeynikovb10308e2007-01-28 13:31:35 +00002003 // If this is is a call to a struct-return function, the callee
2004 // pops the hidden struct pointer, so we have to push it back.
2005 // This is common for Darwin/X86, Linux & Mingw32 targets.
Gordon Henriksenae636f82008-01-03 16:47:34 +00002006 NumBytesForCalleeToPush = 4;
Gordon Henriksen86737662008-01-05 16:56:59 +00002007 else
Gordon Henriksenae636f82008-01-03 16:47:34 +00002008 NumBytesForCalleeToPush = 0; // Callee pops nothing.
Scott Michelfdc40a02009-02-17 22:15:04 +00002009
Gordon Henriksenae636f82008-01-03 16:47:34 +00002010 // Returns a flag for retval copy to use.
Bill Wendling0f8d9c02007-11-13 00:44:25 +00002011 Chain = DAG.getCALLSEQ_END(Chain,
Chris Lattnere563bbc2008-10-11 22:08:30 +00002012 DAG.getIntPtrConstant(NumBytes, true),
2013 DAG.getIntPtrConstant(NumBytesForCalleeToPush,
2014 true),
Bill Wendling0f8d9c02007-11-13 00:44:25 +00002015 InFlag);
Chris Lattner3085e152007-02-25 08:59:22 +00002016 InFlag = Chain.getValue(1);
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00002017
Chris Lattner3085e152007-02-25 08:59:22 +00002018 // Handle result values, copying them out of physregs into vregs that we
2019 // return.
Dan Gohman98ca4f22009-08-05 01:29:28 +00002020 return LowerCallResult(Chain, InFlag, CallConv, isVarArg,
2021 Ins, dl, DAG, InVals);
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00002022}
2023
Evan Cheng25ab6902006-09-08 06:48:29 +00002024
2025//===----------------------------------------------------------------------===//
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00002026// Fast Calling Convention (tail call) implementation
2027//===----------------------------------------------------------------------===//
2028
2029// Like std call, callee cleans arguments, convention except that ECX is
2030// reserved for storing the tail called function address. Only 2 registers are
2031// free for argument passing (inreg). Tail call optimization is performed
2032// provided:
2033// * tailcallopt is enabled
2034// * caller/callee are fastcc
Arnold Schwaighofera2a4b472008-02-26 10:21:54 +00002035// On X86_64 architecture with GOT-style position independent code only local
2036// (within module) calls are supported at the moment.
Arnold Schwaighofer48abc5c2007-10-12 21:30:57 +00002037// To keep the stack aligned according to platform abi the function
2038// GetAlignedArgumentStackSize ensures that argument delta is always multiples
2039// of stack alignment. (Dynamic linkers need this - darwin's dyld for example)
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00002040// If a tail called function callee has more arguments than the caller the
2041// caller needs to make sure that there is room to move the RETADDR to. This is
Arnold Schwaighofer48abc5c2007-10-12 21:30:57 +00002042// achieved by reserving an area the size of the argument delta right after the
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00002043// original REtADDR, but before the saved framepointer or the spilled registers
2044// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4)
2045// stack layout:
2046// arg1
2047// arg2
2048// RETADDR
Scott Michelfdc40a02009-02-17 22:15:04 +00002049// [ new RETADDR
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00002050// move area ]
2051// (possible EBP)
2052// ESI
2053// EDI
2054// local1 ..
2055
2056/// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned
2057/// for a 16 byte align requirement.
Scott Michelfdc40a02009-02-17 22:15:04 +00002058unsigned X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize,
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00002059 SelectionDAG& DAG) {
Evan Chenge9ac9e62008-09-07 09:07:23 +00002060 MachineFunction &MF = DAG.getMachineFunction();
2061 const TargetMachine &TM = MF.getTarget();
2062 const TargetFrameInfo &TFI = *TM.getFrameInfo();
2063 unsigned StackAlignment = TFI.getStackAlignment();
Scott Michelfdc40a02009-02-17 22:15:04 +00002064 uint64_t AlignMask = StackAlignment - 1;
Evan Chenge9ac9e62008-09-07 09:07:23 +00002065 int64_t Offset = StackSize;
Anton Korobeynikovbff66b02008-09-09 18:22:57 +00002066 uint64_t SlotSize = TD->getPointerSize();
Evan Chenge9ac9e62008-09-07 09:07:23 +00002067 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) {
2068 // Number smaller than 12 so just add the difference.
2069 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask));
2070 } else {
2071 // Mask out lower bits, add stackalignment once plus the 12 bytes.
Scott Michelfdc40a02009-02-17 22:15:04 +00002072 Offset = ((~AlignMask) & Offset) + StackAlignment +
Evan Chenge9ac9e62008-09-07 09:07:23 +00002073 (StackAlignment-SlotSize);
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00002074 }
Evan Chenge9ac9e62008-09-07 09:07:23 +00002075 return Offset;
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00002076}
2077
Dan Gohman98ca4f22009-08-05 01:29:28 +00002078/// IsEligibleForTailCallOptimization - Check whether the call is eligible
2079/// for tail call optimization. Targets which want to do tail call
2080/// optimization should implement this function.
2081bool
2082X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
2083 unsigned CalleeCC,
2084 bool isVarArg,
2085 const SmallVectorImpl<ISD::InputArg> &Ins,
2086 SelectionDAG& DAG) const {
2087 MachineFunction &MF = DAG.getMachineFunction();
2088 unsigned CallerCC = MF.getFunction()->getCallingConv();
2089 return CalleeCC == CallingConv::Fast && CallerCC == CalleeCC;
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00002090}
2091
Dan Gohman3df24e62008-09-03 23:12:08 +00002092FastISel *
2093X86TargetLowering::createFastISel(MachineFunction &mf,
Dan Gohmand57dd5f2008-09-23 21:53:34 +00002094 MachineModuleInfo *mmo,
Devang Patel83489bb2009-01-13 00:35:13 +00002095 DwarfWriter *dw,
Dan Gohman3df24e62008-09-03 23:12:08 +00002096 DenseMap<const Value *, unsigned> &vm,
2097 DenseMap<const BasicBlock *,
Dan Gohman0586d912008-09-10 20:11:02 +00002098 MachineBasicBlock *> &bm,
Dan Gohmandd5b58a2008-10-14 23:54:11 +00002099 DenseMap<const AllocaInst *, int> &am
2100#ifndef NDEBUG
2101 , SmallSet<Instruction*, 8> &cil
2102#endif
2103 ) {
Devang Patel83489bb2009-01-13 00:35:13 +00002104 return X86::createFastISel(mf, mmo, dw, vm, bm, am
Dan Gohmandd5b58a2008-10-14 23:54:11 +00002105#ifndef NDEBUG
2106 , cil
2107#endif
2108 );
Dan Gohmand9f3c482008-08-19 21:32:53 +00002109}
2110
2111
Chris Lattnerfcf1a3d2007-02-28 06:10:12 +00002112//===----------------------------------------------------------------------===//
2113// Other Lowering Hooks
2114//===----------------------------------------------------------------------===//
2115
2116
Dan Gohman475871a2008-07-27 21:46:04 +00002117SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) {
Anton Korobeynikova2780e12007-08-15 17:12:32 +00002118 MachineFunction &MF = DAG.getMachineFunction();
2119 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2120 int ReturnAddrIndex = FuncInfo->getRAIndex();
2121
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00002122 if (ReturnAddrIndex == 0) {
2123 // Set up a frame object for the return address.
Bill Wendling64e87322009-01-16 19:25:27 +00002124 uint64_t SlotSize = TD->getPointerSize();
Anton Korobeynikovbff66b02008-09-09 18:22:57 +00002125 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(SlotSize, -SlotSize);
Anton Korobeynikova2780e12007-08-15 17:12:32 +00002126 FuncInfo->setRAIndex(ReturnAddrIndex);
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00002127 }
2128
Evan Cheng25ab6902006-09-08 06:48:29 +00002129 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy());
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00002130}
2131
2132
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00002133bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M,
2134 bool hasSymbolicDisplacement) {
2135 // Offset should fit into 32 bit immediate field.
2136 if (!isInt32(Offset))
2137 return false;
2138
2139 // If we don't have a symbolic displacement - we don't have any extra
2140 // restrictions.
2141 if (!hasSymbolicDisplacement)
2142 return true;
2143
2144 // FIXME: Some tweaks might be needed for medium code model.
2145 if (M != CodeModel::Small && M != CodeModel::Kernel)
2146 return false;
2147
2148 // For small code model we assume that latest object is 16MB before end of 31
2149 // bits boundary. We may also accept pretty large negative constants knowing
2150 // that all objects are in the positive half of address space.
2151 if (M == CodeModel::Small && Offset < 16*1024*1024)
2152 return true;
2153
2154 // For kernel code model we know that all object resist in the negative half
2155 // of 32bits address space. We may not accept negative offsets, since they may
2156 // be just off and we may accept pretty large positive ones.
2157 if (M == CodeModel::Kernel && Offset > 0)
2158 return true;
2159
2160 return false;
2161}
2162
Chris Lattner1c39d4c2008-12-24 23:53:05 +00002163/// TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86
2164/// specific condition code, returning the condition code and the LHS/RHS of the
2165/// comparison to make.
2166static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP,
2167 SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) {
Evan Chengd9558e02006-01-06 00:43:03 +00002168 if (!isFP) {
Chris Lattnerbfd68a72006-09-13 17:04:54 +00002169 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
2170 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) {
2171 // X > -1 -> X == 0, jump !sign.
2172 RHS = DAG.getConstant(0, RHS.getValueType());
Chris Lattner1c39d4c2008-12-24 23:53:05 +00002173 return X86::COND_NS;
Chris Lattnerbfd68a72006-09-13 17:04:54 +00002174 } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) {
2175 // X < 0 -> X == 0, jump on sign.
Chris Lattner1c39d4c2008-12-24 23:53:05 +00002176 return X86::COND_S;
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00002177 } else if (SetCCOpcode == ISD::SETLT && RHSC->getZExtValue() == 1) {
Dan Gohman5f6913c2007-09-17 14:49:27 +00002178 // X < 1 -> X <= 0
2179 RHS = DAG.getConstant(0, RHS.getValueType());
Chris Lattner1c39d4c2008-12-24 23:53:05 +00002180 return X86::COND_LE;
Chris Lattnerbfd68a72006-09-13 17:04:54 +00002181 }
Chris Lattnerf9570512006-09-13 03:22:10 +00002182 }
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00002183
Evan Chengd9558e02006-01-06 00:43:03 +00002184 switch (SetCCOpcode) {
Torok Edwinc23197a2009-07-14 16:55:14 +00002185 default: llvm_unreachable("Invalid integer condition!");
Chris Lattner1c39d4c2008-12-24 23:53:05 +00002186 case ISD::SETEQ: return X86::COND_E;
2187 case ISD::SETGT: return X86::COND_G;
2188 case ISD::SETGE: return X86::COND_GE;
2189 case ISD::SETLT: return X86::COND_L;
2190 case ISD::SETLE: return X86::COND_LE;
2191 case ISD::SETNE: return X86::COND_NE;
2192 case ISD::SETULT: return X86::COND_B;
2193 case ISD::SETUGT: return X86::COND_A;
2194 case ISD::SETULE: return X86::COND_BE;
2195 case ISD::SETUGE: return X86::COND_AE;
Evan Chengd9558e02006-01-06 00:43:03 +00002196 }
Chris Lattner4c78e022008-12-23 23:42:27 +00002197 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002198
Chris Lattner4c78e022008-12-23 23:42:27 +00002199 // First determine if it is required or is profitable to flip the operands.
Duncan Sands4047f4a2008-10-24 13:03:10 +00002200
Chris Lattner4c78e022008-12-23 23:42:27 +00002201 // If LHS is a foldable load, but RHS is not, flip the condition.
2202 if ((ISD::isNON_EXTLoad(LHS.getNode()) && LHS.hasOneUse()) &&
2203 !(ISD::isNON_EXTLoad(RHS.getNode()) && RHS.hasOneUse())) {
2204 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
2205 std::swap(LHS, RHS);
Evan Cheng4d46d0a2008-08-28 23:48:31 +00002206 }
2207
Chris Lattner4c78e022008-12-23 23:42:27 +00002208 switch (SetCCOpcode) {
2209 default: break;
2210 case ISD::SETOLT:
2211 case ISD::SETOLE:
2212 case ISD::SETUGT:
2213 case ISD::SETUGE:
2214 std::swap(LHS, RHS);
2215 break;
2216 }
2217
2218 // On a floating point condition, the flags are set as follows:
2219 // ZF PF CF op
2220 // 0 | 0 | 0 | X > Y
2221 // 0 | 0 | 1 | X < Y
2222 // 1 | 0 | 0 | X == Y
2223 // 1 | 1 | 1 | unordered
2224 switch (SetCCOpcode) {
Torok Edwinc23197a2009-07-14 16:55:14 +00002225 default: llvm_unreachable("Condcode should be pre-legalized away");
Chris Lattner4c78e022008-12-23 23:42:27 +00002226 case ISD::SETUEQ:
Chris Lattner1c39d4c2008-12-24 23:53:05 +00002227 case ISD::SETEQ: return X86::COND_E;
Chris Lattner4c78e022008-12-23 23:42:27 +00002228 case ISD::SETOLT: // flipped
2229 case ISD::SETOGT:
Chris Lattner1c39d4c2008-12-24 23:53:05 +00002230 case ISD::SETGT: return X86::COND_A;
Chris Lattner4c78e022008-12-23 23:42:27 +00002231 case ISD::SETOLE: // flipped
2232 case ISD::SETOGE:
Chris Lattner1c39d4c2008-12-24 23:53:05 +00002233 case ISD::SETGE: return X86::COND_AE;
Chris Lattner4c78e022008-12-23 23:42:27 +00002234 case ISD::SETUGT: // flipped
2235 case ISD::SETULT:
Chris Lattner1c39d4c2008-12-24 23:53:05 +00002236 case ISD::SETLT: return X86::COND_B;
Chris Lattner4c78e022008-12-23 23:42:27 +00002237 case ISD::SETUGE: // flipped
2238 case ISD::SETULE:
Chris Lattner1c39d4c2008-12-24 23:53:05 +00002239 case ISD::SETLE: return X86::COND_BE;
Chris Lattner4c78e022008-12-23 23:42:27 +00002240 case ISD::SETONE:
Chris Lattner1c39d4c2008-12-24 23:53:05 +00002241 case ISD::SETNE: return X86::COND_NE;
2242 case ISD::SETUO: return X86::COND_P;
2243 case ISD::SETO: return X86::COND_NP;
Chris Lattner4c78e022008-12-23 23:42:27 +00002244 }
Evan Chengd9558e02006-01-06 00:43:03 +00002245}
2246
Evan Cheng4a460802006-01-11 00:33:36 +00002247/// hasFPCMov - is there a floating point cmov for the specific X86 condition
2248/// code. Current x86 isa includes the following FP cmov instructions:
Evan Chengaaca22c2006-01-10 20:26:56 +00002249/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
Evan Cheng4a460802006-01-11 00:33:36 +00002250static bool hasFPCMov(unsigned X86CC) {
Evan Chengaaca22c2006-01-10 20:26:56 +00002251 switch (X86CC) {
2252 default:
2253 return false;
Chris Lattner7fbe9722006-10-20 17:42:20 +00002254 case X86::COND_B:
2255 case X86::COND_BE:
2256 case X86::COND_E:
2257 case X86::COND_P:
2258 case X86::COND_A:
2259 case X86::COND_AE:
2260 case X86::COND_NE:
2261 case X86::COND_NP:
Evan Chengaaca22c2006-01-10 20:26:56 +00002262 return true;
2263 }
2264}
2265
Nate Begeman9008ca62009-04-27 18:41:29 +00002266/// isUndefOrInRange - Return true if Val is undef or if its value falls within
2267/// the specified range (L, H].
2268static bool isUndefOrInRange(int Val, int Low, int Hi) {
2269 return (Val < 0) || (Val >= Low && Val < Hi);
2270}
2271
2272/// isUndefOrEqual - Val is either less than zero (undef) or equal to the
2273/// specified value.
2274static bool isUndefOrEqual(int Val, int CmpVal) {
2275 if (Val < 0 || Val == CmpVal)
Evan Cheng5ced1d82006-04-06 23:23:56 +00002276 return true;
Nate Begeman9008ca62009-04-27 18:41:29 +00002277 return false;
Evan Chengc5cdff22006-04-07 21:53:05 +00002278}
2279
Nate Begeman9008ca62009-04-27 18:41:29 +00002280/// isPSHUFDMask - Return true if the node specifies a shuffle of elements that
2281/// is suitable for input to PSHUFD or PSHUFW. That is, it doesn't reference
2282/// the second operand.
Owen Andersone50ed302009-08-10 22:56:29 +00002283static bool isPSHUFDMask(const SmallVectorImpl<int> &Mask, EVT VT) {
Owen Anderson825b72b2009-08-11 20:47:22 +00002284 if (VT == MVT::v4f32 || VT == MVT::v4i32 || VT == MVT::v4i16)
Nate Begeman9008ca62009-04-27 18:41:29 +00002285 return (Mask[0] < 4 && Mask[1] < 4 && Mask[2] < 4 && Mask[3] < 4);
Owen Anderson825b72b2009-08-11 20:47:22 +00002286 if (VT == MVT::v2f64 || VT == MVT::v2i64)
Nate Begeman9008ca62009-04-27 18:41:29 +00002287 return (Mask[0] < 2 && Mask[1] < 2);
2288 return false;
Evan Cheng5ced1d82006-04-06 23:23:56 +00002289}
2290
Nate Begeman9008ca62009-04-27 18:41:29 +00002291bool X86::isPSHUFDMask(ShuffleVectorSDNode *N) {
2292 SmallVector<int, 8> M;
2293 N->getMask(M);
2294 return ::isPSHUFDMask(M, N->getValueType(0));
2295}
Evan Cheng0188ecb2006-03-22 18:59:22 +00002296
Nate Begeman9008ca62009-04-27 18:41:29 +00002297/// isPSHUFHWMask - Return true if the node specifies a shuffle of elements that
2298/// is suitable for input to PSHUFHW.
Owen Andersone50ed302009-08-10 22:56:29 +00002299static bool isPSHUFHWMask(const SmallVectorImpl<int> &Mask, EVT VT) {
Owen Anderson825b72b2009-08-11 20:47:22 +00002300 if (VT != MVT::v8i16)
Evan Cheng0188ecb2006-03-22 18:59:22 +00002301 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002302
2303 // Lower quadword copied in order or undef.
2304 for (int i = 0; i != 4; ++i)
2305 if (Mask[i] >= 0 && Mask[i] != i)
Evan Cheng506d3df2006-03-29 23:07:14 +00002306 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002307
Evan Cheng506d3df2006-03-29 23:07:14 +00002308 // Upper quadword shuffled.
Nate Begeman9008ca62009-04-27 18:41:29 +00002309 for (int i = 4; i != 8; ++i)
2310 if (Mask[i] >= 0 && (Mask[i] < 4 || Mask[i] > 7))
Evan Cheng506d3df2006-03-29 23:07:14 +00002311 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002312
Evan Cheng506d3df2006-03-29 23:07:14 +00002313 return true;
2314}
2315
Nate Begeman9008ca62009-04-27 18:41:29 +00002316bool X86::isPSHUFHWMask(ShuffleVectorSDNode *N) {
2317 SmallVector<int, 8> M;
2318 N->getMask(M);
2319 return ::isPSHUFHWMask(M, N->getValueType(0));
2320}
Evan Cheng506d3df2006-03-29 23:07:14 +00002321
Nate Begeman9008ca62009-04-27 18:41:29 +00002322/// isPSHUFLWMask - Return true if the node specifies a shuffle of elements that
2323/// is suitable for input to PSHUFLW.
Owen Andersone50ed302009-08-10 22:56:29 +00002324static bool isPSHUFLWMask(const SmallVectorImpl<int> &Mask, EVT VT) {
Owen Anderson825b72b2009-08-11 20:47:22 +00002325 if (VT != MVT::v8i16)
Evan Cheng506d3df2006-03-29 23:07:14 +00002326 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002327
Rafael Espindola15684b22009-04-24 12:40:33 +00002328 // Upper quadword copied in order.
Nate Begeman9008ca62009-04-27 18:41:29 +00002329 for (int i = 4; i != 8; ++i)
2330 if (Mask[i] >= 0 && Mask[i] != i)
Rafael Espindola15684b22009-04-24 12:40:33 +00002331 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002332
Rafael Espindola15684b22009-04-24 12:40:33 +00002333 // Lower quadword shuffled.
Nate Begeman9008ca62009-04-27 18:41:29 +00002334 for (int i = 0; i != 4; ++i)
2335 if (Mask[i] >= 4)
Rafael Espindola15684b22009-04-24 12:40:33 +00002336 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002337
Rafael Espindola15684b22009-04-24 12:40:33 +00002338 return true;
Nate Begemanb706d292009-04-24 03:42:54 +00002339}
2340
Nate Begeman9008ca62009-04-27 18:41:29 +00002341bool X86::isPSHUFLWMask(ShuffleVectorSDNode *N) {
2342 SmallVector<int, 8> M;
2343 N->getMask(M);
2344 return ::isPSHUFLWMask(M, N->getValueType(0));
2345}
2346
Evan Cheng14aed5e2006-03-24 01:18:28 +00002347/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand
2348/// specifies a shuffle of elements that is suitable for input to SHUFP*.
Owen Andersone50ed302009-08-10 22:56:29 +00002349static bool isSHUFPMask(const SmallVectorImpl<int> &Mask, EVT VT) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002350 int NumElems = VT.getVectorNumElements();
2351 if (NumElems != 2 && NumElems != 4)
2352 return false;
2353
2354 int Half = NumElems / 2;
2355 for (int i = 0; i < Half; ++i)
2356 if (!isUndefOrInRange(Mask[i], 0, NumElems))
Evan Cheng39623da2006-04-20 08:58:49 +00002357 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002358 for (int i = Half; i < NumElems; ++i)
2359 if (!isUndefOrInRange(Mask[i], NumElems, NumElems*2))
Evan Cheng39623da2006-04-20 08:58:49 +00002360 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002361
Evan Cheng14aed5e2006-03-24 01:18:28 +00002362 return true;
2363}
2364
Nate Begeman9008ca62009-04-27 18:41:29 +00002365bool X86::isSHUFPMask(ShuffleVectorSDNode *N) {
2366 SmallVector<int, 8> M;
2367 N->getMask(M);
2368 return ::isSHUFPMask(M, N->getValueType(0));
Evan Cheng39623da2006-04-20 08:58:49 +00002369}
2370
Evan Cheng213d2cf2007-05-17 18:45:50 +00002371/// isCommutedSHUFP - Returns true if the shuffle mask is exactly
Evan Cheng39623da2006-04-20 08:58:49 +00002372/// the reverse of what x86 shuffles want. x86 shuffles requires the lower
2373/// half elements to come from vector 1 (which would equal the dest.) and
2374/// the upper half to come from vector 2.
Owen Andersone50ed302009-08-10 22:56:29 +00002375static bool isCommutedSHUFPMask(const SmallVectorImpl<int> &Mask, EVT VT) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002376 int NumElems = VT.getVectorNumElements();
2377
2378 if (NumElems != 2 && NumElems != 4)
2379 return false;
2380
2381 int Half = NumElems / 2;
2382 for (int i = 0; i < Half; ++i)
2383 if (!isUndefOrInRange(Mask[i], NumElems, NumElems*2))
Evan Cheng39623da2006-04-20 08:58:49 +00002384 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002385 for (int i = Half; i < NumElems; ++i)
2386 if (!isUndefOrInRange(Mask[i], 0, NumElems))
Evan Cheng39623da2006-04-20 08:58:49 +00002387 return false;
2388 return true;
2389}
2390
Nate Begeman9008ca62009-04-27 18:41:29 +00002391static bool isCommutedSHUFP(ShuffleVectorSDNode *N) {
2392 SmallVector<int, 8> M;
2393 N->getMask(M);
2394 return isCommutedSHUFPMask(M, N->getValueType(0));
Evan Cheng39623da2006-04-20 08:58:49 +00002395}
2396
Evan Cheng2c0dbd02006-03-24 02:58:06 +00002397/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand
2398/// specifies a shuffle of elements that is suitable for input to MOVHLPS.
Nate Begeman9008ca62009-04-27 18:41:29 +00002399bool X86::isMOVHLPSMask(ShuffleVectorSDNode *N) {
2400 if (N->getValueType(0).getVectorNumElements() != 4)
Evan Cheng2c0dbd02006-03-24 02:58:06 +00002401 return false;
2402
Evan Cheng2064a2b2006-03-28 06:50:32 +00002403 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3
Nate Begeman9008ca62009-04-27 18:41:29 +00002404 return isUndefOrEqual(N->getMaskElt(0), 6) &&
2405 isUndefOrEqual(N->getMaskElt(1), 7) &&
2406 isUndefOrEqual(N->getMaskElt(2), 2) &&
2407 isUndefOrEqual(N->getMaskElt(3), 3);
Evan Cheng6e56e2c2006-11-07 22:14:24 +00002408}
2409
Evan Cheng5ced1d82006-04-06 23:23:56 +00002410/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand
2411/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}.
Nate Begeman9008ca62009-04-27 18:41:29 +00002412bool X86::isMOVLPMask(ShuffleVectorSDNode *N) {
2413 unsigned NumElems = N->getValueType(0).getVectorNumElements();
Evan Cheng5ced1d82006-04-06 23:23:56 +00002414
Evan Cheng5ced1d82006-04-06 23:23:56 +00002415 if (NumElems != 2 && NumElems != 4)
2416 return false;
2417
Evan Chengc5cdff22006-04-07 21:53:05 +00002418 for (unsigned i = 0; i < NumElems/2; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00002419 if (!isUndefOrEqual(N->getMaskElt(i), i + NumElems))
Evan Chengc5cdff22006-04-07 21:53:05 +00002420 return false;
Evan Cheng5ced1d82006-04-06 23:23:56 +00002421
Evan Chengc5cdff22006-04-07 21:53:05 +00002422 for (unsigned i = NumElems/2; i < NumElems; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00002423 if (!isUndefOrEqual(N->getMaskElt(i), i))
Evan Chengc5cdff22006-04-07 21:53:05 +00002424 return false;
Evan Cheng5ced1d82006-04-06 23:23:56 +00002425
2426 return true;
2427}
2428
2429/// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand
Evan Cheng533a0aa2006-04-19 20:35:22 +00002430/// specifies a shuffle of elements that is suitable for input to MOVHP{S|D}
2431/// and MOVLHPS.
Nate Begeman9008ca62009-04-27 18:41:29 +00002432bool X86::isMOVHPMask(ShuffleVectorSDNode *N) {
2433 unsigned NumElems = N->getValueType(0).getVectorNumElements();
Evan Cheng5ced1d82006-04-06 23:23:56 +00002434
Evan Cheng5ced1d82006-04-06 23:23:56 +00002435 if (NumElems != 2 && NumElems != 4)
2436 return false;
2437
Evan Chengc5cdff22006-04-07 21:53:05 +00002438 for (unsigned i = 0; i < NumElems/2; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00002439 if (!isUndefOrEqual(N->getMaskElt(i), i))
Evan Chengc5cdff22006-04-07 21:53:05 +00002440 return false;
Evan Cheng5ced1d82006-04-06 23:23:56 +00002441
Nate Begeman9008ca62009-04-27 18:41:29 +00002442 for (unsigned i = 0; i < NumElems/2; ++i)
2443 if (!isUndefOrEqual(N->getMaskElt(i + NumElems/2), i + NumElems))
Evan Chengc5cdff22006-04-07 21:53:05 +00002444 return false;
Evan Cheng5ced1d82006-04-06 23:23:56 +00002445
2446 return true;
2447}
2448
Nate Begeman9008ca62009-04-27 18:41:29 +00002449/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form
2450/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef,
2451/// <2, 3, 2, 3>
2452bool X86::isMOVHLPS_v_undef_Mask(ShuffleVectorSDNode *N) {
2453 unsigned NumElems = N->getValueType(0).getVectorNumElements();
2454
2455 if (NumElems != 4)
2456 return false;
2457
2458 return isUndefOrEqual(N->getMaskElt(0), 2) &&
2459 isUndefOrEqual(N->getMaskElt(1), 3) &&
2460 isUndefOrEqual(N->getMaskElt(2), 2) &&
2461 isUndefOrEqual(N->getMaskElt(3), 3);
2462}
2463
Evan Cheng0038e592006-03-28 00:39:58 +00002464/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand
2465/// specifies a shuffle of elements that is suitable for input to UNPCKL.
Owen Andersone50ed302009-08-10 22:56:29 +00002466static bool isUNPCKLMask(const SmallVectorImpl<int> &Mask, EVT VT,
Rafael Espindola15684b22009-04-24 12:40:33 +00002467 bool V2IsSplat = false) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002468 int NumElts = VT.getVectorNumElements();
Chris Lattner5a88b832007-02-25 07:10:00 +00002469 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
Evan Cheng0038e592006-03-28 00:39:58 +00002470 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002471
2472 for (int i = 0, j = 0; i != NumElts; i += 2, ++j) {
2473 int BitI = Mask[i];
2474 int BitI1 = Mask[i+1];
Evan Chengc5cdff22006-04-07 21:53:05 +00002475 if (!isUndefOrEqual(BitI, j))
2476 return false;
Evan Cheng39623da2006-04-20 08:58:49 +00002477 if (V2IsSplat) {
Mon P Wang7bcaefa2009-02-04 01:16:59 +00002478 if (!isUndefOrEqual(BitI1, NumElts))
Evan Cheng39623da2006-04-20 08:58:49 +00002479 return false;
2480 } else {
Chris Lattner5a88b832007-02-25 07:10:00 +00002481 if (!isUndefOrEqual(BitI1, j + NumElts))
Evan Cheng39623da2006-04-20 08:58:49 +00002482 return false;
2483 }
Evan Cheng0038e592006-03-28 00:39:58 +00002484 }
Evan Cheng0038e592006-03-28 00:39:58 +00002485 return true;
2486}
2487
Nate Begeman9008ca62009-04-27 18:41:29 +00002488bool X86::isUNPCKLMask(ShuffleVectorSDNode *N, bool V2IsSplat) {
2489 SmallVector<int, 8> M;
2490 N->getMask(M);
2491 return ::isUNPCKLMask(M, N->getValueType(0), V2IsSplat);
Evan Cheng39623da2006-04-20 08:58:49 +00002492}
2493
Evan Cheng4fcb9222006-03-28 02:43:26 +00002494/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand
2495/// specifies a shuffle of elements that is suitable for input to UNPCKH.
Owen Andersone50ed302009-08-10 22:56:29 +00002496static bool isUNPCKHMask(const SmallVectorImpl<int> &Mask, EVT VT,
Rafael Espindola15684b22009-04-24 12:40:33 +00002497 bool V2IsSplat = false) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002498 int NumElts = VT.getVectorNumElements();
Chris Lattner5a88b832007-02-25 07:10:00 +00002499 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
Evan Cheng4fcb9222006-03-28 02:43:26 +00002500 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002501
2502 for (int i = 0, j = 0; i != NumElts; i += 2, ++j) {
2503 int BitI = Mask[i];
2504 int BitI1 = Mask[i+1];
Chris Lattner5a88b832007-02-25 07:10:00 +00002505 if (!isUndefOrEqual(BitI, j + NumElts/2))
Evan Chengc5cdff22006-04-07 21:53:05 +00002506 return false;
Evan Cheng39623da2006-04-20 08:58:49 +00002507 if (V2IsSplat) {
Chris Lattner5a88b832007-02-25 07:10:00 +00002508 if (isUndefOrEqual(BitI1, NumElts))
Evan Cheng39623da2006-04-20 08:58:49 +00002509 return false;
2510 } else {
Chris Lattner5a88b832007-02-25 07:10:00 +00002511 if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts))
Evan Cheng39623da2006-04-20 08:58:49 +00002512 return false;
2513 }
Evan Cheng4fcb9222006-03-28 02:43:26 +00002514 }
Evan Cheng4fcb9222006-03-28 02:43:26 +00002515 return true;
2516}
2517
Nate Begeman9008ca62009-04-27 18:41:29 +00002518bool X86::isUNPCKHMask(ShuffleVectorSDNode *N, bool V2IsSplat) {
2519 SmallVector<int, 8> M;
2520 N->getMask(M);
2521 return ::isUNPCKHMask(M, N->getValueType(0), V2IsSplat);
Evan Cheng39623da2006-04-20 08:58:49 +00002522}
2523
Evan Cheng1d5a8cc2006-04-05 07:20:06 +00002524/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form
2525/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef,
2526/// <0, 0, 1, 1>
Owen Andersone50ed302009-08-10 22:56:29 +00002527static bool isUNPCKL_v_undef_Mask(const SmallVectorImpl<int> &Mask, EVT VT) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002528 int NumElems = VT.getVectorNumElements();
Bill Wendling2f9bb1a2007-04-24 21:16:55 +00002529 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16)
Evan Cheng1d5a8cc2006-04-05 07:20:06 +00002530 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002531
2532 for (int i = 0, j = 0; i != NumElems; i += 2, ++j) {
2533 int BitI = Mask[i];
2534 int BitI1 = Mask[i+1];
Evan Chengc5cdff22006-04-07 21:53:05 +00002535 if (!isUndefOrEqual(BitI, j))
2536 return false;
2537 if (!isUndefOrEqual(BitI1, j))
2538 return false;
Evan Cheng1d5a8cc2006-04-05 07:20:06 +00002539 }
Rafael Espindola15684b22009-04-24 12:40:33 +00002540 return true;
Nate Begemanb706d292009-04-24 03:42:54 +00002541}
2542
Nate Begeman9008ca62009-04-27 18:41:29 +00002543bool X86::isUNPCKL_v_undef_Mask(ShuffleVectorSDNode *N) {
2544 SmallVector<int, 8> M;
2545 N->getMask(M);
2546 return ::isUNPCKL_v_undef_Mask(M, N->getValueType(0));
2547}
2548
Bill Wendling2f9bb1a2007-04-24 21:16:55 +00002549/// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form
2550/// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef,
2551/// <2, 2, 3, 3>
Owen Andersone50ed302009-08-10 22:56:29 +00002552static bool isUNPCKH_v_undef_Mask(const SmallVectorImpl<int> &Mask, EVT VT) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002553 int NumElems = VT.getVectorNumElements();
Bill Wendling2f9bb1a2007-04-24 21:16:55 +00002554 if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16)
2555 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002556
2557 for (int i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) {
2558 int BitI = Mask[i];
2559 int BitI1 = Mask[i+1];
Bill Wendling2f9bb1a2007-04-24 21:16:55 +00002560 if (!isUndefOrEqual(BitI, j))
2561 return false;
2562 if (!isUndefOrEqual(BitI1, j))
2563 return false;
2564 }
Rafael Espindola15684b22009-04-24 12:40:33 +00002565 return true;
Nate Begemanb706d292009-04-24 03:42:54 +00002566}
2567
Nate Begeman9008ca62009-04-27 18:41:29 +00002568bool X86::isUNPCKH_v_undef_Mask(ShuffleVectorSDNode *N) {
2569 SmallVector<int, 8> M;
2570 N->getMask(M);
2571 return ::isUNPCKH_v_undef_Mask(M, N->getValueType(0));
2572}
2573
Evan Cheng017dcc62006-04-21 01:05:10 +00002574/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand
2575/// specifies a shuffle of elements that is suitable for input to MOVSS,
2576/// MOVSD, and MOVD, i.e. setting the lowest element.
Owen Andersone50ed302009-08-10 22:56:29 +00002577static bool isMOVLMask(const SmallVectorImpl<int> &Mask, EVT VT) {
Eli Friedman10415532009-06-06 06:05:10 +00002578 if (VT.getVectorElementType().getSizeInBits() < 32)
Evan Chengd6d1cbd2006-04-11 00:19:04 +00002579 return false;
Eli Friedman10415532009-06-06 06:05:10 +00002580
2581 int NumElts = VT.getVectorNumElements();
Nate Begeman9008ca62009-04-27 18:41:29 +00002582
2583 if (!isUndefOrEqual(Mask[0], NumElts))
Evan Chengd6d1cbd2006-04-11 00:19:04 +00002584 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002585
2586 for (int i = 1; i < NumElts; ++i)
2587 if (!isUndefOrEqual(Mask[i], i))
Evan Chengd6d1cbd2006-04-11 00:19:04 +00002588 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002589
Evan Chengd6d1cbd2006-04-11 00:19:04 +00002590 return true;
2591}
Evan Cheng1d5a8cc2006-04-05 07:20:06 +00002592
Nate Begeman9008ca62009-04-27 18:41:29 +00002593bool X86::isMOVLMask(ShuffleVectorSDNode *N) {
2594 SmallVector<int, 8> M;
2595 N->getMask(M);
2596 return ::isMOVLMask(M, N->getValueType(0));
Evan Cheng39623da2006-04-20 08:58:49 +00002597}
2598
Evan Cheng017dcc62006-04-21 01:05:10 +00002599/// isCommutedMOVL - Returns true if the shuffle mask is except the reverse
2600/// of what x86 movss want. X86 movs requires the lowest element to be lowest
Evan Cheng39623da2006-04-20 08:58:49 +00002601/// element of vector 2 and the other elements to come from vector 1 in order.
Owen Andersone50ed302009-08-10 22:56:29 +00002602static bool isCommutedMOVLMask(const SmallVectorImpl<int> &Mask, EVT VT,
Nate Begeman9008ca62009-04-27 18:41:29 +00002603 bool V2IsSplat = false, bool V2IsUndef = false) {
2604 int NumOps = VT.getVectorNumElements();
Chris Lattner5a88b832007-02-25 07:10:00 +00002605 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16)
Evan Cheng39623da2006-04-20 08:58:49 +00002606 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002607
2608 if (!isUndefOrEqual(Mask[0], 0))
Evan Cheng39623da2006-04-20 08:58:49 +00002609 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002610
2611 for (int i = 1; i < NumOps; ++i)
2612 if (!(isUndefOrEqual(Mask[i], i+NumOps) ||
2613 (V2IsUndef && isUndefOrInRange(Mask[i], NumOps, NumOps*2)) ||
2614 (V2IsSplat && isUndefOrEqual(Mask[i], NumOps))))
Evan Cheng8cf723d2006-09-08 01:50:06 +00002615 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002616
Evan Cheng39623da2006-04-20 08:58:49 +00002617 return true;
2618}
2619
Nate Begeman9008ca62009-04-27 18:41:29 +00002620static bool isCommutedMOVL(ShuffleVectorSDNode *N, bool V2IsSplat = false,
Evan Cheng8cf723d2006-09-08 01:50:06 +00002621 bool V2IsUndef = false) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002622 SmallVector<int, 8> M;
2623 N->getMask(M);
2624 return isCommutedMOVLMask(M, N->getValueType(0), V2IsSplat, V2IsUndef);
Evan Cheng39623da2006-04-20 08:58:49 +00002625}
2626
Evan Chengd9539472006-04-14 21:59:03 +00002627/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand
2628/// specifies a shuffle of elements that is suitable for input to MOVSHDUP.
Nate Begeman9008ca62009-04-27 18:41:29 +00002629bool X86::isMOVSHDUPMask(ShuffleVectorSDNode *N) {
2630 if (N->getValueType(0).getVectorNumElements() != 4)
Evan Chengd9539472006-04-14 21:59:03 +00002631 return false;
2632
2633 // Expect 1, 1, 3, 3
Rafael Espindola15684b22009-04-24 12:40:33 +00002634 for (unsigned i = 0; i < 2; ++i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002635 int Elt = N->getMaskElt(i);
2636 if (Elt >= 0 && Elt != 1)
2637 return false;
Rafael Espindola15684b22009-04-24 12:40:33 +00002638 }
Evan Cheng57ebe9f2006-04-15 05:37:34 +00002639
2640 bool HasHi = false;
Evan Chengd9539472006-04-14 21:59:03 +00002641 for (unsigned i = 2; i < 4; ++i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002642 int Elt = N->getMaskElt(i);
2643 if (Elt >= 0 && Elt != 3)
2644 return false;
2645 if (Elt == 3)
2646 HasHi = true;
Evan Chengd9539472006-04-14 21:59:03 +00002647 }
Evan Cheng57ebe9f2006-04-15 05:37:34 +00002648 // Don't use movshdup if it can be done with a shufps.
Nate Begeman9008ca62009-04-27 18:41:29 +00002649 // FIXME: verify that matching u, u, 3, 3 is what we want.
Evan Cheng57ebe9f2006-04-15 05:37:34 +00002650 return HasHi;
Evan Chengd9539472006-04-14 21:59:03 +00002651}
2652
2653/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand
2654/// specifies a shuffle of elements that is suitable for input to MOVSLDUP.
Nate Begeman9008ca62009-04-27 18:41:29 +00002655bool X86::isMOVSLDUPMask(ShuffleVectorSDNode *N) {
2656 if (N->getValueType(0).getVectorNumElements() != 4)
Evan Chengd9539472006-04-14 21:59:03 +00002657 return false;
2658
2659 // Expect 0, 0, 2, 2
Nate Begeman9008ca62009-04-27 18:41:29 +00002660 for (unsigned i = 0; i < 2; ++i)
2661 if (N->getMaskElt(i) > 0)
2662 return false;
Evan Cheng57ebe9f2006-04-15 05:37:34 +00002663
2664 bool HasHi = false;
Evan Chengd9539472006-04-14 21:59:03 +00002665 for (unsigned i = 2; i < 4; ++i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002666 int Elt = N->getMaskElt(i);
2667 if (Elt >= 0 && Elt != 2)
2668 return false;
2669 if (Elt == 2)
2670 HasHi = true;
Evan Chengd9539472006-04-14 21:59:03 +00002671 }
Nate Begeman9008ca62009-04-27 18:41:29 +00002672 // Don't use movsldup if it can be done with a shufps.
Evan Cheng57ebe9f2006-04-15 05:37:34 +00002673 return HasHi;
Evan Chengd9539472006-04-14 21:59:03 +00002674}
2675
Evan Cheng0b457f02008-09-25 20:50:48 +00002676/// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand
2677/// specifies a shuffle of elements that is suitable for input to MOVDDUP.
Nate Begeman9008ca62009-04-27 18:41:29 +00002678bool X86::isMOVDDUPMask(ShuffleVectorSDNode *N) {
2679 int e = N->getValueType(0).getVectorNumElements() / 2;
2680
2681 for (int i = 0; i < e; ++i)
2682 if (!isUndefOrEqual(N->getMaskElt(i), i))
Evan Cheng0b457f02008-09-25 20:50:48 +00002683 return false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002684 for (int i = 0; i < e; ++i)
2685 if (!isUndefOrEqual(N->getMaskElt(e+i), i))
Evan Cheng0b457f02008-09-25 20:50:48 +00002686 return false;
2687 return true;
2688}
2689
Evan Cheng63d33002006-03-22 08:01:21 +00002690/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle
2691/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP*
2692/// instructions.
2693unsigned X86::getShuffleSHUFImmediate(SDNode *N) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002694 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
2695 int NumOperands = SVOp->getValueType(0).getVectorNumElements();
2696
Evan Chengb9df0ca2006-03-22 02:53:00 +00002697 unsigned Shift = (NumOperands == 4) ? 2 : 1;
2698 unsigned Mask = 0;
Nate Begeman9008ca62009-04-27 18:41:29 +00002699 for (int i = 0; i < NumOperands; ++i) {
2700 int Val = SVOp->getMaskElt(NumOperands-i-1);
2701 if (Val < 0) Val = 0;
Evan Cheng14aed5e2006-03-24 01:18:28 +00002702 if (Val >= NumOperands) Val -= NumOperands;
Evan Cheng63d33002006-03-22 08:01:21 +00002703 Mask |= Val;
Evan Cheng36b27f32006-03-28 23:41:33 +00002704 if (i != NumOperands - 1)
2705 Mask <<= Shift;
2706 }
Evan Cheng63d33002006-03-22 08:01:21 +00002707 return Mask;
2708}
2709
Evan Cheng506d3df2006-03-29 23:07:14 +00002710/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle
2711/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW
2712/// instructions.
2713unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002714 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
Evan Cheng506d3df2006-03-29 23:07:14 +00002715 unsigned Mask = 0;
2716 // 8 nodes, but we only care about the last 4.
2717 for (unsigned i = 7; i >= 4; --i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002718 int Val = SVOp->getMaskElt(i);
2719 if (Val >= 0)
Mon P Wang7bcaefa2009-02-04 01:16:59 +00002720 Mask |= (Val - 4);
Evan Cheng506d3df2006-03-29 23:07:14 +00002721 if (i != 4)
2722 Mask <<= 2;
2723 }
Evan Cheng506d3df2006-03-29 23:07:14 +00002724 return Mask;
2725}
2726
2727/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle
2728/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW
2729/// instructions.
2730unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002731 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
Evan Cheng506d3df2006-03-29 23:07:14 +00002732 unsigned Mask = 0;
2733 // 8 nodes, but we only care about the first 4.
2734 for (int i = 3; i >= 0; --i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002735 int Val = SVOp->getMaskElt(i);
2736 if (Val >= 0)
2737 Mask |= Val;
Evan Cheng506d3df2006-03-29 23:07:14 +00002738 if (i != 0)
2739 Mask <<= 2;
2740 }
Evan Cheng506d3df2006-03-29 23:07:14 +00002741 return Mask;
2742}
2743
Evan Cheng37b73872009-07-30 08:33:02 +00002744/// isZeroNode - Returns true if Elt is a constant zero or a floating point
2745/// constant +0.0.
2746bool X86::isZeroNode(SDValue Elt) {
2747 return ((isa<ConstantSDNode>(Elt) &&
2748 cast<ConstantSDNode>(Elt)->getZExtValue() == 0) ||
2749 (isa<ConstantFPSDNode>(Elt) &&
2750 cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero()));
2751}
2752
Nate Begeman9008ca62009-04-27 18:41:29 +00002753/// CommuteVectorShuffle - Swap vector_shuffle operands as well as values in
2754/// their permute mask.
2755static SDValue CommuteVectorShuffle(ShuffleVectorSDNode *SVOp,
2756 SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00002757 EVT VT = SVOp->getValueType(0);
Nate Begeman5a5ca152009-04-29 05:20:52 +00002758 unsigned NumElems = VT.getVectorNumElements();
Nate Begeman9008ca62009-04-27 18:41:29 +00002759 SmallVector<int, 8> MaskVec;
2760
Nate Begeman5a5ca152009-04-29 05:20:52 +00002761 for (unsigned i = 0; i != NumElems; ++i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002762 int idx = SVOp->getMaskElt(i);
2763 if (idx < 0)
2764 MaskVec.push_back(idx);
Nate Begeman5a5ca152009-04-29 05:20:52 +00002765 else if (idx < (int)NumElems)
Nate Begeman9008ca62009-04-27 18:41:29 +00002766 MaskVec.push_back(idx + NumElems);
Evan Cheng5ced1d82006-04-06 23:23:56 +00002767 else
Nate Begeman9008ca62009-04-27 18:41:29 +00002768 MaskVec.push_back(idx - NumElems);
Evan Cheng5ced1d82006-04-06 23:23:56 +00002769 }
Nate Begeman9008ca62009-04-27 18:41:29 +00002770 return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(1),
2771 SVOp->getOperand(0), &MaskVec[0]);
Evan Cheng5ced1d82006-04-06 23:23:56 +00002772}
2773
Evan Cheng779ccea2007-12-07 21:30:01 +00002774/// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming
2775/// the two vector operands have swapped position.
Owen Andersone50ed302009-08-10 22:56:29 +00002776static void CommuteVectorShuffleMask(SmallVectorImpl<int> &Mask, EVT VT) {
Nate Begeman5a5ca152009-04-29 05:20:52 +00002777 unsigned NumElems = VT.getVectorNumElements();
2778 for (unsigned i = 0; i != NumElems; ++i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002779 int idx = Mask[i];
2780 if (idx < 0)
Evan Cheng8a86c3f2007-12-07 08:07:39 +00002781 continue;
Nate Begeman5a5ca152009-04-29 05:20:52 +00002782 else if (idx < (int)NumElems)
Nate Begeman9008ca62009-04-27 18:41:29 +00002783 Mask[i] = idx + NumElems;
Evan Cheng8a86c3f2007-12-07 08:07:39 +00002784 else
Nate Begeman9008ca62009-04-27 18:41:29 +00002785 Mask[i] = idx - NumElems;
Evan Cheng8a86c3f2007-12-07 08:07:39 +00002786 }
Evan Cheng8a86c3f2007-12-07 08:07:39 +00002787}
2788
Evan Cheng533a0aa2006-04-19 20:35:22 +00002789/// ShouldXformToMOVHLPS - Return true if the node should be transformed to
2790/// match movhlps. The lower half elements should come from upper half of
2791/// V1 (and in order), and the upper half elements should come from the upper
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00002792/// half of V2 (and in order).
Nate Begeman9008ca62009-04-27 18:41:29 +00002793static bool ShouldXformToMOVHLPS(ShuffleVectorSDNode *Op) {
2794 if (Op->getValueType(0).getVectorNumElements() != 4)
Evan Cheng533a0aa2006-04-19 20:35:22 +00002795 return false;
2796 for (unsigned i = 0, e = 2; i != e; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00002797 if (!isUndefOrEqual(Op->getMaskElt(i), i+2))
Evan Cheng533a0aa2006-04-19 20:35:22 +00002798 return false;
2799 for (unsigned i = 2; i != 4; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00002800 if (!isUndefOrEqual(Op->getMaskElt(i), i+4))
Evan Cheng533a0aa2006-04-19 20:35:22 +00002801 return false;
2802 return true;
2803}
2804
Evan Cheng5ced1d82006-04-06 23:23:56 +00002805/// isScalarLoadToVector - Returns true if the node is a scalar load that
Evan Cheng7e2ff772008-05-08 00:57:18 +00002806/// is promoted to a vector. It also returns the LoadSDNode by reference if
2807/// required.
2808static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) {
Evan Cheng0b457f02008-09-25 20:50:48 +00002809 if (N->getOpcode() != ISD::SCALAR_TO_VECTOR)
2810 return false;
2811 N = N->getOperand(0).getNode();
2812 if (!ISD::isNON_EXTLoad(N))
2813 return false;
2814 if (LD)
2815 *LD = cast<LoadSDNode>(N);
2816 return true;
Evan Cheng5ced1d82006-04-06 23:23:56 +00002817}
2818
Evan Cheng533a0aa2006-04-19 20:35:22 +00002819/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to
2820/// match movlp{s|d}. The lower half elements should come from lower half of
2821/// V1 (and in order), and the upper half elements should come from the upper
2822/// half of V2 (and in order). And since V1 will become the source of the
2823/// MOVLP, it must be either a vector load or a scalar load to vector.
Nate Begeman9008ca62009-04-27 18:41:29 +00002824static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2,
2825 ShuffleVectorSDNode *Op) {
Evan Cheng466685d2006-10-09 20:57:25 +00002826 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1))
Evan Cheng533a0aa2006-04-19 20:35:22 +00002827 return false;
Evan Cheng23425f52006-10-09 21:39:25 +00002828 // Is V2 is a vector load, don't do this transformation. We will try to use
2829 // load folding shufps op.
2830 if (ISD::isNON_EXTLoad(V2))
2831 return false;
Evan Cheng5ced1d82006-04-06 23:23:56 +00002832
Nate Begeman5a5ca152009-04-29 05:20:52 +00002833 unsigned NumElems = Op->getValueType(0).getVectorNumElements();
Nate Begeman9008ca62009-04-27 18:41:29 +00002834
Evan Cheng533a0aa2006-04-19 20:35:22 +00002835 if (NumElems != 2 && NumElems != 4)
2836 return false;
Nate Begeman5a5ca152009-04-29 05:20:52 +00002837 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00002838 if (!isUndefOrEqual(Op->getMaskElt(i), i))
Evan Cheng533a0aa2006-04-19 20:35:22 +00002839 return false;
Nate Begeman5a5ca152009-04-29 05:20:52 +00002840 for (unsigned i = NumElems/2; i != NumElems; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00002841 if (!isUndefOrEqual(Op->getMaskElt(i), i+NumElems))
Evan Cheng533a0aa2006-04-19 20:35:22 +00002842 return false;
2843 return true;
Evan Cheng5ced1d82006-04-06 23:23:56 +00002844}
2845
Evan Cheng39623da2006-04-20 08:58:49 +00002846/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are
2847/// all the same.
2848static bool isSplatVector(SDNode *N) {
2849 if (N->getOpcode() != ISD::BUILD_VECTOR)
2850 return false;
Evan Cheng5ced1d82006-04-06 23:23:56 +00002851
Dan Gohman475871a2008-07-27 21:46:04 +00002852 SDValue SplatValue = N->getOperand(0);
Evan Cheng39623da2006-04-20 08:58:49 +00002853 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
2854 if (N->getOperand(i) != SplatValue)
Evan Cheng5ced1d82006-04-06 23:23:56 +00002855 return false;
2856 return true;
2857}
2858
Evan Cheng213d2cf2007-05-17 18:45:50 +00002859/// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved
Nate Begeman9008ca62009-04-27 18:41:29 +00002860/// to an zero vector.
Nate Begeman5a5ca152009-04-29 05:20:52 +00002861/// FIXME: move to dag combiner / method on ShuffleVectorSDNode
Nate Begeman9008ca62009-04-27 18:41:29 +00002862static bool isZeroShuffle(ShuffleVectorSDNode *N) {
Dan Gohman475871a2008-07-27 21:46:04 +00002863 SDValue V1 = N->getOperand(0);
2864 SDValue V2 = N->getOperand(1);
Nate Begeman5a5ca152009-04-29 05:20:52 +00002865 unsigned NumElems = N->getValueType(0).getVectorNumElements();
2866 for (unsigned i = 0; i != NumElems; ++i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002867 int Idx = N->getMaskElt(i);
Nate Begeman5a5ca152009-04-29 05:20:52 +00002868 if (Idx >= (int)NumElems) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002869 unsigned Opc = V2.getOpcode();
Rafael Espindola15684b22009-04-24 12:40:33 +00002870 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode()))
2871 continue;
Evan Cheng37b73872009-07-30 08:33:02 +00002872 if (Opc != ISD::BUILD_VECTOR ||
2873 !X86::isZeroNode(V2.getOperand(Idx-NumElems)))
Nate Begeman9008ca62009-04-27 18:41:29 +00002874 return false;
2875 } else if (Idx >= 0) {
2876 unsigned Opc = V1.getOpcode();
2877 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode()))
2878 continue;
Evan Cheng37b73872009-07-30 08:33:02 +00002879 if (Opc != ISD::BUILD_VECTOR ||
2880 !X86::isZeroNode(V1.getOperand(Idx)))
Chris Lattner8a594482007-11-25 00:24:49 +00002881 return false;
Evan Cheng213d2cf2007-05-17 18:45:50 +00002882 }
2883 }
2884 return true;
2885}
2886
2887/// getZeroVector - Returns a vector of specified type with all zero elements.
2888///
Owen Andersone50ed302009-08-10 22:56:29 +00002889static SDValue getZeroVector(EVT VT, bool HasSSE2, SelectionDAG &DAG,
Dale Johannesenace16102009-02-03 19:33:06 +00002890 DebugLoc dl) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00002891 assert(VT.isVector() && "Expected a vector type");
Scott Michelfdc40a02009-02-17 22:15:04 +00002892
Chris Lattner8a594482007-11-25 00:24:49 +00002893 // Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest
2894 // type. This ensures they get CSE'd.
Dan Gohman475871a2008-07-27 21:46:04 +00002895 SDValue Vec;
Duncan Sands83ec4b62008-06-06 12:08:01 +00002896 if (VT.getSizeInBits() == 64) { // MMX
Owen Anderson825b72b2009-08-11 20:47:22 +00002897 SDValue Cst = DAG.getTargetConstant(0, MVT::i32);
2898 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i32, Cst, Cst);
Evan Chengf0df0312008-05-15 08:39:06 +00002899 } else if (HasSSE2) { // SSE2
Owen Anderson825b72b2009-08-11 20:47:22 +00002900 SDValue Cst = DAG.getTargetConstant(0, MVT::i32);
2901 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
Evan Chengf0df0312008-05-15 08:39:06 +00002902 } else { // SSE1
Owen Anderson825b72b2009-08-11 20:47:22 +00002903 SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32);
2904 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst);
Evan Chengf0df0312008-05-15 08:39:06 +00002905 }
Dale Johannesenace16102009-02-03 19:33:06 +00002906 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Vec);
Evan Cheng213d2cf2007-05-17 18:45:50 +00002907}
2908
Chris Lattner8a594482007-11-25 00:24:49 +00002909/// getOnesVector - Returns a vector of specified type with all bits set.
2910///
Owen Andersone50ed302009-08-10 22:56:29 +00002911static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, DebugLoc dl) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00002912 assert(VT.isVector() && "Expected a vector type");
Scott Michelfdc40a02009-02-17 22:15:04 +00002913
Chris Lattner8a594482007-11-25 00:24:49 +00002914 // Always build ones vectors as <4 x i32> or <2 x i32> bitcasted to their dest
2915 // type. This ensures they get CSE'd.
Owen Anderson825b72b2009-08-11 20:47:22 +00002916 SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32);
Dan Gohman475871a2008-07-27 21:46:04 +00002917 SDValue Vec;
Duncan Sands83ec4b62008-06-06 12:08:01 +00002918 if (VT.getSizeInBits() == 64) // MMX
Owen Anderson825b72b2009-08-11 20:47:22 +00002919 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i32, Cst, Cst);
Chris Lattner8a594482007-11-25 00:24:49 +00002920 else // SSE
Owen Anderson825b72b2009-08-11 20:47:22 +00002921 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
Dale Johannesenace16102009-02-03 19:33:06 +00002922 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Vec);
Chris Lattner8a594482007-11-25 00:24:49 +00002923}
2924
2925
Evan Cheng39623da2006-04-20 08:58:49 +00002926/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements
2927/// that point to V2 points to its first element.
Nate Begeman9008ca62009-04-27 18:41:29 +00002928static SDValue NormalizeMask(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00002929 EVT VT = SVOp->getValueType(0);
Nate Begeman5a5ca152009-04-29 05:20:52 +00002930 unsigned NumElems = VT.getVectorNumElements();
Nate Begeman9008ca62009-04-27 18:41:29 +00002931
Evan Cheng39623da2006-04-20 08:58:49 +00002932 bool Changed = false;
Nate Begeman9008ca62009-04-27 18:41:29 +00002933 SmallVector<int, 8> MaskVec;
2934 SVOp->getMask(MaskVec);
2935
Nate Begeman5a5ca152009-04-29 05:20:52 +00002936 for (unsigned i = 0; i != NumElems; ++i) {
2937 if (MaskVec[i] > (int)NumElems) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002938 MaskVec[i] = NumElems;
2939 Changed = true;
Evan Cheng39623da2006-04-20 08:58:49 +00002940 }
Evan Cheng39623da2006-04-20 08:58:49 +00002941 }
Evan Cheng39623da2006-04-20 08:58:49 +00002942 if (Changed)
Nate Begeman9008ca62009-04-27 18:41:29 +00002943 return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(0),
2944 SVOp->getOperand(1), &MaskVec[0]);
2945 return SDValue(SVOp, 0);
Evan Cheng39623da2006-04-20 08:58:49 +00002946}
2947
Evan Cheng017dcc62006-04-21 01:05:10 +00002948/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd
2949/// operation of specified width.
Owen Andersone50ed302009-08-10 22:56:29 +00002950static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
Nate Begeman9008ca62009-04-27 18:41:29 +00002951 SDValue V2) {
2952 unsigned NumElems = VT.getVectorNumElements();
2953 SmallVector<int, 8> Mask;
2954 Mask.push_back(NumElems);
Evan Cheng39623da2006-04-20 08:58:49 +00002955 for (unsigned i = 1; i != NumElems; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00002956 Mask.push_back(i);
2957 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
Evan Cheng39623da2006-04-20 08:58:49 +00002958}
2959
Nate Begeman9008ca62009-04-27 18:41:29 +00002960/// getUnpackl - Returns a vector_shuffle node for an unpackl operation.
Owen Andersone50ed302009-08-10 22:56:29 +00002961static SDValue getUnpackl(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
Nate Begeman9008ca62009-04-27 18:41:29 +00002962 SDValue V2) {
2963 unsigned NumElems = VT.getVectorNumElements();
2964 SmallVector<int, 8> Mask;
Evan Chengc575ca22006-04-17 20:43:08 +00002965 for (unsigned i = 0, e = NumElems/2; i != e; ++i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002966 Mask.push_back(i);
2967 Mask.push_back(i + NumElems);
Evan Chengc575ca22006-04-17 20:43:08 +00002968 }
Nate Begeman9008ca62009-04-27 18:41:29 +00002969 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
Evan Chengc575ca22006-04-17 20:43:08 +00002970}
2971
Nate Begeman9008ca62009-04-27 18:41:29 +00002972/// getUnpackhMask - Returns a vector_shuffle node for an unpackh operation.
Owen Andersone50ed302009-08-10 22:56:29 +00002973static SDValue getUnpackh(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
Nate Begeman9008ca62009-04-27 18:41:29 +00002974 SDValue V2) {
2975 unsigned NumElems = VT.getVectorNumElements();
Evan Cheng39623da2006-04-20 08:58:49 +00002976 unsigned Half = NumElems/2;
Nate Begeman9008ca62009-04-27 18:41:29 +00002977 SmallVector<int, 8> Mask;
Evan Cheng39623da2006-04-20 08:58:49 +00002978 for (unsigned i = 0; i != Half; ++i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00002979 Mask.push_back(i + Half);
2980 Mask.push_back(i + NumElems + Half);
Evan Cheng39623da2006-04-20 08:58:49 +00002981 }
Nate Begeman9008ca62009-04-27 18:41:29 +00002982 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
Chris Lattner62098042008-03-09 01:05:04 +00002983}
2984
Evan Cheng0c0f83f2008-04-05 00:30:36 +00002985/// PromoteSplat - Promote a splat of v4f32, v8i16 or v16i8 to v4i32.
Nate Begeman9008ca62009-04-27 18:41:29 +00002986static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG,
2987 bool HasSSE2) {
2988 if (SV->getValueType(0).getVectorNumElements() <= 4)
2989 return SDValue(SV, 0);
2990
Owen Anderson825b72b2009-08-11 20:47:22 +00002991 EVT PVT = MVT::v4f32;
Owen Andersone50ed302009-08-10 22:56:29 +00002992 EVT VT = SV->getValueType(0);
Nate Begeman9008ca62009-04-27 18:41:29 +00002993 DebugLoc dl = SV->getDebugLoc();
2994 SDValue V1 = SV->getOperand(0);
2995 int NumElems = VT.getVectorNumElements();
2996 int EltNo = SV->getSplatIndex();
Rafael Espindola15684b22009-04-24 12:40:33 +00002997
Nate Begeman9008ca62009-04-27 18:41:29 +00002998 // unpack elements to the correct location
2999 while (NumElems > 4) {
3000 if (EltNo < NumElems/2) {
3001 V1 = getUnpackl(DAG, dl, VT, V1, V1);
3002 } else {
3003 V1 = getUnpackh(DAG, dl, VT, V1, V1);
3004 EltNo -= NumElems/2;
3005 }
3006 NumElems >>= 1;
3007 }
3008
3009 // Perform the splat.
3010 int SplatMask[4] = { EltNo, EltNo, EltNo, EltNo };
Dale Johannesenace16102009-02-03 19:33:06 +00003011 V1 = DAG.getNode(ISD::BIT_CONVERT, dl, PVT, V1);
Nate Begeman9008ca62009-04-27 18:41:29 +00003012 V1 = DAG.getVectorShuffle(PVT, dl, V1, DAG.getUNDEF(PVT), &SplatMask[0]);
3013 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, V1);
Evan Chengc575ca22006-04-17 20:43:08 +00003014}
3015
Evan Chengba05f722006-04-21 23:03:30 +00003016/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified
Chris Lattner8a594482007-11-25 00:24:49 +00003017/// vector of zero or undef vector. This produces a shuffle where the low
3018/// element of V2 is swizzled into the zero/undef vector, landing at element
3019/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
Dan Gohman475871a2008-07-27 21:46:04 +00003020static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx,
Evan Chengf0df0312008-05-15 08:39:06 +00003021 bool isZero, bool HasSSE2,
3022 SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00003023 EVT VT = V2.getValueType();
Dan Gohman475871a2008-07-27 21:46:04 +00003024 SDValue V1 = isZero
Nate Begeman9008ca62009-04-27 18:41:29 +00003025 ? getZeroVector(VT, HasSSE2, DAG, V2.getDebugLoc()) : DAG.getUNDEF(VT);
3026 unsigned NumElems = VT.getVectorNumElements();
3027 SmallVector<int, 16> MaskVec;
Chris Lattner8a594482007-11-25 00:24:49 +00003028 for (unsigned i = 0; i != NumElems; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00003029 // If this is the insertion idx, put the low elt of V2 here.
3030 MaskVec.push_back(i == Idx ? NumElems : i);
3031 return DAG.getVectorShuffle(VT, V2.getDebugLoc(), V1, V2, &MaskVec[0]);
Evan Cheng017dcc62006-04-21 01:05:10 +00003032}
3033
Evan Chengf26ffe92008-05-29 08:22:04 +00003034/// getNumOfConsecutiveZeros - Return the number of elements in a result of
3035/// a shuffle that is zero.
3036static
Nate Begeman9008ca62009-04-27 18:41:29 +00003037unsigned getNumOfConsecutiveZeros(ShuffleVectorSDNode *SVOp, int NumElems,
3038 bool Low, SelectionDAG &DAG) {
Evan Chengf26ffe92008-05-29 08:22:04 +00003039 unsigned NumZeros = 0;
Nate Begeman9008ca62009-04-27 18:41:29 +00003040 for (int i = 0; i < NumElems; ++i) {
Evan Chengab262272008-06-25 20:52:59 +00003041 unsigned Index = Low ? i : NumElems-i-1;
Nate Begeman9008ca62009-04-27 18:41:29 +00003042 int Idx = SVOp->getMaskElt(Index);
3043 if (Idx < 0) {
Evan Chengf26ffe92008-05-29 08:22:04 +00003044 ++NumZeros;
3045 continue;
3046 }
Nate Begeman9008ca62009-04-27 18:41:29 +00003047 SDValue Elt = DAG.getShuffleScalarElt(SVOp, Index);
Evan Cheng37b73872009-07-30 08:33:02 +00003048 if (Elt.getNode() && X86::isZeroNode(Elt))
Evan Chengf26ffe92008-05-29 08:22:04 +00003049 ++NumZeros;
3050 else
3051 break;
3052 }
3053 return NumZeros;
3054}
3055
3056/// isVectorShift - Returns true if the shuffle can be implemented as a
3057/// logical left or right shift of a vector.
Nate Begeman9008ca62009-04-27 18:41:29 +00003058/// FIXME: split into pslldqi, psrldqi, palignr variants.
3059static bool isVectorShift(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG,
Dan Gohman475871a2008-07-27 21:46:04 +00003060 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003061 int NumElems = SVOp->getValueType(0).getVectorNumElements();
Evan Chengf26ffe92008-05-29 08:22:04 +00003062
3063 isLeft = true;
Nate Begeman9008ca62009-04-27 18:41:29 +00003064 unsigned NumZeros = getNumOfConsecutiveZeros(SVOp, NumElems, true, DAG);
Evan Chengf26ffe92008-05-29 08:22:04 +00003065 if (!NumZeros) {
3066 isLeft = false;
Nate Begeman9008ca62009-04-27 18:41:29 +00003067 NumZeros = getNumOfConsecutiveZeros(SVOp, NumElems, false, DAG);
Evan Chengf26ffe92008-05-29 08:22:04 +00003068 if (!NumZeros)
3069 return false;
3070 }
Evan Chengf26ffe92008-05-29 08:22:04 +00003071 bool SeenV1 = false;
3072 bool SeenV2 = false;
Nate Begeman9008ca62009-04-27 18:41:29 +00003073 for (int i = NumZeros; i < NumElems; ++i) {
3074 int Val = isLeft ? (i - NumZeros) : i;
3075 int Idx = SVOp->getMaskElt(isLeft ? i : (i - NumZeros));
3076 if (Idx < 0)
Evan Chengf26ffe92008-05-29 08:22:04 +00003077 continue;
Nate Begeman9008ca62009-04-27 18:41:29 +00003078 if (Idx < NumElems)
Evan Chengf26ffe92008-05-29 08:22:04 +00003079 SeenV1 = true;
3080 else {
Nate Begeman9008ca62009-04-27 18:41:29 +00003081 Idx -= NumElems;
Evan Chengf26ffe92008-05-29 08:22:04 +00003082 SeenV2 = true;
3083 }
Nate Begeman9008ca62009-04-27 18:41:29 +00003084 if (Idx != Val)
Evan Chengf26ffe92008-05-29 08:22:04 +00003085 return false;
3086 }
3087 if (SeenV1 && SeenV2)
3088 return false;
3089
Nate Begeman9008ca62009-04-27 18:41:29 +00003090 ShVal = SeenV1 ? SVOp->getOperand(0) : SVOp->getOperand(1);
Evan Chengf26ffe92008-05-29 08:22:04 +00003091 ShAmt = NumZeros;
3092 return true;
3093}
3094
3095
Evan Chengc78d3b42006-04-24 18:01:45 +00003096/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8.
3097///
Dan Gohman475871a2008-07-27 21:46:04 +00003098static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
Evan Chengc78d3b42006-04-24 18:01:45 +00003099 unsigned NumNonZero, unsigned NumZero,
Evan Cheng25ab6902006-09-08 06:48:29 +00003100 SelectionDAG &DAG, TargetLowering &TLI) {
Evan Chengc78d3b42006-04-24 18:01:45 +00003101 if (NumNonZero > 8)
Dan Gohman475871a2008-07-27 21:46:04 +00003102 return SDValue();
Evan Chengc78d3b42006-04-24 18:01:45 +00003103
Dale Johannesen6f38cb62009-02-07 19:59:05 +00003104 DebugLoc dl = Op.getDebugLoc();
Dan Gohman475871a2008-07-27 21:46:04 +00003105 SDValue V(0, 0);
Evan Chengc78d3b42006-04-24 18:01:45 +00003106 bool First = true;
3107 for (unsigned i = 0; i < 16; ++i) {
3108 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0;
3109 if (ThisIsNonZero && First) {
3110 if (NumZero)
Owen Anderson825b72b2009-08-11 20:47:22 +00003111 V = getZeroVector(MVT::v8i16, true, DAG, dl);
Evan Chengc78d3b42006-04-24 18:01:45 +00003112 else
Owen Anderson825b72b2009-08-11 20:47:22 +00003113 V = DAG.getUNDEF(MVT::v8i16);
Evan Chengc78d3b42006-04-24 18:01:45 +00003114 First = false;
3115 }
3116
3117 if ((i & 1) != 0) {
Dan Gohman475871a2008-07-27 21:46:04 +00003118 SDValue ThisElt(0, 0), LastElt(0, 0);
Evan Chengc78d3b42006-04-24 18:01:45 +00003119 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0;
3120 if (LastIsNonZero) {
Scott Michelfdc40a02009-02-17 22:15:04 +00003121 LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00003122 MVT::i16, Op.getOperand(i-1));
Evan Chengc78d3b42006-04-24 18:01:45 +00003123 }
3124 if (ThisIsNonZero) {
Owen Anderson825b72b2009-08-11 20:47:22 +00003125 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i));
3126 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16,
3127 ThisElt, DAG.getConstant(8, MVT::i8));
Evan Chengc78d3b42006-04-24 18:01:45 +00003128 if (LastIsNonZero)
Owen Anderson825b72b2009-08-11 20:47:22 +00003129 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt);
Evan Chengc78d3b42006-04-24 18:01:45 +00003130 } else
3131 ThisElt = LastElt;
3132
Gabor Greifba36cb52008-08-28 21:40:38 +00003133 if (ThisElt.getNode())
Owen Anderson825b72b2009-08-11 20:47:22 +00003134 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt,
Chris Lattner0bd48932008-01-17 07:00:52 +00003135 DAG.getIntPtrConstant(i/2));
Evan Chengc78d3b42006-04-24 18:01:45 +00003136 }
3137 }
3138
Owen Anderson825b72b2009-08-11 20:47:22 +00003139 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, V);
Evan Chengc78d3b42006-04-24 18:01:45 +00003140}
3141
Bill Wendlinga348c562007-03-22 18:42:45 +00003142/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16.
Evan Chengc78d3b42006-04-24 18:01:45 +00003143///
Dan Gohman475871a2008-07-27 21:46:04 +00003144static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
Evan Chengc78d3b42006-04-24 18:01:45 +00003145 unsigned NumNonZero, unsigned NumZero,
Evan Cheng25ab6902006-09-08 06:48:29 +00003146 SelectionDAG &DAG, TargetLowering &TLI) {
Evan Chengc78d3b42006-04-24 18:01:45 +00003147 if (NumNonZero > 4)
Dan Gohman475871a2008-07-27 21:46:04 +00003148 return SDValue();
Evan Chengc78d3b42006-04-24 18:01:45 +00003149
Dale Johannesen6f38cb62009-02-07 19:59:05 +00003150 DebugLoc dl = Op.getDebugLoc();
Dan Gohman475871a2008-07-27 21:46:04 +00003151 SDValue V(0, 0);
Evan Chengc78d3b42006-04-24 18:01:45 +00003152 bool First = true;
3153 for (unsigned i = 0; i < 8; ++i) {
3154 bool isNonZero = (NonZeros & (1 << i)) != 0;
3155 if (isNonZero) {
3156 if (First) {
3157 if (NumZero)
Owen Anderson825b72b2009-08-11 20:47:22 +00003158 V = getZeroVector(MVT::v8i16, true, DAG, dl);
Evan Chengc78d3b42006-04-24 18:01:45 +00003159 else
Owen Anderson825b72b2009-08-11 20:47:22 +00003160 V = DAG.getUNDEF(MVT::v8i16);
Evan Chengc78d3b42006-04-24 18:01:45 +00003161 First = false;
3162 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003163 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00003164 MVT::v8i16, V, Op.getOperand(i),
Chris Lattner0bd48932008-01-17 07:00:52 +00003165 DAG.getIntPtrConstant(i));
Evan Chengc78d3b42006-04-24 18:01:45 +00003166 }
3167 }
3168
3169 return V;
3170}
3171
Evan Chengf26ffe92008-05-29 08:22:04 +00003172/// getVShift - Return a vector logical shift node.
3173///
Owen Andersone50ed302009-08-10 22:56:29 +00003174static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp,
Nate Begeman9008ca62009-04-27 18:41:29 +00003175 unsigned NumBits, SelectionDAG &DAG,
3176 const TargetLowering &TLI, DebugLoc dl) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00003177 bool isMMX = VT.getSizeInBits() == 64;
Owen Anderson825b72b2009-08-11 20:47:22 +00003178 EVT ShVT = isMMX ? MVT::v1i64 : MVT::v2i64;
Evan Chengf26ffe92008-05-29 08:22:04 +00003179 unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL;
Dale Johannesenace16102009-02-03 19:33:06 +00003180 SrcOp = DAG.getNode(ISD::BIT_CONVERT, dl, ShVT, SrcOp);
3181 return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
3182 DAG.getNode(Opc, dl, ShVT, SrcOp,
Gabor Greif327ef032008-08-28 23:19:51 +00003183 DAG.getConstant(NumBits, TLI.getShiftAmountTy())));
Evan Chengf26ffe92008-05-29 08:22:04 +00003184}
3185
Dan Gohman475871a2008-07-27 21:46:04 +00003186SDValue
3187X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) {
Dale Johannesen6f38cb62009-02-07 19:59:05 +00003188 DebugLoc dl = Op.getDebugLoc();
Chris Lattner8a594482007-11-25 00:24:49 +00003189 // All zero's are handled with pxor, all one's are handled with pcmpeqd.
Gabor Greif327ef032008-08-28 23:19:51 +00003190 if (ISD::isBuildVectorAllZeros(Op.getNode())
3191 || ISD::isBuildVectorAllOnes(Op.getNode())) {
Chris Lattner8a594482007-11-25 00:24:49 +00003192 // Canonicalize this to either <4 x i32> or <2 x i32> (SSE vs MMX) to
3193 // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are
3194 // eliminated on x86-32 hosts.
Owen Anderson825b72b2009-08-11 20:47:22 +00003195 if (Op.getValueType() == MVT::v4i32 || Op.getValueType() == MVT::v2i32)
Chris Lattner8a594482007-11-25 00:24:49 +00003196 return Op;
Evan Cheng0db9fe62006-04-25 20:13:52 +00003197
Gabor Greifba36cb52008-08-28 21:40:38 +00003198 if (ISD::isBuildVectorAllOnes(Op.getNode()))
Dale Johannesenace16102009-02-03 19:33:06 +00003199 return getOnesVector(Op.getValueType(), DAG, dl);
3200 return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG, dl);
Chris Lattner8a594482007-11-25 00:24:49 +00003201 }
Evan Cheng0db9fe62006-04-25 20:13:52 +00003202
Owen Andersone50ed302009-08-10 22:56:29 +00003203 EVT VT = Op.getValueType();
3204 EVT ExtVT = VT.getVectorElementType();
3205 unsigned EVTBits = ExtVT.getSizeInBits();
Evan Cheng0db9fe62006-04-25 20:13:52 +00003206
3207 unsigned NumElems = Op.getNumOperands();
3208 unsigned NumZero = 0;
3209 unsigned NumNonZero = 0;
3210 unsigned NonZeros = 0;
Chris Lattnerc9517fb2008-03-08 22:48:29 +00003211 bool IsAllConstants = true;
Dan Gohman475871a2008-07-27 21:46:04 +00003212 SmallSet<SDValue, 8> Values;
Evan Cheng0db9fe62006-04-25 20:13:52 +00003213 for (unsigned i = 0; i < NumElems; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00003214 SDValue Elt = Op.getOperand(i);
Evan Chengdb2d5242007-12-12 06:45:40 +00003215 if (Elt.getOpcode() == ISD::UNDEF)
3216 continue;
3217 Values.insert(Elt);
3218 if (Elt.getOpcode() != ISD::Constant &&
3219 Elt.getOpcode() != ISD::ConstantFP)
Chris Lattnerc9517fb2008-03-08 22:48:29 +00003220 IsAllConstants = false;
Evan Cheng37b73872009-07-30 08:33:02 +00003221 if (X86::isZeroNode(Elt))
Evan Chengdb2d5242007-12-12 06:45:40 +00003222 NumZero++;
3223 else {
3224 NonZeros |= (1 << i);
3225 NumNonZero++;
Evan Cheng0db9fe62006-04-25 20:13:52 +00003226 }
3227 }
3228
Dan Gohman7f321562007-06-25 16:23:39 +00003229 if (NumNonZero == 0) {
Chris Lattner8a594482007-11-25 00:24:49 +00003230 // All undef vector. Return an UNDEF. All zero vectors were handled above.
Dale Johannesene8d72302009-02-06 23:05:02 +00003231 return DAG.getUNDEF(VT);
Dan Gohman7f321562007-06-25 16:23:39 +00003232 }
Evan Cheng0db9fe62006-04-25 20:13:52 +00003233
Chris Lattner67f453a2008-03-09 05:42:06 +00003234 // Special case for single non-zero, non-undef, element.
Eli Friedman10415532009-06-06 06:05:10 +00003235 if (NumNonZero == 1) {
Evan Cheng0db9fe62006-04-25 20:13:52 +00003236 unsigned Idx = CountTrailingZeros_32(NonZeros);
Dan Gohman475871a2008-07-27 21:46:04 +00003237 SDValue Item = Op.getOperand(Idx);
Scott Michelfdc40a02009-02-17 22:15:04 +00003238
Chris Lattner62098042008-03-09 01:05:04 +00003239 // If this is an insertion of an i64 value on x86-32, and if the top bits of
3240 // the value are obviously zero, truncate the value to i32 and do the
3241 // insertion that way. Only do this if the value is non-constant or if the
3242 // value is a constant being inserted into element 0. It is cheaper to do
3243 // a constant pool load than it is to do a movd + shuffle.
Owen Anderson825b72b2009-08-11 20:47:22 +00003244 if (ExtVT == MVT::i64 && !Subtarget->is64Bit() &&
Chris Lattner62098042008-03-09 01:05:04 +00003245 (!IsAllConstants || Idx == 0)) {
3246 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) {
3247 // Handle MMX and SSE both.
Owen Anderson825b72b2009-08-11 20:47:22 +00003248 EVT VecVT = VT == MVT::v2i64 ? MVT::v4i32 : MVT::v2i32;
3249 unsigned VecElts = VT == MVT::v2i64 ? 4 : 2;
Scott Michelfdc40a02009-02-17 22:15:04 +00003250
Chris Lattner62098042008-03-09 01:05:04 +00003251 // Truncate the value (which may itself be a constant) to i32, and
3252 // convert it to a vector with movd (S2V+shuffle to zero extend).
Owen Anderson825b72b2009-08-11 20:47:22 +00003253 Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item);
Dale Johannesenace16102009-02-03 19:33:06 +00003254 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item);
Evan Chengf0df0312008-05-15 08:39:06 +00003255 Item = getShuffleVectorZeroOrUndef(Item, 0, true,
3256 Subtarget->hasSSE2(), DAG);
Scott Michelfdc40a02009-02-17 22:15:04 +00003257
Chris Lattner62098042008-03-09 01:05:04 +00003258 // Now we have our 32-bit value zero extended in the low element of
3259 // a vector. If Idx != 0, swizzle it into place.
3260 if (Idx != 0) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003261 SmallVector<int, 4> Mask;
3262 Mask.push_back(Idx);
3263 for (unsigned i = 1; i != VecElts; ++i)
3264 Mask.push_back(i);
3265 Item = DAG.getVectorShuffle(VecVT, dl, Item,
3266 DAG.getUNDEF(Item.getValueType()),
3267 &Mask[0]);
Chris Lattner62098042008-03-09 01:05:04 +00003268 }
Dale Johannesenace16102009-02-03 19:33:06 +00003269 return DAG.getNode(ISD::BIT_CONVERT, dl, Op.getValueType(), Item);
Chris Lattner62098042008-03-09 01:05:04 +00003270 }
3271 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003272
Chris Lattner19f79692008-03-08 22:59:52 +00003273 // If we have a constant or non-constant insertion into the low element of
3274 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into
3275 // the rest of the elements. This will be matched as movd/movq/movss/movsd
Eli Friedman10415532009-06-06 06:05:10 +00003276 // depending on what the source datatype is.
3277 if (Idx == 0) {
3278 if (NumZero == 0) {
3279 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
Owen Anderson825b72b2009-08-11 20:47:22 +00003280 } else if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 ||
3281 (ExtVT == MVT::i64 && Subtarget->is64Bit())) {
Eli Friedman10415532009-06-06 06:05:10 +00003282 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
3283 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector.
3284 return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget->hasSSE2(),
3285 DAG);
Owen Anderson825b72b2009-08-11 20:47:22 +00003286 } else if (ExtVT == MVT::i16 || ExtVT == MVT::i8) {
3287 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item);
3288 EVT MiddleVT = VT.getSizeInBits() == 64 ? MVT::v2i32 : MVT::v4i32;
Eli Friedman10415532009-06-06 06:05:10 +00003289 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MiddleVT, Item);
3290 Item = getShuffleVectorZeroOrUndef(Item, 0, true,
3291 Subtarget->hasSSE2(), DAG);
3292 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Item);
3293 }
Chris Lattnerc9517fb2008-03-08 22:48:29 +00003294 }
Evan Chengf26ffe92008-05-29 08:22:04 +00003295
3296 // Is it a vector logical left shift?
3297 if (NumElems == 2 && Idx == 1 &&
Evan Cheng37b73872009-07-30 08:33:02 +00003298 X86::isZeroNode(Op.getOperand(0)) &&
3299 !X86::isZeroNode(Op.getOperand(1))) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00003300 unsigned NumBits = VT.getSizeInBits();
Evan Chengf26ffe92008-05-29 08:22:04 +00003301 return getVShift(true, VT,
Scott Michelfdc40a02009-02-17 22:15:04 +00003302 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
Dale Johannesenb300d2a2009-02-07 00:55:49 +00003303 VT, Op.getOperand(1)),
Dale Johannesenace16102009-02-03 19:33:06 +00003304 NumBits/2, DAG, *this, dl);
Evan Chengf26ffe92008-05-29 08:22:04 +00003305 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003306
Chris Lattnerc9517fb2008-03-08 22:48:29 +00003307 if (IsAllConstants) // Otherwise, it's better to do a constpool load.
Dan Gohman475871a2008-07-27 21:46:04 +00003308 return SDValue();
Evan Cheng0db9fe62006-04-25 20:13:52 +00003309
Chris Lattner19f79692008-03-08 22:59:52 +00003310 // Otherwise, if this is a vector with i32 or f32 elements, and the element
3311 // is a non-constant being inserted into an element other than the low one,
3312 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka
3313 // movd/movss) to move this into the low element, then shuffle it into
3314 // place.
Evan Cheng0db9fe62006-04-25 20:13:52 +00003315 if (EVTBits == 32) {
Dale Johannesenace16102009-02-03 19:33:06 +00003316 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
Scott Michelfdc40a02009-02-17 22:15:04 +00003317
Evan Cheng0db9fe62006-04-25 20:13:52 +00003318 // Turn it into a shuffle of zero and zero-extended scalar to vector.
Evan Chengf0df0312008-05-15 08:39:06 +00003319 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0,
3320 Subtarget->hasSSE2(), DAG);
Nate Begeman9008ca62009-04-27 18:41:29 +00003321 SmallVector<int, 8> MaskVec;
Evan Cheng0db9fe62006-04-25 20:13:52 +00003322 for (unsigned i = 0; i < NumElems; i++)
Nate Begeman9008ca62009-04-27 18:41:29 +00003323 MaskVec.push_back(i == Idx ? 0 : 1);
3324 return DAG.getVectorShuffle(VT, dl, Item, DAG.getUNDEF(VT), &MaskVec[0]);
Evan Cheng0db9fe62006-04-25 20:13:52 +00003325 }
3326 }
3327
Chris Lattner67f453a2008-03-09 05:42:06 +00003328 // Splat is obviously ok. Let legalizer expand it to a shuffle.
3329 if (Values.size() == 1)
Dan Gohman475871a2008-07-27 21:46:04 +00003330 return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00003331
Dan Gohmana3941172007-07-24 22:55:08 +00003332 // A vector full of immediates; various special cases are already
3333 // handled, so this is best done with a single constant-pool load.
Chris Lattnerc9517fb2008-03-08 22:48:29 +00003334 if (IsAllConstants)
Dan Gohman475871a2008-07-27 21:46:04 +00003335 return SDValue();
Dan Gohmana3941172007-07-24 22:55:08 +00003336
Bill Wendling2f9bb1a2007-04-24 21:16:55 +00003337 // Let legalizer expand 2-wide build_vectors.
Evan Cheng7e2ff772008-05-08 00:57:18 +00003338 if (EVTBits == 64) {
3339 if (NumNonZero == 1) {
3340 // One half is zero or undef.
3341 unsigned Idx = CountTrailingZeros_32(NonZeros);
Dale Johannesenace16102009-02-03 19:33:06 +00003342 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
Evan Cheng7e2ff772008-05-08 00:57:18 +00003343 Op.getOperand(Idx));
Evan Chengf0df0312008-05-15 08:39:06 +00003344 return getShuffleVectorZeroOrUndef(V2, Idx, true,
3345 Subtarget->hasSSE2(), DAG);
Evan Cheng7e2ff772008-05-08 00:57:18 +00003346 }
Dan Gohman475871a2008-07-27 21:46:04 +00003347 return SDValue();
Evan Cheng7e2ff772008-05-08 00:57:18 +00003348 }
Evan Cheng0db9fe62006-04-25 20:13:52 +00003349
3350 // If element VT is < 32 bits, convert it to inserts into a zero vector.
Bill Wendling826f36f2007-03-28 00:57:11 +00003351 if (EVTBits == 8 && NumElems == 16) {
Dan Gohman475871a2008-07-27 21:46:04 +00003352 SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG,
Evan Cheng25ab6902006-09-08 06:48:29 +00003353 *this);
Gabor Greifba36cb52008-08-28 21:40:38 +00003354 if (V.getNode()) return V;
Evan Cheng0db9fe62006-04-25 20:13:52 +00003355 }
3356
Bill Wendling826f36f2007-03-28 00:57:11 +00003357 if (EVTBits == 16 && NumElems == 8) {
Dan Gohman475871a2008-07-27 21:46:04 +00003358 SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG,
Evan Cheng25ab6902006-09-08 06:48:29 +00003359 *this);
Gabor Greifba36cb52008-08-28 21:40:38 +00003360 if (V.getNode()) return V;
Evan Cheng0db9fe62006-04-25 20:13:52 +00003361 }
3362
3363 // If element VT is == 32 bits, turn it into a number of shuffles.
Dan Gohman475871a2008-07-27 21:46:04 +00003364 SmallVector<SDValue, 8> V;
Chris Lattner5a88b832007-02-25 07:10:00 +00003365 V.resize(NumElems);
Evan Cheng0db9fe62006-04-25 20:13:52 +00003366 if (NumElems == 4 && NumZero > 0) {
3367 for (unsigned i = 0; i < 4; ++i) {
3368 bool isZero = !(NonZeros & (1 << i));
3369 if (isZero)
Dale Johannesenace16102009-02-03 19:33:06 +00003370 V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl);
Evan Cheng0db9fe62006-04-25 20:13:52 +00003371 else
Dale Johannesenace16102009-02-03 19:33:06 +00003372 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
Evan Cheng0db9fe62006-04-25 20:13:52 +00003373 }
3374
3375 for (unsigned i = 0; i < 2; ++i) {
3376 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) {
3377 default: break;
3378 case 0:
3379 V[i] = V[i*2]; // Must be a zero vector.
3380 break;
3381 case 1:
Nate Begeman9008ca62009-04-27 18:41:29 +00003382 V[i] = getMOVL(DAG, dl, VT, V[i*2+1], V[i*2]);
Evan Cheng0db9fe62006-04-25 20:13:52 +00003383 break;
3384 case 2:
Nate Begeman9008ca62009-04-27 18:41:29 +00003385 V[i] = getMOVL(DAG, dl, VT, V[i*2], V[i*2+1]);
Evan Cheng0db9fe62006-04-25 20:13:52 +00003386 break;
3387 case 3:
Nate Begeman9008ca62009-04-27 18:41:29 +00003388 V[i] = getUnpackl(DAG, dl, VT, V[i*2], V[i*2+1]);
Evan Cheng0db9fe62006-04-25 20:13:52 +00003389 break;
3390 }
3391 }
3392
Nate Begeman9008ca62009-04-27 18:41:29 +00003393 SmallVector<int, 8> MaskVec;
Evan Cheng0db9fe62006-04-25 20:13:52 +00003394 bool Reverse = (NonZeros & 0x3) == 2;
3395 for (unsigned i = 0; i < 2; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00003396 MaskVec.push_back(Reverse ? 1-i : i);
Evan Cheng0db9fe62006-04-25 20:13:52 +00003397 Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2;
3398 for (unsigned i = 0; i < 2; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00003399 MaskVec.push_back(Reverse ? 1-i+NumElems : i+NumElems);
3400 return DAG.getVectorShuffle(VT, dl, V[0], V[1], &MaskVec[0]);
Evan Cheng0db9fe62006-04-25 20:13:52 +00003401 }
3402
3403 if (Values.size() > 2) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003404 // If we have SSE 4.1, Expand into a number of inserts unless the number of
3405 // values to be inserted is equal to the number of elements, in which case
3406 // use the unpack code below in the hopes of matching the consecutive elts
3407 // load merge pattern for shuffles.
3408 // FIXME: We could probably just check that here directly.
3409 if (Values.size() < NumElems && VT.getSizeInBits() == 128 &&
3410 getSubtarget()->hasSSE41()) {
3411 V[0] = DAG.getUNDEF(VT);
3412 for (unsigned i = 0; i < NumElems; ++i)
3413 if (Op.getOperand(i).getOpcode() != ISD::UNDEF)
3414 V[0] = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, V[0],
3415 Op.getOperand(i), DAG.getIntPtrConstant(i));
3416 return V[0];
3417 }
Evan Cheng0db9fe62006-04-25 20:13:52 +00003418 // Expand into a number of unpckl*.
3419 // e.g. for v4f32
3420 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0>
3421 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1>
3422 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0>
Evan Cheng0db9fe62006-04-25 20:13:52 +00003423 for (unsigned i = 0; i < NumElems; ++i)
Dale Johannesenace16102009-02-03 19:33:06 +00003424 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
Evan Cheng0db9fe62006-04-25 20:13:52 +00003425 NumElems >>= 1;
3426 while (NumElems != 0) {
3427 for (unsigned i = 0; i < NumElems; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00003428 V[i] = getUnpackl(DAG, dl, VT, V[i], V[i + NumElems]);
Evan Cheng0db9fe62006-04-25 20:13:52 +00003429 NumElems >>= 1;
3430 }
3431 return V[0];
3432 }
3433
Dan Gohman475871a2008-07-27 21:46:04 +00003434 return SDValue();
Evan Cheng0db9fe62006-04-25 20:13:52 +00003435}
3436
Nate Begemanb9a47b82009-02-23 08:49:38 +00003437// v8i16 shuffles - Prefer shuffles in the following order:
3438// 1. [all] pshuflw, pshufhw, optional move
3439// 2. [ssse3] 1 x pshufb
3440// 3. [ssse3] 2 x pshufb + 1 x por
3441// 4. [all] mov + pshuflw + pshufhw + N x (pextrw + pinsrw)
Evan Cheng8a86c3f2007-12-07 08:07:39 +00003442static
Nate Begeman9008ca62009-04-27 18:41:29 +00003443SDValue LowerVECTOR_SHUFFLEv8i16(ShuffleVectorSDNode *SVOp,
3444 SelectionDAG &DAG, X86TargetLowering &TLI) {
3445 SDValue V1 = SVOp->getOperand(0);
3446 SDValue V2 = SVOp->getOperand(1);
3447 DebugLoc dl = SVOp->getDebugLoc();
Nate Begemanb9a47b82009-02-23 08:49:38 +00003448 SmallVector<int, 8> MaskVals;
Evan Cheng14b32e12007-12-11 01:46:18 +00003449
Nate Begemanb9a47b82009-02-23 08:49:38 +00003450 // Determine if more than 1 of the words in each of the low and high quadwords
3451 // of the result come from the same quadword of one of the two inputs. Undef
3452 // mask values count as coming from any quadword, for better codegen.
3453 SmallVector<unsigned, 4> LoQuad(4);
3454 SmallVector<unsigned, 4> HiQuad(4);
3455 BitVector InputQuads(4);
3456 for (unsigned i = 0; i < 8; ++i) {
3457 SmallVectorImpl<unsigned> &Quad = i < 4 ? LoQuad : HiQuad;
Nate Begeman9008ca62009-04-27 18:41:29 +00003458 int EltIdx = SVOp->getMaskElt(i);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003459 MaskVals.push_back(EltIdx);
3460 if (EltIdx < 0) {
3461 ++Quad[0];
3462 ++Quad[1];
3463 ++Quad[2];
3464 ++Quad[3];
Evan Cheng14b32e12007-12-11 01:46:18 +00003465 continue;
Nate Begemanb9a47b82009-02-23 08:49:38 +00003466 }
3467 ++Quad[EltIdx / 4];
3468 InputQuads.set(EltIdx / 4);
Evan Cheng14b32e12007-12-11 01:46:18 +00003469 }
Bill Wendlinge85dc492008-08-21 22:35:37 +00003470
Nate Begemanb9a47b82009-02-23 08:49:38 +00003471 int BestLoQuad = -1;
Evan Cheng14b32e12007-12-11 01:46:18 +00003472 unsigned MaxQuad = 1;
3473 for (unsigned i = 0; i < 4; ++i) {
Nate Begemanb9a47b82009-02-23 08:49:38 +00003474 if (LoQuad[i] > MaxQuad) {
3475 BestLoQuad = i;
3476 MaxQuad = LoQuad[i];
Evan Cheng14b32e12007-12-11 01:46:18 +00003477 }
Evan Cheng8a86c3f2007-12-07 08:07:39 +00003478 }
3479
Nate Begemanb9a47b82009-02-23 08:49:38 +00003480 int BestHiQuad = -1;
Evan Cheng14b32e12007-12-11 01:46:18 +00003481 MaxQuad = 1;
3482 for (unsigned i = 0; i < 4; ++i) {
Nate Begemanb9a47b82009-02-23 08:49:38 +00003483 if (HiQuad[i] > MaxQuad) {
3484 BestHiQuad = i;
3485 MaxQuad = HiQuad[i];
Evan Cheng14b32e12007-12-11 01:46:18 +00003486 }
3487 }
3488
Nate Begemanb9a47b82009-02-23 08:49:38 +00003489 // For SSSE3, If all 8 words of the result come from only 1 quadword of each
3490 // of the two input vectors, shuffle them into one input vector so only a
3491 // single pshufb instruction is necessary. If There are more than 2 input
3492 // quads, disable the next transformation since it does not help SSSE3.
3493 bool V1Used = InputQuads[0] || InputQuads[1];
3494 bool V2Used = InputQuads[2] || InputQuads[3];
3495 if (TLI.getSubtarget()->hasSSSE3()) {
3496 if (InputQuads.count() == 2 && V1Used && V2Used) {
3497 BestLoQuad = InputQuads.find_first();
3498 BestHiQuad = InputQuads.find_next(BestLoQuad);
3499 }
3500 if (InputQuads.count() > 2) {
3501 BestLoQuad = -1;
3502 BestHiQuad = -1;
3503 }
3504 }
Bill Wendlinge85dc492008-08-21 22:35:37 +00003505
Nate Begemanb9a47b82009-02-23 08:49:38 +00003506 // If BestLoQuad or BestHiQuad are set, shuffle the quads together and update
3507 // the shuffle mask. If a quad is scored as -1, that means that it contains
3508 // words from all 4 input quadwords.
3509 SDValue NewV;
3510 if (BestLoQuad >= 0 || BestHiQuad >= 0) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003511 SmallVector<int, 8> MaskV;
3512 MaskV.push_back(BestLoQuad < 0 ? 0 : BestLoQuad);
3513 MaskV.push_back(BestHiQuad < 0 ? 1 : BestHiQuad);
Owen Anderson825b72b2009-08-11 20:47:22 +00003514 NewV = DAG.getVectorShuffle(MVT::v2i64, dl,
3515 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, V1),
3516 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, V2), &MaskV[0]);
3517 NewV = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, NewV);
Evan Cheng14b32e12007-12-11 01:46:18 +00003518
Nate Begemanb9a47b82009-02-23 08:49:38 +00003519 // Rewrite the MaskVals and assign NewV to V1 if NewV now contains all the
3520 // source words for the shuffle, to aid later transformations.
3521 bool AllWordsInNewV = true;
Mon P Wang37b9a192009-03-11 06:35:11 +00003522 bool InOrder[2] = { true, true };
Evan Cheng14b32e12007-12-11 01:46:18 +00003523 for (unsigned i = 0; i != 8; ++i) {
Nate Begemanb9a47b82009-02-23 08:49:38 +00003524 int idx = MaskVals[i];
Mon P Wang37b9a192009-03-11 06:35:11 +00003525 if (idx != (int)i)
3526 InOrder[i/4] = false;
Nate Begemanb9a47b82009-02-23 08:49:38 +00003527 if (idx < 0 || (idx/4) == BestLoQuad || (idx/4) == BestHiQuad)
Evan Cheng14b32e12007-12-11 01:46:18 +00003528 continue;
Nate Begemanb9a47b82009-02-23 08:49:38 +00003529 AllWordsInNewV = false;
3530 break;
Evan Cheng14b32e12007-12-11 01:46:18 +00003531 }
Bill Wendlinge85dc492008-08-21 22:35:37 +00003532
Nate Begemanb9a47b82009-02-23 08:49:38 +00003533 bool pshuflw = AllWordsInNewV, pshufhw = AllWordsInNewV;
3534 if (AllWordsInNewV) {
3535 for (int i = 0; i != 8; ++i) {
3536 int idx = MaskVals[i];
3537 if (idx < 0)
Evan Cheng14b32e12007-12-11 01:46:18 +00003538 continue;
Nate Begemanb9a47b82009-02-23 08:49:38 +00003539 idx = MaskVals[i] = (idx / 4) == BestLoQuad ? (idx & 3) : (idx & 3) + 4;
3540 if ((idx != i) && idx < 4)
3541 pshufhw = false;
3542 if ((idx != i) && idx > 3)
3543 pshuflw = false;
Evan Cheng14b32e12007-12-11 01:46:18 +00003544 }
Nate Begemanb9a47b82009-02-23 08:49:38 +00003545 V1 = NewV;
3546 V2Used = false;
3547 BestLoQuad = 0;
3548 BestHiQuad = 1;
Evan Cheng8a86c3f2007-12-07 08:07:39 +00003549 }
Evan Cheng14b32e12007-12-11 01:46:18 +00003550
Nate Begemanb9a47b82009-02-23 08:49:38 +00003551 // If we've eliminated the use of V2, and the new mask is a pshuflw or
3552 // pshufhw, that's as cheap as it gets. Return the new shuffle.
Mon P Wang37b9a192009-03-11 06:35:11 +00003553 if ((pshufhw && InOrder[0]) || (pshuflw && InOrder[1])) {
Owen Anderson825b72b2009-08-11 20:47:22 +00003554 return DAG.getVectorShuffle(MVT::v8i16, dl, NewV,
3555 DAG.getUNDEF(MVT::v8i16), &MaskVals[0]);
Evan Cheng14b32e12007-12-11 01:46:18 +00003556 }
Evan Cheng14b32e12007-12-11 01:46:18 +00003557 }
Nate Begemanb9a47b82009-02-23 08:49:38 +00003558
3559 // If we have SSSE3, and all words of the result are from 1 input vector,
3560 // case 2 is generated, otherwise case 3 is generated. If no SSSE3
3561 // is present, fall back to case 4.
3562 if (TLI.getSubtarget()->hasSSSE3()) {
3563 SmallVector<SDValue,16> pshufbMask;
3564
3565 // If we have elements from both input vectors, set the high bit of the
3566 // shuffle mask element to zero out elements that come from V2 in the V1
3567 // mask, and elements that come from V1 in the V2 mask, so that the two
3568 // results can be OR'd together.
3569 bool TwoInputs = V1Used && V2Used;
3570 for (unsigned i = 0; i != 8; ++i) {
3571 int EltIdx = MaskVals[i] * 2;
3572 if (TwoInputs && (EltIdx >= 16)) {
Owen Anderson825b72b2009-08-11 20:47:22 +00003573 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
3574 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
Nate Begemanb9a47b82009-02-23 08:49:38 +00003575 continue;
3576 }
Owen Anderson825b72b2009-08-11 20:47:22 +00003577 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8));
3578 pshufbMask.push_back(DAG.getConstant(EltIdx+1, MVT::i8));
Nate Begemanb9a47b82009-02-23 08:49:38 +00003579 }
Owen Anderson825b72b2009-08-11 20:47:22 +00003580 V1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, V1);
3581 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1,
Evan Chenga87008d2009-02-25 22:49:59 +00003582 DAG.getNode(ISD::BUILD_VECTOR, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00003583 MVT::v16i8, &pshufbMask[0], 16));
Nate Begemanb9a47b82009-02-23 08:49:38 +00003584 if (!TwoInputs)
Owen Anderson825b72b2009-08-11 20:47:22 +00003585 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V1);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003586
3587 // Calculate the shuffle mask for the second input, shuffle it, and
3588 // OR it with the first shuffled input.
3589 pshufbMask.clear();
3590 for (unsigned i = 0; i != 8; ++i) {
3591 int EltIdx = MaskVals[i] * 2;
3592 if (EltIdx < 16) {
Owen Anderson825b72b2009-08-11 20:47:22 +00003593 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
3594 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
Nate Begemanb9a47b82009-02-23 08:49:38 +00003595 continue;
3596 }
Owen Anderson825b72b2009-08-11 20:47:22 +00003597 pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8));
3598 pshufbMask.push_back(DAG.getConstant(EltIdx - 15, MVT::i8));
Nate Begemanb9a47b82009-02-23 08:49:38 +00003599 }
Owen Anderson825b72b2009-08-11 20:47:22 +00003600 V2 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, V2);
3601 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2,
Evan Chenga87008d2009-02-25 22:49:59 +00003602 DAG.getNode(ISD::BUILD_VECTOR, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00003603 MVT::v16i8, &pshufbMask[0], 16));
3604 V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2);
3605 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V1);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003606 }
3607
3608 // If BestLoQuad >= 0, generate a pshuflw to put the low elements in order,
3609 // and update MaskVals with new element order.
3610 BitVector InOrder(8);
3611 if (BestLoQuad >= 0) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003612 SmallVector<int, 8> MaskV;
Nate Begemanb9a47b82009-02-23 08:49:38 +00003613 for (int i = 0; i != 4; ++i) {
3614 int idx = MaskVals[i];
3615 if (idx < 0) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003616 MaskV.push_back(-1);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003617 InOrder.set(i);
3618 } else if ((idx / 4) == BestLoQuad) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003619 MaskV.push_back(idx & 3);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003620 InOrder.set(i);
3621 } else {
Nate Begeman9008ca62009-04-27 18:41:29 +00003622 MaskV.push_back(-1);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003623 }
3624 }
3625 for (unsigned i = 4; i != 8; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00003626 MaskV.push_back(i);
Owen Anderson825b72b2009-08-11 20:47:22 +00003627 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
Nate Begeman9008ca62009-04-27 18:41:29 +00003628 &MaskV[0]);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003629 }
3630
3631 // If BestHi >= 0, generate a pshufhw to put the high elements in order,
3632 // and update MaskVals with the new element order.
3633 if (BestHiQuad >= 0) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003634 SmallVector<int, 8> MaskV;
Nate Begemanb9a47b82009-02-23 08:49:38 +00003635 for (unsigned i = 0; i != 4; ++i)
Nate Begeman9008ca62009-04-27 18:41:29 +00003636 MaskV.push_back(i);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003637 for (unsigned i = 4; i != 8; ++i) {
3638 int idx = MaskVals[i];
3639 if (idx < 0) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003640 MaskV.push_back(-1);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003641 InOrder.set(i);
3642 } else if ((idx / 4) == BestHiQuad) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003643 MaskV.push_back((idx & 3) + 4);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003644 InOrder.set(i);
3645 } else {
Nate Begeman9008ca62009-04-27 18:41:29 +00003646 MaskV.push_back(-1);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003647 }
3648 }
Owen Anderson825b72b2009-08-11 20:47:22 +00003649 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
Nate Begeman9008ca62009-04-27 18:41:29 +00003650 &MaskV[0]);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003651 }
3652
3653 // In case BestHi & BestLo were both -1, which means each quadword has a word
3654 // from each of the four input quadwords, calculate the InOrder bitvector now
3655 // before falling through to the insert/extract cleanup.
3656 if (BestLoQuad == -1 && BestHiQuad == -1) {
3657 NewV = V1;
3658 for (int i = 0; i != 8; ++i)
3659 if (MaskVals[i] < 0 || MaskVals[i] == i)
3660 InOrder.set(i);
3661 }
3662
3663 // The other elements are put in the right place using pextrw and pinsrw.
3664 for (unsigned i = 0; i != 8; ++i) {
3665 if (InOrder[i])
3666 continue;
3667 int EltIdx = MaskVals[i];
3668 if (EltIdx < 0)
3669 continue;
3670 SDValue ExtOp = (EltIdx < 8)
Owen Anderson825b72b2009-08-11 20:47:22 +00003671 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V1,
Nate Begemanb9a47b82009-02-23 08:49:38 +00003672 DAG.getIntPtrConstant(EltIdx))
Owen Anderson825b72b2009-08-11 20:47:22 +00003673 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V2,
Nate Begemanb9a47b82009-02-23 08:49:38 +00003674 DAG.getIntPtrConstant(EltIdx - 8));
Owen Anderson825b72b2009-08-11 20:47:22 +00003675 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, ExtOp,
Nate Begemanb9a47b82009-02-23 08:49:38 +00003676 DAG.getIntPtrConstant(i));
3677 }
3678 return NewV;
3679}
3680
3681// v16i8 shuffles - Prefer shuffles in the following order:
3682// 1. [ssse3] 1 x pshufb
3683// 2. [ssse3] 2 x pshufb + 1 x por
3684// 3. [all] v8i16 shuffle + N x pextrw + rotate + pinsrw
3685static
Nate Begeman9008ca62009-04-27 18:41:29 +00003686SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp,
3687 SelectionDAG &DAG, X86TargetLowering &TLI) {
3688 SDValue V1 = SVOp->getOperand(0);
3689 SDValue V2 = SVOp->getOperand(1);
3690 DebugLoc dl = SVOp->getDebugLoc();
Nate Begemanb9a47b82009-02-23 08:49:38 +00003691 SmallVector<int, 16> MaskVals;
Nate Begeman9008ca62009-04-27 18:41:29 +00003692 SVOp->getMask(MaskVals);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003693
3694 // If we have SSSE3, case 1 is generated when all result bytes come from
3695 // one of the inputs. Otherwise, case 2 is generated. If no SSSE3 is
3696 // present, fall back to case 3.
3697 // FIXME: kill V2Only once shuffles are canonizalized by getNode.
3698 bool V1Only = true;
3699 bool V2Only = true;
3700 for (unsigned i = 0; i < 16; ++i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003701 int EltIdx = MaskVals[i];
Nate Begemanb9a47b82009-02-23 08:49:38 +00003702 if (EltIdx < 0)
3703 continue;
3704 if (EltIdx < 16)
3705 V2Only = false;
3706 else
3707 V1Only = false;
3708 }
3709
3710 // If SSSE3, use 1 pshufb instruction per vector with elements in the result.
3711 if (TLI.getSubtarget()->hasSSSE3()) {
3712 SmallVector<SDValue,16> pshufbMask;
3713
3714 // If all result elements are from one input vector, then only translate
3715 // undef mask values to 0x80 (zero out result) in the pshufb mask.
3716 //
3717 // Otherwise, we have elements from both input vectors, and must zero out
3718 // elements that come from V2 in the first mask, and V1 in the second mask
3719 // so that we can OR them together.
3720 bool TwoInputs = !(V1Only || V2Only);
3721 for (unsigned i = 0; i != 16; ++i) {
3722 int EltIdx = MaskVals[i];
3723 if (EltIdx < 0 || (TwoInputs && EltIdx >= 16)) {
Owen Anderson825b72b2009-08-11 20:47:22 +00003724 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
Nate Begemanb9a47b82009-02-23 08:49:38 +00003725 continue;
3726 }
Owen Anderson825b72b2009-08-11 20:47:22 +00003727 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8));
Nate Begemanb9a47b82009-02-23 08:49:38 +00003728 }
3729 // If all the elements are from V2, assign it to V1 and return after
3730 // building the first pshufb.
3731 if (V2Only)
3732 V1 = V2;
Owen Anderson825b72b2009-08-11 20:47:22 +00003733 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1,
Evan Chenga87008d2009-02-25 22:49:59 +00003734 DAG.getNode(ISD::BUILD_VECTOR, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00003735 MVT::v16i8, &pshufbMask[0], 16));
Nate Begemanb9a47b82009-02-23 08:49:38 +00003736 if (!TwoInputs)
3737 return V1;
3738
3739 // Calculate the shuffle mask for the second input, shuffle it, and
3740 // OR it with the first shuffled input.
3741 pshufbMask.clear();
3742 for (unsigned i = 0; i != 16; ++i) {
3743 int EltIdx = MaskVals[i];
3744 if (EltIdx < 16) {
Owen Anderson825b72b2009-08-11 20:47:22 +00003745 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
Nate Begemanb9a47b82009-02-23 08:49:38 +00003746 continue;
3747 }
Owen Anderson825b72b2009-08-11 20:47:22 +00003748 pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8));
Nate Begemanb9a47b82009-02-23 08:49:38 +00003749 }
Owen Anderson825b72b2009-08-11 20:47:22 +00003750 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2,
Evan Chenga87008d2009-02-25 22:49:59 +00003751 DAG.getNode(ISD::BUILD_VECTOR, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00003752 MVT::v16i8, &pshufbMask[0], 16));
3753 return DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003754 }
3755
3756 // No SSSE3 - Calculate in place words and then fix all out of place words
3757 // With 0-16 extracts & inserts. Worst case is 16 bytes out of order from
3758 // the 16 different words that comprise the two doublequadword input vectors.
Owen Anderson825b72b2009-08-11 20:47:22 +00003759 V1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V1);
3760 V2 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V2);
Nate Begemanb9a47b82009-02-23 08:49:38 +00003761 SDValue NewV = V2Only ? V2 : V1;
3762 for (int i = 0; i != 8; ++i) {
3763 int Elt0 = MaskVals[i*2];
3764 int Elt1 = MaskVals[i*2+1];
3765
3766 // This word of the result is all undef, skip it.
3767 if (Elt0 < 0 && Elt1 < 0)
3768 continue;
3769
3770 // This word of the result is already in the correct place, skip it.
3771 if (V1Only && (Elt0 == i*2) && (Elt1 == i*2+1))
3772 continue;
3773 if (V2Only && (Elt0 == i*2+16) && (Elt1 == i*2+17))
3774 continue;
3775
3776 SDValue Elt0Src = Elt0 < 16 ? V1 : V2;
3777 SDValue Elt1Src = Elt1 < 16 ? V1 : V2;
3778 SDValue InsElt;
Mon P Wang6b3ef692009-03-11 18:47:57 +00003779
3780 // If Elt0 and Elt1 are defined, are consecutive, and can be load
3781 // using a single extract together, load it and store it.
3782 if ((Elt0 >= 0) && ((Elt0 + 1) == Elt1) && ((Elt0 & 1) == 0)) {
Owen Anderson825b72b2009-08-11 20:47:22 +00003783 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src,
Mon P Wang6b3ef692009-03-11 18:47:57 +00003784 DAG.getIntPtrConstant(Elt1 / 2));
Owen Anderson825b72b2009-08-11 20:47:22 +00003785 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt,
Mon P Wang6b3ef692009-03-11 18:47:57 +00003786 DAG.getIntPtrConstant(i));
3787 continue;
3788 }
3789
Nate Begemanb9a47b82009-02-23 08:49:38 +00003790 // If Elt1 is defined, extract it from the appropriate source. If the
Mon P Wang6b3ef692009-03-11 18:47:57 +00003791 // source byte is not also odd, shift the extracted word left 8 bits
3792 // otherwise clear the bottom 8 bits if we need to do an or.
Nate Begemanb9a47b82009-02-23 08:49:38 +00003793 if (Elt1 >= 0) {
Owen Anderson825b72b2009-08-11 20:47:22 +00003794 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src,
Nate Begemanb9a47b82009-02-23 08:49:38 +00003795 DAG.getIntPtrConstant(Elt1 / 2));
3796 if ((Elt1 & 1) == 0)
Owen Anderson825b72b2009-08-11 20:47:22 +00003797 InsElt = DAG.getNode(ISD::SHL, dl, MVT::i16, InsElt,
Nate Begemanb9a47b82009-02-23 08:49:38 +00003798 DAG.getConstant(8, TLI.getShiftAmountTy()));
Mon P Wang6b3ef692009-03-11 18:47:57 +00003799 else if (Elt0 >= 0)
Owen Anderson825b72b2009-08-11 20:47:22 +00003800 InsElt = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt,
3801 DAG.getConstant(0xFF00, MVT::i16));
Nate Begemanb9a47b82009-02-23 08:49:38 +00003802 }
3803 // If Elt0 is defined, extract it from the appropriate source. If the
3804 // source byte is not also even, shift the extracted word right 8 bits. If
3805 // Elt1 was also defined, OR the extracted values together before
3806 // inserting them in the result.
3807 if (Elt0 >= 0) {
Owen Anderson825b72b2009-08-11 20:47:22 +00003808 SDValue InsElt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
Nate Begemanb9a47b82009-02-23 08:49:38 +00003809 Elt0Src, DAG.getIntPtrConstant(Elt0 / 2));
3810 if ((Elt0 & 1) != 0)
Owen Anderson825b72b2009-08-11 20:47:22 +00003811 InsElt0 = DAG.getNode(ISD::SRL, dl, MVT::i16, InsElt0,
Nate Begemanb9a47b82009-02-23 08:49:38 +00003812 DAG.getConstant(8, TLI.getShiftAmountTy()));
Mon P Wang6b3ef692009-03-11 18:47:57 +00003813 else if (Elt1 >= 0)
Owen Anderson825b72b2009-08-11 20:47:22 +00003814 InsElt0 = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt0,
3815 DAG.getConstant(0x00FF, MVT::i16));
3816 InsElt = Elt1 >= 0 ? DAG.getNode(ISD::OR, dl, MVT::i16, InsElt, InsElt0)
Nate Begemanb9a47b82009-02-23 08:49:38 +00003817 : InsElt0;
3818 }
Owen Anderson825b72b2009-08-11 20:47:22 +00003819 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt,
Nate Begemanb9a47b82009-02-23 08:49:38 +00003820 DAG.getIntPtrConstant(i));
3821 }
Owen Anderson825b72b2009-08-11 20:47:22 +00003822 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, NewV);
Evan Cheng14b32e12007-12-11 01:46:18 +00003823}
3824
Evan Cheng7a831ce2007-12-15 03:00:47 +00003825/// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide
3826/// ones, or rewriting v4i32 / v2f32 as 2 wide ones if possible. This can be
3827/// done when every pair / quad of shuffle mask elements point to elements in
3828/// the right sequence. e.g.
Evan Cheng14b32e12007-12-11 01:46:18 +00003829/// vector_shuffle <>, <>, < 3, 4, | 10, 11, | 0, 1, | 14, 15>
3830static
Nate Begeman9008ca62009-04-27 18:41:29 +00003831SDValue RewriteAsNarrowerShuffle(ShuffleVectorSDNode *SVOp,
3832 SelectionDAG &DAG,
3833 TargetLowering &TLI, DebugLoc dl) {
Owen Andersone50ed302009-08-10 22:56:29 +00003834 EVT VT = SVOp->getValueType(0);
Nate Begeman9008ca62009-04-27 18:41:29 +00003835 SDValue V1 = SVOp->getOperand(0);
3836 SDValue V2 = SVOp->getOperand(1);
3837 unsigned NumElems = VT.getVectorNumElements();
Evan Cheng7a831ce2007-12-15 03:00:47 +00003838 unsigned NewWidth = (NumElems == 4) ? 2 : 4;
Owen Anderson825b72b2009-08-11 20:47:22 +00003839 EVT MaskVT = MVT::getIntVectorWithNumElements(NewWidth);
Owen Andersone50ed302009-08-10 22:56:29 +00003840 EVT MaskEltVT = MaskVT.getVectorElementType();
3841 EVT NewVT = MaskVT;
Owen Anderson825b72b2009-08-11 20:47:22 +00003842 switch (VT.getSimpleVT().SimpleTy) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00003843 default: assert(false && "Unexpected!");
Owen Anderson825b72b2009-08-11 20:47:22 +00003844 case MVT::v4f32: NewVT = MVT::v2f64; break;
3845 case MVT::v4i32: NewVT = MVT::v2i64; break;
3846 case MVT::v8i16: NewVT = MVT::v4i32; break;
3847 case MVT::v16i8: NewVT = MVT::v4i32; break;
Evan Cheng7a831ce2007-12-15 03:00:47 +00003848 }
3849
Anton Korobeynikov7c1c2612008-02-20 11:22:39 +00003850 if (NewWidth == 2) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00003851 if (VT.isInteger())
Owen Anderson825b72b2009-08-11 20:47:22 +00003852 NewVT = MVT::v2i64;
Evan Cheng7a831ce2007-12-15 03:00:47 +00003853 else
Owen Anderson825b72b2009-08-11 20:47:22 +00003854 NewVT = MVT::v2f64;
Anton Korobeynikov7c1c2612008-02-20 11:22:39 +00003855 }
Nate Begeman9008ca62009-04-27 18:41:29 +00003856 int Scale = NumElems / NewWidth;
3857 SmallVector<int, 8> MaskVec;
Evan Cheng14b32e12007-12-11 01:46:18 +00003858 for (unsigned i = 0; i < NumElems; i += Scale) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003859 int StartIdx = -1;
3860 for (int j = 0; j < Scale; ++j) {
3861 int EltIdx = SVOp->getMaskElt(i+j);
3862 if (EltIdx < 0)
Evan Cheng14b32e12007-12-11 01:46:18 +00003863 continue;
Nate Begeman9008ca62009-04-27 18:41:29 +00003864 if (StartIdx == -1)
Evan Cheng14b32e12007-12-11 01:46:18 +00003865 StartIdx = EltIdx - (EltIdx % Scale);
3866 if (EltIdx != StartIdx + j)
Dan Gohman475871a2008-07-27 21:46:04 +00003867 return SDValue();
Evan Cheng14b32e12007-12-11 01:46:18 +00003868 }
Nate Begeman9008ca62009-04-27 18:41:29 +00003869 if (StartIdx == -1)
3870 MaskVec.push_back(-1);
Evan Cheng14b32e12007-12-11 01:46:18 +00003871 else
Nate Begeman9008ca62009-04-27 18:41:29 +00003872 MaskVec.push_back(StartIdx / Scale);
Evan Cheng8a86c3f2007-12-07 08:07:39 +00003873 }
3874
Dale Johannesenace16102009-02-03 19:33:06 +00003875 V1 = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, V1);
3876 V2 = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, V2);
Nate Begeman9008ca62009-04-27 18:41:29 +00003877 return DAG.getVectorShuffle(NewVT, dl, V1, V2, &MaskVec[0]);
Evan Cheng8a86c3f2007-12-07 08:07:39 +00003878}
3879
Evan Chengd880b972008-05-09 21:53:03 +00003880/// getVZextMovL - Return a zero-extending vector move low node.
Evan Cheng7e2ff772008-05-08 00:57:18 +00003881///
Owen Andersone50ed302009-08-10 22:56:29 +00003882static SDValue getVZextMovL(EVT VT, EVT OpVT,
Nate Begeman9008ca62009-04-27 18:41:29 +00003883 SDValue SrcOp, SelectionDAG &DAG,
3884 const X86Subtarget *Subtarget, DebugLoc dl) {
Owen Anderson825b72b2009-08-11 20:47:22 +00003885 if (VT == MVT::v2f64 || VT == MVT::v4f32) {
Evan Cheng7e2ff772008-05-08 00:57:18 +00003886 LoadSDNode *LD = NULL;
Gabor Greifba36cb52008-08-28 21:40:38 +00003887 if (!isScalarLoadToVector(SrcOp.getNode(), &LD))
Evan Cheng7e2ff772008-05-08 00:57:18 +00003888 LD = dyn_cast<LoadSDNode>(SrcOp);
3889 if (!LD) {
3890 // movssrr and movsdrr do not clear top bits. Try to use movd, movq
3891 // instead.
Owen Anderson766b5ef2009-08-11 21:59:30 +00003892 MVT ExtVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32;
3893 if ((ExtVT.SimpleTy != MVT::i64 || Subtarget->is64Bit()) &&
Evan Cheng7e2ff772008-05-08 00:57:18 +00003894 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR &&
3895 SrcOp.getOperand(0).getOpcode() == ISD::BIT_CONVERT &&
Owen Anderson766b5ef2009-08-11 21:59:30 +00003896 SrcOp.getOperand(0).getOperand(0).getValueType() == ExtVT) {
Evan Cheng7e2ff772008-05-08 00:57:18 +00003897 // PR2108
Owen Anderson825b72b2009-08-11 20:47:22 +00003898 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32;
Dale Johannesenace16102009-02-03 19:33:06 +00003899 return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
3900 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
3901 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
3902 OpVT,
Gabor Greif327ef032008-08-28 23:19:51 +00003903 SrcOp.getOperand(0)
3904 .getOperand(0))));
Evan Cheng7e2ff772008-05-08 00:57:18 +00003905 }
3906 }
3907 }
3908
Dale Johannesenace16102009-02-03 19:33:06 +00003909 return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
3910 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
Scott Michelfdc40a02009-02-17 22:15:04 +00003911 DAG.getNode(ISD::BIT_CONVERT, dl,
Dale Johannesenace16102009-02-03 19:33:06 +00003912 OpVT, SrcOp)));
Evan Cheng7e2ff772008-05-08 00:57:18 +00003913}
3914
Evan Chengace3c172008-07-22 21:13:36 +00003915/// LowerVECTOR_SHUFFLE_4wide - Handle all 4 wide cases with a number of
3916/// shuffles.
Dan Gohman475871a2008-07-27 21:46:04 +00003917static SDValue
Nate Begeman9008ca62009-04-27 18:41:29 +00003918LowerVECTOR_SHUFFLE_4wide(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
3919 SDValue V1 = SVOp->getOperand(0);
3920 SDValue V2 = SVOp->getOperand(1);
3921 DebugLoc dl = SVOp->getDebugLoc();
Owen Andersone50ed302009-08-10 22:56:29 +00003922 EVT VT = SVOp->getValueType(0);
Nate Begeman9008ca62009-04-27 18:41:29 +00003923
Evan Chengace3c172008-07-22 21:13:36 +00003924 SmallVector<std::pair<int, int>, 8> Locs;
Rafael Espindola833a9902008-08-28 18:32:53 +00003925 Locs.resize(4);
Nate Begeman9008ca62009-04-27 18:41:29 +00003926 SmallVector<int, 8> Mask1(4U, -1);
3927 SmallVector<int, 8> PermMask;
3928 SVOp->getMask(PermMask);
3929
Evan Chengace3c172008-07-22 21:13:36 +00003930 unsigned NumHi = 0;
3931 unsigned NumLo = 0;
Evan Chengace3c172008-07-22 21:13:36 +00003932 for (unsigned i = 0; i != 4; ++i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003933 int Idx = PermMask[i];
3934 if (Idx < 0) {
Evan Chengace3c172008-07-22 21:13:36 +00003935 Locs[i] = std::make_pair(-1, -1);
3936 } else {
Nate Begeman9008ca62009-04-27 18:41:29 +00003937 assert(Idx < 8 && "Invalid VECTOR_SHUFFLE index!");
3938 if (Idx < 4) {
Evan Chengace3c172008-07-22 21:13:36 +00003939 Locs[i] = std::make_pair(0, NumLo);
Nate Begeman9008ca62009-04-27 18:41:29 +00003940 Mask1[NumLo] = Idx;
Evan Chengace3c172008-07-22 21:13:36 +00003941 NumLo++;
3942 } else {
3943 Locs[i] = std::make_pair(1, NumHi);
3944 if (2+NumHi < 4)
Nate Begeman9008ca62009-04-27 18:41:29 +00003945 Mask1[2+NumHi] = Idx;
Evan Chengace3c172008-07-22 21:13:36 +00003946 NumHi++;
3947 }
3948 }
3949 }
Evan Cheng5e6ebaf2008-07-23 00:22:17 +00003950
Evan Chengace3c172008-07-22 21:13:36 +00003951 if (NumLo <= 2 && NumHi <= 2) {
Evan Cheng5e6ebaf2008-07-23 00:22:17 +00003952 // If no more than two elements come from either vector. This can be
3953 // implemented with two shuffles. First shuffle gather the elements.
3954 // The second shuffle, which takes the first shuffle as both of its
3955 // vector operands, put the elements into the right order.
Nate Begeman9008ca62009-04-27 18:41:29 +00003956 V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
Evan Cheng5e6ebaf2008-07-23 00:22:17 +00003957
Nate Begeman9008ca62009-04-27 18:41:29 +00003958 SmallVector<int, 8> Mask2(4U, -1);
3959
Evan Chengace3c172008-07-22 21:13:36 +00003960 for (unsigned i = 0; i != 4; ++i) {
3961 if (Locs[i].first == -1)
3962 continue;
3963 else {
3964 unsigned Idx = (i < 2) ? 0 : 4;
3965 Idx += Locs[i].first * 2 + Locs[i].second;
Nate Begeman9008ca62009-04-27 18:41:29 +00003966 Mask2[i] = Idx;
Evan Chengace3c172008-07-22 21:13:36 +00003967 }
3968 }
3969
Nate Begeman9008ca62009-04-27 18:41:29 +00003970 return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]);
Evan Cheng5e6ebaf2008-07-23 00:22:17 +00003971 } else if (NumLo == 3 || NumHi == 3) {
3972 // Otherwise, we must have three elements from one vector, call it X, and
3973 // one element from the other, call it Y. First, use a shufps to build an
3974 // intermediate vector with the one element from Y and the element from X
3975 // that will be in the same half in the final destination (the indexes don't
3976 // matter). Then, use a shufps to build the final vector, taking the half
3977 // containing the element from Y from the intermediate, and the other half
3978 // from X.
3979 if (NumHi == 3) {
3980 // Normalize it so the 3 elements come from V1.
Nate Begeman9008ca62009-04-27 18:41:29 +00003981 CommuteVectorShuffleMask(PermMask, VT);
Evan Cheng5e6ebaf2008-07-23 00:22:17 +00003982 std::swap(V1, V2);
3983 }
3984
3985 // Find the element from V2.
3986 unsigned HiIndex;
3987 for (HiIndex = 0; HiIndex < 3; ++HiIndex) {
Nate Begeman9008ca62009-04-27 18:41:29 +00003988 int Val = PermMask[HiIndex];
3989 if (Val < 0)
Evan Cheng5e6ebaf2008-07-23 00:22:17 +00003990 continue;
Evan Cheng5e6ebaf2008-07-23 00:22:17 +00003991 if (Val >= 4)
3992 break;
3993 }
3994
Nate Begeman9008ca62009-04-27 18:41:29 +00003995 Mask1[0] = PermMask[HiIndex];
3996 Mask1[1] = -1;
3997 Mask1[2] = PermMask[HiIndex^1];
3998 Mask1[3] = -1;
3999 V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
Evan Cheng5e6ebaf2008-07-23 00:22:17 +00004000
4001 if (HiIndex >= 2) {
Nate Begeman9008ca62009-04-27 18:41:29 +00004002 Mask1[0] = PermMask[0];
4003 Mask1[1] = PermMask[1];
4004 Mask1[2] = HiIndex & 1 ? 6 : 4;
4005 Mask1[3] = HiIndex & 1 ? 4 : 6;
4006 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
Evan Cheng5e6ebaf2008-07-23 00:22:17 +00004007 } else {
Nate Begeman9008ca62009-04-27 18:41:29 +00004008 Mask1[0] = HiIndex & 1 ? 2 : 0;
4009 Mask1[1] = HiIndex & 1 ? 0 : 2;
4010 Mask1[2] = PermMask[2];
4011 Mask1[3] = PermMask[3];
4012 if (Mask1[2] >= 0)
4013 Mask1[2] += 4;
4014 if (Mask1[3] >= 0)
4015 Mask1[3] += 4;
4016 return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]);
Evan Cheng5e6ebaf2008-07-23 00:22:17 +00004017 }
Evan Chengace3c172008-07-22 21:13:36 +00004018 }
4019
4020 // Break it into (shuffle shuffle_hi, shuffle_lo).
4021 Locs.clear();
Nate Begeman9008ca62009-04-27 18:41:29 +00004022 SmallVector<int,8> LoMask(4U, -1);
4023 SmallVector<int,8> HiMask(4U, -1);
4024
4025 SmallVector<int,8> *MaskPtr = &LoMask;
Evan Chengace3c172008-07-22 21:13:36 +00004026 unsigned MaskIdx = 0;
4027 unsigned LoIdx = 0;
4028 unsigned HiIdx = 2;
4029 for (unsigned i = 0; i != 4; ++i) {
4030 if (i == 2) {
4031 MaskPtr = &HiMask;
4032 MaskIdx = 1;
4033 LoIdx = 0;
4034 HiIdx = 2;
4035 }
Nate Begeman9008ca62009-04-27 18:41:29 +00004036 int Idx = PermMask[i];
4037 if (Idx < 0) {
Evan Chengace3c172008-07-22 21:13:36 +00004038 Locs[i] = std::make_pair(-1, -1);
Nate Begeman9008ca62009-04-27 18:41:29 +00004039 } else if (Idx < 4) {
Evan Chengace3c172008-07-22 21:13:36 +00004040 Locs[i] = std::make_pair(MaskIdx, LoIdx);
Nate Begeman9008ca62009-04-27 18:41:29 +00004041 (*MaskPtr)[LoIdx] = Idx;
Evan Chengace3c172008-07-22 21:13:36 +00004042 LoIdx++;
4043 } else {
4044 Locs[i] = std::make_pair(MaskIdx, HiIdx);
Nate Begeman9008ca62009-04-27 18:41:29 +00004045 (*MaskPtr)[HiIdx] = Idx;
Evan Chengace3c172008-07-22 21:13:36 +00004046 HiIdx++;
4047 }
4048 }
4049
Nate Begeman9008ca62009-04-27 18:41:29 +00004050 SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]);
4051 SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]);
4052 SmallVector<int, 8> MaskOps;
Evan Chengace3c172008-07-22 21:13:36 +00004053 for (unsigned i = 0; i != 4; ++i) {
4054 if (Locs[i].first == -1) {
Nate Begeman9008ca62009-04-27 18:41:29 +00004055 MaskOps.push_back(-1);
Evan Chengace3c172008-07-22 21:13:36 +00004056 } else {
4057 unsigned Idx = Locs[i].first * 4 + Locs[i].second;
Nate Begeman9008ca62009-04-27 18:41:29 +00004058 MaskOps.push_back(Idx);
Evan Chengace3c172008-07-22 21:13:36 +00004059 }
4060 }
Nate Begeman9008ca62009-04-27 18:41:29 +00004061 return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]);
Evan Chengace3c172008-07-22 21:13:36 +00004062}
4063
Dan Gohman475871a2008-07-27 21:46:04 +00004064SDValue
4065X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
Nate Begeman9008ca62009-04-27 18:41:29 +00004066 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
Dan Gohman475871a2008-07-27 21:46:04 +00004067 SDValue V1 = Op.getOperand(0);
4068 SDValue V2 = Op.getOperand(1);
Owen Andersone50ed302009-08-10 22:56:29 +00004069 EVT VT = Op.getValueType();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00004070 DebugLoc dl = Op.getDebugLoc();
Nate Begeman9008ca62009-04-27 18:41:29 +00004071 unsigned NumElems = VT.getVectorNumElements();
Duncan Sands83ec4b62008-06-06 12:08:01 +00004072 bool isMMX = VT.getSizeInBits() == 64;
Evan Cheng0db9fe62006-04-25 20:13:52 +00004073 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF;
4074 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
Evan Chengd9b8e402006-10-16 06:36:00 +00004075 bool V1IsSplat = false;
4076 bool V2IsSplat = false;
Evan Cheng0db9fe62006-04-25 20:13:52 +00004077
Nate Begeman9008ca62009-04-27 18:41:29 +00004078 if (isZeroShuffle(SVOp))
Dale Johannesenace16102009-02-03 19:33:06 +00004079 return getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl);
Evan Cheng213d2cf2007-05-17 18:45:50 +00004080
Nate Begeman9008ca62009-04-27 18:41:29 +00004081 // Promote splats to v4f32.
4082 if (SVOp->isSplat()) {
4083 if (isMMX || NumElems < 4)
4084 return Op;
4085 return PromoteSplat(SVOp, DAG, Subtarget->hasSSE2());
Evan Cheng0db9fe62006-04-25 20:13:52 +00004086 }
4087
Evan Cheng7a831ce2007-12-15 03:00:47 +00004088 // If the shuffle can be profitably rewritten as a narrower shuffle, then
4089 // do it!
Owen Anderson825b72b2009-08-11 20:47:22 +00004090 if (VT == MVT::v8i16 || VT == MVT::v16i8) {
Nate Begeman9008ca62009-04-27 18:41:29 +00004091 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, *this, dl);
Gabor Greifba36cb52008-08-28 21:40:38 +00004092 if (NewOp.getNode())
Scott Michelfdc40a02009-02-17 22:15:04 +00004093 return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
Dale Johannesenace16102009-02-03 19:33:06 +00004094 LowerVECTOR_SHUFFLE(NewOp, DAG));
Owen Anderson825b72b2009-08-11 20:47:22 +00004095 } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) {
Evan Cheng7a831ce2007-12-15 03:00:47 +00004096 // FIXME: Figure out a cleaner way to do this.
4097 // Try to make use of movq to zero out the top part.
Gabor Greifba36cb52008-08-28 21:40:38 +00004098 if (ISD::isBuildVectorAllZeros(V2.getNode())) {
Nate Begeman9008ca62009-04-27 18:41:29 +00004099 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, *this, dl);
Gabor Greifba36cb52008-08-28 21:40:38 +00004100 if (NewOp.getNode()) {
Nate Begeman9008ca62009-04-27 18:41:29 +00004101 if (isCommutedMOVL(cast<ShuffleVectorSDNode>(NewOp), true, false))
4102 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(0),
4103 DAG, Subtarget, dl);
Evan Cheng7a831ce2007-12-15 03:00:47 +00004104 }
Gabor Greifba36cb52008-08-28 21:40:38 +00004105 } else if (ISD::isBuildVectorAllZeros(V1.getNode())) {
Nate Begeman9008ca62009-04-27 18:41:29 +00004106 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, *this, dl);
4107 if (NewOp.getNode() && X86::isMOVLMask(cast<ShuffleVectorSDNode>(NewOp)))
Evan Chengd880b972008-05-09 21:53:03 +00004108 return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(1),
Nate Begeman9008ca62009-04-27 18:41:29 +00004109 DAG, Subtarget, dl);
Evan Cheng7a831ce2007-12-15 03:00:47 +00004110 }
4111 }
Nate Begeman9008ca62009-04-27 18:41:29 +00004112
4113 if (X86::isPSHUFDMask(SVOp))
4114 return Op;
4115
Evan Chengf26ffe92008-05-29 08:22:04 +00004116 // Check if this can be converted into a logical shift.
4117 bool isLeft = false;
4118 unsigned ShAmt = 0;
Dan Gohman475871a2008-07-27 21:46:04 +00004119 SDValue ShVal;
Nate Begeman9008ca62009-04-27 18:41:29 +00004120 bool isShift = getSubtarget()->hasSSE2() &&
4121 isVectorShift(SVOp, DAG, isLeft, ShVal, ShAmt);
Evan Chengf26ffe92008-05-29 08:22:04 +00004122 if (isShift && ShVal.hasOneUse()) {
Scott Michelfdc40a02009-02-17 22:15:04 +00004123 // If the shifted value has multiple uses, it may be cheaper to use
Evan Chengf26ffe92008-05-29 08:22:04 +00004124 // v_set0 + movlhps or movhlps, etc.
Owen Andersone50ed302009-08-10 22:56:29 +00004125 EVT EVT = VT.getVectorElementType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00004126 ShAmt *= EVT.getSizeInBits();
Dale Johannesenace16102009-02-03 19:33:06 +00004127 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl);
Evan Chengf26ffe92008-05-29 08:22:04 +00004128 }
Nate Begeman9008ca62009-04-27 18:41:29 +00004129
4130 if (X86::isMOVLMask(SVOp)) {
Evan Cheng7e2ff772008-05-08 00:57:18 +00004131 if (V1IsUndef)
4132 return V2;
Gabor Greifba36cb52008-08-28 21:40:38 +00004133 if (ISD::isBuildVectorAllZeros(V1.getNode()))
Dale Johannesenace16102009-02-03 19:33:06 +00004134 return getVZextMovL(VT, VT, V2, DAG, Subtarget, dl);
Nate Begemanfb8ead02008-07-25 19:05:58 +00004135 if (!isMMX)
4136 return Op;
Evan Cheng7e2ff772008-05-08 00:57:18 +00004137 }
Nate Begeman9008ca62009-04-27 18:41:29 +00004138
4139 // FIXME: fold these into legal mask.
4140 if (!isMMX && (X86::isMOVSHDUPMask(SVOp) ||
4141 X86::isMOVSLDUPMask(SVOp) ||
4142 X86::isMOVHLPSMask(SVOp) ||
4143 X86::isMOVHPMask(SVOp) ||
4144 X86::isMOVLPMask(SVOp)))
Evan Cheng9bbbb982006-10-25 20:48:19 +00004145 return Op;
Evan Cheng0db9fe62006-04-25 20:13:52 +00004146
Nate Begeman9008ca62009-04-27 18:41:29 +00004147 if (ShouldXformToMOVHLPS(SVOp) ||
4148 ShouldXformToMOVLP(V1.getNode(), V2.getNode(), SVOp))
4149 return CommuteVectorShuffle(SVOp, DAG);
Evan Cheng0db9fe62006-04-25 20:13:52 +00004150
Evan Chengf26ffe92008-05-29 08:22:04 +00004151 if (isShift) {
4152 // No better options. Use a vshl / vsrl.
Owen Andersone50ed302009-08-10 22:56:29 +00004153 EVT EVT = VT.getVectorElementType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00004154 ShAmt *= EVT.getSizeInBits();
Dale Johannesenace16102009-02-03 19:33:06 +00004155 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl);
Evan Chengf26ffe92008-05-29 08:22:04 +00004156 }
Nate Begeman9008ca62009-04-27 18:41:29 +00004157
Evan Cheng9eca5e82006-10-25 21:49:50 +00004158 bool Commuted = false;
Chris Lattner8a594482007-11-25 00:24:49 +00004159 // FIXME: This should also accept a bitcast of a splat? Be careful, not
4160 // 1,1,1,1 -> v8i16 though.
Gabor Greifba36cb52008-08-28 21:40:38 +00004161 V1IsSplat = isSplatVector(V1.getNode());
4162 V2IsSplat = isSplatVector(V2.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00004163
Chris Lattner8a594482007-11-25 00:24:49 +00004164 // Canonicalize the splat or undef, if present, to be on the RHS.
Evan Cheng9bbbb982006-10-25 20:48:19 +00004165 if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) {
Nate Begeman9008ca62009-04-27 18:41:29 +00004166 Op = CommuteVectorShuffle(SVOp, DAG);
4167 SVOp = cast<ShuffleVectorSDNode>(Op);
4168 V1 = SVOp->getOperand(0);
4169 V2 = SVOp->getOperand(1);
Evan Cheng9bbbb982006-10-25 20:48:19 +00004170 std::swap(V1IsSplat, V2IsSplat);
4171 std::swap(V1IsUndef, V2IsUndef);
Evan Cheng9eca5e82006-10-25 21:49:50 +00004172 Commuted = true;
Evan Cheng9bbbb982006-10-25 20:48:19 +00004173 }
4174
Nate Begeman9008ca62009-04-27 18:41:29 +00004175 if (isCommutedMOVL(SVOp, V2IsSplat, V2IsUndef)) {
4176 // Shuffling low element of v1 into undef, just return v1.
4177 if (V2IsUndef)
4178 return V1;
4179 // If V2 is a splat, the mask may be malformed such as <4,3,3,3>, which
4180 // the instruction selector will not match, so get a canonical MOVL with
4181 // swapped operands to undo the commute.
4182 return getMOVL(DAG, dl, VT, V2, V1);
Evan Chengd9b8e402006-10-16 06:36:00 +00004183 }
Evan Cheng0db9fe62006-04-25 20:13:52 +00004184
Nate Begeman9008ca62009-04-27 18:41:29 +00004185 if (X86::isUNPCKL_v_undef_Mask(SVOp) ||
4186 X86::isUNPCKH_v_undef_Mask(SVOp) ||
4187 X86::isUNPCKLMask(SVOp) ||
4188 X86::isUNPCKHMask(SVOp))
Evan Chengd9b8e402006-10-16 06:36:00 +00004189 return Op;
Evan Chenge1113032006-10-04 18:33:38 +00004190
Evan Cheng9bbbb982006-10-25 20:48:19 +00004191 if (V2IsSplat) {
4192 // Normalize mask so all entries that point to V2 points to its first
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00004193 // element then try to match unpck{h|l} again. If match, return a
Evan Cheng9bbbb982006-10-25 20:48:19 +00004194 // new vector_shuffle with the corrected mask.
Nate Begeman9008ca62009-04-27 18:41:29 +00004195 SDValue NewMask = NormalizeMask(SVOp, DAG);
4196 ShuffleVectorSDNode *NSVOp = cast<ShuffleVectorSDNode>(NewMask);
4197 if (NSVOp != SVOp) {
4198 if (X86::isUNPCKLMask(NSVOp, true)) {
4199 return NewMask;
4200 } else if (X86::isUNPCKHMask(NSVOp, true)) {
4201 return NewMask;
Evan Cheng0db9fe62006-04-25 20:13:52 +00004202 }
4203 }
4204 }
4205
Evan Cheng9eca5e82006-10-25 21:49:50 +00004206 if (Commuted) {
4207 // Commute is back and try unpck* again.
Nate Begeman9008ca62009-04-27 18:41:29 +00004208 // FIXME: this seems wrong.
4209 SDValue NewOp = CommuteVectorShuffle(SVOp, DAG);
4210 ShuffleVectorSDNode *NewSVOp = cast<ShuffleVectorSDNode>(NewOp);
4211 if (X86::isUNPCKL_v_undef_Mask(NewSVOp) ||
4212 X86::isUNPCKH_v_undef_Mask(NewSVOp) ||
4213 X86::isUNPCKLMask(NewSVOp) ||
4214 X86::isUNPCKHMask(NewSVOp))
4215 return NewOp;
Evan Cheng9eca5e82006-10-25 21:49:50 +00004216 }
Evan Cheng0db9fe62006-04-25 20:13:52 +00004217
Nate Begemanb9a47b82009-02-23 08:49:38 +00004218 // FIXME: for mmx, bitcast v2i32 to v4i16 for shuffle.
Nate Begeman9008ca62009-04-27 18:41:29 +00004219
4220 // Normalize the node to match x86 shuffle ops if needed
4221 if (!isMMX && V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(SVOp))
4222 return CommuteVectorShuffle(SVOp, DAG);
4223
4224 // Check for legal shuffle and return?
4225 SmallVector<int, 16> PermMask;
4226 SVOp->getMask(PermMask);
4227 if (isShuffleMaskLegal(PermMask, VT))
Evan Cheng0c0f83f2008-04-05 00:30:36 +00004228 return Op;
Nate Begeman9008ca62009-04-27 18:41:29 +00004229
Evan Cheng14b32e12007-12-11 01:46:18 +00004230 // Handle v8i16 specifically since SSE can do byte extraction and insertion.
Owen Anderson825b72b2009-08-11 20:47:22 +00004231 if (VT == MVT::v8i16) {
Nate Begeman9008ca62009-04-27 18:41:29 +00004232 SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(SVOp, DAG, *this);
Gabor Greifba36cb52008-08-28 21:40:38 +00004233 if (NewOp.getNode())
Evan Cheng14b32e12007-12-11 01:46:18 +00004234 return NewOp;
4235 }
4236
Owen Anderson825b72b2009-08-11 20:47:22 +00004237 if (VT == MVT::v16i8) {
Nate Begeman9008ca62009-04-27 18:41:29 +00004238 SDValue NewOp = LowerVECTOR_SHUFFLEv16i8(SVOp, DAG, *this);
Nate Begemanb9a47b82009-02-23 08:49:38 +00004239 if (NewOp.getNode())
4240 return NewOp;
4241 }
4242
Evan Chengace3c172008-07-22 21:13:36 +00004243 // Handle all 4 wide cases with a number of shuffles except for MMX.
4244 if (NumElems == 4 && !isMMX)
Nate Begeman9008ca62009-04-27 18:41:29 +00004245 return LowerVECTOR_SHUFFLE_4wide(SVOp, DAG);
Evan Cheng0db9fe62006-04-25 20:13:52 +00004246
Dan Gohman475871a2008-07-27 21:46:04 +00004247 return SDValue();
Evan Cheng0db9fe62006-04-25 20:13:52 +00004248}
4249
Dan Gohman475871a2008-07-27 21:46:04 +00004250SDValue
4251X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op,
Nate Begeman14d12ca2008-02-11 04:19:36 +00004252 SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00004253 EVT VT = Op.getValueType();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00004254 DebugLoc dl = Op.getDebugLoc();
Duncan Sands83ec4b62008-06-06 12:08:01 +00004255 if (VT.getSizeInBits() == 8) {
Owen Anderson825b72b2009-08-11 20:47:22 +00004256 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
Nate Begeman14d12ca2008-02-11 04:19:36 +00004257 Op.getOperand(0), Op.getOperand(1));
Owen Anderson825b72b2009-08-11 20:47:22 +00004258 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
Nate Begeman14d12ca2008-02-11 04:19:36 +00004259 DAG.getValueType(VT));
Dale Johannesenace16102009-02-03 19:33:06 +00004260 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
Duncan Sands83ec4b62008-06-06 12:08:01 +00004261 } else if (VT.getSizeInBits() == 16) {
Evan Cheng52ceafa2009-01-02 05:29:08 +00004262 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
4263 // If Idx is 0, it's cheaper to do a move instead of a pextrw.
4264 if (Idx == 0)
Owen Anderson825b72b2009-08-11 20:47:22 +00004265 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
4266 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
Dale Johannesenace16102009-02-03 19:33:06 +00004267 DAG.getNode(ISD::BIT_CONVERT, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00004268 MVT::v4i32,
Evan Cheng52ceafa2009-01-02 05:29:08 +00004269 Op.getOperand(0)),
4270 Op.getOperand(1)));
Owen Anderson825b72b2009-08-11 20:47:22 +00004271 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
Nate Begeman14d12ca2008-02-11 04:19:36 +00004272 Op.getOperand(0), Op.getOperand(1));
Owen Anderson825b72b2009-08-11 20:47:22 +00004273 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
Nate Begeman14d12ca2008-02-11 04:19:36 +00004274 DAG.getValueType(VT));
Dale Johannesenace16102009-02-03 19:33:06 +00004275 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
Owen Anderson825b72b2009-08-11 20:47:22 +00004276 } else if (VT == MVT::f32) {
Evan Cheng62a3f152008-03-24 21:52:23 +00004277 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy
4278 // the result back to FR32 register. It's only worth matching if the
Dan Gohmand17cfbe2008-10-31 00:57:24 +00004279 // result has a single use which is a store or a bitcast to i32. And in
4280 // the case of a store, it's not worth it if the index is a constant 0,
4281 // because a MOVSSmr can be used instead, which is smaller and faster.
Evan Cheng62a3f152008-03-24 21:52:23 +00004282 if (!Op.hasOneUse())
Dan Gohman475871a2008-07-27 21:46:04 +00004283 return SDValue();
Gabor Greifba36cb52008-08-28 21:40:38 +00004284 SDNode *User = *Op.getNode()->use_begin();
Dan Gohmand17cfbe2008-10-31 00:57:24 +00004285 if ((User->getOpcode() != ISD::STORE ||
4286 (isa<ConstantSDNode>(Op.getOperand(1)) &&
4287 cast<ConstantSDNode>(Op.getOperand(1))->isNullValue())) &&
Dan Gohman171c11e2008-04-16 02:32:24 +00004288 (User->getOpcode() != ISD::BIT_CONVERT ||
Owen Anderson825b72b2009-08-11 20:47:22 +00004289 User->getValueType(0) != MVT::i32))
Dan Gohman475871a2008-07-27 21:46:04 +00004290 return SDValue();
Owen Anderson825b72b2009-08-11 20:47:22 +00004291 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
4292 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v4i32,
Dale Johannesenace16102009-02-03 19:33:06 +00004293 Op.getOperand(0)),
4294 Op.getOperand(1));
Owen Anderson825b72b2009-08-11 20:47:22 +00004295 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, Extract);
4296 } else if (VT == MVT::i32) {
Mon P Wangf0fcdd82009-01-15 21:10:20 +00004297 // ExtractPS works with constant index.
4298 if (isa<ConstantSDNode>(Op.getOperand(1)))
4299 return Op;
Nate Begeman14d12ca2008-02-11 04:19:36 +00004300 }
Dan Gohman475871a2008-07-27 21:46:04 +00004301 return SDValue();
Nate Begeman14d12ca2008-02-11 04:19:36 +00004302}
4303
4304
Dan Gohman475871a2008-07-27 21:46:04 +00004305SDValue
4306X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) {
Evan Cheng0db9fe62006-04-25 20:13:52 +00004307 if (!isa<ConstantSDNode>(Op.getOperand(1)))
Dan Gohman475871a2008-07-27 21:46:04 +00004308 return SDValue();
Evan Cheng0db9fe62006-04-25 20:13:52 +00004309
Evan Cheng62a3f152008-03-24 21:52:23 +00004310 if (Subtarget->hasSSE41()) {
Dan Gohman475871a2008-07-27 21:46:04 +00004311 SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG);
Gabor Greifba36cb52008-08-28 21:40:38 +00004312 if (Res.getNode())
Evan Cheng62a3f152008-03-24 21:52:23 +00004313 return Res;
4314 }
Nate Begeman14d12ca2008-02-11 04:19:36 +00004315
Owen Andersone50ed302009-08-10 22:56:29 +00004316 EVT VT = Op.getValueType();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00004317 DebugLoc dl = Op.getDebugLoc();
Evan Cheng0db9fe62006-04-25 20:13:52 +00004318 // TODO: handle v16i8.
Duncan Sands83ec4b62008-06-06 12:08:01 +00004319 if (VT.getSizeInBits() == 16) {
Dan Gohman475871a2008-07-27 21:46:04 +00004320 SDValue Vec = Op.getOperand(0);
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00004321 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
Evan Cheng14b32e12007-12-11 01:46:18 +00004322 if (Idx == 0)
Owen Anderson825b72b2009-08-11 20:47:22 +00004323 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
4324 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
Scott Michelfdc40a02009-02-17 22:15:04 +00004325 DAG.getNode(ISD::BIT_CONVERT, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00004326 MVT::v4i32, Vec),
Evan Cheng14b32e12007-12-11 01:46:18 +00004327 Op.getOperand(1)));
Evan Cheng0db9fe62006-04-25 20:13:52 +00004328 // Transform it so it match pextrw which produces a 32-bit result.
Owen Anderson825b72b2009-08-11 20:47:22 +00004329 EVT EVT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy+1);
Dale Johannesenace16102009-02-03 19:33:06 +00004330 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EVT,
Evan Cheng0db9fe62006-04-25 20:13:52 +00004331 Op.getOperand(0), Op.getOperand(1));
Dale Johannesenace16102009-02-03 19:33:06 +00004332 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, EVT, Extract,
Evan Cheng0db9fe62006-04-25 20:13:52 +00004333 DAG.getValueType(VT));
Dale Johannesenace16102009-02-03 19:33:06 +00004334 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
Duncan Sands83ec4b62008-06-06 12:08:01 +00004335 } else if (VT.getSizeInBits() == 32) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00004336 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
Evan Cheng0db9fe62006-04-25 20:13:52 +00004337 if (Idx == 0)
4338 return Op;
Nate Begeman9008ca62009-04-27 18:41:29 +00004339
Evan Cheng0db9fe62006-04-25 20:13:52 +00004340 // SHUFPS the element to the lowest double word, then movss.
Nate Begeman9008ca62009-04-27 18:41:29 +00004341 int Mask[4] = { Idx, -1, -1, -1 };
Owen Andersone50ed302009-08-10 22:56:29 +00004342 EVT VVT = Op.getOperand(0).getValueType();
Nate Begeman9008ca62009-04-27 18:41:29 +00004343 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
4344 DAG.getUNDEF(VVT), Mask);
Dale Johannesenace16102009-02-03 19:33:06 +00004345 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
Chris Lattner0bd48932008-01-17 07:00:52 +00004346 DAG.getIntPtrConstant(0));
Duncan Sands83ec4b62008-06-06 12:08:01 +00004347 } else if (VT.getSizeInBits() == 64) {
Nate Begeman14d12ca2008-02-11 04:19:36 +00004348 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b
4349 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught
4350 // to match extract_elt for f64.
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00004351 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
Evan Cheng0db9fe62006-04-25 20:13:52 +00004352 if (Idx == 0)
4353 return Op;
4354
4355 // UNPCKHPD the element to the lowest double word, then movsd.
4356 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored
4357 // to a f64mem, the whole operation is folded into a single MOVHPDmr.
Nate Begeman9008ca62009-04-27 18:41:29 +00004358 int Mask[2] = { 1, -1 };
Owen Andersone50ed302009-08-10 22:56:29 +00004359 EVT VVT = Op.getOperand(0).getValueType();
Nate Begeman9008ca62009-04-27 18:41:29 +00004360 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
4361 DAG.getUNDEF(VVT), Mask);
Dale Johannesenace16102009-02-03 19:33:06 +00004362 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
Chris Lattner0bd48932008-01-17 07:00:52 +00004363 DAG.getIntPtrConstant(0));
Evan Cheng0db9fe62006-04-25 20:13:52 +00004364 }
4365
Dan Gohman475871a2008-07-27 21:46:04 +00004366 return SDValue();
Evan Cheng0db9fe62006-04-25 20:13:52 +00004367}
4368
Dan Gohman475871a2008-07-27 21:46:04 +00004369SDValue
4370X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG){
Owen Andersone50ed302009-08-10 22:56:29 +00004371 EVT VT = Op.getValueType();
4372 EVT EVT = VT.getVectorElementType();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00004373 DebugLoc dl = Op.getDebugLoc();
Nate Begeman14d12ca2008-02-11 04:19:36 +00004374
Dan Gohman475871a2008-07-27 21:46:04 +00004375 SDValue N0 = Op.getOperand(0);
4376 SDValue N1 = Op.getOperand(1);
4377 SDValue N2 = Op.getOperand(2);
Nate Begeman14d12ca2008-02-11 04:19:36 +00004378
Dan Gohmanef521f12008-08-14 22:53:18 +00004379 if ((EVT.getSizeInBits() == 8 || EVT.getSizeInBits() == 16) &&
4380 isa<ConstantSDNode>(N2)) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00004381 unsigned Opc = (EVT.getSizeInBits() == 8) ? X86ISD::PINSRB
Nate Begemanb9a47b82009-02-23 08:49:38 +00004382 : X86ISD::PINSRW;
Nate Begeman14d12ca2008-02-11 04:19:36 +00004383 // Transform it so it match pinsr{b,w} which expects a GR32 as its second
4384 // argument.
Owen Anderson825b72b2009-08-11 20:47:22 +00004385 if (N1.getValueType() != MVT::i32)
4386 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
4387 if (N2.getValueType() != MVT::i32)
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00004388 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue());
Dale Johannesenace16102009-02-03 19:33:06 +00004389 return DAG.getNode(Opc, dl, VT, N0, N1, N2);
Owen Anderson825b72b2009-08-11 20:47:22 +00004390 } else if (EVT == MVT::f32 && isa<ConstantSDNode>(N2)) {
Nate Begeman14d12ca2008-02-11 04:19:36 +00004391 // Bits [7:6] of the constant are the source select. This will always be
4392 // zero here. The DAG Combiner may combine an extract_elt index into these
4393 // bits. For example (insert (extract, 3), 2) could be matched by putting
4394 // the '3' into bits [7:6] of X86ISD::INSERTPS.
Scott Michelfdc40a02009-02-17 22:15:04 +00004395 // Bits [5:4] of the constant are the destination select. This is the
Nate Begeman14d12ca2008-02-11 04:19:36 +00004396 // value of the incoming immediate.
Scott Michelfdc40a02009-02-17 22:15:04 +00004397 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may
Nate Begeman14d12ca2008-02-11 04:19:36 +00004398 // combine either bitwise AND or insert of float 0.0 to set these bits.
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00004399 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue() << 4);
Eric Christopherfbd66872009-07-24 00:33:09 +00004400 // Create this as a scalar to vector..
Owen Anderson825b72b2009-08-11 20:47:22 +00004401 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
Dale Johannesenace16102009-02-03 19:33:06 +00004402 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);
Owen Anderson825b72b2009-08-11 20:47:22 +00004403 } else if (EVT == MVT::i32 && isa<ConstantSDNode>(N2)) {
Eric Christopherfbd66872009-07-24 00:33:09 +00004404 // PINSR* works with constant index.
4405 return Op;
Nate Begeman14d12ca2008-02-11 04:19:36 +00004406 }
Dan Gohman475871a2008-07-27 21:46:04 +00004407 return SDValue();
Nate Begeman14d12ca2008-02-11 04:19:36 +00004408}
4409
Dan Gohman475871a2008-07-27 21:46:04 +00004410SDValue
4411X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00004412 EVT VT = Op.getValueType();
4413 EVT EVT = VT.getVectorElementType();
Nate Begeman14d12ca2008-02-11 04:19:36 +00004414
4415 if (Subtarget->hasSSE41())
4416 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG);
4417
Owen Anderson825b72b2009-08-11 20:47:22 +00004418 if (EVT == MVT::i8)
Dan Gohman475871a2008-07-27 21:46:04 +00004419 return SDValue();
Evan Cheng794405e2007-12-12 07:55:34 +00004420
Dale Johannesen6f38cb62009-02-07 19:59:05 +00004421 DebugLoc dl = Op.getDebugLoc();
Dan Gohman475871a2008-07-27 21:46:04 +00004422 SDValue N0 = Op.getOperand(0);
4423 SDValue N1 = Op.getOperand(1);
4424 SDValue N2 = Op.getOperand(2);
Evan Cheng794405e2007-12-12 07:55:34 +00004425
Eli Friedman30e71eb2009-06-06 06:32:50 +00004426 if (EVT.getSizeInBits() == 16 && isa<ConstantSDNode>(N2)) {
Evan Cheng794405e2007-12-12 07:55:34 +00004427 // Transform it so it match pinsrw which expects a 16-bit value in a GR32
4428 // as its second argument.
Owen Anderson825b72b2009-08-11 20:47:22 +00004429 if (N1.getValueType() != MVT::i32)
4430 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
4431 if (N2.getValueType() != MVT::i32)
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00004432 N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue());
Dale Johannesenace16102009-02-03 19:33:06 +00004433 return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2);
Evan Cheng0db9fe62006-04-25 20:13:52 +00004434 }
Dan Gohman475871a2008-07-27 21:46:04 +00004435 return SDValue();
Evan Cheng0db9fe62006-04-25 20:13:52 +00004436}
4437
Dan Gohman475871a2008-07-27 21:46:04 +00004438SDValue
4439X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) {
Dale Johannesen6f38cb62009-02-07 19:59:05 +00004440 DebugLoc dl = Op.getDebugLoc();
Owen Anderson825b72b2009-08-11 20:47:22 +00004441 if (Op.getValueType() == MVT::v2f32)
4442 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f32,
4443 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i32,
4444 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32,
Evan Cheng52672b82008-07-22 18:39:19 +00004445 Op.getOperand(0))));
4446
Owen Anderson825b72b2009-08-11 20:47:22 +00004447 if (Op.getValueType() == MVT::v1i64 && Op.getOperand(0).getValueType() == MVT::i64)
4448 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0));
Rafael Espindoladef390a2009-08-03 02:45:34 +00004449
Owen Anderson825b72b2009-08-11 20:47:22 +00004450 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
4451 EVT VT = MVT::v2i32;
4452 switch (Op.getValueType().getSimpleVT().SimpleTy) {
Evan Chengefec7512008-02-18 23:04:32 +00004453 default: break;
Owen Anderson825b72b2009-08-11 20:47:22 +00004454 case MVT::v16i8:
4455 case MVT::v8i16:
4456 VT = MVT::v4i32;
Evan Chengefec7512008-02-18 23:04:32 +00004457 break;
4458 }
Dale Johannesenace16102009-02-03 19:33:06 +00004459 return DAG.getNode(ISD::BIT_CONVERT, dl, Op.getValueType(),
4460 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, AnyExt));
Evan Cheng0db9fe62006-04-25 20:13:52 +00004461}
4462
Bill Wendling056292f2008-09-16 21:48:12 +00004463// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
4464// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is
4465// one of the above mentioned nodes. It has to be wrapped because otherwise
4466// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
4467// be used to form addressing mode. These wrapped nodes will be selected
4468// into MOV32ri.
Dan Gohman475871a2008-07-27 21:46:04 +00004469SDValue
4470X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) {
Evan Cheng0db9fe62006-04-25 20:13:52 +00004471 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
Chris Lattner41621a22009-06-26 19:22:52 +00004472
4473 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
4474 // global base reg.
4475 unsigned char OpFlag = 0;
Chris Lattner18c59872009-06-27 04:16:01 +00004476 unsigned WrapperKind = X86ISD::Wrapper;
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00004477 CodeModel::Model M = getTargetMachine().getCodeModel();
4478
Chris Lattner4f066492009-07-11 20:29:19 +00004479 if (Subtarget->isPICStyleRIPRel() &&
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00004480 (M == CodeModel::Small || M == CodeModel::Kernel))
Chris Lattnere4df7562009-07-09 03:15:51 +00004481 WrapperKind = X86ISD::WrapperRIP;
Chris Lattner3b67e9b2009-07-10 20:47:30 +00004482 else if (Subtarget->isPICStyleGOT())
Chris Lattner88e1fd52009-07-09 04:24:46 +00004483 OpFlag = X86II::MO_GOTOFF;
Chris Lattnere2c92082009-07-10 21:00:45 +00004484 else if (Subtarget->isPICStyleStubPIC())
Chris Lattner88e1fd52009-07-09 04:24:46 +00004485 OpFlag = X86II::MO_PIC_BASE_OFFSET;
Chris Lattner41621a22009-06-26 19:22:52 +00004486
Evan Cheng1606e8e2009-03-13 07:51:59 +00004487 SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), getPointerTy(),
Chris Lattner41621a22009-06-26 19:22:52 +00004488 CP->getAlignment(),
4489 CP->getOffset(), OpFlag);
4490 DebugLoc DL = CP->getDebugLoc();
Chris Lattner18c59872009-06-27 04:16:01 +00004491 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
Anton Korobeynikov7f705592007-01-12 19:20:47 +00004492 // With PIC, the address is actually $g + Offset.
Chris Lattner41621a22009-06-26 19:22:52 +00004493 if (OpFlag) {
4494 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
Dale Johannesenb300d2a2009-02-07 00:55:49 +00004495 DAG.getNode(X86ISD::GlobalBaseReg,
Chris Lattner41621a22009-06-26 19:22:52 +00004496 DebugLoc::getUnknownLoc(), getPointerTy()),
Anton Korobeynikov7f705592007-01-12 19:20:47 +00004497 Result);
Evan Cheng0db9fe62006-04-25 20:13:52 +00004498 }
4499
4500 return Result;
4501}
4502
Chris Lattner18c59872009-06-27 04:16:01 +00004503SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) {
4504 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
4505
4506 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
4507 // global base reg.
4508 unsigned char OpFlag = 0;
4509 unsigned WrapperKind = X86ISD::Wrapper;
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00004510 CodeModel::Model M = getTargetMachine().getCodeModel();
4511
Chris Lattner4f066492009-07-11 20:29:19 +00004512 if (Subtarget->isPICStyleRIPRel() &&
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00004513 (M == CodeModel::Small || M == CodeModel::Kernel))
Chris Lattnere4df7562009-07-09 03:15:51 +00004514 WrapperKind = X86ISD::WrapperRIP;
Chris Lattner3b67e9b2009-07-10 20:47:30 +00004515 else if (Subtarget->isPICStyleGOT())
Chris Lattner88e1fd52009-07-09 04:24:46 +00004516 OpFlag = X86II::MO_GOTOFF;
Chris Lattnere2c92082009-07-10 21:00:45 +00004517 else if (Subtarget->isPICStyleStubPIC())
Chris Lattner88e1fd52009-07-09 04:24:46 +00004518 OpFlag = X86II::MO_PIC_BASE_OFFSET;
Chris Lattner18c59872009-06-27 04:16:01 +00004519
4520 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy(),
4521 OpFlag);
4522 DebugLoc DL = JT->getDebugLoc();
4523 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
4524
4525 // With PIC, the address is actually $g + Offset.
4526 if (OpFlag) {
4527 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
4528 DAG.getNode(X86ISD::GlobalBaseReg,
4529 DebugLoc::getUnknownLoc(), getPointerTy()),
4530 Result);
4531 }
4532
4533 return Result;
4534}
4535
4536SDValue
4537X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) {
4538 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol();
4539
4540 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
4541 // global base reg.
4542 unsigned char OpFlag = 0;
4543 unsigned WrapperKind = X86ISD::Wrapper;
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00004544 CodeModel::Model M = getTargetMachine().getCodeModel();
4545
Chris Lattner4f066492009-07-11 20:29:19 +00004546 if (Subtarget->isPICStyleRIPRel() &&
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00004547 (M == CodeModel::Small || M == CodeModel::Kernel))
Chris Lattnere4df7562009-07-09 03:15:51 +00004548 WrapperKind = X86ISD::WrapperRIP;
Chris Lattner3b67e9b2009-07-10 20:47:30 +00004549 else if (Subtarget->isPICStyleGOT())
Chris Lattner88e1fd52009-07-09 04:24:46 +00004550 OpFlag = X86II::MO_GOTOFF;
Chris Lattnere2c92082009-07-10 21:00:45 +00004551 else if (Subtarget->isPICStyleStubPIC())
Chris Lattner88e1fd52009-07-09 04:24:46 +00004552 OpFlag = X86II::MO_PIC_BASE_OFFSET;
Chris Lattner18c59872009-06-27 04:16:01 +00004553
4554 SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy(), OpFlag);
4555
4556 DebugLoc DL = Op.getDebugLoc();
4557 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
4558
4559
4560 // With PIC, the address is actually $g + Offset.
4561 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
Chris Lattnere4df7562009-07-09 03:15:51 +00004562 !Subtarget->is64Bit()) {
Chris Lattner18c59872009-06-27 04:16:01 +00004563 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
4564 DAG.getNode(X86ISD::GlobalBaseReg,
4565 DebugLoc::getUnknownLoc(),
4566 getPointerTy()),
4567 Result);
4568 }
4569
4570 return Result;
4571}
4572
Dan Gohman475871a2008-07-27 21:46:04 +00004573SDValue
Dale Johannesen33c960f2009-02-04 20:06:27 +00004574X86TargetLowering::LowerGlobalAddress(const GlobalValue *GV, DebugLoc dl,
Dan Gohman6520e202008-10-18 02:06:02 +00004575 int64_t Offset,
Evan Chengda43bcf2008-09-24 00:05:32 +00004576 SelectionDAG &DAG) const {
Dan Gohman6520e202008-10-18 02:06:02 +00004577 // Create the TargetGlobalAddress node, folding in the constant
4578 // offset if it is legal.
Chris Lattnerd392bd92009-07-10 07:20:05 +00004579 unsigned char OpFlags =
4580 Subtarget->ClassifyGlobalReference(GV, getTargetMachine());
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00004581 CodeModel::Model M = getTargetMachine().getCodeModel();
Dan Gohman6520e202008-10-18 02:06:02 +00004582 SDValue Result;
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00004583 if (OpFlags == X86II::MO_NO_FLAG &&
4584 X86::isOffsetSuitableForCodeModel(Offset, M)) {
Chris Lattner4aa21aa2009-07-09 00:58:53 +00004585 // A direct static reference to a global.
Dale Johannesen60b3ba02009-07-21 00:12:29 +00004586 Result = DAG.getTargetGlobalAddress(GV, getPointerTy(), Offset);
Dan Gohman6520e202008-10-18 02:06:02 +00004587 Offset = 0;
Chris Lattner18c59872009-06-27 04:16:01 +00004588 } else {
Chris Lattnerb1acd682009-06-27 05:39:56 +00004589 Result = DAG.getTargetGlobalAddress(GV, getPointerTy(), 0, OpFlags);
Chris Lattner18c59872009-06-27 04:16:01 +00004590 }
4591
Chris Lattner4f066492009-07-11 20:29:19 +00004592 if (Subtarget->isPICStyleRIPRel() &&
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00004593 (M == CodeModel::Small || M == CodeModel::Kernel))
Chris Lattner18c59872009-06-27 04:16:01 +00004594 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
4595 else
4596 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
Dan Gohman6520e202008-10-18 02:06:02 +00004597
Anton Korobeynikov7f705592007-01-12 19:20:47 +00004598 // With PIC, the address is actually $g + Offset.
Chris Lattner36c25012009-07-10 07:34:39 +00004599 if (isGlobalRelativeToPICBase(OpFlags)) {
Dale Johannesen33c960f2009-02-04 20:06:27 +00004600 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(),
4601 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
Anton Korobeynikov7f705592007-01-12 19:20:47 +00004602 Result);
Evan Cheng0db9fe62006-04-25 20:13:52 +00004603 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004604
Chris Lattner36c25012009-07-10 07:34:39 +00004605 // For globals that require a load from a stub to get the address, emit the
4606 // load.
4607 if (isGlobalStubReference(OpFlags))
Dale Johannesen33c960f2009-02-04 20:06:27 +00004608 Result = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Result,
Dan Gohman3069b872008-02-07 18:41:25 +00004609 PseudoSourceValue::getGOT(), 0);
Evan Cheng0db9fe62006-04-25 20:13:52 +00004610
Dan Gohman6520e202008-10-18 02:06:02 +00004611 // If there was a non-zero offset that we didn't fold, create an explicit
4612 // addition for it.
4613 if (Offset != 0)
Dale Johannesen33c960f2009-02-04 20:06:27 +00004614 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), Result,
Dan Gohman6520e202008-10-18 02:06:02 +00004615 DAG.getConstant(Offset, getPointerTy()));
4616
Evan Cheng0db9fe62006-04-25 20:13:52 +00004617 return Result;
4618}
4619
Evan Chengda43bcf2008-09-24 00:05:32 +00004620SDValue
4621X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) {
4622 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
Dan Gohman6520e202008-10-18 02:06:02 +00004623 int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00004624 return LowerGlobalAddress(GV, Op.getDebugLoc(), Offset, DAG);
Evan Chengda43bcf2008-09-24 00:05:32 +00004625}
4626
Rafael Espindola2ee3db32009-04-17 14:35:58 +00004627static SDValue
4628GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
Owen Andersone50ed302009-08-10 22:56:29 +00004629 SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg,
Chris Lattnerb903bed2009-06-26 21:20:29 +00004630 unsigned char OperandFlags) {
Owen Anderson825b72b2009-08-11 20:47:22 +00004631 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Rafael Espindola2ee3db32009-04-17 14:35:58 +00004632 DebugLoc dl = GA->getDebugLoc();
4633 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(),
4634 GA->getValueType(0),
Chris Lattnerb903bed2009-06-26 21:20:29 +00004635 GA->getOffset(),
4636 OperandFlags);
Rafael Espindola2ee3db32009-04-17 14:35:58 +00004637 if (InFlag) {
4638 SDValue Ops[] = { Chain, TGA, *InFlag };
Rafael Espindola15f1b662009-04-24 12:59:40 +00004639 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 3);
Rafael Espindola2ee3db32009-04-17 14:35:58 +00004640 } else {
4641 SDValue Ops[] = { Chain, TGA };
Rafael Espindola15f1b662009-04-24 12:59:40 +00004642 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 2);
Rafael Espindola2ee3db32009-04-17 14:35:58 +00004643 }
Rafael Espindola15f1b662009-04-24 12:59:40 +00004644 SDValue Flag = Chain.getValue(1);
4645 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Flag);
Rafael Espindola2ee3db32009-04-17 14:35:58 +00004646}
4647
Anton Korobeynikov6625eff2008-05-04 21:36:32 +00004648// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit
Dan Gohman475871a2008-07-27 21:46:04 +00004649static SDValue
Anton Korobeynikov6625eff2008-05-04 21:36:32 +00004650LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
Owen Andersone50ed302009-08-10 22:56:29 +00004651 const EVT PtrVT) {
Dan Gohman475871a2008-07-27 21:46:04 +00004652 SDValue InFlag;
Dale Johannesendd64c412009-02-04 00:33:20 +00004653 DebugLoc dl = GA->getDebugLoc(); // ? function entry point might be better
4654 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00004655 DAG.getNode(X86ISD::GlobalBaseReg,
Dale Johannesenb300d2a2009-02-07 00:55:49 +00004656 DebugLoc::getUnknownLoc(),
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00004657 PtrVT), InFlag);
4658 InFlag = Chain.getValue(1);
4659
Chris Lattnerb903bed2009-06-26 21:20:29 +00004660 return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD);
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00004661}
4662
Anton Korobeynikov6625eff2008-05-04 21:36:32 +00004663// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit
Dan Gohman475871a2008-07-27 21:46:04 +00004664static SDValue
Anton Korobeynikov6625eff2008-05-04 21:36:32 +00004665LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
Owen Andersone50ed302009-08-10 22:56:29 +00004666 const EVT PtrVT) {
Chris Lattnerb903bed2009-06-26 21:20:29 +00004667 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, NULL, PtrVT,
4668 X86::RAX, X86II::MO_TLSGD);
Anton Korobeynikov6625eff2008-05-04 21:36:32 +00004669}
4670
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00004671// Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or
4672// "local exec" model.
Dan Gohman475871a2008-07-27 21:46:04 +00004673static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
Owen Andersone50ed302009-08-10 22:56:29 +00004674 const EVT PtrVT, TLSModel::Model model,
Rafael Espindola7ff5bff2009-04-13 13:02:49 +00004675 bool is64Bit) {
Dale Johannesen33c960f2009-02-04 20:06:27 +00004676 DebugLoc dl = GA->getDebugLoc();
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00004677 // Get the Thread Pointer
Rafael Espindola094fad32009-04-08 21:14:34 +00004678 SDValue Base = DAG.getNode(X86ISD::SegmentBaseAddress,
4679 DebugLoc::getUnknownLoc(), PtrVT,
Rafael Espindola7ff5bff2009-04-13 13:02:49 +00004680 DAG.getRegister(is64Bit? X86::FS : X86::GS,
Owen Anderson825b72b2009-08-11 20:47:22 +00004681 MVT::i32));
Rafael Espindola094fad32009-04-08 21:14:34 +00004682
4683 SDValue ThreadPointer = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Base,
4684 NULL, 0);
4685
Chris Lattnerb903bed2009-06-26 21:20:29 +00004686 unsigned char OperandFlags = 0;
Chris Lattner18c59872009-06-27 04:16:01 +00004687 // Most TLS accesses are not RIP relative, even on x86-64. One exception is
4688 // initialexec.
4689 unsigned WrapperKind = X86ISD::Wrapper;
4690 if (model == TLSModel::LocalExec) {
Chris Lattnerb903bed2009-06-26 21:20:29 +00004691 OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF;
Chris Lattner18c59872009-06-27 04:16:01 +00004692 } else if (is64Bit) {
4693 assert(model == TLSModel::InitialExec);
4694 OperandFlags = X86II::MO_GOTTPOFF;
4695 WrapperKind = X86ISD::WrapperRIP;
4696 } else {
4697 assert(model == TLSModel::InitialExec);
4698 OperandFlags = X86II::MO_INDNTPOFF;
Chris Lattnerb903bed2009-06-26 21:20:29 +00004699 }
Chris Lattnerb903bed2009-06-26 21:20:29 +00004700
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00004701 // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial
4702 // exec)
Chris Lattner4150c082009-06-21 02:22:34 +00004703 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0),
Chris Lattnerb903bed2009-06-26 21:20:29 +00004704 GA->getOffset(), OperandFlags);
Chris Lattner18c59872009-06-27 04:16:01 +00004705 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
Lauro Ramos Venancio7d2cc2b2007-04-22 22:50:52 +00004706
Rafael Espindola9a580232009-02-27 13:37:18 +00004707 if (model == TLSModel::InitialExec)
Dale Johannesen33c960f2009-02-04 20:06:27 +00004708 Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset,
Dan Gohman3069b872008-02-07 18:41:25 +00004709 PseudoSourceValue::getGOT(), 0);
Lauro Ramos Venancio7d2cc2b2007-04-22 22:50:52 +00004710
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00004711 // The address of the thread local variable is the add of the thread
4712 // pointer with the offset of the variable.
Dale Johannesen33c960f2009-02-04 20:06:27 +00004713 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00004714}
4715
Dan Gohman475871a2008-07-27 21:46:04 +00004716SDValue
4717X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) {
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00004718 // TODO: implement the "local dynamic" model
Lauro Ramos Venancio2c5c1112007-04-21 20:56:26 +00004719 // TODO: implement the "initial exec"model for pic executables
Anton Korobeynikov6625eff2008-05-04 21:36:32 +00004720 assert(Subtarget->isTargetELF() &&
4721 "TLS not implemented for non-ELF targets");
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00004722 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
Chris Lattnerb903bed2009-06-26 21:20:29 +00004723 const GlobalValue *GV = GA->getGlobal();
4724
4725 // If GV is an alias then use the aliasee for determining
4726 // thread-localness.
4727 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
4728 GV = GA->resolveAliasedGlobal(false);
4729
4730 TLSModel::Model model = getTLSModel(GV,
4731 getTargetMachine().getRelocationModel());
4732
4733 switch (model) {
4734 case TLSModel::GeneralDynamic:
4735 case TLSModel::LocalDynamic: // not implemented
4736 if (Subtarget->is64Bit())
Rafael Espindola9a580232009-02-27 13:37:18 +00004737 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy());
Chris Lattnerb903bed2009-06-26 21:20:29 +00004738 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy());
4739
4740 case TLSModel::InitialExec:
4741 case TLSModel::LocalExec:
4742 return LowerToTLSExecModel(GA, DAG, getPointerTy(), model,
4743 Subtarget->is64Bit());
Anton Korobeynikov6625eff2008-05-04 21:36:32 +00004744 }
Chris Lattnerb903bed2009-06-26 21:20:29 +00004745
Torok Edwinc23197a2009-07-14 16:55:14 +00004746 llvm_unreachable("Unreachable");
Chris Lattner5867de12009-04-01 22:14:45 +00004747 return SDValue();
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00004748}
4749
Evan Cheng0db9fe62006-04-25 20:13:52 +00004750
Chris Lattner2ff75ee2007-10-17 06:02:13 +00004751/// LowerShift - Lower SRA_PARTS and friends, which return two i32 values and
Scott Michelfdc40a02009-02-17 22:15:04 +00004752/// take a 2 x i32 value to shift plus a shift amount.
Dan Gohman475871a2008-07-27 21:46:04 +00004753SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) {
Dan Gohman4c1fa612008-03-03 22:22:09 +00004754 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
Owen Andersone50ed302009-08-10 22:56:29 +00004755 EVT VT = Op.getValueType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00004756 unsigned VTBits = VT.getSizeInBits();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00004757 DebugLoc dl = Op.getDebugLoc();
Chris Lattner2ff75ee2007-10-17 06:02:13 +00004758 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
Dan Gohman475871a2008-07-27 21:46:04 +00004759 SDValue ShOpLo = Op.getOperand(0);
4760 SDValue ShOpHi = Op.getOperand(1);
4761 SDValue ShAmt = Op.getOperand(2);
Chris Lattner31dcfe62009-07-29 05:48:09 +00004762 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
Owen Anderson825b72b2009-08-11 20:47:22 +00004763 DAG.getConstant(VTBits - 1, MVT::i8))
Chris Lattner31dcfe62009-07-29 05:48:09 +00004764 : DAG.getConstant(0, VT);
Evan Chenge3413162006-01-09 18:33:28 +00004765
Dan Gohman475871a2008-07-27 21:46:04 +00004766 SDValue Tmp2, Tmp3;
Chris Lattner2ff75ee2007-10-17 06:02:13 +00004767 if (Op.getOpcode() == ISD::SHL_PARTS) {
Dale Johannesenace16102009-02-03 19:33:06 +00004768 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt);
4769 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
Chris Lattner2ff75ee2007-10-17 06:02:13 +00004770 } else {
Dale Johannesenace16102009-02-03 19:33:06 +00004771 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt);
4772 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, ShAmt);
Chris Lattner2ff75ee2007-10-17 06:02:13 +00004773 }
Evan Chenge3413162006-01-09 18:33:28 +00004774
Owen Anderson825b72b2009-08-11 20:47:22 +00004775 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
4776 DAG.getConstant(VTBits, MVT::i8));
Dale Johannesenace16102009-02-03 19:33:06 +00004777 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00004778 AndNode, DAG.getConstant(0, MVT::i8));
Evan Chenge3413162006-01-09 18:33:28 +00004779
Dan Gohman475871a2008-07-27 21:46:04 +00004780 SDValue Hi, Lo;
Owen Anderson825b72b2009-08-11 20:47:22 +00004781 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8);
Dan Gohman475871a2008-07-27 21:46:04 +00004782 SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond };
4783 SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond };
Duncan Sandsf9516202008-06-30 10:19:09 +00004784
Chris Lattner2ff75ee2007-10-17 06:02:13 +00004785 if (Op.getOpcode() == ISD::SHL_PARTS) {
Dale Johannesenace16102009-02-03 19:33:06 +00004786 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4);
4787 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4);
Chris Lattner2ff75ee2007-10-17 06:02:13 +00004788 } else {
Dale Johannesenace16102009-02-03 19:33:06 +00004789 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4);
4790 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4);
Chris Lattner2ff75ee2007-10-17 06:02:13 +00004791 }
4792
Dan Gohman475871a2008-07-27 21:46:04 +00004793 SDValue Ops[2] = { Lo, Hi };
Dale Johannesenace16102009-02-03 19:33:06 +00004794 return DAG.getMergeValues(Ops, 2, dl);
Evan Cheng0db9fe62006-04-25 20:13:52 +00004795}
Evan Chenga3195e82006-01-12 22:54:21 +00004796
Dan Gohman475871a2008-07-27 21:46:04 +00004797SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00004798 EVT SrcVT = Op.getOperand(0).getValueType();
Eli Friedman23ef1052009-06-06 03:57:58 +00004799
4800 if (SrcVT.isVector()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00004801 if (SrcVT == MVT::v2i32 && Op.getValueType() == MVT::v2f64) {
Eli Friedman23ef1052009-06-06 03:57:58 +00004802 return Op;
4803 }
4804 return SDValue();
4805 }
4806
Owen Anderson825b72b2009-08-11 20:47:22 +00004807 assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 &&
Chris Lattnerb09916b2008-02-27 05:57:41 +00004808 "Unknown SINT_TO_FP to lower!");
Scott Michelfdc40a02009-02-17 22:15:04 +00004809
Eli Friedman36df4992009-05-27 00:47:34 +00004810 // These are really Legal; return the operand so the caller accepts it as
4811 // Legal.
Owen Anderson825b72b2009-08-11 20:47:22 +00004812 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType()))
Eli Friedman36df4992009-05-27 00:47:34 +00004813 return Op;
Owen Anderson825b72b2009-08-11 20:47:22 +00004814 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) &&
Eli Friedman36df4992009-05-27 00:47:34 +00004815 Subtarget->is64Bit()) {
4816 return Op;
4817 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004818
Dale Johannesen6f38cb62009-02-07 19:59:05 +00004819 DebugLoc dl = Op.getDebugLoc();
Duncan Sands83ec4b62008-06-06 12:08:01 +00004820 unsigned Size = SrcVT.getSizeInBits()/8;
Evan Cheng0db9fe62006-04-25 20:13:52 +00004821 MachineFunction &MF = DAG.getMachineFunction();
4822 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
Dan Gohman475871a2008-07-27 21:46:04 +00004823 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
Dale Johannesenace16102009-02-03 19:33:06 +00004824 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
Bill Wendling105be5a2009-03-13 08:41:47 +00004825 StackSlot,
4826 PseudoSourceValue::getFixedStack(SSFI), 0);
Eli Friedman948e95a2009-05-23 09:59:16 +00004827 return BuildFILD(Op, SrcVT, Chain, StackSlot, DAG);
4828}
Evan Cheng0db9fe62006-04-25 20:13:52 +00004829
Owen Andersone50ed302009-08-10 22:56:29 +00004830SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
Eli Friedman948e95a2009-05-23 09:59:16 +00004831 SDValue StackSlot,
4832 SelectionDAG &DAG) {
Evan Cheng0db9fe62006-04-25 20:13:52 +00004833 // Build the FILD
Eli Friedman948e95a2009-05-23 09:59:16 +00004834 DebugLoc dl = Op.getDebugLoc();
Chris Lattner5a88b832007-02-25 07:10:00 +00004835 SDVTList Tys;
Chris Lattner78631162008-01-16 06:24:21 +00004836 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType());
Dale Johannesen9e3d3ab2007-09-14 22:26:36 +00004837 if (useSSE)
Owen Anderson825b72b2009-08-11 20:47:22 +00004838 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag);
Chris Lattner5a88b832007-02-25 07:10:00 +00004839 else
Owen Anderson825b72b2009-08-11 20:47:22 +00004840 Tys = DAG.getVTList(Op.getValueType(), MVT::Other);
Dan Gohman475871a2008-07-27 21:46:04 +00004841 SmallVector<SDValue, 8> Ops;
Evan Cheng0db9fe62006-04-25 20:13:52 +00004842 Ops.push_back(Chain);
4843 Ops.push_back(StackSlot);
4844 Ops.push_back(DAG.getValueType(SrcVT));
Dale Johannesenace16102009-02-03 19:33:06 +00004845 SDValue Result = DAG.getNode(useSSE ? X86ISD::FILD_FLAG : X86ISD::FILD, dl,
Chris Lattnerb09916b2008-02-27 05:57:41 +00004846 Tys, &Ops[0], Ops.size());
Evan Cheng0db9fe62006-04-25 20:13:52 +00004847
Dale Johannesen9e3d3ab2007-09-14 22:26:36 +00004848 if (useSSE) {
Evan Cheng0db9fe62006-04-25 20:13:52 +00004849 Chain = Result.getValue(1);
Dan Gohman475871a2008-07-27 21:46:04 +00004850 SDValue InFlag = Result.getValue(2);
Evan Cheng0db9fe62006-04-25 20:13:52 +00004851
4852 // FIXME: Currently the FST is flagged to the FILD_FLAG. This
4853 // shouldn't be necessary except that RFP cannot be live across
4854 // multiple blocks. When stackifier is fixed, they can be uncoupled.
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00004855 MachineFunction &MF = DAG.getMachineFunction();
Evan Cheng0db9fe62006-04-25 20:13:52 +00004856 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8);
Dan Gohman475871a2008-07-27 21:46:04 +00004857 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
Owen Anderson825b72b2009-08-11 20:47:22 +00004858 Tys = DAG.getVTList(MVT::Other);
Dan Gohman475871a2008-07-27 21:46:04 +00004859 SmallVector<SDValue, 8> Ops;
Evan Chenga3195e82006-01-12 22:54:21 +00004860 Ops.push_back(Chain);
Evan Cheng0db9fe62006-04-25 20:13:52 +00004861 Ops.push_back(Result);
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00004862 Ops.push_back(StackSlot);
Evan Cheng0db9fe62006-04-25 20:13:52 +00004863 Ops.push_back(DAG.getValueType(Op.getValueType()));
4864 Ops.push_back(InFlag);
Dale Johannesenace16102009-02-03 19:33:06 +00004865 Chain = DAG.getNode(X86ISD::FST, dl, Tys, &Ops[0], Ops.size());
4866 Result = DAG.getLoad(Op.getValueType(), dl, Chain, StackSlot,
Dan Gohmana54cf172008-07-11 22:44:52 +00004867 PseudoSourceValue::getFixedStack(SSFI), 0);
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00004868 }
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00004869
Evan Cheng0db9fe62006-04-25 20:13:52 +00004870 return Result;
4871}
4872
Bill Wendling8b8a6362009-01-17 03:56:04 +00004873// LowerUINT_TO_FP_i64 - 64-bit unsigned integer to double expansion.
4874SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG) {
4875 // This algorithm is not obvious. Here it is in C code, more or less:
4876 /*
4877 double uint64_to_double( uint32_t hi, uint32_t lo ) {
4878 static const __m128i exp = { 0x4330000045300000ULL, 0 };
4879 static const __m128d bias = { 0x1.0p84, 0x1.0p52 };
Dale Johannesen040225f2008-10-21 23:07:49 +00004880
Bill Wendling8b8a6362009-01-17 03:56:04 +00004881 // Copy ints to xmm registers.
4882 __m128i xh = _mm_cvtsi32_si128( hi );
4883 __m128i xl = _mm_cvtsi32_si128( lo );
Dale Johannesen040225f2008-10-21 23:07:49 +00004884
Bill Wendling8b8a6362009-01-17 03:56:04 +00004885 // Combine into low half of a single xmm register.
4886 __m128i x = _mm_unpacklo_epi32( xh, xl );
4887 __m128d d;
4888 double sd;
Dale Johannesen040225f2008-10-21 23:07:49 +00004889
Bill Wendling8b8a6362009-01-17 03:56:04 +00004890 // Merge in appropriate exponents to give the integer bits the right
4891 // magnitude.
4892 x = _mm_unpacklo_epi32( x, exp );
Dale Johannesen040225f2008-10-21 23:07:49 +00004893
Bill Wendling8b8a6362009-01-17 03:56:04 +00004894 // Subtract away the biases to deal with the IEEE-754 double precision
4895 // implicit 1.
4896 d = _mm_sub_pd( (__m128d) x, bias );
Dale Johannesen040225f2008-10-21 23:07:49 +00004897
Bill Wendling8b8a6362009-01-17 03:56:04 +00004898 // All conversions up to here are exact. The correctly rounded result is
4899 // calculated using the current rounding mode using the following
4900 // horizontal add.
4901 d = _mm_add_sd( d, _mm_unpackhi_pd( d, d ) );
4902 _mm_store_sd( &sd, d ); // Because we are returning doubles in XMM, this
4903 // store doesn't really need to be here (except
4904 // maybe to zero the other double)
4905 return sd;
4906 }
4907 */
Dale Johannesen040225f2008-10-21 23:07:49 +00004908
Dale Johannesen6f38cb62009-02-07 19:59:05 +00004909 DebugLoc dl = Op.getDebugLoc();
Owen Andersona90b3dc2009-07-15 21:51:10 +00004910 LLVMContext *Context = DAG.getContext();
Dale Johannesenace16102009-02-03 19:33:06 +00004911
Dale Johannesen1c15bf52008-10-21 20:50:01 +00004912 // Build some magic constants.
Bill Wendling8b8a6362009-01-17 03:56:04 +00004913 std::vector<Constant*> CV0;
Owen Andersoneed707b2009-07-24 23:12:02 +00004914 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0x45300000)));
4915 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0x43300000)));
4916 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0)));
4917 CV0.push_back(ConstantInt::get(*Context, APInt(32, 0)));
Owen Andersonaf7ec972009-07-28 21:19:26 +00004918 Constant *C0 = ConstantVector::get(CV0);
Evan Cheng1606e8e2009-03-13 07:51:59 +00004919 SDValue CPIdx0 = DAG.getConstantPool(C0, getPointerTy(), 16);
Dale Johannesen1c15bf52008-10-21 20:50:01 +00004920
Bill Wendling8b8a6362009-01-17 03:56:04 +00004921 std::vector<Constant*> CV1;
Owen Andersona90b3dc2009-07-15 21:51:10 +00004922 CV1.push_back(
Owen Anderson6f83c9c2009-07-27 20:59:43 +00004923 ConstantFP::get(*Context, APFloat(APInt(64, 0x4530000000000000ULL))));
Owen Andersona90b3dc2009-07-15 21:51:10 +00004924 CV1.push_back(
Owen Anderson6f83c9c2009-07-27 20:59:43 +00004925 ConstantFP::get(*Context, APFloat(APInt(64, 0x4330000000000000ULL))));
Owen Andersonaf7ec972009-07-28 21:19:26 +00004926 Constant *C1 = ConstantVector::get(CV1);
Evan Cheng1606e8e2009-03-13 07:51:59 +00004927 SDValue CPIdx1 = DAG.getConstantPool(C1, getPointerTy(), 16);
Dale Johannesen1c15bf52008-10-21 20:50:01 +00004928
Owen Anderson825b72b2009-08-11 20:47:22 +00004929 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,
4930 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
Duncan Sands6b6aeb32008-10-22 11:24:12 +00004931 Op.getOperand(0),
4932 DAG.getIntPtrConstant(1)));
Owen Anderson825b72b2009-08-11 20:47:22 +00004933 SDValue XR2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,
4934 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
Duncan Sands6b6aeb32008-10-22 11:24:12 +00004935 Op.getOperand(0),
4936 DAG.getIntPtrConstant(0)));
Owen Anderson825b72b2009-08-11 20:47:22 +00004937 SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32, XR1, XR2);
4938 SDValue CLod0 = DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0,
Bill Wendling8b8a6362009-01-17 03:56:04 +00004939 PseudoSourceValue::getConstantPool(), 0,
4940 false, 16);
Owen Anderson825b72b2009-08-11 20:47:22 +00004941 SDValue Unpck2 = getUnpackl(DAG, dl, MVT::v4i32, Unpck1, CLod0);
4942 SDValue XR2F = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f64, Unpck2);
4943 SDValue CLod1 = DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1,
Bill Wendling8b8a6362009-01-17 03:56:04 +00004944 PseudoSourceValue::getConstantPool(), 0,
4945 false, 16);
Owen Anderson825b72b2009-08-11 20:47:22 +00004946 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
Bill Wendling8b8a6362009-01-17 03:56:04 +00004947
Dale Johannesen1c15bf52008-10-21 20:50:01 +00004948 // Add the halves; easiest way is to swap them into another reg first.
Nate Begeman9008ca62009-04-27 18:41:29 +00004949 int ShufMask[2] = { 1, -1 };
Owen Anderson825b72b2009-08-11 20:47:22 +00004950 SDValue Shuf = DAG.getVectorShuffle(MVT::v2f64, dl, Sub,
4951 DAG.getUNDEF(MVT::v2f64), ShufMask);
4952 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuf, Sub);
4953 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Add,
Dale Johannesen1c15bf52008-10-21 20:50:01 +00004954 DAG.getIntPtrConstant(0));
4955}
4956
Bill Wendling8b8a6362009-01-17 03:56:04 +00004957// LowerUINT_TO_FP_i32 - 32-bit unsigned integer to float expansion.
4958SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op, SelectionDAG &DAG) {
Dale Johannesen6f38cb62009-02-07 19:59:05 +00004959 DebugLoc dl = Op.getDebugLoc();
Bill Wendling8b8a6362009-01-17 03:56:04 +00004960 // FP constant to bias correct the final result.
4961 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL),
Owen Anderson825b72b2009-08-11 20:47:22 +00004962 MVT::f64);
Bill Wendling8b8a6362009-01-17 03:56:04 +00004963
4964 // Load the 32-bit value into an XMM register.
Owen Anderson825b72b2009-08-11 20:47:22 +00004965 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,
4966 DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
Bill Wendling8b8a6362009-01-17 03:56:04 +00004967 Op.getOperand(0),
4968 DAG.getIntPtrConstant(0)));
4969
Owen Anderson825b72b2009-08-11 20:47:22 +00004970 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
4971 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f64, Load),
Bill Wendling8b8a6362009-01-17 03:56:04 +00004972 DAG.getIntPtrConstant(0));
4973
4974 // Or the load with the bias.
Owen Anderson825b72b2009-08-11 20:47:22 +00004975 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64,
4976 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64,
Dale Johannesenace16102009-02-03 19:33:06 +00004977 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00004978 MVT::v2f64, Load)),
4979 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64,
Dale Johannesenace16102009-02-03 19:33:06 +00004980 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00004981 MVT::v2f64, Bias)));
4982 Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
4983 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f64, Or),
Bill Wendling8b8a6362009-01-17 03:56:04 +00004984 DAG.getIntPtrConstant(0));
4985
4986 // Subtract the bias.
Owen Anderson825b72b2009-08-11 20:47:22 +00004987 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
Bill Wendling8b8a6362009-01-17 03:56:04 +00004988
4989 // Handle final rounding.
Owen Andersone50ed302009-08-10 22:56:29 +00004990 EVT DestVT = Op.getValueType();
Bill Wendling030939c2009-01-17 07:40:19 +00004991
Owen Anderson825b72b2009-08-11 20:47:22 +00004992 if (DestVT.bitsLT(MVT::f64)) {
Dale Johannesenace16102009-02-03 19:33:06 +00004993 return DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
Bill Wendling030939c2009-01-17 07:40:19 +00004994 DAG.getIntPtrConstant(0));
Owen Anderson825b72b2009-08-11 20:47:22 +00004995 } else if (DestVT.bitsGT(MVT::f64)) {
Dale Johannesenace16102009-02-03 19:33:06 +00004996 return DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
Bill Wendling030939c2009-01-17 07:40:19 +00004997 }
4998
4999 // Handle final rounding.
5000 return Sub;
Bill Wendling8b8a6362009-01-17 03:56:04 +00005001}
5002
5003SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
Evan Chenga06ec9e2009-01-19 08:08:22 +00005004 SDValue N0 = Op.getOperand(0);
Dale Johannesen6f38cb62009-02-07 19:59:05 +00005005 DebugLoc dl = Op.getDebugLoc();
Bill Wendling8b8a6362009-01-17 03:56:04 +00005006
Evan Chenga06ec9e2009-01-19 08:08:22 +00005007 // Now not UINT_TO_FP is legal (it's marked custom), dag combiner won't
5008 // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform
5009 // the optimization here.
5010 if (DAG.SignBitIsZero(N0))
Dale Johannesenace16102009-02-03 19:33:06 +00005011 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0);
Evan Chenga06ec9e2009-01-19 08:08:22 +00005012
Owen Andersone50ed302009-08-10 22:56:29 +00005013 EVT SrcVT = N0.getValueType();
Owen Anderson825b72b2009-08-11 20:47:22 +00005014 if (SrcVT == MVT::i64) {
Eli Friedman36df4992009-05-27 00:47:34 +00005015 // We only handle SSE2 f64 target here; caller can expand the rest.
Owen Anderson825b72b2009-08-11 20:47:22 +00005016 if (Op.getValueType() != MVT::f64 || !X86ScalarSSEf64)
Daniel Dunbar82205572009-05-26 21:27:02 +00005017 return SDValue();
Bill Wendling030939c2009-01-17 07:40:19 +00005018
Bill Wendling8b8a6362009-01-17 03:56:04 +00005019 return LowerUINT_TO_FP_i64(Op, DAG);
Owen Anderson825b72b2009-08-11 20:47:22 +00005020 } else if (SrcVT == MVT::i32 && X86ScalarSSEf64) {
Bill Wendling8b8a6362009-01-17 03:56:04 +00005021 return LowerUINT_TO_FP_i32(Op, DAG);
5022 }
5023
Owen Anderson825b72b2009-08-11 20:47:22 +00005024 assert(SrcVT == MVT::i32 && "Unknown UINT_TO_FP to lower!");
Eli Friedman948e95a2009-05-23 09:59:16 +00005025
5026 // Make a 64-bit buffer, and use it to build an FILD.
Owen Anderson825b72b2009-08-11 20:47:22 +00005027 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64);
Eli Friedman948e95a2009-05-23 09:59:16 +00005028 SDValue WordOff = DAG.getConstant(4, getPointerTy());
5029 SDValue OffsetSlot = DAG.getNode(ISD::ADD, dl,
5030 getPointerTy(), StackSlot, WordOff);
5031 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
5032 StackSlot, NULL, 0);
Owen Anderson825b72b2009-08-11 20:47:22 +00005033 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, MVT::i32),
Eli Friedman948e95a2009-05-23 09:59:16 +00005034 OffsetSlot, NULL, 0);
Owen Anderson825b72b2009-08-11 20:47:22 +00005035 return BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG);
Bill Wendling8b8a6362009-01-17 03:56:04 +00005036}
5037
Dan Gohman475871a2008-07-27 21:46:04 +00005038std::pair<SDValue,SDValue> X86TargetLowering::
Eli Friedman948e95a2009-05-23 09:59:16 +00005039FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned) {
Dale Johannesen6f38cb62009-02-07 19:59:05 +00005040 DebugLoc dl = Op.getDebugLoc();
Eli Friedman948e95a2009-05-23 09:59:16 +00005041
Owen Andersone50ed302009-08-10 22:56:29 +00005042 EVT DstTy = Op.getValueType();
Eli Friedman948e95a2009-05-23 09:59:16 +00005043
5044 if (!IsSigned) {
Owen Anderson825b72b2009-08-11 20:47:22 +00005045 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT");
5046 DstTy = MVT::i64;
Eli Friedman948e95a2009-05-23 09:59:16 +00005047 }
5048
Owen Anderson825b72b2009-08-11 20:47:22 +00005049 assert(DstTy.getSimpleVT() <= MVT::i64 &&
5050 DstTy.getSimpleVT() >= MVT::i16 &&
Evan Cheng0db9fe62006-04-25 20:13:52 +00005051 "Unknown FP_TO_SINT to lower!");
Evan Cheng0db9fe62006-04-25 20:13:52 +00005052
Dale Johannesen9e3d3ab2007-09-14 22:26:36 +00005053 // These are really Legal.
Owen Anderson825b72b2009-08-11 20:47:22 +00005054 if (DstTy == MVT::i32 &&
Chris Lattner78631162008-01-16 06:24:21 +00005055 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
Dan Gohman475871a2008-07-27 21:46:04 +00005056 return std::make_pair(SDValue(), SDValue());
Dale Johannesen73328d12007-09-19 23:55:34 +00005057 if (Subtarget->is64Bit() &&
Owen Anderson825b72b2009-08-11 20:47:22 +00005058 DstTy == MVT::i64 &&
Eli Friedman36df4992009-05-27 00:47:34 +00005059 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
Dan Gohman475871a2008-07-27 21:46:04 +00005060 return std::make_pair(SDValue(), SDValue());
Dale Johannesen9e3d3ab2007-09-14 22:26:36 +00005061
Evan Cheng87c89352007-10-15 20:11:21 +00005062 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary
5063 // stack slot.
5064 MachineFunction &MF = DAG.getMachineFunction();
Eli Friedman948e95a2009-05-23 09:59:16 +00005065 unsigned MemSize = DstTy.getSizeInBits()/8;
Evan Cheng87c89352007-10-15 20:11:21 +00005066 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize);
Dan Gohman475871a2008-07-27 21:46:04 +00005067 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
Eli Friedman948e95a2009-05-23 09:59:16 +00005068
Evan Cheng0db9fe62006-04-25 20:13:52 +00005069 unsigned Opc;
Owen Anderson825b72b2009-08-11 20:47:22 +00005070 switch (DstTy.getSimpleVT().SimpleTy) {
Torok Edwinc23197a2009-07-14 16:55:14 +00005071 default: llvm_unreachable("Invalid FP_TO_SINT to lower!");
Owen Anderson825b72b2009-08-11 20:47:22 +00005072 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
5073 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
5074 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
Evan Cheng0db9fe62006-04-25 20:13:52 +00005075 }
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00005076
Dan Gohman475871a2008-07-27 21:46:04 +00005077 SDValue Chain = DAG.getEntryNode();
5078 SDValue Value = Op.getOperand(0);
Chris Lattner78631162008-01-16 06:24:21 +00005079 if (isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) {
Owen Anderson825b72b2009-08-11 20:47:22 +00005080 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!");
Dale Johannesenace16102009-02-03 19:33:06 +00005081 Chain = DAG.getStore(Chain, dl, Value, StackSlot,
Dan Gohmana54cf172008-07-11 22:44:52 +00005082 PseudoSourceValue::getFixedStack(SSFI), 0);
Owen Anderson825b72b2009-08-11 20:47:22 +00005083 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other);
Dan Gohman475871a2008-07-27 21:46:04 +00005084 SDValue Ops[] = {
Chris Lattner5a88b832007-02-25 07:10:00 +00005085 Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType())
5086 };
Dale Johannesenace16102009-02-03 19:33:06 +00005087 Value = DAG.getNode(X86ISD::FLD, dl, Tys, Ops, 3);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005088 Chain = Value.getValue(1);
5089 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize);
5090 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
5091 }
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00005092
Evan Cheng0db9fe62006-04-25 20:13:52 +00005093 // Build the FP_TO_INT*_IN_MEM
Dan Gohman475871a2008-07-27 21:46:04 +00005094 SDValue Ops[] = { Chain, Value, StackSlot };
Owen Anderson825b72b2009-08-11 20:47:22 +00005095 SDValue FIST = DAG.getNode(Opc, dl, MVT::Other, Ops, 3);
Evan Chengd9558e02006-01-06 00:43:03 +00005096
Chris Lattner27a6c732007-11-24 07:07:01 +00005097 return std::make_pair(FIST, StackSlot);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005098}
5099
Dan Gohman475871a2008-07-27 21:46:04 +00005100SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) {
Eli Friedman23ef1052009-06-06 03:57:58 +00005101 if (Op.getValueType().isVector()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00005102 if (Op.getValueType() == MVT::v2i32 &&
5103 Op.getOperand(0).getValueType() == MVT::v2f64) {
Eli Friedman23ef1052009-06-06 03:57:58 +00005104 return Op;
5105 }
5106 return SDValue();
5107 }
5108
Eli Friedman948e95a2009-05-23 09:59:16 +00005109 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, true);
Dan Gohman475871a2008-07-27 21:46:04 +00005110 SDValue FIST = Vals.first, StackSlot = Vals.second;
Eli Friedman36df4992009-05-27 00:47:34 +00005111 // If FP_TO_INTHelper failed, the node is actually supposed to be Legal.
5112 if (FIST.getNode() == 0) return Op;
Scott Michelfdc40a02009-02-17 22:15:04 +00005113
Chris Lattner27a6c732007-11-24 07:07:01 +00005114 // Load the result.
Dale Johannesen6f38cb62009-02-07 19:59:05 +00005115 return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(),
Dale Johannesenace16102009-02-03 19:33:06 +00005116 FIST, StackSlot, NULL, 0);
Chris Lattner27a6c732007-11-24 07:07:01 +00005117}
5118
Eli Friedman948e95a2009-05-23 09:59:16 +00005119SDValue X86TargetLowering::LowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) {
5120 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, false);
5121 SDValue FIST = Vals.first, StackSlot = Vals.second;
5122 assert(FIST.getNode() && "Unexpected failure");
5123
5124 // Load the result.
5125 return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(),
5126 FIST, StackSlot, NULL, 0);
5127}
5128
Dan Gohman475871a2008-07-27 21:46:04 +00005129SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) {
Owen Andersona90b3dc2009-07-15 21:51:10 +00005130 LLVMContext *Context = DAG.getContext();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00005131 DebugLoc dl = Op.getDebugLoc();
Owen Andersone50ed302009-08-10 22:56:29 +00005132 EVT VT = Op.getValueType();
5133 EVT EltVT = VT;
Duncan Sands83ec4b62008-06-06 12:08:01 +00005134 if (VT.isVector())
5135 EltVT = VT.getVectorElementType();
Evan Cheng0db9fe62006-04-25 20:13:52 +00005136 std::vector<Constant*> CV;
Owen Anderson825b72b2009-08-11 20:47:22 +00005137 if (EltVT == MVT::f64) {
Owen Anderson6f83c9c2009-07-27 20:59:43 +00005138 Constant *C = ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63))));
Dan Gohman20382522007-07-10 00:05:58 +00005139 CV.push_back(C);
5140 CV.push_back(C);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005141 } else {
Owen Anderson6f83c9c2009-07-27 20:59:43 +00005142 Constant *C = ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31))));
Dan Gohman20382522007-07-10 00:05:58 +00005143 CV.push_back(C);
5144 CV.push_back(C);
5145 CV.push_back(C);
5146 CV.push_back(C);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005147 }
Owen Andersonaf7ec972009-07-28 21:19:26 +00005148 Constant *C = ConstantVector::get(CV);
Evan Cheng1606e8e2009-03-13 07:51:59 +00005149 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
Dale Johannesenace16102009-02-03 19:33:06 +00005150 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
Dan Gohman3069b872008-02-07 18:41:25 +00005151 PseudoSourceValue::getConstantPool(), 0,
Dan Gohmand3006222007-07-27 17:16:43 +00005152 false, 16);
Dale Johannesenace16102009-02-03 19:33:06 +00005153 return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005154}
5155
Dan Gohman475871a2008-07-27 21:46:04 +00005156SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) {
Owen Andersona90b3dc2009-07-15 21:51:10 +00005157 LLVMContext *Context = DAG.getContext();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00005158 DebugLoc dl = Op.getDebugLoc();
Owen Andersone50ed302009-08-10 22:56:29 +00005159 EVT VT = Op.getValueType();
5160 EVT EltVT = VT;
Evan Chengd4d01b72007-07-19 23:36:01 +00005161 unsigned EltNum = 1;
Duncan Sands83ec4b62008-06-06 12:08:01 +00005162 if (VT.isVector()) {
5163 EltVT = VT.getVectorElementType();
5164 EltNum = VT.getVectorNumElements();
Evan Chengd4d01b72007-07-19 23:36:01 +00005165 }
Evan Cheng0db9fe62006-04-25 20:13:52 +00005166 std::vector<Constant*> CV;
Owen Anderson825b72b2009-08-11 20:47:22 +00005167 if (EltVT == MVT::f64) {
Owen Anderson6f83c9c2009-07-27 20:59:43 +00005168 Constant *C = ConstantFP::get(*Context, APFloat(APInt(64, 1ULL << 63)));
Dan Gohman20382522007-07-10 00:05:58 +00005169 CV.push_back(C);
5170 CV.push_back(C);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005171 } else {
Owen Anderson6f83c9c2009-07-27 20:59:43 +00005172 Constant *C = ConstantFP::get(*Context, APFloat(APInt(32, 1U << 31)));
Dan Gohman20382522007-07-10 00:05:58 +00005173 CV.push_back(C);
5174 CV.push_back(C);
5175 CV.push_back(C);
5176 CV.push_back(C);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005177 }
Owen Andersonaf7ec972009-07-28 21:19:26 +00005178 Constant *C = ConstantVector::get(CV);
Evan Cheng1606e8e2009-03-13 07:51:59 +00005179 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
Dale Johannesenace16102009-02-03 19:33:06 +00005180 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
Dan Gohman3069b872008-02-07 18:41:25 +00005181 PseudoSourceValue::getConstantPool(), 0,
Dan Gohmand3006222007-07-27 17:16:43 +00005182 false, 16);
Duncan Sands83ec4b62008-06-06 12:08:01 +00005183 if (VT.isVector()) {
Dale Johannesenace16102009-02-03 19:33:06 +00005184 return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00005185 DAG.getNode(ISD::XOR, dl, MVT::v2i64,
5186 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64,
Dale Johannesenace16102009-02-03 19:33:06 +00005187 Op.getOperand(0)),
Owen Anderson825b72b2009-08-11 20:47:22 +00005188 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, Mask)));
Evan Chengd4d01b72007-07-19 23:36:01 +00005189 } else {
Dale Johannesenace16102009-02-03 19:33:06 +00005190 return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask);
Evan Chengd4d01b72007-07-19 23:36:01 +00005191 }
Evan Cheng0db9fe62006-04-25 20:13:52 +00005192}
5193
Dan Gohman475871a2008-07-27 21:46:04 +00005194SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
Owen Andersona90b3dc2009-07-15 21:51:10 +00005195 LLVMContext *Context = DAG.getContext();
Dan Gohman475871a2008-07-27 21:46:04 +00005196 SDValue Op0 = Op.getOperand(0);
5197 SDValue Op1 = Op.getOperand(1);
Dale Johannesen6f38cb62009-02-07 19:59:05 +00005198 DebugLoc dl = Op.getDebugLoc();
Owen Andersone50ed302009-08-10 22:56:29 +00005199 EVT VT = Op.getValueType();
5200 EVT SrcVT = Op1.getValueType();
Evan Cheng73d6cf12007-01-05 21:37:56 +00005201
5202 // If second operand is smaller, extend it first.
Duncan Sands8e4eb092008-06-08 20:54:56 +00005203 if (SrcVT.bitsLT(VT)) {
Dale Johannesenace16102009-02-03 19:33:06 +00005204 Op1 = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op1);
Evan Cheng73d6cf12007-01-05 21:37:56 +00005205 SrcVT = VT;
5206 }
Dale Johannesen61c7ef32007-10-21 01:07:44 +00005207 // And if it is bigger, shrink it first.
Duncan Sands8e4eb092008-06-08 20:54:56 +00005208 if (SrcVT.bitsGT(VT)) {
Dale Johannesenace16102009-02-03 19:33:06 +00005209 Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1));
Dale Johannesen61c7ef32007-10-21 01:07:44 +00005210 SrcVT = VT;
Dale Johannesen61c7ef32007-10-21 01:07:44 +00005211 }
5212
5213 // At this point the operands and the result should have the same
5214 // type, and that won't be f80 since that is not custom lowered.
Evan Cheng73d6cf12007-01-05 21:37:56 +00005215
Evan Cheng68c47cb2007-01-05 07:55:56 +00005216 // First get the sign bit of second operand.
5217 std::vector<Constant*> CV;
Owen Anderson825b72b2009-08-11 20:47:22 +00005218 if (SrcVT == MVT::f64) {
Owen Anderson6f83c9c2009-07-27 20:59:43 +00005219 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 1ULL << 63))));
5220 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 0))));
Evan Cheng68c47cb2007-01-05 07:55:56 +00005221 } else {
Owen Anderson6f83c9c2009-07-27 20:59:43 +00005222 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 1U << 31))));
5223 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
5224 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
5225 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
Evan Cheng68c47cb2007-01-05 07:55:56 +00005226 }
Owen Andersonaf7ec972009-07-28 21:19:26 +00005227 Constant *C = ConstantVector::get(CV);
Evan Cheng1606e8e2009-03-13 07:51:59 +00005228 SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
Dale Johannesenace16102009-02-03 19:33:06 +00005229 SDValue Mask1 = DAG.getLoad(SrcVT, dl, DAG.getEntryNode(), CPIdx,
Dan Gohman3069b872008-02-07 18:41:25 +00005230 PseudoSourceValue::getConstantPool(), 0,
Dan Gohmand3006222007-07-27 17:16:43 +00005231 false, 16);
Dale Johannesenace16102009-02-03 19:33:06 +00005232 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1);
Evan Cheng68c47cb2007-01-05 07:55:56 +00005233
5234 // Shift sign bit right or left if the two operands have different types.
Duncan Sands8e4eb092008-06-08 20:54:56 +00005235 if (SrcVT.bitsGT(VT)) {
Owen Anderson825b72b2009-08-11 20:47:22 +00005236 // Op0 is MVT::f32, Op1 is MVT::f64.
5237 SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, SignBit);
5238 SignBit = DAG.getNode(X86ISD::FSRL, dl, MVT::v2f64, SignBit,
5239 DAG.getConstant(32, MVT::i32));
5240 SignBit = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v4f32, SignBit);
5241 SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, SignBit,
Chris Lattner0bd48932008-01-17 07:00:52 +00005242 DAG.getIntPtrConstant(0));
Evan Cheng68c47cb2007-01-05 07:55:56 +00005243 }
5244
Evan Cheng73d6cf12007-01-05 21:37:56 +00005245 // Clear first operand sign bit.
5246 CV.clear();
Owen Anderson825b72b2009-08-11 20:47:22 +00005247 if (VT == MVT::f64) {
Owen Anderson6f83c9c2009-07-27 20:59:43 +00005248 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63)))));
5249 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 0))));
Evan Cheng73d6cf12007-01-05 21:37:56 +00005250 } else {
Owen Anderson6f83c9c2009-07-27 20:59:43 +00005251 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31)))));
5252 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
5253 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
5254 CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
Evan Cheng73d6cf12007-01-05 21:37:56 +00005255 }
Owen Andersonaf7ec972009-07-28 21:19:26 +00005256 C = ConstantVector::get(CV);
Evan Cheng1606e8e2009-03-13 07:51:59 +00005257 CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
Dale Johannesenace16102009-02-03 19:33:06 +00005258 SDValue Mask2 = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
Dan Gohman3069b872008-02-07 18:41:25 +00005259 PseudoSourceValue::getConstantPool(), 0,
Dan Gohmand3006222007-07-27 17:16:43 +00005260 false, 16);
Dale Johannesenace16102009-02-03 19:33:06 +00005261 SDValue Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Mask2);
Evan Cheng73d6cf12007-01-05 21:37:56 +00005262
5263 // Or the value with the sign bit.
Dale Johannesenace16102009-02-03 19:33:06 +00005264 return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit);
Evan Cheng68c47cb2007-01-05 07:55:56 +00005265}
5266
Dan Gohman076aee32009-03-04 19:44:21 +00005267/// Emit nodes that will be selected as "test Op0,Op0", or something
5268/// equivalent.
Dan Gohman31125812009-03-07 01:58:32 +00005269SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC,
5270 SelectionDAG &DAG) {
Dan Gohman076aee32009-03-04 19:44:21 +00005271 DebugLoc dl = Op.getDebugLoc();
5272
Dan Gohman31125812009-03-07 01:58:32 +00005273 // CF and OF aren't always set the way we want. Determine which
5274 // of these we need.
5275 bool NeedCF = false;
5276 bool NeedOF = false;
5277 switch (X86CC) {
5278 case X86::COND_A: case X86::COND_AE:
5279 case X86::COND_B: case X86::COND_BE:
5280 NeedCF = true;
5281 break;
5282 case X86::COND_G: case X86::COND_GE:
5283 case X86::COND_L: case X86::COND_LE:
5284 case X86::COND_O: case X86::COND_NO:
5285 NeedOF = true;
5286 break;
5287 default: break;
5288 }
5289
Dan Gohman076aee32009-03-04 19:44:21 +00005290 // See if we can use the EFLAGS value from the operand instead of
Dan Gohman31125812009-03-07 01:58:32 +00005291 // doing a separate TEST. TEST always sets OF and CF to 0, so unless
5292 // we prove that the arithmetic won't overflow, we can't use OF or CF.
5293 if (Op.getResNo() == 0 && !NeedOF && !NeedCF) {
Dan Gohman076aee32009-03-04 19:44:21 +00005294 unsigned Opcode = 0;
Dan Gohman51bb4742009-03-05 21:29:28 +00005295 unsigned NumOperands = 0;
Dan Gohman076aee32009-03-04 19:44:21 +00005296 switch (Op.getNode()->getOpcode()) {
5297 case ISD::ADD:
5298 // Due to an isel shortcoming, be conservative if this add is likely to
5299 // be selected as part of a load-modify-store instruction. When the root
5300 // node in a match is a store, isel doesn't know how to remap non-chain
5301 // non-flag uses of other nodes in the match, such as the ADD in this
5302 // case. This leads to the ADD being left around and reselected, with
5303 // the result being two adds in the output.
5304 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
5305 UE = Op.getNode()->use_end(); UI != UE; ++UI)
5306 if (UI->getOpcode() == ISD::STORE)
5307 goto default_case;
Dan Gohman076aee32009-03-04 19:44:21 +00005308 if (ConstantSDNode *C =
Dan Gohman4bfcf2a2009-03-05 19:32:48 +00005309 dyn_cast<ConstantSDNode>(Op.getNode()->getOperand(1))) {
5310 // An add of one will be selected as an INC.
Dan Gohman076aee32009-03-04 19:44:21 +00005311 if (C->getAPIntValue() == 1) {
5312 Opcode = X86ISD::INC;
Dan Gohman51bb4742009-03-05 21:29:28 +00005313 NumOperands = 1;
Dan Gohman076aee32009-03-04 19:44:21 +00005314 break;
5315 }
Dan Gohman4bfcf2a2009-03-05 19:32:48 +00005316 // An add of negative one (subtract of one) will be selected as a DEC.
5317 if (C->getAPIntValue().isAllOnesValue()) {
5318 Opcode = X86ISD::DEC;
Dan Gohman51bb4742009-03-05 21:29:28 +00005319 NumOperands = 1;
Dan Gohman4bfcf2a2009-03-05 19:32:48 +00005320 break;
5321 }
5322 }
Dan Gohman076aee32009-03-04 19:44:21 +00005323 // Otherwise use a regular EFLAGS-setting add.
5324 Opcode = X86ISD::ADD;
Dan Gohman51bb4742009-03-05 21:29:28 +00005325 NumOperands = 2;
Dan Gohman076aee32009-03-04 19:44:21 +00005326 break;
5327 case ISD::SUB:
5328 // Due to the ISEL shortcoming noted above, be conservative if this sub is
5329 // likely to be selected as part of a load-modify-store instruction.
5330 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
5331 UE = Op.getNode()->use_end(); UI != UE; ++UI)
5332 if (UI->getOpcode() == ISD::STORE)
5333 goto default_case;
Dan Gohman076aee32009-03-04 19:44:21 +00005334 // Otherwise use a regular EFLAGS-setting sub.
5335 Opcode = X86ISD::SUB;
Dan Gohman51bb4742009-03-05 21:29:28 +00005336 NumOperands = 2;
Dan Gohman076aee32009-03-04 19:44:21 +00005337 break;
5338 case X86ISD::ADD:
5339 case X86ISD::SUB:
5340 case X86ISD::INC:
5341 case X86ISD::DEC:
5342 return SDValue(Op.getNode(), 1);
5343 default:
5344 default_case:
5345 break;
5346 }
5347 if (Opcode != 0) {
Owen Anderson825b72b2009-08-11 20:47:22 +00005348 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
Dan Gohman076aee32009-03-04 19:44:21 +00005349 SmallVector<SDValue, 4> Ops;
Dan Gohman31125812009-03-07 01:58:32 +00005350 for (unsigned i = 0; i != NumOperands; ++i)
Dan Gohman076aee32009-03-04 19:44:21 +00005351 Ops.push_back(Op.getOperand(i));
Dan Gohmanfc166572009-04-09 23:54:40 +00005352 SDValue New = DAG.getNode(Opcode, dl, VTs, &Ops[0], NumOperands);
Dan Gohman076aee32009-03-04 19:44:21 +00005353 DAG.ReplaceAllUsesWith(Op, New);
5354 return SDValue(New.getNode(), 1);
5355 }
5356 }
5357
5358 // Otherwise just emit a CMP with 0, which is the TEST pattern.
Owen Anderson825b72b2009-08-11 20:47:22 +00005359 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
Dan Gohman076aee32009-03-04 19:44:21 +00005360 DAG.getConstant(0, Op.getValueType()));
5361}
5362
5363/// Emit nodes that will be selected as "cmp Op0,Op1", or something
5364/// equivalent.
Dan Gohman31125812009-03-07 01:58:32 +00005365SDValue X86TargetLowering::EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
5366 SelectionDAG &DAG) {
Dan Gohman076aee32009-03-04 19:44:21 +00005367 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op1))
5368 if (C->getAPIntValue() == 0)
Dan Gohman31125812009-03-07 01:58:32 +00005369 return EmitTest(Op0, X86CC, DAG);
Dan Gohman076aee32009-03-04 19:44:21 +00005370
5371 DebugLoc dl = Op0.getDebugLoc();
Owen Anderson825b72b2009-08-11 20:47:22 +00005372 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1);
Dan Gohman076aee32009-03-04 19:44:21 +00005373}
5374
Dan Gohman475871a2008-07-27 21:46:04 +00005375SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) {
Owen Anderson825b72b2009-08-11 20:47:22 +00005376 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer");
Dan Gohman475871a2008-07-27 21:46:04 +00005377 SDValue Op0 = Op.getOperand(0);
5378 SDValue Op1 = Op.getOperand(1);
Dale Johannesen6f38cb62009-02-07 19:59:05 +00005379 DebugLoc dl = Op.getDebugLoc();
Chris Lattnere55484e2008-12-25 05:34:37 +00005380 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
Scott Michelfdc40a02009-02-17 22:15:04 +00005381
Dan Gohmane5af2d32009-01-29 01:59:02 +00005382 // Lower (X & (1 << N)) == 0 to BT(X, N).
5383 // Lower ((X >>u N) & 1) != 0 to BT(X, N).
5384 // Lower ((X >>s N) & 1) != 0 to BT(X, N).
Dan Gohman286575c2009-01-13 23:25:30 +00005385 if (Op0.getOpcode() == ISD::AND &&
5386 Op0.hasOneUse() &&
5387 Op1.getOpcode() == ISD::Constant &&
Dan Gohmane5af2d32009-01-29 01:59:02 +00005388 cast<ConstantSDNode>(Op1)->getZExtValue() == 0 &&
Chris Lattnere55484e2008-12-25 05:34:37 +00005389 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
Dan Gohmane5af2d32009-01-29 01:59:02 +00005390 SDValue LHS, RHS;
5391 if (Op0.getOperand(1).getOpcode() == ISD::SHL) {
5392 if (ConstantSDNode *Op010C =
5393 dyn_cast<ConstantSDNode>(Op0.getOperand(1).getOperand(0)))
5394 if (Op010C->getZExtValue() == 1) {
5395 LHS = Op0.getOperand(0);
5396 RHS = Op0.getOperand(1).getOperand(1);
5397 }
5398 } else if (Op0.getOperand(0).getOpcode() == ISD::SHL) {
5399 if (ConstantSDNode *Op000C =
5400 dyn_cast<ConstantSDNode>(Op0.getOperand(0).getOperand(0)))
5401 if (Op000C->getZExtValue() == 1) {
5402 LHS = Op0.getOperand(1);
5403 RHS = Op0.getOperand(0).getOperand(1);
5404 }
5405 } else if (Op0.getOperand(1).getOpcode() == ISD::Constant) {
5406 ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op0.getOperand(1));
5407 SDValue AndLHS = Op0.getOperand(0);
5408 if (AndRHS->getZExtValue() == 1 && AndLHS.getOpcode() == ISD::SRL) {
5409 LHS = AndLHS.getOperand(0);
5410 RHS = AndLHS.getOperand(1);
5411 }
5412 }
Evan Cheng0488db92007-09-25 01:57:46 +00005413
Dan Gohmane5af2d32009-01-29 01:59:02 +00005414 if (LHS.getNode()) {
Chris Lattnere55484e2008-12-25 05:34:37 +00005415 // If LHS is i8, promote it to i16 with any_extend. There is no i8 BT
5416 // instruction. Since the shift amount is in-range-or-undefined, we know
5417 // that doing a bittest on the i16 value is ok. We extend to i32 because
5418 // the encoding for the i16 version is larger than the i32 version.
Owen Anderson825b72b2009-08-11 20:47:22 +00005419 if (LHS.getValueType() == MVT::i8)
5420 LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS);
Chris Lattnere55484e2008-12-25 05:34:37 +00005421
5422 // If the operand types disagree, extend the shift amount to match. Since
5423 // BT ignores high bits (like shifts) we can use anyextend.
5424 if (LHS.getValueType() != RHS.getValueType())
Dale Johannesenace16102009-02-03 19:33:06 +00005425 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LHS.getValueType(), RHS);
Dan Gohmane5af2d32009-01-29 01:59:02 +00005426
Owen Anderson825b72b2009-08-11 20:47:22 +00005427 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS);
Dan Gohman653456c2009-01-07 00:15:08 +00005428 unsigned Cond = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
Owen Anderson825b72b2009-08-11 20:47:22 +00005429 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
5430 DAG.getConstant(Cond, MVT::i8), BT);
Chris Lattnere55484e2008-12-25 05:34:37 +00005431 }
5432 }
5433
5434 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint();
5435 unsigned X86CC = TranslateX86CC(CC, isFP, Op0, Op1, DAG);
Scott Michelfdc40a02009-02-17 22:15:04 +00005436
Dan Gohman31125812009-03-07 01:58:32 +00005437 SDValue Cond = EmitCmp(Op0, Op1, X86CC, DAG);
Owen Anderson825b72b2009-08-11 20:47:22 +00005438 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
5439 DAG.getConstant(X86CC, MVT::i8), Cond);
Evan Cheng0488db92007-09-25 01:57:46 +00005440}
5441
Dan Gohman475871a2008-07-27 21:46:04 +00005442SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) {
5443 SDValue Cond;
5444 SDValue Op0 = Op.getOperand(0);
5445 SDValue Op1 = Op.getOperand(1);
5446 SDValue CC = Op.getOperand(2);
Owen Andersone50ed302009-08-10 22:56:29 +00005447 EVT VT = Op.getValueType();
Nate Begeman30a0de92008-07-17 16:51:19 +00005448 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
5449 bool isFP = Op.getOperand(1).getValueType().isFloatingPoint();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00005450 DebugLoc dl = Op.getDebugLoc();
Nate Begeman30a0de92008-07-17 16:51:19 +00005451
5452 if (isFP) {
5453 unsigned SSECC = 8;
Owen Andersone50ed302009-08-10 22:56:29 +00005454 EVT VT0 = Op0.getValueType();
Owen Anderson825b72b2009-08-11 20:47:22 +00005455 assert(VT0 == MVT::v4f32 || VT0 == MVT::v2f64);
5456 unsigned Opc = VT0 == MVT::v4f32 ? X86ISD::CMPPS : X86ISD::CMPPD;
Nate Begeman30a0de92008-07-17 16:51:19 +00005457 bool Swap = false;
5458
5459 switch (SetCCOpcode) {
5460 default: break;
Nate Begemanfb8ead02008-07-25 19:05:58 +00005461 case ISD::SETOEQ:
Nate Begeman30a0de92008-07-17 16:51:19 +00005462 case ISD::SETEQ: SSECC = 0; break;
Scott Michelfdc40a02009-02-17 22:15:04 +00005463 case ISD::SETOGT:
Nate Begeman30a0de92008-07-17 16:51:19 +00005464 case ISD::SETGT: Swap = true; // Fallthrough
5465 case ISD::SETLT:
5466 case ISD::SETOLT: SSECC = 1; break;
5467 case ISD::SETOGE:
5468 case ISD::SETGE: Swap = true; // Fallthrough
5469 case ISD::SETLE:
5470 case ISD::SETOLE: SSECC = 2; break;
5471 case ISD::SETUO: SSECC = 3; break;
Nate Begemanfb8ead02008-07-25 19:05:58 +00005472 case ISD::SETUNE:
Nate Begeman30a0de92008-07-17 16:51:19 +00005473 case ISD::SETNE: SSECC = 4; break;
5474 case ISD::SETULE: Swap = true;
5475 case ISD::SETUGE: SSECC = 5; break;
5476 case ISD::SETULT: Swap = true;
5477 case ISD::SETUGT: SSECC = 6; break;
5478 case ISD::SETO: SSECC = 7; break;
5479 }
5480 if (Swap)
5481 std::swap(Op0, Op1);
5482
Nate Begemanfb8ead02008-07-25 19:05:58 +00005483 // In the two special cases we can't handle, emit two comparisons.
Nate Begeman30a0de92008-07-17 16:51:19 +00005484 if (SSECC == 8) {
Nate Begemanfb8ead02008-07-25 19:05:58 +00005485 if (SetCCOpcode == ISD::SETUEQ) {
Dan Gohman475871a2008-07-27 21:46:04 +00005486 SDValue UNORD, EQ;
Owen Anderson825b72b2009-08-11 20:47:22 +00005487 UNORD = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(3, MVT::i8));
5488 EQ = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(0, MVT::i8));
Dale Johannesenace16102009-02-03 19:33:06 +00005489 return DAG.getNode(ISD::OR, dl, VT, UNORD, EQ);
Nate Begemanfb8ead02008-07-25 19:05:58 +00005490 }
5491 else if (SetCCOpcode == ISD::SETONE) {
Dan Gohman475871a2008-07-27 21:46:04 +00005492 SDValue ORD, NEQ;
Owen Anderson825b72b2009-08-11 20:47:22 +00005493 ORD = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(7, MVT::i8));
5494 NEQ = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(4, MVT::i8));
Dale Johannesenace16102009-02-03 19:33:06 +00005495 return DAG.getNode(ISD::AND, dl, VT, ORD, NEQ);
Nate Begemanfb8ead02008-07-25 19:05:58 +00005496 }
Torok Edwinc23197a2009-07-14 16:55:14 +00005497 llvm_unreachable("Illegal FP comparison");
Nate Begeman30a0de92008-07-17 16:51:19 +00005498 }
5499 // Handle all other FP comparisons here.
Owen Anderson825b72b2009-08-11 20:47:22 +00005500 return DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8));
Nate Begeman30a0de92008-07-17 16:51:19 +00005501 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005502
Nate Begeman30a0de92008-07-17 16:51:19 +00005503 // We are handling one of the integer comparisons here. Since SSE only has
5504 // GT and EQ comparisons for integer, swapping operands and multiple
5505 // operations may be required for some comparisons.
5506 unsigned Opc = 0, EQOpc = 0, GTOpc = 0;
5507 bool Swap = false, Invert = false, FlipSigns = false;
Scott Michelfdc40a02009-02-17 22:15:04 +00005508
Owen Anderson825b72b2009-08-11 20:47:22 +00005509 switch (VT.getSimpleVT().SimpleTy) {
Nate Begeman30a0de92008-07-17 16:51:19 +00005510 default: break;
Owen Anderson825b72b2009-08-11 20:47:22 +00005511 case MVT::v8i8:
5512 case MVT::v16i8: EQOpc = X86ISD::PCMPEQB; GTOpc = X86ISD::PCMPGTB; break;
5513 case MVT::v4i16:
5514 case MVT::v8i16: EQOpc = X86ISD::PCMPEQW; GTOpc = X86ISD::PCMPGTW; break;
5515 case MVT::v2i32:
5516 case MVT::v4i32: EQOpc = X86ISD::PCMPEQD; GTOpc = X86ISD::PCMPGTD; break;
5517 case MVT::v2i64: EQOpc = X86ISD::PCMPEQQ; GTOpc = X86ISD::PCMPGTQ; break;
Nate Begeman30a0de92008-07-17 16:51:19 +00005518 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005519
Nate Begeman30a0de92008-07-17 16:51:19 +00005520 switch (SetCCOpcode) {
5521 default: break;
5522 case ISD::SETNE: Invert = true;
5523 case ISD::SETEQ: Opc = EQOpc; break;
5524 case ISD::SETLT: Swap = true;
5525 case ISD::SETGT: Opc = GTOpc; break;
5526 case ISD::SETGE: Swap = true;
5527 case ISD::SETLE: Opc = GTOpc; Invert = true; break;
5528 case ISD::SETULT: Swap = true;
5529 case ISD::SETUGT: Opc = GTOpc; FlipSigns = true; break;
5530 case ISD::SETUGE: Swap = true;
5531 case ISD::SETULE: Opc = GTOpc; FlipSigns = true; Invert = true; break;
5532 }
5533 if (Swap)
5534 std::swap(Op0, Op1);
Scott Michelfdc40a02009-02-17 22:15:04 +00005535
Nate Begeman30a0de92008-07-17 16:51:19 +00005536 // Since SSE has no unsigned integer comparisons, we need to flip the sign
5537 // bits of the inputs before performing those operations.
5538 if (FlipSigns) {
Owen Andersone50ed302009-08-10 22:56:29 +00005539 EVT EltVT = VT.getVectorElementType();
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00005540 SDValue SignBit = DAG.getConstant(APInt::getSignBit(EltVT.getSizeInBits()),
5541 EltVT);
Dan Gohman475871a2008-07-27 21:46:04 +00005542 std::vector<SDValue> SignBits(VT.getVectorNumElements(), SignBit);
Evan Chenga87008d2009-02-25 22:49:59 +00005543 SDValue SignVec = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &SignBits[0],
5544 SignBits.size());
Dale Johannesenace16102009-02-03 19:33:06 +00005545 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SignVec);
5546 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SignVec);
Nate Begeman30a0de92008-07-17 16:51:19 +00005547 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005548
Dale Johannesenace16102009-02-03 19:33:06 +00005549 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
Nate Begeman30a0de92008-07-17 16:51:19 +00005550
5551 // If the logical-not of the result is required, perform that now.
Bob Wilson4c245462009-01-22 17:39:32 +00005552 if (Invert)
Dale Johannesenace16102009-02-03 19:33:06 +00005553 Result = DAG.getNOT(dl, Result, VT);
Bob Wilson4c245462009-01-22 17:39:32 +00005554
Nate Begeman30a0de92008-07-17 16:51:19 +00005555 return Result;
5556}
Evan Cheng0488db92007-09-25 01:57:46 +00005557
Evan Cheng370e5342008-12-03 08:38:43 +00005558// isX86LogicalCmp - Return true if opcode is a X86 logical comparison.
Dan Gohman076aee32009-03-04 19:44:21 +00005559static bool isX86LogicalCmp(SDValue Op) {
5560 unsigned Opc = Op.getNode()->getOpcode();
5561 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI)
5562 return true;
5563 if (Op.getResNo() == 1 &&
5564 (Opc == X86ISD::ADD ||
5565 Opc == X86ISD::SUB ||
5566 Opc == X86ISD::SMUL ||
5567 Opc == X86ISD::UMUL ||
5568 Opc == X86ISD::INC ||
5569 Opc == X86ISD::DEC))
5570 return true;
5571
5572 return false;
Evan Cheng370e5342008-12-03 08:38:43 +00005573}
5574
Dan Gohman475871a2008-07-27 21:46:04 +00005575SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) {
Evan Cheng734503b2006-09-11 02:19:56 +00005576 bool addTest = true;
Dan Gohman475871a2008-07-27 21:46:04 +00005577 SDValue Cond = Op.getOperand(0);
Dale Johannesen6f38cb62009-02-07 19:59:05 +00005578 DebugLoc dl = Op.getDebugLoc();
Dan Gohman475871a2008-07-27 21:46:04 +00005579 SDValue CC;
Evan Cheng9bba8942006-01-26 02:13:10 +00005580
Evan Cheng734503b2006-09-11 02:19:56 +00005581 if (Cond.getOpcode() == ISD::SETCC)
Evan Chenge5f62042007-09-29 00:00:36 +00005582 Cond = LowerSETCC(Cond, DAG);
Evan Cheng734503b2006-09-11 02:19:56 +00005583
Evan Cheng3f41d662007-10-08 22:16:29 +00005584 // If condition flag is set by a X86ISD::CMP, then use it as the condition
5585 // setting operand in place of the X86ISD::SETCC.
Evan Cheng734503b2006-09-11 02:19:56 +00005586 if (Cond.getOpcode() == X86ISD::SETCC) {
5587 CC = Cond.getOperand(0);
5588
Dan Gohman475871a2008-07-27 21:46:04 +00005589 SDValue Cmp = Cond.getOperand(1);
Evan Cheng734503b2006-09-11 02:19:56 +00005590 unsigned Opc = Cmp.getOpcode();
Owen Andersone50ed302009-08-10 22:56:29 +00005591 EVT VT = Op.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00005592
Evan Cheng3f41d662007-10-08 22:16:29 +00005593 bool IllegalFPCMov = false;
Duncan Sands83ec4b62008-06-06 12:08:01 +00005594 if (VT.isFloatingPoint() && !VT.isVector() &&
Chris Lattner78631162008-01-16 06:24:21 +00005595 !isScalarFPTypeInSSEReg(VT)) // FPStack?
Dan Gohman7810bfe2008-09-26 21:54:37 +00005596 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue());
Scott Michelfdc40a02009-02-17 22:15:04 +00005597
Chris Lattnerd1980a52009-03-12 06:52:53 +00005598 if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) ||
5599 Opc == X86ISD::BT) { // FIXME
Evan Cheng3f41d662007-10-08 22:16:29 +00005600 Cond = Cmp;
Evan Cheng0488db92007-09-25 01:57:46 +00005601 addTest = false;
5602 }
5603 }
5604
5605 if (addTest) {
Owen Anderson825b72b2009-08-11 20:47:22 +00005606 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
Dan Gohman31125812009-03-07 01:58:32 +00005607 Cond = EmitTest(Cond, X86::COND_NE, DAG);
Evan Cheng0488db92007-09-25 01:57:46 +00005608 }
5609
Owen Anderson825b72b2009-08-11 20:47:22 +00005610 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Flag);
Dan Gohman475871a2008-07-27 21:46:04 +00005611 SmallVector<SDValue, 4> Ops;
Evan Cheng0488db92007-09-25 01:57:46 +00005612 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
5613 // condition is true.
5614 Ops.push_back(Op.getOperand(2));
5615 Ops.push_back(Op.getOperand(1));
5616 Ops.push_back(CC);
5617 Ops.push_back(Cond);
Dan Gohmanfc166572009-04-09 23:54:40 +00005618 return DAG.getNode(X86ISD::CMOV, dl, VTs, &Ops[0], Ops.size());
Evan Cheng0488db92007-09-25 01:57:46 +00005619}
5620
Evan Cheng370e5342008-12-03 08:38:43 +00005621// isAndOrOfSingleUseSetCCs - Return true if node is an ISD::AND or
5622// ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart
5623// from the AND / OR.
5624static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
5625 Opc = Op.getOpcode();
5626 if (Opc != ISD::OR && Opc != ISD::AND)
5627 return false;
5628 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
5629 Op.getOperand(0).hasOneUse() &&
5630 Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
5631 Op.getOperand(1).hasOneUse());
5632}
5633
Evan Cheng961d6d42009-02-02 08:19:07 +00005634// isXor1OfSetCC - Return true if node is an ISD::XOR of a X86ISD::SETCC and
5635// 1 and that the SETCC node has a single use.
Evan Cheng67ad9db2009-02-02 08:07:36 +00005636static bool isXor1OfSetCC(SDValue Op) {
5637 if (Op.getOpcode() != ISD::XOR)
5638 return false;
5639 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
5640 if (N1C && N1C->getAPIntValue() == 1) {
5641 return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
5642 Op.getOperand(0).hasOneUse();
5643 }
5644 return false;
5645}
5646
Dan Gohman475871a2008-07-27 21:46:04 +00005647SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) {
Evan Cheng734503b2006-09-11 02:19:56 +00005648 bool addTest = true;
Dan Gohman475871a2008-07-27 21:46:04 +00005649 SDValue Chain = Op.getOperand(0);
5650 SDValue Cond = Op.getOperand(1);
5651 SDValue Dest = Op.getOperand(2);
Dale Johannesen6f38cb62009-02-07 19:59:05 +00005652 DebugLoc dl = Op.getDebugLoc();
Dan Gohman475871a2008-07-27 21:46:04 +00005653 SDValue CC;
Evan Cheng734503b2006-09-11 02:19:56 +00005654
Evan Cheng0db9fe62006-04-25 20:13:52 +00005655 if (Cond.getOpcode() == ISD::SETCC)
Evan Chenge5f62042007-09-29 00:00:36 +00005656 Cond = LowerSETCC(Cond, DAG);
Chris Lattnere55484e2008-12-25 05:34:37 +00005657#if 0
5658 // FIXME: LowerXALUO doesn't handle these!!
Bill Wendlingd350e022008-12-12 21:15:41 +00005659 else if (Cond.getOpcode() == X86ISD::ADD ||
5660 Cond.getOpcode() == X86ISD::SUB ||
5661 Cond.getOpcode() == X86ISD::SMUL ||
5662 Cond.getOpcode() == X86ISD::UMUL)
Bill Wendling74c37652008-12-09 22:08:41 +00005663 Cond = LowerXALUO(Cond, DAG);
Chris Lattnere55484e2008-12-25 05:34:37 +00005664#endif
Scott Michelfdc40a02009-02-17 22:15:04 +00005665
Evan Cheng3f41d662007-10-08 22:16:29 +00005666 // If condition flag is set by a X86ISD::CMP, then use it as the condition
5667 // setting operand in place of the X86ISD::SETCC.
Evan Cheng0db9fe62006-04-25 20:13:52 +00005668 if (Cond.getOpcode() == X86ISD::SETCC) {
Evan Cheng734503b2006-09-11 02:19:56 +00005669 CC = Cond.getOperand(0);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005670
Dan Gohman475871a2008-07-27 21:46:04 +00005671 SDValue Cmp = Cond.getOperand(1);
Evan Cheng734503b2006-09-11 02:19:56 +00005672 unsigned Opc = Cmp.getOpcode();
Chris Lattnere55484e2008-12-25 05:34:37 +00005673 // FIXME: WHY THE SPECIAL CASING OF LogicalCmp??
Dan Gohman076aee32009-03-04 19:44:21 +00005674 if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) {
Evan Cheng3f41d662007-10-08 22:16:29 +00005675 Cond = Cmp;
Evan Cheng0488db92007-09-25 01:57:46 +00005676 addTest = false;
Bill Wendling61edeb52008-12-02 01:06:39 +00005677 } else {
Evan Cheng370e5342008-12-03 08:38:43 +00005678 switch (cast<ConstantSDNode>(CC)->getZExtValue()) {
Bill Wendling0ea25cb2008-12-03 08:32:02 +00005679 default: break;
5680 case X86::COND_O:
Dan Gohman653456c2009-01-07 00:15:08 +00005681 case X86::COND_B:
Chris Lattnere55484e2008-12-25 05:34:37 +00005682 // These can only come from an arithmetic instruction with overflow,
5683 // e.g. SADDO, UADDO.
Bill Wendling0ea25cb2008-12-03 08:32:02 +00005684 Cond = Cond.getNode()->getOperand(1);
5685 addTest = false;
5686 break;
Bill Wendling61edeb52008-12-02 01:06:39 +00005687 }
Evan Cheng0488db92007-09-25 01:57:46 +00005688 }
Evan Cheng370e5342008-12-03 08:38:43 +00005689 } else {
5690 unsigned CondOpc;
5691 if (Cond.hasOneUse() && isAndOrOfSetCCs(Cond, CondOpc)) {
5692 SDValue Cmp = Cond.getOperand(0).getOperand(1);
Evan Cheng370e5342008-12-03 08:38:43 +00005693 if (CondOpc == ISD::OR) {
5694 // Also, recognize the pattern generated by an FCMP_UNE. We can emit
5695 // two branches instead of an explicit OR instruction with a
5696 // separate test.
5697 if (Cmp == Cond.getOperand(1).getOperand(1) &&
Dan Gohman076aee32009-03-04 19:44:21 +00005698 isX86LogicalCmp(Cmp)) {
Evan Cheng370e5342008-12-03 08:38:43 +00005699 CC = Cond.getOperand(0).getOperand(0);
Dale Johannesene4d209d2009-02-03 20:21:25 +00005700 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
Evan Cheng370e5342008-12-03 08:38:43 +00005701 Chain, Dest, CC, Cmp);
5702 CC = Cond.getOperand(1).getOperand(0);
5703 Cond = Cmp;
5704 addTest = false;
5705 }
5706 } else { // ISD::AND
5707 // Also, recognize the pattern generated by an FCMP_OEQ. We can emit
5708 // two branches instead of an explicit AND instruction with a
5709 // separate test. However, we only do this if this block doesn't
5710 // have a fall-through edge, because this requires an explicit
5711 // jmp when the condition is false.
5712 if (Cmp == Cond.getOperand(1).getOperand(1) &&
Dan Gohman076aee32009-03-04 19:44:21 +00005713 isX86LogicalCmp(Cmp) &&
Evan Cheng370e5342008-12-03 08:38:43 +00005714 Op.getNode()->hasOneUse()) {
5715 X86::CondCode CCode =
5716 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
5717 CCode = X86::GetOppositeBranchCondition(CCode);
Owen Anderson825b72b2009-08-11 20:47:22 +00005718 CC = DAG.getConstant(CCode, MVT::i8);
Evan Cheng370e5342008-12-03 08:38:43 +00005719 SDValue User = SDValue(*Op.getNode()->use_begin(), 0);
5720 // Look for an unconditional branch following this conditional branch.
5721 // We need this because we need to reverse the successors in order
5722 // to implement FCMP_OEQ.
5723 if (User.getOpcode() == ISD::BR) {
5724 SDValue FalseBB = User.getOperand(1);
5725 SDValue NewBR =
5726 DAG.UpdateNodeOperands(User, User.getOperand(0), Dest);
5727 assert(NewBR == User);
5728 Dest = FalseBB;
Dan Gohman279c22e2008-10-21 03:29:32 +00005729
Dale Johannesene4d209d2009-02-03 20:21:25 +00005730 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
Evan Cheng370e5342008-12-03 08:38:43 +00005731 Chain, Dest, CC, Cmp);
5732 X86::CondCode CCode =
5733 (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0);
5734 CCode = X86::GetOppositeBranchCondition(CCode);
Owen Anderson825b72b2009-08-11 20:47:22 +00005735 CC = DAG.getConstant(CCode, MVT::i8);
Evan Cheng370e5342008-12-03 08:38:43 +00005736 Cond = Cmp;
5737 addTest = false;
5738 }
5739 }
Dan Gohman279c22e2008-10-21 03:29:32 +00005740 }
Evan Cheng67ad9db2009-02-02 08:07:36 +00005741 } else if (Cond.hasOneUse() && isXor1OfSetCC(Cond)) {
5742 // Recognize for xorb (setcc), 1 patterns. The xor inverts the condition.
5743 // It should be transformed during dag combiner except when the condition
5744 // is set by a arithmetics with overflow node.
5745 X86::CondCode CCode =
5746 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
5747 CCode = X86::GetOppositeBranchCondition(CCode);
Owen Anderson825b72b2009-08-11 20:47:22 +00005748 CC = DAG.getConstant(CCode, MVT::i8);
Evan Cheng67ad9db2009-02-02 08:07:36 +00005749 Cond = Cond.getOperand(0).getOperand(1);
5750 addTest = false;
Dan Gohman279c22e2008-10-21 03:29:32 +00005751 }
Evan Cheng0488db92007-09-25 01:57:46 +00005752 }
5753
5754 if (addTest) {
Owen Anderson825b72b2009-08-11 20:47:22 +00005755 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
Dan Gohman31125812009-03-07 01:58:32 +00005756 Cond = EmitTest(Cond, X86::COND_NE, DAG);
Evan Cheng0488db92007-09-25 01:57:46 +00005757 }
Dale Johannesene4d209d2009-02-03 20:21:25 +00005758 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
Dan Gohman279c22e2008-10-21 03:29:32 +00005759 Chain, Dest, CC, Cond);
Evan Cheng0488db92007-09-25 01:57:46 +00005760}
5761
Anton Korobeynikove060b532007-04-17 19:34:00 +00005762
5763// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.
5764// Calls to _alloca is needed to probe the stack when allocating more than 4k
5765// bytes in one go. Touching the stack at 4K increments is necessary to ensure
5766// that the guard pages used by the OS virtual memory manager are allocated in
5767// correct sequence.
Dan Gohman475871a2008-07-27 21:46:04 +00005768SDValue
5769X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
Anton Korobeynikov4304bcc2007-07-05 20:36:08 +00005770 SelectionDAG &DAG) {
Anton Korobeynikove060b532007-04-17 19:34:00 +00005771 assert(Subtarget->isTargetCygMing() &&
5772 "This should be used only on Cygwin/Mingw targets");
Dale Johannesen6f38cb62009-02-07 19:59:05 +00005773 DebugLoc dl = Op.getDebugLoc();
Anton Korobeynikov096b4612008-06-11 20:16:42 +00005774
Anton Korobeynikov57fc00d2007-04-17 09:20:00 +00005775 // Get the inputs.
Dan Gohman475871a2008-07-27 21:46:04 +00005776 SDValue Chain = Op.getOperand(0);
5777 SDValue Size = Op.getOperand(1);
Anton Korobeynikov57fc00d2007-04-17 09:20:00 +00005778 // FIXME: Ensure alignment here
5779
Dan Gohman475871a2008-07-27 21:46:04 +00005780 SDValue Flag;
Anton Korobeynikov096b4612008-06-11 20:16:42 +00005781
Owen Andersone50ed302009-08-10 22:56:29 +00005782 EVT IntPtr = getPointerTy();
Owen Anderson825b72b2009-08-11 20:47:22 +00005783 EVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32;
Anton Korobeynikov57fc00d2007-04-17 09:20:00 +00005784
Chris Lattnere563bbc2008-10-11 22:08:30 +00005785 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true));
Anton Korobeynikov096b4612008-06-11 20:16:42 +00005786
Dale Johannesendd64c412009-02-04 00:33:20 +00005787 Chain = DAG.getCopyToReg(Chain, dl, X86::EAX, Size, Flag);
Anton Korobeynikov4304bcc2007-07-05 20:36:08 +00005788 Flag = Chain.getValue(1);
5789
Owen Anderson825b72b2009-08-11 20:47:22 +00005790 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
Dan Gohman475871a2008-07-27 21:46:04 +00005791 SDValue Ops[] = { Chain,
Bill Wendling056292f2008-09-16 21:48:12 +00005792 DAG.getTargetExternalSymbol("_alloca", IntPtr),
Anton Korobeynikov4304bcc2007-07-05 20:36:08 +00005793 DAG.getRegister(X86::EAX, IntPtr),
Anton Korobeynikov096b4612008-06-11 20:16:42 +00005794 DAG.getRegister(X86StackPtr, SPTy),
Anton Korobeynikov4304bcc2007-07-05 20:36:08 +00005795 Flag };
Dale Johannesene4d209d2009-02-03 20:21:25 +00005796 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops, 5);
Anton Korobeynikov4304bcc2007-07-05 20:36:08 +00005797 Flag = Chain.getValue(1);
5798
Anton Korobeynikov096b4612008-06-11 20:16:42 +00005799 Chain = DAG.getCALLSEQ_END(Chain,
Chris Lattnere563bbc2008-10-11 22:08:30 +00005800 DAG.getIntPtrConstant(0, true),
5801 DAG.getIntPtrConstant(0, true),
Anton Korobeynikov096b4612008-06-11 20:16:42 +00005802 Flag);
5803
Dale Johannesendd64c412009-02-04 00:33:20 +00005804 Chain = DAG.getCopyFromReg(Chain, dl, X86StackPtr, SPTy).getValue(1);
Anton Korobeynikov096b4612008-06-11 20:16:42 +00005805
Dan Gohman475871a2008-07-27 21:46:04 +00005806 SDValue Ops1[2] = { Chain.getValue(0), Chain };
Dale Johannesene4d209d2009-02-03 20:21:25 +00005807 return DAG.getMergeValues(Ops1, 2, dl);
Anton Korobeynikov57fc00d2007-04-17 09:20:00 +00005808}
5809
Dan Gohman475871a2008-07-27 21:46:04 +00005810SDValue
Dale Johannesen0f502f62009-02-03 22:26:09 +00005811X86TargetLowering::EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
Bill Wendling6f287b22008-09-30 21:22:07 +00005812 SDValue Chain,
5813 SDValue Dst, SDValue Src,
5814 SDValue Size, unsigned Align,
5815 const Value *DstSV,
Bill Wendling6158d842008-10-01 00:59:58 +00005816 uint64_t DstSVOff) {
Dan Gohman707e0182008-04-12 04:36:06 +00005817 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005818
Bill Wendling6f287b22008-09-30 21:22:07 +00005819 // If not DWORD aligned or size is more than the threshold, call the library.
5820 // The libc version is likely to be faster for these cases. It can use the
5821 // address value and run time information about the CPU.
Evan Cheng1887c1c2008-08-21 21:00:15 +00005822 if ((Align & 3) != 0 ||
Dan Gohman707e0182008-04-12 04:36:06 +00005823 !ConstantSize ||
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00005824 ConstantSize->getZExtValue() >
5825 getSubtarget()->getMaxInlineSizeThreshold()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005826 SDValue InFlag(0, 0);
Dan Gohman68d599d2008-04-01 20:38:36 +00005827
5828 // Check to see if there is a specialized entry-point for memory zeroing.
Dan Gohman707e0182008-04-12 04:36:06 +00005829 ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
Bill Wendling6f287b22008-09-30 21:22:07 +00005830
Bill Wendling6158d842008-10-01 00:59:58 +00005831 if (const char *bzeroEntry = V &&
5832 V->isNullValue() ? Subtarget->getBZeroEntry() : 0) {
Owen Andersone50ed302009-08-10 22:56:29 +00005833 EVT IntPtr = getPointerTy();
Owen Anderson1d0be152009-08-13 21:58:54 +00005834 const Type *IntPtrTy = TD->getIntPtrType(*DAG.getContext());
Scott Michelfdc40a02009-02-17 22:15:04 +00005835 TargetLowering::ArgListTy Args;
Bill Wendling6158d842008-10-01 00:59:58 +00005836 TargetLowering::ArgListEntry Entry;
5837 Entry.Node = Dst;
5838 Entry.Ty = IntPtrTy;
5839 Args.push_back(Entry);
5840 Entry.Node = Size;
5841 Args.push_back(Entry);
5842 std::pair<SDValue,SDValue> CallResult =
Owen Anderson1d0be152009-08-13 21:58:54 +00005843 LowerCallTo(Chain, Type::getVoidTy(*DAG.getContext()),
5844 false, false, false, false,
Dan Gohman98ca4f22009-08-05 01:29:28 +00005845 0, CallingConv::C, false, /*isReturnValueUsed=*/false,
Dale Johannesen0f502f62009-02-03 22:26:09 +00005846 DAG.getExternalSymbol(bzeroEntry, IntPtr), Args, DAG, dl);
Bill Wendling6158d842008-10-01 00:59:58 +00005847 return CallResult.second;
Dan Gohman68d599d2008-04-01 20:38:36 +00005848 }
5849
Dan Gohman707e0182008-04-12 04:36:06 +00005850 // Otherwise have the target-independent code call memset.
Dan Gohman475871a2008-07-27 21:46:04 +00005851 return SDValue();
Evan Cheng48090aa2006-03-21 23:01:21 +00005852 }
Evan Chengb9df0ca2006-03-22 02:53:00 +00005853
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00005854 uint64_t SizeVal = ConstantSize->getZExtValue();
Dan Gohman475871a2008-07-27 21:46:04 +00005855 SDValue InFlag(0, 0);
Owen Andersone50ed302009-08-10 22:56:29 +00005856 EVT AVT;
Dan Gohman475871a2008-07-27 21:46:04 +00005857 SDValue Count;
Dan Gohman707e0182008-04-12 04:36:06 +00005858 ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005859 unsigned BytesLeft = 0;
5860 bool TwoRepStos = false;
5861 if (ValC) {
5862 unsigned ValReg;
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00005863 uint64_t Val = ValC->getZExtValue() & 255;
Evan Cheng5ced1d82006-04-06 23:23:56 +00005864
Evan Cheng0db9fe62006-04-25 20:13:52 +00005865 // If the value is a constant, then we can potentially use larger sets.
5866 switch (Align & 3) {
Evan Cheng1887c1c2008-08-21 21:00:15 +00005867 case 2: // WORD aligned
Owen Anderson825b72b2009-08-11 20:47:22 +00005868 AVT = MVT::i16;
Evan Cheng1887c1c2008-08-21 21:00:15 +00005869 ValReg = X86::AX;
5870 Val = (Val << 8) | Val;
5871 break;
5872 case 0: // DWORD aligned
Owen Anderson825b72b2009-08-11 20:47:22 +00005873 AVT = MVT::i32;
Evan Cheng1887c1c2008-08-21 21:00:15 +00005874 ValReg = X86::EAX;
5875 Val = (Val << 8) | Val;
5876 Val = (Val << 16) | Val;
5877 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned
Owen Anderson825b72b2009-08-11 20:47:22 +00005878 AVT = MVT::i64;
Evan Cheng1887c1c2008-08-21 21:00:15 +00005879 ValReg = X86::RAX;
5880 Val = (Val << 32) | Val;
5881 }
5882 break;
5883 default: // Byte aligned
Owen Anderson825b72b2009-08-11 20:47:22 +00005884 AVT = MVT::i8;
Evan Cheng1887c1c2008-08-21 21:00:15 +00005885 ValReg = X86::AL;
5886 Count = DAG.getIntPtrConstant(SizeVal);
5887 break;
Evan Cheng80d428c2006-04-19 22:48:17 +00005888 }
5889
Owen Anderson825b72b2009-08-11 20:47:22 +00005890 if (AVT.bitsGT(MVT::i8)) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00005891 unsigned UBytes = AVT.getSizeInBits() / 8;
Dan Gohman707e0182008-04-12 04:36:06 +00005892 Count = DAG.getIntPtrConstant(SizeVal / UBytes);
5893 BytesLeft = SizeVal % UBytes;
Evan Cheng25ab6902006-09-08 06:48:29 +00005894 }
5895
Dale Johannesen0f502f62009-02-03 22:26:09 +00005896 Chain = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, AVT),
Evan Cheng0db9fe62006-04-25 20:13:52 +00005897 InFlag);
5898 InFlag = Chain.getValue(1);
5899 } else {
Owen Anderson825b72b2009-08-11 20:47:22 +00005900 AVT = MVT::i8;
Dan Gohmanbcda2852008-04-16 01:32:32 +00005901 Count = DAG.getIntPtrConstant(SizeVal);
Dale Johannesen0f502f62009-02-03 22:26:09 +00005902 Chain = DAG.getCopyToReg(Chain, dl, X86::AL, Src, InFlag);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005903 InFlag = Chain.getValue(1);
Evan Chengb9df0ca2006-03-22 02:53:00 +00005904 }
Evan Chengc78d3b42006-04-24 18:01:45 +00005905
Scott Michelfdc40a02009-02-17 22:15:04 +00005906 Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RCX :
Dale Johannesen0f502f62009-02-03 22:26:09 +00005907 X86::ECX,
Evan Cheng25ab6902006-09-08 06:48:29 +00005908 Count, InFlag);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005909 InFlag = Chain.getValue(1);
Scott Michelfdc40a02009-02-17 22:15:04 +00005910 Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RDI :
Dale Johannesen0f502f62009-02-03 22:26:09 +00005911 X86::EDI,
Dan Gohman707e0182008-04-12 04:36:06 +00005912 Dst, InFlag);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005913 InFlag = Chain.getValue(1);
Evan Chenga0b3afb2006-03-27 07:00:16 +00005914
Owen Anderson825b72b2009-08-11 20:47:22 +00005915 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Dan Gohman475871a2008-07-27 21:46:04 +00005916 SmallVector<SDValue, 8> Ops;
Evan Cheng0db9fe62006-04-25 20:13:52 +00005917 Ops.push_back(Chain);
5918 Ops.push_back(DAG.getValueType(AVT));
5919 Ops.push_back(InFlag);
Dale Johannesen0f502f62009-02-03 22:26:09 +00005920 Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, &Ops[0], Ops.size());
Evan Chengc78d3b42006-04-24 18:01:45 +00005921
Evan Cheng0db9fe62006-04-25 20:13:52 +00005922 if (TwoRepStos) {
5923 InFlag = Chain.getValue(1);
Dan Gohman707e0182008-04-12 04:36:06 +00005924 Count = Size;
Owen Andersone50ed302009-08-10 22:56:29 +00005925 EVT CVT = Count.getValueType();
Dale Johannesen0f502f62009-02-03 22:26:09 +00005926 SDValue Left = DAG.getNode(ISD::AND, dl, CVT, Count,
Owen Anderson825b72b2009-08-11 20:47:22 +00005927 DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT));
5928 Chain = DAG.getCopyToReg(Chain, dl, (CVT == MVT::i64) ? X86::RCX :
Dale Johannesen0f502f62009-02-03 22:26:09 +00005929 X86::ECX,
Evan Cheng25ab6902006-09-08 06:48:29 +00005930 Left, InFlag);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005931 InFlag = Chain.getValue(1);
Owen Anderson825b72b2009-08-11 20:47:22 +00005932 Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005933 Ops.clear();
5934 Ops.push_back(Chain);
Owen Anderson825b72b2009-08-11 20:47:22 +00005935 Ops.push_back(DAG.getValueType(MVT::i8));
Evan Cheng0db9fe62006-04-25 20:13:52 +00005936 Ops.push_back(InFlag);
Dale Johannesen0f502f62009-02-03 22:26:09 +00005937 Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, &Ops[0], Ops.size());
Evan Cheng0db9fe62006-04-25 20:13:52 +00005938 } else if (BytesLeft) {
Dan Gohman707e0182008-04-12 04:36:06 +00005939 // Handle the last 1 - 7 bytes.
5940 unsigned Offset = SizeVal - BytesLeft;
Owen Andersone50ed302009-08-10 22:56:29 +00005941 EVT AddrVT = Dst.getValueType();
5942 EVT SizeVT = Size.getValueType();
Dan Gohman707e0182008-04-12 04:36:06 +00005943
Dale Johannesen0f502f62009-02-03 22:26:09 +00005944 Chain = DAG.getMemset(Chain, dl,
5945 DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
Dan Gohman707e0182008-04-12 04:36:06 +00005946 DAG.getConstant(Offset, AddrVT)),
5947 Src,
5948 DAG.getConstant(BytesLeft, SizeVT),
Dan Gohman1f13c682008-04-28 17:15:20 +00005949 Align, DstSV, DstSVOff + Offset);
Evan Cheng386031a2006-03-24 07:29:27 +00005950 }
Evan Cheng11e15b32006-04-03 20:53:28 +00005951
Dan Gohman707e0182008-04-12 04:36:06 +00005952 // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain.
Evan Cheng0db9fe62006-04-25 20:13:52 +00005953 return Chain;
5954}
Evan Cheng11e15b32006-04-03 20:53:28 +00005955
Dan Gohman475871a2008-07-27 21:46:04 +00005956SDValue
Dale Johannesen0f502f62009-02-03 22:26:09 +00005957X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, DebugLoc dl,
Evan Cheng1887c1c2008-08-21 21:00:15 +00005958 SDValue Chain, SDValue Dst, SDValue Src,
5959 SDValue Size, unsigned Align,
5960 bool AlwaysInline,
5961 const Value *DstSV, uint64_t DstSVOff,
Scott Michelfdc40a02009-02-17 22:15:04 +00005962 const Value *SrcSV, uint64_t SrcSVOff) {
Dan Gohman707e0182008-04-12 04:36:06 +00005963 // This requires the copy size to be a constant, preferrably
5964 // within a subtarget-specific limit.
5965 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
5966 if (!ConstantSize)
Dan Gohman475871a2008-07-27 21:46:04 +00005967 return SDValue();
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00005968 uint64_t SizeVal = ConstantSize->getZExtValue();
Dan Gohman707e0182008-04-12 04:36:06 +00005969 if (!AlwaysInline && SizeVal > getSubtarget()->getMaxInlineSizeThreshold())
Dan Gohman475871a2008-07-27 21:46:04 +00005970 return SDValue();
Dan Gohman707e0182008-04-12 04:36:06 +00005971
Evan Cheng1887c1c2008-08-21 21:00:15 +00005972 /// If not DWORD aligned, call the library.
5973 if ((Align & 3) != 0)
5974 return SDValue();
5975
5976 // DWORD aligned
Owen Anderson825b72b2009-08-11 20:47:22 +00005977 EVT AVT = MVT::i32;
Evan Cheng1887c1c2008-08-21 21:00:15 +00005978 if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) // QWORD aligned
Owen Anderson825b72b2009-08-11 20:47:22 +00005979 AVT = MVT::i64;
Evan Cheng0db9fe62006-04-25 20:13:52 +00005980
Duncan Sands83ec4b62008-06-06 12:08:01 +00005981 unsigned UBytes = AVT.getSizeInBits() / 8;
Dan Gohman707e0182008-04-12 04:36:06 +00005982 unsigned CountVal = SizeVal / UBytes;
Dan Gohman475871a2008-07-27 21:46:04 +00005983 SDValue Count = DAG.getIntPtrConstant(CountVal);
Evan Cheng1887c1c2008-08-21 21:00:15 +00005984 unsigned BytesLeft = SizeVal % UBytes;
Evan Cheng25ab6902006-09-08 06:48:29 +00005985
Dan Gohman475871a2008-07-27 21:46:04 +00005986 SDValue InFlag(0, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005987 Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RCX :
Dale Johannesen0f502f62009-02-03 22:26:09 +00005988 X86::ECX,
Evan Cheng25ab6902006-09-08 06:48:29 +00005989 Count, InFlag);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005990 InFlag = Chain.getValue(1);
Scott Michelfdc40a02009-02-17 22:15:04 +00005991 Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RDI :
Dale Johannesen0f502f62009-02-03 22:26:09 +00005992 X86::EDI,
Dan Gohman707e0182008-04-12 04:36:06 +00005993 Dst, InFlag);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005994 InFlag = Chain.getValue(1);
Scott Michelfdc40a02009-02-17 22:15:04 +00005995 Chain = DAG.getCopyToReg(Chain, dl, Subtarget->is64Bit() ? X86::RSI :
Dale Johannesen0f502f62009-02-03 22:26:09 +00005996 X86::ESI,
Dan Gohman707e0182008-04-12 04:36:06 +00005997 Src, InFlag);
Evan Cheng0db9fe62006-04-25 20:13:52 +00005998 InFlag = Chain.getValue(1);
5999
Owen Anderson825b72b2009-08-11 20:47:22 +00006000 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Dan Gohman475871a2008-07-27 21:46:04 +00006001 SmallVector<SDValue, 8> Ops;
Evan Cheng0db9fe62006-04-25 20:13:52 +00006002 Ops.push_back(Chain);
6003 Ops.push_back(DAG.getValueType(AVT));
6004 Ops.push_back(InFlag);
Dale Johannesen0f502f62009-02-03 22:26:09 +00006005 SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, dl, Tys, &Ops[0], Ops.size());
Evan Cheng0db9fe62006-04-25 20:13:52 +00006006
Dan Gohman475871a2008-07-27 21:46:04 +00006007 SmallVector<SDValue, 4> Results;
Evan Cheng2749c722008-04-25 00:26:43 +00006008 Results.push_back(RepMovs);
Rafael Espindola068317b2007-09-28 12:53:01 +00006009 if (BytesLeft) {
Dan Gohman707e0182008-04-12 04:36:06 +00006010 // Handle the last 1 - 7 bytes.
6011 unsigned Offset = SizeVal - BytesLeft;
Owen Andersone50ed302009-08-10 22:56:29 +00006012 EVT DstVT = Dst.getValueType();
6013 EVT SrcVT = Src.getValueType();
6014 EVT SizeVT = Size.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00006015 Results.push_back(DAG.getMemcpy(Chain, dl,
Dale Johannesen0f502f62009-02-03 22:26:09 +00006016 DAG.getNode(ISD::ADD, dl, DstVT, Dst,
Evan Cheng2749c722008-04-25 00:26:43 +00006017 DAG.getConstant(Offset, DstVT)),
Dale Johannesen0f502f62009-02-03 22:26:09 +00006018 DAG.getNode(ISD::ADD, dl, SrcVT, Src,
Evan Cheng2749c722008-04-25 00:26:43 +00006019 DAG.getConstant(Offset, SrcVT)),
Dan Gohman707e0182008-04-12 04:36:06 +00006020 DAG.getConstant(BytesLeft, SizeVT),
6021 Align, AlwaysInline,
Dan Gohman1f13c682008-04-28 17:15:20 +00006022 DstSV, DstSVOff + Offset,
6023 SrcSV, SrcSVOff + Offset));
Evan Chengb067a1e2006-03-31 19:22:53 +00006024 }
Evan Cheng0db9fe62006-04-25 20:13:52 +00006025
Owen Anderson825b72b2009-08-11 20:47:22 +00006026 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
Dale Johannesen0f502f62009-02-03 22:26:09 +00006027 &Results[0], Results.size());
Evan Cheng0db9fe62006-04-25 20:13:52 +00006028}
6029
Dan Gohman475871a2008-07-27 21:46:04 +00006030SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) {
Dan Gohman69de1932008-02-06 22:27:42 +00006031 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006032 DebugLoc dl = Op.getDebugLoc();
Evan Cheng8b2794a2006-10-13 21:14:26 +00006033
Evan Cheng25ab6902006-09-08 06:48:29 +00006034 if (!Subtarget->is64Bit()) {
6035 // vastart just stores the address of the VarArgsFrameIndex slot into the
6036 // memory location argument.
Dan Gohman475871a2008-07-27 21:46:04 +00006037 SDValue FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy());
Dale Johannesene4d209d2009-02-03 20:21:25 +00006038 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), SV, 0);
Evan Cheng25ab6902006-09-08 06:48:29 +00006039 }
6040
6041 // __va_list_tag:
6042 // gp_offset (0 - 6 * 8)
6043 // fp_offset (48 - 48 + 8 * 16)
6044 // overflow_arg_area (point to parameters coming in memory).
6045 // reg_save_area
Dan Gohman475871a2008-07-27 21:46:04 +00006046 SmallVector<SDValue, 8> MemOps;
6047 SDValue FIN = Op.getOperand(1);
Evan Cheng25ab6902006-09-08 06:48:29 +00006048 // Store gp_offset
Dale Johannesene4d209d2009-02-03 20:21:25 +00006049 SDValue Store = DAG.getStore(Op.getOperand(0), dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00006050 DAG.getConstant(VarArgsGPOffset, MVT::i32),
Dan Gohman69de1932008-02-06 22:27:42 +00006051 FIN, SV, 0);
Evan Cheng25ab6902006-09-08 06:48:29 +00006052 MemOps.push_back(Store);
6053
6054 // Store fp_offset
Scott Michelfdc40a02009-02-17 22:15:04 +00006055 FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(),
Dale Johannesene4d209d2009-02-03 20:21:25 +00006056 FIN, DAG.getIntPtrConstant(4));
6057 Store = DAG.getStore(Op.getOperand(0), dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00006058 DAG.getConstant(VarArgsFPOffset, MVT::i32),
Dan Gohman69de1932008-02-06 22:27:42 +00006059 FIN, SV, 0);
Evan Cheng25ab6902006-09-08 06:48:29 +00006060 MemOps.push_back(Store);
6061
6062 // Store ptr to overflow_arg_area
Scott Michelfdc40a02009-02-17 22:15:04 +00006063 FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(),
Dale Johannesene4d209d2009-02-03 20:21:25 +00006064 FIN, DAG.getIntPtrConstant(4));
Dan Gohman475871a2008-07-27 21:46:04 +00006065 SDValue OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy());
Dale Johannesene4d209d2009-02-03 20:21:25 +00006066 Store = DAG.getStore(Op.getOperand(0), dl, OVFIN, FIN, SV, 0);
Evan Cheng25ab6902006-09-08 06:48:29 +00006067 MemOps.push_back(Store);
6068
6069 // Store ptr to reg_save_area.
Scott Michelfdc40a02009-02-17 22:15:04 +00006070 FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(),
Dale Johannesene4d209d2009-02-03 20:21:25 +00006071 FIN, DAG.getIntPtrConstant(8));
Dan Gohman475871a2008-07-27 21:46:04 +00006072 SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy());
Dale Johannesene4d209d2009-02-03 20:21:25 +00006073 Store = DAG.getStore(Op.getOperand(0), dl, RSFIN, FIN, SV, 0);
Evan Cheng25ab6902006-09-08 06:48:29 +00006074 MemOps.push_back(Store);
Owen Anderson825b72b2009-08-11 20:47:22 +00006075 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
Dale Johannesene4d209d2009-02-03 20:21:25 +00006076 &MemOps[0], MemOps.size());
Evan Cheng0db9fe62006-04-25 20:13:52 +00006077}
6078
Dan Gohman475871a2008-07-27 21:46:04 +00006079SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) {
Dan Gohman9018e832008-05-10 01:26:14 +00006080 // X86-64 va_list is a struct { i32, i32, i8*, i8* }.
6081 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_arg!");
Dan Gohman475871a2008-07-27 21:46:04 +00006082 SDValue Chain = Op.getOperand(0);
6083 SDValue SrcPtr = Op.getOperand(1);
6084 SDValue SrcSV = Op.getOperand(2);
Dan Gohman9018e832008-05-10 01:26:14 +00006085
Torok Edwindac237e2009-07-08 20:53:28 +00006086 llvm_report_error("VAArgInst is not yet implemented for x86-64!");
Dan Gohman475871a2008-07-27 21:46:04 +00006087 return SDValue();
Dan Gohman9018e832008-05-10 01:26:14 +00006088}
6089
Dan Gohman475871a2008-07-27 21:46:04 +00006090SDValue X86TargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) {
Evan Chengae642192007-03-02 23:16:35 +00006091 // X86-64 va_list is a struct { i32, i32, i8*, i8* }.
Dan Gohman28269132008-04-18 20:55:41 +00006092 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!");
Dan Gohman475871a2008-07-27 21:46:04 +00006093 SDValue Chain = Op.getOperand(0);
6094 SDValue DstPtr = Op.getOperand(1);
6095 SDValue SrcPtr = Op.getOperand(2);
Dan Gohman69de1932008-02-06 22:27:42 +00006096 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
6097 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006098 DebugLoc dl = Op.getDebugLoc();
Evan Chengae642192007-03-02 23:16:35 +00006099
Dale Johannesendd64c412009-02-04 00:33:20 +00006100 return DAG.getMemcpy(Chain, dl, DstPtr, SrcPtr,
Dan Gohman28269132008-04-18 20:55:41 +00006101 DAG.getIntPtrConstant(24), 8, false,
6102 DstSV, 0, SrcSV, 0);
Evan Chengae642192007-03-02 23:16:35 +00006103}
6104
Dan Gohman475871a2008-07-27 21:46:04 +00006105SDValue
6106X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) {
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006107 DebugLoc dl = Op.getDebugLoc();
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00006108 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
Evan Cheng0db9fe62006-04-25 20:13:52 +00006109 switch (IntNo) {
Dan Gohman475871a2008-07-27 21:46:04 +00006110 default: return SDValue(); // Don't custom lower most intrinsics.
Evan Cheng5759f972008-05-04 09:15:50 +00006111 // Comparison intrinsics.
Evan Cheng0db9fe62006-04-25 20:13:52 +00006112 case Intrinsic::x86_sse_comieq_ss:
6113 case Intrinsic::x86_sse_comilt_ss:
6114 case Intrinsic::x86_sse_comile_ss:
6115 case Intrinsic::x86_sse_comigt_ss:
6116 case Intrinsic::x86_sse_comige_ss:
6117 case Intrinsic::x86_sse_comineq_ss:
6118 case Intrinsic::x86_sse_ucomieq_ss:
6119 case Intrinsic::x86_sse_ucomilt_ss:
6120 case Intrinsic::x86_sse_ucomile_ss:
6121 case Intrinsic::x86_sse_ucomigt_ss:
6122 case Intrinsic::x86_sse_ucomige_ss:
6123 case Intrinsic::x86_sse_ucomineq_ss:
6124 case Intrinsic::x86_sse2_comieq_sd:
6125 case Intrinsic::x86_sse2_comilt_sd:
6126 case Intrinsic::x86_sse2_comile_sd:
6127 case Intrinsic::x86_sse2_comigt_sd:
6128 case Intrinsic::x86_sse2_comige_sd:
6129 case Intrinsic::x86_sse2_comineq_sd:
6130 case Intrinsic::x86_sse2_ucomieq_sd:
6131 case Intrinsic::x86_sse2_ucomilt_sd:
6132 case Intrinsic::x86_sse2_ucomile_sd:
6133 case Intrinsic::x86_sse2_ucomigt_sd:
6134 case Intrinsic::x86_sse2_ucomige_sd:
6135 case Intrinsic::x86_sse2_ucomineq_sd: {
6136 unsigned Opc = 0;
6137 ISD::CondCode CC = ISD::SETCC_INVALID;
6138 switch (IntNo) {
6139 default: break;
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00006140 case Intrinsic::x86_sse_comieq_ss:
6141 case Intrinsic::x86_sse2_comieq_sd:
Evan Cheng0db9fe62006-04-25 20:13:52 +00006142 Opc = X86ISD::COMI;
6143 CC = ISD::SETEQ;
6144 break;
Evan Cheng6be2c582006-04-05 23:38:46 +00006145 case Intrinsic::x86_sse_comilt_ss:
Evan Cheng6be2c582006-04-05 23:38:46 +00006146 case Intrinsic::x86_sse2_comilt_sd:
Evan Cheng0db9fe62006-04-25 20:13:52 +00006147 Opc = X86ISD::COMI;
6148 CC = ISD::SETLT;
6149 break;
6150 case Intrinsic::x86_sse_comile_ss:
Evan Cheng6be2c582006-04-05 23:38:46 +00006151 case Intrinsic::x86_sse2_comile_sd:
Evan Cheng0db9fe62006-04-25 20:13:52 +00006152 Opc = X86ISD::COMI;
6153 CC = ISD::SETLE;
6154 break;
6155 case Intrinsic::x86_sse_comigt_ss:
Evan Cheng6be2c582006-04-05 23:38:46 +00006156 case Intrinsic::x86_sse2_comigt_sd:
Evan Cheng0db9fe62006-04-25 20:13:52 +00006157 Opc = X86ISD::COMI;
6158 CC = ISD::SETGT;
6159 break;
6160 case Intrinsic::x86_sse_comige_ss:
Evan Cheng6be2c582006-04-05 23:38:46 +00006161 case Intrinsic::x86_sse2_comige_sd:
Evan Cheng0db9fe62006-04-25 20:13:52 +00006162 Opc = X86ISD::COMI;
6163 CC = ISD::SETGE;
6164 break;
6165 case Intrinsic::x86_sse_comineq_ss:
Evan Cheng6be2c582006-04-05 23:38:46 +00006166 case Intrinsic::x86_sse2_comineq_sd:
Evan Cheng0db9fe62006-04-25 20:13:52 +00006167 Opc = X86ISD::COMI;
6168 CC = ISD::SETNE;
6169 break;
6170 case Intrinsic::x86_sse_ucomieq_ss:
Evan Cheng6be2c582006-04-05 23:38:46 +00006171 case Intrinsic::x86_sse2_ucomieq_sd:
Evan Cheng0db9fe62006-04-25 20:13:52 +00006172 Opc = X86ISD::UCOMI;
6173 CC = ISD::SETEQ;
6174 break;
6175 case Intrinsic::x86_sse_ucomilt_ss:
Evan Cheng6be2c582006-04-05 23:38:46 +00006176 case Intrinsic::x86_sse2_ucomilt_sd:
Evan Cheng0db9fe62006-04-25 20:13:52 +00006177 Opc = X86ISD::UCOMI;
6178 CC = ISD::SETLT;
6179 break;
6180 case Intrinsic::x86_sse_ucomile_ss:
Evan Cheng6be2c582006-04-05 23:38:46 +00006181 case Intrinsic::x86_sse2_ucomile_sd:
Evan Cheng0db9fe62006-04-25 20:13:52 +00006182 Opc = X86ISD::UCOMI;
6183 CC = ISD::SETLE;
6184 break;
6185 case Intrinsic::x86_sse_ucomigt_ss:
Evan Cheng6be2c582006-04-05 23:38:46 +00006186 case Intrinsic::x86_sse2_ucomigt_sd:
Evan Cheng0db9fe62006-04-25 20:13:52 +00006187 Opc = X86ISD::UCOMI;
6188 CC = ISD::SETGT;
6189 break;
6190 case Intrinsic::x86_sse_ucomige_ss:
Evan Cheng6be2c582006-04-05 23:38:46 +00006191 case Intrinsic::x86_sse2_ucomige_sd:
Evan Cheng0db9fe62006-04-25 20:13:52 +00006192 Opc = X86ISD::UCOMI;
6193 CC = ISD::SETGE;
6194 break;
6195 case Intrinsic::x86_sse_ucomineq_ss:
6196 case Intrinsic::x86_sse2_ucomineq_sd:
6197 Opc = X86ISD::UCOMI;
6198 CC = ISD::SETNE;
6199 break;
Evan Cheng6be2c582006-04-05 23:38:46 +00006200 }
Evan Cheng734503b2006-09-11 02:19:56 +00006201
Dan Gohman475871a2008-07-27 21:46:04 +00006202 SDValue LHS = Op.getOperand(1);
6203 SDValue RHS = Op.getOperand(2);
Chris Lattner1c39d4c2008-12-24 23:53:05 +00006204 unsigned X86CC = TranslateX86CC(CC, true, LHS, RHS, DAG);
Owen Anderson825b72b2009-08-11 20:47:22 +00006205 SDValue Cond = DAG.getNode(Opc, dl, MVT::i32, LHS, RHS);
6206 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
6207 DAG.getConstant(X86CC, MVT::i8), Cond);
6208 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
Evan Cheng6be2c582006-04-05 23:38:46 +00006209 }
Eric Christopher71c67532009-07-29 00:28:05 +00006210 // ptest intrinsics. The intrinsic these come from are designed to return
Eric Christopher794bfed2009-07-29 01:01:19 +00006211 // an integer value, not just an instruction so lower it to the ptest
6212 // pattern and a setcc for the result.
Eric Christopher71c67532009-07-29 00:28:05 +00006213 case Intrinsic::x86_sse41_ptestz:
6214 case Intrinsic::x86_sse41_ptestc:
6215 case Intrinsic::x86_sse41_ptestnzc:{
6216 unsigned X86CC = 0;
6217 switch (IntNo) {
Eric Christopher978dae32009-07-29 18:14:04 +00006218 default: llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
Eric Christopher71c67532009-07-29 00:28:05 +00006219 case Intrinsic::x86_sse41_ptestz:
6220 // ZF = 1
6221 X86CC = X86::COND_E;
6222 break;
6223 case Intrinsic::x86_sse41_ptestc:
6224 // CF = 1
6225 X86CC = X86::COND_B;
6226 break;
6227 case Intrinsic::x86_sse41_ptestnzc:
6228 // ZF and CF = 0
6229 X86CC = X86::COND_A;
6230 break;
6231 }
6232
6233 SDValue LHS = Op.getOperand(1);
6234 SDValue RHS = Op.getOperand(2);
Owen Anderson825b72b2009-08-11 20:47:22 +00006235 SDValue Test = DAG.getNode(X86ISD::PTEST, dl, MVT::i32, LHS, RHS);
6236 SDValue CC = DAG.getConstant(X86CC, MVT::i8);
6237 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test);
6238 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
Eric Christopher71c67532009-07-29 00:28:05 +00006239 }
Evan Cheng5759f972008-05-04 09:15:50 +00006240
6241 // Fix vector shift instructions where the last operand is a non-immediate
6242 // i32 value.
6243 case Intrinsic::x86_sse2_pslli_w:
6244 case Intrinsic::x86_sse2_pslli_d:
6245 case Intrinsic::x86_sse2_pslli_q:
6246 case Intrinsic::x86_sse2_psrli_w:
6247 case Intrinsic::x86_sse2_psrli_d:
6248 case Intrinsic::x86_sse2_psrli_q:
6249 case Intrinsic::x86_sse2_psrai_w:
6250 case Intrinsic::x86_sse2_psrai_d:
6251 case Intrinsic::x86_mmx_pslli_w:
6252 case Intrinsic::x86_mmx_pslli_d:
6253 case Intrinsic::x86_mmx_pslli_q:
6254 case Intrinsic::x86_mmx_psrli_w:
6255 case Intrinsic::x86_mmx_psrli_d:
6256 case Intrinsic::x86_mmx_psrli_q:
6257 case Intrinsic::x86_mmx_psrai_w:
6258 case Intrinsic::x86_mmx_psrai_d: {
Dan Gohman475871a2008-07-27 21:46:04 +00006259 SDValue ShAmt = Op.getOperand(2);
Evan Cheng5759f972008-05-04 09:15:50 +00006260 if (isa<ConstantSDNode>(ShAmt))
Dan Gohman475871a2008-07-27 21:46:04 +00006261 return SDValue();
Evan Cheng5759f972008-05-04 09:15:50 +00006262
6263 unsigned NewIntNo = 0;
Owen Anderson825b72b2009-08-11 20:47:22 +00006264 EVT ShAmtVT = MVT::v4i32;
Evan Cheng5759f972008-05-04 09:15:50 +00006265 switch (IntNo) {
6266 case Intrinsic::x86_sse2_pslli_w:
6267 NewIntNo = Intrinsic::x86_sse2_psll_w;
6268 break;
6269 case Intrinsic::x86_sse2_pslli_d:
6270 NewIntNo = Intrinsic::x86_sse2_psll_d;
6271 break;
6272 case Intrinsic::x86_sse2_pslli_q:
6273 NewIntNo = Intrinsic::x86_sse2_psll_q;
6274 break;
6275 case Intrinsic::x86_sse2_psrli_w:
6276 NewIntNo = Intrinsic::x86_sse2_psrl_w;
6277 break;
6278 case Intrinsic::x86_sse2_psrli_d:
6279 NewIntNo = Intrinsic::x86_sse2_psrl_d;
6280 break;
6281 case Intrinsic::x86_sse2_psrli_q:
6282 NewIntNo = Intrinsic::x86_sse2_psrl_q;
6283 break;
6284 case Intrinsic::x86_sse2_psrai_w:
6285 NewIntNo = Intrinsic::x86_sse2_psra_w;
6286 break;
6287 case Intrinsic::x86_sse2_psrai_d:
6288 NewIntNo = Intrinsic::x86_sse2_psra_d;
6289 break;
6290 default: {
Owen Anderson825b72b2009-08-11 20:47:22 +00006291 ShAmtVT = MVT::v2i32;
Evan Cheng5759f972008-05-04 09:15:50 +00006292 switch (IntNo) {
6293 case Intrinsic::x86_mmx_pslli_w:
6294 NewIntNo = Intrinsic::x86_mmx_psll_w;
6295 break;
6296 case Intrinsic::x86_mmx_pslli_d:
6297 NewIntNo = Intrinsic::x86_mmx_psll_d;
6298 break;
6299 case Intrinsic::x86_mmx_pslli_q:
6300 NewIntNo = Intrinsic::x86_mmx_psll_q;
6301 break;
6302 case Intrinsic::x86_mmx_psrli_w:
6303 NewIntNo = Intrinsic::x86_mmx_psrl_w;
6304 break;
6305 case Intrinsic::x86_mmx_psrli_d:
6306 NewIntNo = Intrinsic::x86_mmx_psrl_d;
6307 break;
6308 case Intrinsic::x86_mmx_psrli_q:
6309 NewIntNo = Intrinsic::x86_mmx_psrl_q;
6310 break;
6311 case Intrinsic::x86_mmx_psrai_w:
6312 NewIntNo = Intrinsic::x86_mmx_psra_w;
6313 break;
6314 case Intrinsic::x86_mmx_psrai_d:
6315 NewIntNo = Intrinsic::x86_mmx_psra_d;
6316 break;
Torok Edwinc23197a2009-07-14 16:55:14 +00006317 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
Evan Cheng5759f972008-05-04 09:15:50 +00006318 }
6319 break;
6320 }
6321 }
Owen Andersone50ed302009-08-10 22:56:29 +00006322 EVT VT = Op.getValueType();
Dale Johannesene4d209d2009-02-03 20:21:25 +00006323 ShAmt = DAG.getNode(ISD::BIT_CONVERT, dl, VT,
6324 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShAmtVT, ShAmt));
6325 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00006326 DAG.getConstant(NewIntNo, MVT::i32),
Evan Cheng5759f972008-05-04 09:15:50 +00006327 Op.getOperand(1), ShAmt);
6328 }
Evan Cheng38bcbaf2005-12-23 07:31:11 +00006329 }
Chris Lattnerdbdbf0c2005-11-15 00:40:23 +00006330}
Evan Cheng72261582005-12-20 06:22:03 +00006331
Dan Gohman475871a2008-07-27 21:46:04 +00006332SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) {
Bill Wendling64e87322009-01-16 19:25:27 +00006333 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006334 DebugLoc dl = Op.getDebugLoc();
Bill Wendling64e87322009-01-16 19:25:27 +00006335
6336 if (Depth > 0) {
6337 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
6338 SDValue Offset =
6339 DAG.getConstant(TD->getPointerSize(),
Owen Anderson825b72b2009-08-11 20:47:22 +00006340 Subtarget->is64Bit() ? MVT::i64 : MVT::i32);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006341 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
Scott Michelfdc40a02009-02-17 22:15:04 +00006342 DAG.getNode(ISD::ADD, dl, getPointerTy(),
Dale Johannesene4d209d2009-02-03 20:21:25 +00006343 FrameAddr, Offset),
Bill Wendling64e87322009-01-16 19:25:27 +00006344 NULL, 0);
6345 }
6346
6347 // Just load the return address.
Dan Gohman475871a2008-07-27 21:46:04 +00006348 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
Scott Michelfdc40a02009-02-17 22:15:04 +00006349 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
Dale Johannesene4d209d2009-02-03 20:21:25 +00006350 RetAddrFI, NULL, 0);
Nate Begemanbcc5f362007-01-29 22:58:52 +00006351}
6352
Dan Gohman475871a2008-07-27 21:46:04 +00006353SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) {
Evan Cheng184793f2008-09-27 01:56:22 +00006354 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
6355 MFI->setFrameAddressIsTaken(true);
Owen Andersone50ed302009-08-10 22:56:29 +00006356 EVT VT = Op.getValueType();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006357 DebugLoc dl = Op.getDebugLoc(); // FIXME probably not meaningful
Evan Cheng184793f2008-09-27 01:56:22 +00006358 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
6359 unsigned FrameReg = Subtarget->is64Bit() ? X86::RBP : X86::EBP;
Dale Johannesendd64c412009-02-04 00:33:20 +00006360 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
Evan Cheng184793f2008-09-27 01:56:22 +00006361 while (Depth--)
Dale Johannesendd64c412009-02-04 00:33:20 +00006362 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, NULL, 0);
Evan Cheng184793f2008-09-27 01:56:22 +00006363 return FrameAddr;
Nate Begemanbcc5f362007-01-29 22:58:52 +00006364}
6365
Dan Gohman475871a2008-07-27 21:46:04 +00006366SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
Anton Korobeynikov260a6b82008-09-08 21:12:11 +00006367 SelectionDAG &DAG) {
Anton Korobeynikovbff66b02008-09-09 18:22:57 +00006368 return DAG.getIntPtrConstant(2*TD->getPointerSize());
Anton Korobeynikov2365f512007-07-14 14:06:15 +00006369}
6370
Dan Gohman475871a2008-07-27 21:46:04 +00006371SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG)
Anton Korobeynikov2365f512007-07-14 14:06:15 +00006372{
Anton Korobeynikov2365f512007-07-14 14:06:15 +00006373 MachineFunction &MF = DAG.getMachineFunction();
Dan Gohman475871a2008-07-27 21:46:04 +00006374 SDValue Chain = Op.getOperand(0);
6375 SDValue Offset = Op.getOperand(1);
6376 SDValue Handler = Op.getOperand(2);
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006377 DebugLoc dl = Op.getDebugLoc();
Anton Korobeynikov2365f512007-07-14 14:06:15 +00006378
Anton Korobeynikovb84c1672008-09-08 21:12:47 +00006379 SDValue Frame = DAG.getRegister(Subtarget->is64Bit() ? X86::RBP : X86::EBP,
6380 getPointerTy());
6381 unsigned StoreAddrReg = (Subtarget->is64Bit() ? X86::RCX : X86::ECX);
Anton Korobeynikov2365f512007-07-14 14:06:15 +00006382
Dale Johannesene4d209d2009-02-03 20:21:25 +00006383 SDValue StoreAddr = DAG.getNode(ISD::SUB, dl, getPointerTy(), Frame,
Anton Korobeynikovbff66b02008-09-09 18:22:57 +00006384 DAG.getIntPtrConstant(-TD->getPointerSize()));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006385 StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), StoreAddr, Offset);
6386 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, NULL, 0);
Dale Johannesendd64c412009-02-04 00:33:20 +00006387 Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr);
Anton Korobeynikovb84c1672008-09-08 21:12:47 +00006388 MF.getRegInfo().addLiveOut(StoreAddrReg);
Anton Korobeynikov2365f512007-07-14 14:06:15 +00006389
Dale Johannesene4d209d2009-02-03 20:21:25 +00006390 return DAG.getNode(X86ISD::EH_RETURN, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00006391 MVT::Other,
Anton Korobeynikovb84c1672008-09-08 21:12:47 +00006392 Chain, DAG.getRegister(StoreAddrReg, getPointerTy()));
Anton Korobeynikov2365f512007-07-14 14:06:15 +00006393}
6394
Dan Gohman475871a2008-07-27 21:46:04 +00006395SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op,
Duncan Sandsb116fac2007-07-27 20:02:49 +00006396 SelectionDAG &DAG) {
Dan Gohman475871a2008-07-27 21:46:04 +00006397 SDValue Root = Op.getOperand(0);
6398 SDValue Trmp = Op.getOperand(1); // trampoline
6399 SDValue FPtr = Op.getOperand(2); // nested function
6400 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006401 DebugLoc dl = Op.getDebugLoc();
Duncan Sandsb116fac2007-07-27 20:02:49 +00006402
Dan Gohman69de1932008-02-06 22:27:42 +00006403 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
Duncan Sandsb116fac2007-07-27 20:02:49 +00006404
Duncan Sands339e14f2008-01-16 22:55:25 +00006405 const X86InstrInfo *TII =
6406 ((X86TargetMachine&)getTargetMachine()).getInstrInfo();
6407
Duncan Sandsb116fac2007-07-27 20:02:49 +00006408 if (Subtarget->is64Bit()) {
Dan Gohman475871a2008-07-27 21:46:04 +00006409 SDValue OutChains[6];
Duncan Sands339e14f2008-01-16 22:55:25 +00006410
6411 // Large code-model.
6412
6413 const unsigned char JMP64r = TII->getBaseOpcodeFor(X86::JMP64r);
6414 const unsigned char MOV64ri = TII->getBaseOpcodeFor(X86::MOV64ri);
6415
Dan Gohmanc9f5f3f2008-05-14 01:58:56 +00006416 const unsigned char N86R10 = RegInfo->getX86RegNum(X86::R10);
6417 const unsigned char N86R11 = RegInfo->getX86RegNum(X86::R11);
Duncan Sands339e14f2008-01-16 22:55:25 +00006418
6419 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix
6420
6421 // Load the pointer to the nested function into R11.
6422 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11
Dan Gohman475871a2008-07-27 21:46:04 +00006423 SDValue Addr = Trmp;
Owen Anderson825b72b2009-08-11 20:47:22 +00006424 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
Dale Johannesene4d209d2009-02-03 20:21:25 +00006425 Addr, TrmpAddr, 0);
Duncan Sands339e14f2008-01-16 22:55:25 +00006426
Owen Anderson825b72b2009-08-11 20:47:22 +00006427 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
6428 DAG.getConstant(2, MVT::i64));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006429 OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr, TrmpAddr, 2, false, 2);
Duncan Sands339e14f2008-01-16 22:55:25 +00006430
6431 // Load the 'nest' parameter value into R10.
6432 // R10 is specified in X86CallingConv.td
6433 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10
Owen Anderson825b72b2009-08-11 20:47:22 +00006434 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
6435 DAG.getConstant(10, MVT::i64));
6436 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
Dale Johannesene4d209d2009-02-03 20:21:25 +00006437 Addr, TrmpAddr, 10);
Duncan Sands339e14f2008-01-16 22:55:25 +00006438
Owen Anderson825b72b2009-08-11 20:47:22 +00006439 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
6440 DAG.getConstant(12, MVT::i64));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006441 OutChains[3] = DAG.getStore(Root, dl, Nest, Addr, TrmpAddr, 12, false, 2);
Duncan Sands339e14f2008-01-16 22:55:25 +00006442
6443 // Jump to the nested function.
6444 OpCode = (JMP64r << 8) | REX_WB; // jmpq *...
Owen Anderson825b72b2009-08-11 20:47:22 +00006445 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
6446 DAG.getConstant(20, MVT::i64));
6447 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
Dale Johannesene4d209d2009-02-03 20:21:25 +00006448 Addr, TrmpAddr, 20);
Duncan Sands339e14f2008-01-16 22:55:25 +00006449
6450 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11
Owen Anderson825b72b2009-08-11 20:47:22 +00006451 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
6452 DAG.getConstant(22, MVT::i64));
6453 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, MVT::i8), Addr,
Dan Gohman69de1932008-02-06 22:27:42 +00006454 TrmpAddr, 22);
Duncan Sands339e14f2008-01-16 22:55:25 +00006455
Dan Gohman475871a2008-07-27 21:46:04 +00006456 SDValue Ops[] =
Owen Anderson825b72b2009-08-11 20:47:22 +00006457 { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 6) };
Dale Johannesene4d209d2009-02-03 20:21:25 +00006458 return DAG.getMergeValues(Ops, 2, dl);
Duncan Sandsb116fac2007-07-27 20:02:49 +00006459 } else {
Dan Gohmanbbfb9c52008-01-31 01:01:48 +00006460 const Function *Func =
Duncan Sandsb116fac2007-07-27 20:02:49 +00006461 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
6462 unsigned CC = Func->getCallingConv();
Duncan Sandsee465742007-08-29 19:01:20 +00006463 unsigned NestReg;
Duncan Sandsb116fac2007-07-27 20:02:49 +00006464
6465 switch (CC) {
6466 default:
Torok Edwinc23197a2009-07-14 16:55:14 +00006467 llvm_unreachable("Unsupported calling convention");
Duncan Sandsb116fac2007-07-27 20:02:49 +00006468 case CallingConv::C:
Duncan Sandsb116fac2007-07-27 20:02:49 +00006469 case CallingConv::X86_StdCall: {
6470 // Pass 'nest' parameter in ECX.
6471 // Must be kept in sync with X86CallingConv.td
Duncan Sandsee465742007-08-29 19:01:20 +00006472 NestReg = X86::ECX;
Duncan Sandsb116fac2007-07-27 20:02:49 +00006473
6474 // Check that ECX wasn't needed by an 'inreg' parameter.
6475 const FunctionType *FTy = Func->getFunctionType();
Devang Patel05988662008-09-25 21:00:45 +00006476 const AttrListPtr &Attrs = Func->getAttributes();
Duncan Sandsb116fac2007-07-27 20:02:49 +00006477
Chris Lattner58d74912008-03-12 17:45:29 +00006478 if (!Attrs.isEmpty() && !Func->isVarArg()) {
Duncan Sandsb116fac2007-07-27 20:02:49 +00006479 unsigned InRegCount = 0;
6480 unsigned Idx = 1;
6481
6482 for (FunctionType::param_iterator I = FTy->param_begin(),
6483 E = FTy->param_end(); I != E; ++I, ++Idx)
Devang Patel05988662008-09-25 21:00:45 +00006484 if (Attrs.paramHasAttr(Idx, Attribute::InReg))
Duncan Sandsb116fac2007-07-27 20:02:49 +00006485 // FIXME: should only count parameters that are lowered to integers.
Anton Korobeynikovbff66b02008-09-09 18:22:57 +00006486 InRegCount += (TD->getTypeSizeInBits(*I) + 31) / 32;
Duncan Sandsb116fac2007-07-27 20:02:49 +00006487
6488 if (InRegCount > 2) {
Torok Edwinab7c09b2009-07-08 18:01:40 +00006489 llvm_report_error("Nest register in use - reduce number of inreg parameters!");
Duncan Sandsb116fac2007-07-27 20:02:49 +00006490 }
6491 }
6492 break;
6493 }
6494 case CallingConv::X86_FastCall:
Duncan Sandsbf53c292008-09-10 13:22:10 +00006495 case CallingConv::Fast:
Duncan Sandsb116fac2007-07-27 20:02:49 +00006496 // Pass 'nest' parameter in EAX.
6497 // Must be kept in sync with X86CallingConv.td
Duncan Sandsee465742007-08-29 19:01:20 +00006498 NestReg = X86::EAX;
Duncan Sandsb116fac2007-07-27 20:02:49 +00006499 break;
6500 }
6501
Dan Gohman475871a2008-07-27 21:46:04 +00006502 SDValue OutChains[4];
6503 SDValue Addr, Disp;
Duncan Sandsb116fac2007-07-27 20:02:49 +00006504
Owen Anderson825b72b2009-08-11 20:47:22 +00006505 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
6506 DAG.getConstant(10, MVT::i32));
6507 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr);
Duncan Sandsb116fac2007-07-27 20:02:49 +00006508
Duncan Sands339e14f2008-01-16 22:55:25 +00006509 const unsigned char MOV32ri = TII->getBaseOpcodeFor(X86::MOV32ri);
Dan Gohmanc9f5f3f2008-05-14 01:58:56 +00006510 const unsigned char N86Reg = RegInfo->getX86RegNum(NestReg);
Scott Michelfdc40a02009-02-17 22:15:04 +00006511 OutChains[0] = DAG.getStore(Root, dl,
Owen Anderson825b72b2009-08-11 20:47:22 +00006512 DAG.getConstant(MOV32ri|N86Reg, MVT::i8),
Dan Gohman69de1932008-02-06 22:27:42 +00006513 Trmp, TrmpAddr, 0);
Duncan Sandsb116fac2007-07-27 20:02:49 +00006514
Owen Anderson825b72b2009-08-11 20:47:22 +00006515 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
6516 DAG.getConstant(1, MVT::i32));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006517 OutChains[1] = DAG.getStore(Root, dl, Nest, Addr, TrmpAddr, 1, false, 1);
Duncan Sandsb116fac2007-07-27 20:02:49 +00006518
Duncan Sands339e14f2008-01-16 22:55:25 +00006519 const unsigned char JMP = TII->getBaseOpcodeFor(X86::JMP);
Owen Anderson825b72b2009-08-11 20:47:22 +00006520 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
6521 DAG.getConstant(5, MVT::i32));
6522 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, MVT::i8), Addr,
Dan Gohman69de1932008-02-06 22:27:42 +00006523 TrmpAddr, 5, false, 1);
Duncan Sandsb116fac2007-07-27 20:02:49 +00006524
Owen Anderson825b72b2009-08-11 20:47:22 +00006525 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
6526 DAG.getConstant(6, MVT::i32));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006527 OutChains[3] = DAG.getStore(Root, dl, Disp, Addr, TrmpAddr, 6, false, 1);
Duncan Sandsb116fac2007-07-27 20:02:49 +00006528
Dan Gohman475871a2008-07-27 21:46:04 +00006529 SDValue Ops[] =
Owen Anderson825b72b2009-08-11 20:47:22 +00006530 { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 4) };
Dale Johannesene4d209d2009-02-03 20:21:25 +00006531 return DAG.getMergeValues(Ops, 2, dl);
Duncan Sandsb116fac2007-07-27 20:02:49 +00006532 }
6533}
6534
Dan Gohman475871a2008-07-27 21:46:04 +00006535SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) {
Anton Korobeynikov45b22fa2007-11-16 01:31:51 +00006536 /*
6537 The rounding mode is in bits 11:10 of FPSR, and has the following
6538 settings:
6539 00 Round to nearest
6540 01 Round to -inf
6541 10 Round to +inf
6542 11 Round to 0
6543
6544 FLT_ROUNDS, on the other hand, expects the following:
6545 -1 Undefined
6546 0 Round to 0
6547 1 Round to nearest
6548 2 Round to +inf
6549 3 Round to -inf
6550
6551 To perform the conversion, we do:
6552 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3)
6553 */
6554
6555 MachineFunction &MF = DAG.getMachineFunction();
6556 const TargetMachine &TM = MF.getTarget();
6557 const TargetFrameInfo &TFI = *TM.getFrameInfo();
6558 unsigned StackAlignment = TFI.getStackAlignment();
Owen Andersone50ed302009-08-10 22:56:29 +00006559 EVT VT = Op.getValueType();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006560 DebugLoc dl = Op.getDebugLoc();
Anton Korobeynikov45b22fa2007-11-16 01:31:51 +00006561
6562 // Save FP Control Word to stack slot
6563 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment);
Dan Gohman475871a2008-07-27 21:46:04 +00006564 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
Anton Korobeynikov45b22fa2007-11-16 01:31:51 +00006565
Owen Anderson825b72b2009-08-11 20:47:22 +00006566 SDValue Chain = DAG.getNode(X86ISD::FNSTCW16m, dl, MVT::Other,
Evan Cheng8a186ae2008-09-24 23:26:36 +00006567 DAG.getEntryNode(), StackSlot);
Anton Korobeynikov45b22fa2007-11-16 01:31:51 +00006568
6569 // Load FP Control Word from stack slot
Owen Anderson825b72b2009-08-11 20:47:22 +00006570 SDValue CWD = DAG.getLoad(MVT::i16, dl, Chain, StackSlot, NULL, 0);
Anton Korobeynikov45b22fa2007-11-16 01:31:51 +00006571
6572 // Transform as necessary
Dan Gohman475871a2008-07-27 21:46:04 +00006573 SDValue CWD1 =
Owen Anderson825b72b2009-08-11 20:47:22 +00006574 DAG.getNode(ISD::SRL, dl, MVT::i16,
6575 DAG.getNode(ISD::AND, dl, MVT::i16,
6576 CWD, DAG.getConstant(0x800, MVT::i16)),
6577 DAG.getConstant(11, MVT::i8));
Dan Gohman475871a2008-07-27 21:46:04 +00006578 SDValue CWD2 =
Owen Anderson825b72b2009-08-11 20:47:22 +00006579 DAG.getNode(ISD::SRL, dl, MVT::i16,
6580 DAG.getNode(ISD::AND, dl, MVT::i16,
6581 CWD, DAG.getConstant(0x400, MVT::i16)),
6582 DAG.getConstant(9, MVT::i8));
Anton Korobeynikov45b22fa2007-11-16 01:31:51 +00006583
Dan Gohman475871a2008-07-27 21:46:04 +00006584 SDValue RetVal =
Owen Anderson825b72b2009-08-11 20:47:22 +00006585 DAG.getNode(ISD::AND, dl, MVT::i16,
6586 DAG.getNode(ISD::ADD, dl, MVT::i16,
6587 DAG.getNode(ISD::OR, dl, MVT::i16, CWD1, CWD2),
6588 DAG.getConstant(1, MVT::i16)),
6589 DAG.getConstant(3, MVT::i16));
Anton Korobeynikov45b22fa2007-11-16 01:31:51 +00006590
6591
Duncan Sands83ec4b62008-06-06 12:08:01 +00006592 return DAG.getNode((VT.getSizeInBits() < 16 ?
Dale Johannesenb300d2a2009-02-07 00:55:49 +00006593 ISD::TRUNCATE : ISD::ZERO_EXTEND), dl, VT, RetVal);
Anton Korobeynikov45b22fa2007-11-16 01:31:51 +00006594}
6595
Dan Gohman475871a2008-07-27 21:46:04 +00006596SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00006597 EVT VT = Op.getValueType();
6598 EVT OpVT = VT;
Duncan Sands83ec4b62008-06-06 12:08:01 +00006599 unsigned NumBits = VT.getSizeInBits();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006600 DebugLoc dl = Op.getDebugLoc();
Evan Cheng18efe262007-12-14 02:13:44 +00006601
6602 Op = Op.getOperand(0);
Owen Anderson825b72b2009-08-11 20:47:22 +00006603 if (VT == MVT::i8) {
Evan Cheng152804e2007-12-14 08:30:15 +00006604 // Zero extend to i32 since there is not an i8 bsr.
Owen Anderson825b72b2009-08-11 20:47:22 +00006605 OpVT = MVT::i32;
Dale Johannesene4d209d2009-02-03 20:21:25 +00006606 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
Evan Cheng18efe262007-12-14 02:13:44 +00006607 }
Evan Cheng18efe262007-12-14 02:13:44 +00006608
Evan Cheng152804e2007-12-14 08:30:15 +00006609 // Issue a bsr (scan bits in reverse) which also sets EFLAGS.
Owen Anderson825b72b2009-08-11 20:47:22 +00006610 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006611 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
Evan Cheng152804e2007-12-14 08:30:15 +00006612
6613 // If src is zero (i.e. bsr sets ZF), returns NumBits.
Dan Gohman475871a2008-07-27 21:46:04 +00006614 SmallVector<SDValue, 4> Ops;
Evan Cheng152804e2007-12-14 08:30:15 +00006615 Ops.push_back(Op);
6616 Ops.push_back(DAG.getConstant(NumBits+NumBits-1, OpVT));
Owen Anderson825b72b2009-08-11 20:47:22 +00006617 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8));
Evan Cheng152804e2007-12-14 08:30:15 +00006618 Ops.push_back(Op.getValue(1));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006619 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, &Ops[0], 4);
Evan Cheng152804e2007-12-14 08:30:15 +00006620
6621 // Finally xor with NumBits-1.
Dale Johannesene4d209d2009-02-03 20:21:25 +00006622 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
Evan Cheng152804e2007-12-14 08:30:15 +00006623
Owen Anderson825b72b2009-08-11 20:47:22 +00006624 if (VT == MVT::i8)
6625 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
Evan Cheng18efe262007-12-14 02:13:44 +00006626 return Op;
6627}
6628
Dan Gohman475871a2008-07-27 21:46:04 +00006629SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00006630 EVT VT = Op.getValueType();
6631 EVT OpVT = VT;
Duncan Sands83ec4b62008-06-06 12:08:01 +00006632 unsigned NumBits = VT.getSizeInBits();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006633 DebugLoc dl = Op.getDebugLoc();
Evan Cheng18efe262007-12-14 02:13:44 +00006634
6635 Op = Op.getOperand(0);
Owen Anderson825b72b2009-08-11 20:47:22 +00006636 if (VT == MVT::i8) {
6637 OpVT = MVT::i32;
Dale Johannesene4d209d2009-02-03 20:21:25 +00006638 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
Evan Cheng18efe262007-12-14 02:13:44 +00006639 }
Evan Cheng152804e2007-12-14 08:30:15 +00006640
6641 // Issue a bsf (scan bits forward) which also sets EFLAGS.
Owen Anderson825b72b2009-08-11 20:47:22 +00006642 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006643 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op);
Evan Cheng152804e2007-12-14 08:30:15 +00006644
6645 // If src is zero (i.e. bsf sets ZF), returns NumBits.
Dan Gohman475871a2008-07-27 21:46:04 +00006646 SmallVector<SDValue, 4> Ops;
Evan Cheng152804e2007-12-14 08:30:15 +00006647 Ops.push_back(Op);
6648 Ops.push_back(DAG.getConstant(NumBits, OpVT));
Owen Anderson825b72b2009-08-11 20:47:22 +00006649 Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8));
Evan Cheng152804e2007-12-14 08:30:15 +00006650 Ops.push_back(Op.getValue(1));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006651 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, &Ops[0], 4);
Evan Cheng152804e2007-12-14 08:30:15 +00006652
Owen Anderson825b72b2009-08-11 20:47:22 +00006653 if (VT == MVT::i8)
6654 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
Evan Cheng18efe262007-12-14 02:13:44 +00006655 return Op;
6656}
6657
Mon P Wangaf9b9522008-12-18 21:42:19 +00006658SDValue X86TargetLowering::LowerMUL_V2I64(SDValue Op, SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00006659 EVT VT = Op.getValueType();
Owen Anderson825b72b2009-08-11 20:47:22 +00006660 assert(VT == MVT::v2i64 && "Only know how to lower V2I64 multiply");
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006661 DebugLoc dl = Op.getDebugLoc();
Scott Michelfdc40a02009-02-17 22:15:04 +00006662
Mon P Wangaf9b9522008-12-18 21:42:19 +00006663 // ulong2 Ahi = __builtin_ia32_psrlqi128( a, 32);
6664 // ulong2 Bhi = __builtin_ia32_psrlqi128( b, 32);
6665 // ulong2 AloBlo = __builtin_ia32_pmuludq128( a, b );
6666 // ulong2 AloBhi = __builtin_ia32_pmuludq128( a, Bhi );
6667 // ulong2 AhiBlo = __builtin_ia32_pmuludq128( Ahi, b );
6668 //
6669 // AloBhi = __builtin_ia32_psllqi128( AloBhi, 32 );
6670 // AhiBlo = __builtin_ia32_psllqi128( AhiBlo, 32 );
6671 // return AloBlo + AloBhi + AhiBlo;
6672
6673 SDValue A = Op.getOperand(0);
6674 SDValue B = Op.getOperand(1);
Scott Michelfdc40a02009-02-17 22:15:04 +00006675
Dale Johannesene4d209d2009-02-03 20:21:25 +00006676 SDValue Ahi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00006677 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32),
6678 A, DAG.getConstant(32, MVT::i32));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006679 SDValue Bhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00006680 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32),
6681 B, DAG.getConstant(32, MVT::i32));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006682 SDValue AloBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00006683 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32),
Mon P Wangaf9b9522008-12-18 21:42:19 +00006684 A, B);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006685 SDValue AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00006686 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32),
Mon P Wangaf9b9522008-12-18 21:42:19 +00006687 A, Bhi);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006688 SDValue AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00006689 DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32),
Mon P Wangaf9b9522008-12-18 21:42:19 +00006690 Ahi, B);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006691 AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00006692 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32),
6693 AloBhi, DAG.getConstant(32, MVT::i32));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006694 AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00006695 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32),
6696 AhiBlo, DAG.getConstant(32, MVT::i32));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006697 SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi);
6698 Res = DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo);
Mon P Wangaf9b9522008-12-18 21:42:19 +00006699 return Res;
6700}
6701
6702
Bill Wendling74c37652008-12-09 22:08:41 +00006703SDValue X86TargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) {
6704 // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
6705 // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
Bill Wendling61edeb52008-12-02 01:06:39 +00006706 // looks for this combo and may remove the "setcc" instruction if the "setcc"
6707 // has only one use.
Bill Wendling3fafd932008-11-26 22:37:40 +00006708 SDNode *N = Op.getNode();
Bill Wendling61edeb52008-12-02 01:06:39 +00006709 SDValue LHS = N->getOperand(0);
6710 SDValue RHS = N->getOperand(1);
Bill Wendling74c37652008-12-09 22:08:41 +00006711 unsigned BaseOp = 0;
6712 unsigned Cond = 0;
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006713 DebugLoc dl = Op.getDebugLoc();
Bill Wendling74c37652008-12-09 22:08:41 +00006714
6715 switch (Op.getOpcode()) {
Torok Edwinc23197a2009-07-14 16:55:14 +00006716 default: llvm_unreachable("Unknown ovf instruction!");
Bill Wendling74c37652008-12-09 22:08:41 +00006717 case ISD::SADDO:
Dan Gohman076aee32009-03-04 19:44:21 +00006718 // A subtract of one will be selected as a INC. Note that INC doesn't
6719 // set CF, so we can't do this for UADDO.
6720 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
6721 if (C->getAPIntValue() == 1) {
6722 BaseOp = X86ISD::INC;
6723 Cond = X86::COND_O;
6724 break;
6725 }
Bill Wendlingab55ebd2008-12-12 00:56:36 +00006726 BaseOp = X86ISD::ADD;
Bill Wendling74c37652008-12-09 22:08:41 +00006727 Cond = X86::COND_O;
6728 break;
6729 case ISD::UADDO:
Bill Wendlingab55ebd2008-12-12 00:56:36 +00006730 BaseOp = X86ISD::ADD;
Dan Gohman653456c2009-01-07 00:15:08 +00006731 Cond = X86::COND_B;
Bill Wendling74c37652008-12-09 22:08:41 +00006732 break;
6733 case ISD::SSUBO:
Dan Gohman076aee32009-03-04 19:44:21 +00006734 // A subtract of one will be selected as a DEC. Note that DEC doesn't
6735 // set CF, so we can't do this for USUBO.
6736 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
6737 if (C->getAPIntValue() == 1) {
6738 BaseOp = X86ISD::DEC;
6739 Cond = X86::COND_O;
6740 break;
6741 }
Bill Wendlingab55ebd2008-12-12 00:56:36 +00006742 BaseOp = X86ISD::SUB;
Bill Wendling74c37652008-12-09 22:08:41 +00006743 Cond = X86::COND_O;
6744 break;
6745 case ISD::USUBO:
Bill Wendlingab55ebd2008-12-12 00:56:36 +00006746 BaseOp = X86ISD::SUB;
Dan Gohman653456c2009-01-07 00:15:08 +00006747 Cond = X86::COND_B;
Bill Wendling74c37652008-12-09 22:08:41 +00006748 break;
6749 case ISD::SMULO:
Bill Wendlingd350e022008-12-12 21:15:41 +00006750 BaseOp = X86ISD::SMUL;
Bill Wendling74c37652008-12-09 22:08:41 +00006751 Cond = X86::COND_O;
6752 break;
6753 case ISD::UMULO:
Bill Wendlingd350e022008-12-12 21:15:41 +00006754 BaseOp = X86ISD::UMUL;
Dan Gohman653456c2009-01-07 00:15:08 +00006755 Cond = X86::COND_B;
Bill Wendling74c37652008-12-09 22:08:41 +00006756 break;
6757 }
Bill Wendling3fafd932008-11-26 22:37:40 +00006758
Bill Wendling61edeb52008-12-02 01:06:39 +00006759 // Also sets EFLAGS.
Owen Anderson825b72b2009-08-11 20:47:22 +00006760 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006761 SDValue Sum = DAG.getNode(BaseOp, dl, VTs, LHS, RHS);
Bill Wendling3fafd932008-11-26 22:37:40 +00006762
Bill Wendling61edeb52008-12-02 01:06:39 +00006763 SDValue SetCC =
Dale Johannesene4d209d2009-02-03 20:21:25 +00006764 DAG.getNode(X86ISD::SETCC, dl, N->getValueType(1),
Owen Anderson825b72b2009-08-11 20:47:22 +00006765 DAG.getConstant(Cond, MVT::i32), SDValue(Sum.getNode(), 1));
Bill Wendling3fafd932008-11-26 22:37:40 +00006766
Bill Wendling61edeb52008-12-02 01:06:39 +00006767 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SetCC);
6768 return Sum;
Bill Wendling41ea7e72008-11-24 19:21:46 +00006769}
6770
Dan Gohman475871a2008-07-27 21:46:04 +00006771SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00006772 EVT T = Op.getValueType();
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006773 DebugLoc dl = Op.getDebugLoc();
Andrew Lenhartha76e2f02008-03-04 21:13:33 +00006774 unsigned Reg = 0;
6775 unsigned size = 0;
Owen Anderson825b72b2009-08-11 20:47:22 +00006776 switch(T.getSimpleVT().SimpleTy) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00006777 default:
6778 assert(false && "Invalid value type!");
Owen Anderson825b72b2009-08-11 20:47:22 +00006779 case MVT::i8: Reg = X86::AL; size = 1; break;
6780 case MVT::i16: Reg = X86::AX; size = 2; break;
6781 case MVT::i32: Reg = X86::EAX; size = 4; break;
6782 case MVT::i64:
Duncan Sands1607f052008-12-01 11:39:25 +00006783 assert(Subtarget->is64Bit() && "Node not type legal!");
6784 Reg = X86::RAX; size = 8;
Andrew Lenharthd19189e2008-03-05 01:15:49 +00006785 break;
Bill Wendling61edeb52008-12-02 01:06:39 +00006786 }
Dale Johannesendd64c412009-02-04 00:33:20 +00006787 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), dl, Reg,
Dale Johannesend18a4622008-09-11 03:12:59 +00006788 Op.getOperand(2), SDValue());
Dan Gohman475871a2008-07-27 21:46:04 +00006789 SDValue Ops[] = { cpIn.getValue(0),
Evan Cheng8a186ae2008-09-24 23:26:36 +00006790 Op.getOperand(1),
6791 Op.getOperand(3),
Owen Anderson825b72b2009-08-11 20:47:22 +00006792 DAG.getTargetConstant(size, MVT::i8),
Evan Cheng8a186ae2008-09-24 23:26:36 +00006793 cpIn.getValue(1) };
Owen Anderson825b72b2009-08-11 20:47:22 +00006794 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006795 SDValue Result = DAG.getNode(X86ISD::LCMPXCHG_DAG, dl, Tys, Ops, 5);
Scott Michelfdc40a02009-02-17 22:15:04 +00006796 SDValue cpOut =
Dale Johannesendd64c412009-02-04 00:33:20 +00006797 DAG.getCopyFromReg(Result.getValue(0), dl, Reg, T, Result.getValue(1));
Andrew Lenharth26ed8692008-03-01 21:52:34 +00006798 return cpOut;
6799}
6800
Duncan Sands1607f052008-12-01 11:39:25 +00006801SDValue X86TargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
Gabor Greif327ef032008-08-28 23:19:51 +00006802 SelectionDAG &DAG) {
Duncan Sands1607f052008-12-01 11:39:25 +00006803 assert(Subtarget->is64Bit() && "Result not type legalized?");
Owen Anderson825b72b2009-08-11 20:47:22 +00006804 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Duncan Sands1607f052008-12-01 11:39:25 +00006805 SDValue TheChain = Op.getOperand(0);
Dale Johannesen6f38cb62009-02-07 19:59:05 +00006806 DebugLoc dl = Op.getDebugLoc();
Dale Johannesene4d209d2009-02-03 20:21:25 +00006807 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1);
Owen Anderson825b72b2009-08-11 20:47:22 +00006808 SDValue rax = DAG.getCopyFromReg(rd, dl, X86::RAX, MVT::i64, rd.getValue(1));
6809 SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), dl, X86::RDX, MVT::i64,
Duncan Sands1607f052008-12-01 11:39:25 +00006810 rax.getValue(2));
Owen Anderson825b72b2009-08-11 20:47:22 +00006811 SDValue Tmp = DAG.getNode(ISD::SHL, dl, MVT::i64, rdx,
6812 DAG.getConstant(32, MVT::i8));
Duncan Sands1607f052008-12-01 11:39:25 +00006813 SDValue Ops[] = {
Owen Anderson825b72b2009-08-11 20:47:22 +00006814 DAG.getNode(ISD::OR, dl, MVT::i64, rax, Tmp),
Duncan Sands1607f052008-12-01 11:39:25 +00006815 rdx.getValue(1)
6816 };
Dale Johannesene4d209d2009-02-03 20:21:25 +00006817 return DAG.getMergeValues(Ops, 2, dl);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00006818}
6819
Dale Johannesen71d1bf52008-09-29 22:25:26 +00006820SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) {
6821 SDNode *Node = Op.getNode();
Dale Johannesene4d209d2009-02-03 20:21:25 +00006822 DebugLoc dl = Node->getDebugLoc();
Owen Andersone50ed302009-08-10 22:56:29 +00006823 EVT T = Node->getValueType(0);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006824 SDValue negOp = DAG.getNode(ISD::SUB, dl, T,
Evan Cheng242b38b2009-02-23 09:03:22 +00006825 DAG.getConstant(0, T), Node->getOperand(2));
Dale Johannesene4d209d2009-02-03 20:21:25 +00006826 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl,
Dan Gohman0b1d4a72008-12-23 21:37:04 +00006827 cast<AtomicSDNode>(Node)->getMemoryVT(),
Dale Johannesen71d1bf52008-09-29 22:25:26 +00006828 Node->getOperand(0),
6829 Node->getOperand(1), negOp,
6830 cast<AtomicSDNode>(Node)->getSrcValue(),
6831 cast<AtomicSDNode>(Node)->getAlignment());
Mon P Wang63307c32008-05-05 19:05:59 +00006832}
6833
Evan Cheng0db9fe62006-04-25 20:13:52 +00006834/// LowerOperation - Provide custom lowering hooks for some operations.
6835///
Dan Gohman475871a2008-07-27 21:46:04 +00006836SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
Evan Cheng0db9fe62006-04-25 20:13:52 +00006837 switch (Op.getOpcode()) {
Torok Edwinc23197a2009-07-14 16:55:14 +00006838 default: llvm_unreachable("Should not custom lower this!");
Dan Gohman0b1d4a72008-12-23 21:37:04 +00006839 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG);
6840 case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG);
Evan Cheng0db9fe62006-04-25 20:13:52 +00006841 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
6842 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
6843 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
6844 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
6845 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
6846 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
6847 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00006848 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
Bill Wendling056292f2008-09-16 21:48:12 +00006849 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG);
Evan Cheng0db9fe62006-04-25 20:13:52 +00006850 case ISD::SHL_PARTS:
6851 case ISD::SRA_PARTS:
6852 case ISD::SRL_PARTS: return LowerShift(Op, DAG);
6853 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
Dale Johannesen1c15bf52008-10-21 20:50:01 +00006854 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
Evan Cheng0db9fe62006-04-25 20:13:52 +00006855 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG);
Eli Friedman948e95a2009-05-23 09:59:16 +00006856 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG);
Evan Cheng0db9fe62006-04-25 20:13:52 +00006857 case ISD::FABS: return LowerFABS(Op, DAG);
6858 case ISD::FNEG: return LowerFNEG(Op, DAG);
Evan Cheng68c47cb2007-01-05 07:55:56 +00006859 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
Evan Chenge5f62042007-09-29 00:00:36 +00006860 case ISD::SETCC: return LowerSETCC(Op, DAG);
Nate Begeman30a0de92008-07-17 16:51:19 +00006861 case ISD::VSETCC: return LowerVSETCC(Op, DAG);
Evan Chenge5f62042007-09-29 00:00:36 +00006862 case ISD::SELECT: return LowerSELECT(Op, DAG);
6863 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
Evan Cheng0db9fe62006-04-25 20:13:52 +00006864 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
Evan Cheng0db9fe62006-04-25 20:13:52 +00006865 case ISD::VASTART: return LowerVASTART(Op, DAG);
Dan Gohman9018e832008-05-10 01:26:14 +00006866 case ISD::VAARG: return LowerVAARG(Op, DAG);
Evan Chengae642192007-03-02 23:16:35 +00006867 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
Evan Cheng0db9fe62006-04-25 20:13:52 +00006868 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
Nate Begemanbcc5f362007-01-29 22:58:52 +00006869 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
6870 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
Anton Korobeynikov2365f512007-07-14 14:06:15 +00006871 case ISD::FRAME_TO_ARGS_OFFSET:
6872 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
Anton Korobeynikov57fc00d2007-04-17 09:20:00 +00006873 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
Anton Korobeynikov2365f512007-07-14 14:06:15 +00006874 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
Duncan Sandsb116fac2007-07-27 20:02:49 +00006875 case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG);
Dan Gohman1a024862008-01-31 00:41:03 +00006876 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG);
Evan Cheng18efe262007-12-14 02:13:44 +00006877 case ISD::CTLZ: return LowerCTLZ(Op, DAG);
6878 case ISD::CTTZ: return LowerCTTZ(Op, DAG);
Mon P Wangaf9b9522008-12-18 21:42:19 +00006879 case ISD::MUL: return LowerMUL_V2I64(Op, DAG);
Bill Wendling74c37652008-12-09 22:08:41 +00006880 case ISD::SADDO:
6881 case ISD::UADDO:
6882 case ISD::SSUBO:
6883 case ISD::USUBO:
6884 case ISD::SMULO:
6885 case ISD::UMULO: return LowerXALUO(Op, DAG);
Duncan Sands1607f052008-12-01 11:39:25 +00006886 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG);
Evan Cheng0db9fe62006-04-25 20:13:52 +00006887 }
Chris Lattner27a6c732007-11-24 07:07:01 +00006888}
6889
Duncan Sands1607f052008-12-01 11:39:25 +00006890void X86TargetLowering::
6891ReplaceATOMIC_BINARY_64(SDNode *Node, SmallVectorImpl<SDValue>&Results,
6892 SelectionDAG &DAG, unsigned NewOp) {
Owen Andersone50ed302009-08-10 22:56:29 +00006893 EVT T = Node->getValueType(0);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006894 DebugLoc dl = Node->getDebugLoc();
Owen Anderson825b72b2009-08-11 20:47:22 +00006895 assert (T == MVT::i64 && "Only know how to expand i64 atomics");
Duncan Sands1607f052008-12-01 11:39:25 +00006896
6897 SDValue Chain = Node->getOperand(0);
6898 SDValue In1 = Node->getOperand(1);
Owen Anderson825b72b2009-08-11 20:47:22 +00006899 SDValue In2L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
Duncan Sands1607f052008-12-01 11:39:25 +00006900 Node->getOperand(2), DAG.getIntPtrConstant(0));
Owen Anderson825b72b2009-08-11 20:47:22 +00006901 SDValue In2H = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
Duncan Sands1607f052008-12-01 11:39:25 +00006902 Node->getOperand(2), DAG.getIntPtrConstant(1));
6903 // This is a generalized SDNode, not an AtomicSDNode, so it doesn't
6904 // have a MemOperand. Pass the info through as a normal operand.
6905 SDValue LSI = DAG.getMemOperand(cast<MemSDNode>(Node)->getMemOperand());
6906 SDValue Ops[] = { Chain, In1, In2L, In2H, LSI };
Owen Anderson825b72b2009-08-11 20:47:22 +00006907 SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006908 SDValue Result = DAG.getNode(NewOp, dl, Tys, Ops, 5);
Duncan Sands1607f052008-12-01 11:39:25 +00006909 SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)};
Owen Anderson825b72b2009-08-11 20:47:22 +00006910 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2));
Duncan Sands1607f052008-12-01 11:39:25 +00006911 Results.push_back(Result.getValue(2));
6912}
6913
Duncan Sands126d9072008-07-04 11:47:58 +00006914/// ReplaceNodeResults - Replace a node with an illegal result type
6915/// with a new node built out of custom code.
Duncan Sands1607f052008-12-01 11:39:25 +00006916void X86TargetLowering::ReplaceNodeResults(SDNode *N,
6917 SmallVectorImpl<SDValue>&Results,
6918 SelectionDAG &DAG) {
Dale Johannesene4d209d2009-02-03 20:21:25 +00006919 DebugLoc dl = N->getDebugLoc();
Chris Lattner27a6c732007-11-24 07:07:01 +00006920 switch (N->getOpcode()) {
Duncan Sandsed294c42008-10-20 15:56:33 +00006921 default:
Duncan Sands1607f052008-12-01 11:39:25 +00006922 assert(false && "Do not know how to custom type legalize this operation!");
6923 return;
6924 case ISD::FP_TO_SINT: {
Eli Friedman948e95a2009-05-23 09:59:16 +00006925 std::pair<SDValue,SDValue> Vals =
6926 FP_TO_INTHelper(SDValue(N, 0), DAG, true);
Duncan Sands1607f052008-12-01 11:39:25 +00006927 SDValue FIST = Vals.first, StackSlot = Vals.second;
6928 if (FIST.getNode() != 0) {
Owen Andersone50ed302009-08-10 22:56:29 +00006929 EVT VT = N->getValueType(0);
Duncan Sands1607f052008-12-01 11:39:25 +00006930 // Return a load from the stack slot.
Dale Johannesene4d209d2009-02-03 20:21:25 +00006931 Results.push_back(DAG.getLoad(VT, dl, FIST, StackSlot, NULL, 0));
Duncan Sands1607f052008-12-01 11:39:25 +00006932 }
6933 return;
6934 }
6935 case ISD::READCYCLECOUNTER: {
Owen Anderson825b72b2009-08-11 20:47:22 +00006936 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Duncan Sands1607f052008-12-01 11:39:25 +00006937 SDValue TheChain = N->getOperand(0);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006938 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1);
Owen Anderson825b72b2009-08-11 20:47:22 +00006939 SDValue eax = DAG.getCopyFromReg(rd, dl, X86::EAX, MVT::i32,
Dale Johannesendd64c412009-02-04 00:33:20 +00006940 rd.getValue(1));
Owen Anderson825b72b2009-08-11 20:47:22 +00006941 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), dl, X86::EDX, MVT::i32,
Duncan Sands1607f052008-12-01 11:39:25 +00006942 eax.getValue(2));
6943 // Use a buildpair to merge the two 32-bit values into a 64-bit one.
6944 SDValue Ops[] = { eax, edx };
Owen Anderson825b72b2009-08-11 20:47:22 +00006945 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Ops, 2));
Duncan Sands1607f052008-12-01 11:39:25 +00006946 Results.push_back(edx.getValue(1));
6947 return;
6948 }
Dan Gohman0b1d4a72008-12-23 21:37:04 +00006949 case ISD::ATOMIC_CMP_SWAP: {
Owen Andersone50ed302009-08-10 22:56:29 +00006950 EVT T = N->getValueType(0);
Owen Anderson825b72b2009-08-11 20:47:22 +00006951 assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap");
Duncan Sands1607f052008-12-01 11:39:25 +00006952 SDValue cpInL, cpInH;
Owen Anderson825b72b2009-08-11 20:47:22 +00006953 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2),
6954 DAG.getConstant(0, MVT::i32));
6955 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2),
6956 DAG.getConstant(1, MVT::i32));
Dale Johannesendd64c412009-02-04 00:33:20 +00006957 cpInL = DAG.getCopyToReg(N->getOperand(0), dl, X86::EAX, cpInL, SDValue());
6958 cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl, X86::EDX, cpInH,
Duncan Sands1607f052008-12-01 11:39:25 +00006959 cpInL.getValue(1));
6960 SDValue swapInL, swapInH;
Owen Anderson825b72b2009-08-11 20:47:22 +00006961 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3),
6962 DAG.getConstant(0, MVT::i32));
6963 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3),
6964 DAG.getConstant(1, MVT::i32));
Dale Johannesendd64c412009-02-04 00:33:20 +00006965 swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl, X86::EBX, swapInL,
Duncan Sands1607f052008-12-01 11:39:25 +00006966 cpInH.getValue(1));
Dale Johannesendd64c412009-02-04 00:33:20 +00006967 swapInH = DAG.getCopyToReg(swapInL.getValue(0), dl, X86::ECX, swapInH,
Duncan Sands1607f052008-12-01 11:39:25 +00006968 swapInL.getValue(1));
6969 SDValue Ops[] = { swapInH.getValue(0),
6970 N->getOperand(1),
6971 swapInH.getValue(1) };
Owen Anderson825b72b2009-08-11 20:47:22 +00006972 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
Dale Johannesene4d209d2009-02-03 20:21:25 +00006973 SDValue Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, dl, Tys, Ops, 3);
Dale Johannesendd64c412009-02-04 00:33:20 +00006974 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, X86::EAX,
Owen Anderson825b72b2009-08-11 20:47:22 +00006975 MVT::i32, Result.getValue(1));
Dale Johannesendd64c412009-02-04 00:33:20 +00006976 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl, X86::EDX,
Owen Anderson825b72b2009-08-11 20:47:22 +00006977 MVT::i32, cpOutL.getValue(2));
Duncan Sands1607f052008-12-01 11:39:25 +00006978 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
Owen Anderson825b72b2009-08-11 20:47:22 +00006979 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2));
Duncan Sands1607f052008-12-01 11:39:25 +00006980 Results.push_back(cpOutH.getValue(1));
6981 return;
6982 }
Dan Gohman0b1d4a72008-12-23 21:37:04 +00006983 case ISD::ATOMIC_LOAD_ADD:
Duncan Sands1607f052008-12-01 11:39:25 +00006984 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMADD64_DAG);
6985 return;
Dan Gohman0b1d4a72008-12-23 21:37:04 +00006986 case ISD::ATOMIC_LOAD_AND:
Duncan Sands1607f052008-12-01 11:39:25 +00006987 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMAND64_DAG);
6988 return;
Dan Gohman0b1d4a72008-12-23 21:37:04 +00006989 case ISD::ATOMIC_LOAD_NAND:
Duncan Sands1607f052008-12-01 11:39:25 +00006990 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMNAND64_DAG);
6991 return;
Dan Gohman0b1d4a72008-12-23 21:37:04 +00006992 case ISD::ATOMIC_LOAD_OR:
Duncan Sands1607f052008-12-01 11:39:25 +00006993 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMOR64_DAG);
6994 return;
Dan Gohman0b1d4a72008-12-23 21:37:04 +00006995 case ISD::ATOMIC_LOAD_SUB:
Duncan Sands1607f052008-12-01 11:39:25 +00006996 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSUB64_DAG);
6997 return;
Dan Gohman0b1d4a72008-12-23 21:37:04 +00006998 case ISD::ATOMIC_LOAD_XOR:
Duncan Sands1607f052008-12-01 11:39:25 +00006999 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMXOR64_DAG);
7000 return;
Dan Gohman0b1d4a72008-12-23 21:37:04 +00007001 case ISD::ATOMIC_SWAP:
Duncan Sands1607f052008-12-01 11:39:25 +00007002 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSWAP64_DAG);
7003 return;
Chris Lattner27a6c732007-11-24 07:07:01 +00007004 }
Evan Cheng0db9fe62006-04-25 20:13:52 +00007005}
7006
Evan Cheng72261582005-12-20 06:22:03 +00007007const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
7008 switch (Opcode) {
7009 default: return NULL;
Evan Cheng18efe262007-12-14 02:13:44 +00007010 case X86ISD::BSF: return "X86ISD::BSF";
7011 case X86ISD::BSR: return "X86ISD::BSR";
Evan Chenge3413162006-01-09 18:33:28 +00007012 case X86ISD::SHLD: return "X86ISD::SHLD";
7013 case X86ISD::SHRD: return "X86ISD::SHRD";
Evan Chengef6ffb12006-01-31 03:14:29 +00007014 case X86ISD::FAND: return "X86ISD::FAND";
Evan Cheng68c47cb2007-01-05 07:55:56 +00007015 case X86ISD::FOR: return "X86ISD::FOR";
Evan Cheng223547a2006-01-31 22:28:30 +00007016 case X86ISD::FXOR: return "X86ISD::FXOR";
Evan Cheng68c47cb2007-01-05 07:55:56 +00007017 case X86ISD::FSRL: return "X86ISD::FSRL";
Evan Chenga3195e82006-01-12 22:54:21 +00007018 case X86ISD::FILD: return "X86ISD::FILD";
Evan Chenge3de85b2006-02-04 02:20:30 +00007019 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
Evan Cheng72261582005-12-20 06:22:03 +00007020 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
7021 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
7022 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
Evan Chengb077b842005-12-21 02:39:21 +00007023 case X86ISD::FLD: return "X86ISD::FLD";
Evan Chengd90eb7f2006-01-05 00:27:02 +00007024 case X86ISD::FST: return "X86ISD::FST";
Evan Cheng72261582005-12-20 06:22:03 +00007025 case X86ISD::CALL: return "X86ISD::CALL";
Evan Cheng72261582005-12-20 06:22:03 +00007026 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG";
Dan Gohmanc7a37d42008-12-23 22:45:23 +00007027 case X86ISD::BT: return "X86ISD::BT";
Evan Cheng72261582005-12-20 06:22:03 +00007028 case X86ISD::CMP: return "X86ISD::CMP";
Evan Cheng6be2c582006-04-05 23:38:46 +00007029 case X86ISD::COMI: return "X86ISD::COMI";
7030 case X86ISD::UCOMI: return "X86ISD::UCOMI";
Evan Chengd5781fc2005-12-21 20:21:51 +00007031 case X86ISD::SETCC: return "X86ISD::SETCC";
Evan Cheng72261582005-12-20 06:22:03 +00007032 case X86ISD::CMOV: return "X86ISD::CMOV";
7033 case X86ISD::BRCOND: return "X86ISD::BRCOND";
Evan Chengb077b842005-12-21 02:39:21 +00007034 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
Evan Cheng8df346b2006-03-04 01:12:00 +00007035 case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
7036 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
Evan Cheng7ccced62006-02-18 00:15:05 +00007037 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
Evan Cheng020d2e82006-02-23 20:41:18 +00007038 case X86ISD::Wrapper: return "X86ISD::Wrapper";
Chris Lattner18c59872009-06-27 04:16:01 +00007039 case X86ISD::WrapperRIP: return "X86ISD::WrapperRIP";
Nate Begeman14d12ca2008-02-11 04:19:36 +00007040 case X86ISD::PEXTRB: return "X86ISD::PEXTRB";
Evan Chengb067a1e2006-03-31 19:22:53 +00007041 case X86ISD::PEXTRW: return "X86ISD::PEXTRW";
Nate Begeman14d12ca2008-02-11 04:19:36 +00007042 case X86ISD::INSERTPS: return "X86ISD::INSERTPS";
7043 case X86ISD::PINSRB: return "X86ISD::PINSRB";
Evan Cheng653159f2006-03-31 21:55:24 +00007044 case X86ISD::PINSRW: return "X86ISD::PINSRW";
Nate Begemanb9a47b82009-02-23 08:49:38 +00007045 case X86ISD::PSHUFB: return "X86ISD::PSHUFB";
Evan Cheng8ca29322006-11-10 21:43:37 +00007046 case X86ISD::FMAX: return "X86ISD::FMAX";
7047 case X86ISD::FMIN: return "X86ISD::FMIN";
Dan Gohman20382522007-07-10 00:05:58 +00007048 case X86ISD::FRSQRT: return "X86ISD::FRSQRT";
7049 case X86ISD::FRCP: return "X86ISD::FRCP";
Lauro Ramos Venanciob3a04172007-04-20 21:38:10 +00007050 case X86ISD::TLSADDR: return "X86ISD::TLSADDR";
Rafael Espindola094fad32009-04-08 21:14:34 +00007051 case X86ISD::SegmentBaseAddress: return "X86ISD::SegmentBaseAddress";
Anton Korobeynikov2365f512007-07-14 14:06:15 +00007052 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN";
Arnold Schwaighoferc85e1712007-10-11 19:40:01 +00007053 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN";
Anton Korobeynikov45b22fa2007-11-16 01:31:51 +00007054 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m";
Evan Cheng7e2ff772008-05-08 00:57:18 +00007055 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG";
7056 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG";
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007057 case X86ISD::ATOMADD64_DAG: return "X86ISD::ATOMADD64_DAG";
7058 case X86ISD::ATOMSUB64_DAG: return "X86ISD::ATOMSUB64_DAG";
7059 case X86ISD::ATOMOR64_DAG: return "X86ISD::ATOMOR64_DAG";
7060 case X86ISD::ATOMXOR64_DAG: return "X86ISD::ATOMXOR64_DAG";
7061 case X86ISD::ATOMAND64_DAG: return "X86ISD::ATOMAND64_DAG";
7062 case X86ISD::ATOMNAND64_DAG: return "X86ISD::ATOMNAND64_DAG";
Evan Chengd880b972008-05-09 21:53:03 +00007063 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL";
7064 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD";
Evan Chengf26ffe92008-05-29 08:22:04 +00007065 case X86ISD::VSHL: return "X86ISD::VSHL";
7066 case X86ISD::VSRL: return "X86ISD::VSRL";
Nate Begeman30a0de92008-07-17 16:51:19 +00007067 case X86ISD::CMPPD: return "X86ISD::CMPPD";
7068 case X86ISD::CMPPS: return "X86ISD::CMPPS";
7069 case X86ISD::PCMPEQB: return "X86ISD::PCMPEQB";
7070 case X86ISD::PCMPEQW: return "X86ISD::PCMPEQW";
7071 case X86ISD::PCMPEQD: return "X86ISD::PCMPEQD";
7072 case X86ISD::PCMPEQQ: return "X86ISD::PCMPEQQ";
7073 case X86ISD::PCMPGTB: return "X86ISD::PCMPGTB";
7074 case X86ISD::PCMPGTW: return "X86ISD::PCMPGTW";
7075 case X86ISD::PCMPGTD: return "X86ISD::PCMPGTD";
7076 case X86ISD::PCMPGTQ: return "X86ISD::PCMPGTQ";
Bill Wendlingab55ebd2008-12-12 00:56:36 +00007077 case X86ISD::ADD: return "X86ISD::ADD";
7078 case X86ISD::SUB: return "X86ISD::SUB";
Bill Wendlingd350e022008-12-12 21:15:41 +00007079 case X86ISD::SMUL: return "X86ISD::SMUL";
7080 case X86ISD::UMUL: return "X86ISD::UMUL";
Dan Gohman076aee32009-03-04 19:44:21 +00007081 case X86ISD::INC: return "X86ISD::INC";
7082 case X86ISD::DEC: return "X86ISD::DEC";
Evan Cheng73f24c92009-03-30 21:36:47 +00007083 case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM";
Eric Christopher71c67532009-07-29 00:28:05 +00007084 case X86ISD::PTEST: return "X86ISD::PTEST";
Evan Cheng72261582005-12-20 06:22:03 +00007085 }
7086}
Evan Cheng3a03ebb2005-12-21 23:05:39 +00007087
Chris Lattnerc9addb72007-03-30 23:15:24 +00007088// isLegalAddressingMode - Return true if the addressing mode represented
7089// by AM is legal for this target, for a load/store of the specified type.
Scott Michelfdc40a02009-02-17 22:15:04 +00007090bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM,
Chris Lattnerc9addb72007-03-30 23:15:24 +00007091 const Type *Ty) const {
7092 // X86 supports extremely general addressing modes.
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00007093 CodeModel::Model M = getTargetMachine().getCodeModel();
Scott Michelfdc40a02009-02-17 22:15:04 +00007094
Chris Lattnerc9addb72007-03-30 23:15:24 +00007095 // X86 allows a sign-extended 32-bit immediate field as a displacement.
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00007096 if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != NULL))
Chris Lattnerc9addb72007-03-30 23:15:24 +00007097 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +00007098
Chris Lattnerc9addb72007-03-30 23:15:24 +00007099 if (AM.BaseGV) {
Chris Lattnerdfed4132009-07-10 07:38:24 +00007100 unsigned GVFlags =
7101 Subtarget->ClassifyGlobalReference(AM.BaseGV, getTargetMachine());
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00007102
Chris Lattnerdfed4132009-07-10 07:38:24 +00007103 // If a reference to this global requires an extra load, we can't fold it.
7104 if (isGlobalStubReference(GVFlags))
Chris Lattnerc9addb72007-03-30 23:15:24 +00007105 return false;
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00007106
Chris Lattnerdfed4132009-07-10 07:38:24 +00007107 // If BaseGV requires a register for the PIC base, we cannot also have a
7108 // BaseReg specified.
7109 if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags))
Dale Johannesen203af582008-12-05 21:47:27 +00007110 return false;
Evan Cheng52787842007-08-01 23:46:47 +00007111
Anton Korobeynikovb5e01722009-08-05 23:01:26 +00007112 // If lower 4G is not available, then we must use rip-relative addressing.
7113 if (Subtarget->is64Bit() && (AM.BaseOffs || AM.Scale > 1))
7114 return false;
Chris Lattnerc9addb72007-03-30 23:15:24 +00007115 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007116
Chris Lattnerc9addb72007-03-30 23:15:24 +00007117 switch (AM.Scale) {
7118 case 0:
7119 case 1:
7120 case 2:
7121 case 4:
7122 case 8:
7123 // These scales always work.
7124 break;
7125 case 3:
7126 case 5:
7127 case 9:
7128 // These scales are formed with basereg+scalereg. Only accept if there is
7129 // no basereg yet.
7130 if (AM.HasBaseReg)
7131 return false;
7132 break;
7133 default: // Other stuff never works.
7134 return false;
7135 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007136
Chris Lattnerc9addb72007-03-30 23:15:24 +00007137 return true;
7138}
7139
7140
Evan Cheng2bd122c2007-10-26 01:56:11 +00007141bool X86TargetLowering::isTruncateFree(const Type *Ty1, const Type *Ty2) const {
7142 if (!Ty1->isInteger() || !Ty2->isInteger())
7143 return false;
Evan Chenge127a732007-10-29 07:57:50 +00007144 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
7145 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
Evan Cheng260e07e2008-03-20 02:18:41 +00007146 if (NumBits1 <= NumBits2)
Evan Chenge127a732007-10-29 07:57:50 +00007147 return false;
7148 return Subtarget->is64Bit() || NumBits1 < 64;
Evan Cheng2bd122c2007-10-26 01:56:11 +00007149}
7150
Owen Andersone50ed302009-08-10 22:56:29 +00007151bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
Duncan Sands83ec4b62008-06-06 12:08:01 +00007152 if (!VT1.isInteger() || !VT2.isInteger())
Evan Cheng3c3ddb32007-10-29 19:58:20 +00007153 return false;
Duncan Sands83ec4b62008-06-06 12:08:01 +00007154 unsigned NumBits1 = VT1.getSizeInBits();
7155 unsigned NumBits2 = VT2.getSizeInBits();
Evan Cheng260e07e2008-03-20 02:18:41 +00007156 if (NumBits1 <= NumBits2)
Evan Cheng3c3ddb32007-10-29 19:58:20 +00007157 return false;
7158 return Subtarget->is64Bit() || NumBits1 < 64;
7159}
Evan Cheng2bd122c2007-10-26 01:56:11 +00007160
Dan Gohman97121ba2009-04-08 00:15:30 +00007161bool X86TargetLowering::isZExtFree(const Type *Ty1, const Type *Ty2) const {
Dan Gohman349ba492009-04-09 02:06:09 +00007162 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
Owen Anderson1d0be152009-08-13 21:58:54 +00007163 return Ty1 == Type::getInt32Ty(Ty1->getContext()) &&
7164 Ty2 == Type::getInt64Ty(Ty1->getContext()) && Subtarget->is64Bit();
Dan Gohman97121ba2009-04-08 00:15:30 +00007165}
7166
Owen Andersone50ed302009-08-10 22:56:29 +00007167bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
Dan Gohman349ba492009-04-09 02:06:09 +00007168 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
Owen Anderson825b72b2009-08-11 20:47:22 +00007169 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget->is64Bit();
Dan Gohman97121ba2009-04-08 00:15:30 +00007170}
7171
Owen Andersone50ed302009-08-10 22:56:29 +00007172bool X86TargetLowering::isNarrowingProfitable(EVT VT1, EVT VT2) const {
Evan Cheng8b944d32009-05-28 00:35:15 +00007173 // i16 instructions are longer (0x66 prefix) and potentially slower.
Owen Anderson825b72b2009-08-11 20:47:22 +00007174 return !(VT1 == MVT::i32 && VT2 == MVT::i16);
Evan Cheng8b944d32009-05-28 00:35:15 +00007175}
7176
Evan Cheng60c07e12006-07-05 22:17:51 +00007177/// isShuffleMaskLegal - Targets can use this to indicate that they only
7178/// support *some* VECTOR_SHUFFLE operations, those with specific masks.
7179/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
7180/// are assumed to be legal.
7181bool
Nate Begeman5a5ca152009-04-29 05:20:52 +00007182X86TargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
Owen Andersone50ed302009-08-10 22:56:29 +00007183 EVT VT) const {
Evan Cheng60c07e12006-07-05 22:17:51 +00007184 // Only do shuffles on 128-bit vector types for now.
Nate Begeman9008ca62009-04-27 18:41:29 +00007185 if (VT.getSizeInBits() == 64)
7186 return false;
7187
7188 // FIXME: pshufb, blends, palignr, shifts.
7189 return (VT.getVectorNumElements() == 2 ||
7190 ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
7191 isMOVLMask(M, VT) ||
7192 isSHUFPMask(M, VT) ||
7193 isPSHUFDMask(M, VT) ||
7194 isPSHUFHWMask(M, VT) ||
7195 isPSHUFLWMask(M, VT) ||
7196 isUNPCKLMask(M, VT) ||
7197 isUNPCKHMask(M, VT) ||
7198 isUNPCKL_v_undef_Mask(M, VT) ||
7199 isUNPCKH_v_undef_Mask(M, VT));
Evan Cheng60c07e12006-07-05 22:17:51 +00007200}
7201
Dan Gohman7d8143f2008-04-09 20:09:42 +00007202bool
Nate Begeman5a5ca152009-04-29 05:20:52 +00007203X86TargetLowering::isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask,
Owen Andersone50ed302009-08-10 22:56:29 +00007204 EVT VT) const {
Nate Begeman9008ca62009-04-27 18:41:29 +00007205 unsigned NumElts = VT.getVectorNumElements();
7206 // FIXME: This collection of masks seems suspect.
7207 if (NumElts == 2)
7208 return true;
7209 if (NumElts == 4 && VT.getSizeInBits() == 128) {
7210 return (isMOVLMask(Mask, VT) ||
7211 isCommutedMOVLMask(Mask, VT, true) ||
7212 isSHUFPMask(Mask, VT) ||
7213 isCommutedSHUFPMask(Mask, VT));
Evan Cheng60c07e12006-07-05 22:17:51 +00007214 }
7215 return false;
7216}
7217
7218//===----------------------------------------------------------------------===//
7219// X86 Scheduler Hooks
7220//===----------------------------------------------------------------------===//
7221
Mon P Wang63307c32008-05-05 19:05:59 +00007222// private utility function
7223MachineBasicBlock *
7224X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr,
7225 MachineBasicBlock *MBB,
7226 unsigned regOpc,
Andrew Lenharth507a58a2008-06-14 05:48:15 +00007227 unsigned immOpc,
Dale Johannesen140be2d2008-08-19 18:47:28 +00007228 unsigned LoadOpc,
7229 unsigned CXchgOpc,
7230 unsigned copyOpc,
7231 unsigned notOpc,
7232 unsigned EAXreg,
7233 TargetRegisterClass *RC,
Dan Gohman1fdbc1d2009-02-07 16:15:20 +00007234 bool invSrc) const {
Mon P Wang63307c32008-05-05 19:05:59 +00007235 // For the atomic bitwise operator, we generate
7236 // thisMBB:
7237 // newMBB:
Mon P Wangab3e7472008-05-05 22:56:23 +00007238 // ld t1 = [bitinstr.addr]
7239 // op t2 = t1, [bitinstr.val]
7240 // mov EAX = t1
Mon P Wang63307c32008-05-05 19:05:59 +00007241 // lcs dest = [bitinstr.addr], t2 [EAX is implicit]
7242 // bz newMBB
7243 // fallthrough -->nextMBB
7244 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
7245 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
Dan Gohman8e5f2c62008-07-07 23:14:23 +00007246 MachineFunction::iterator MBBIter = MBB;
Mon P Wang63307c32008-05-05 19:05:59 +00007247 ++MBBIter;
Scott Michelfdc40a02009-02-17 22:15:04 +00007248
Mon P Wang63307c32008-05-05 19:05:59 +00007249 /// First build the CFG
7250 MachineFunction *F = MBB->getParent();
7251 MachineBasicBlock *thisMBB = MBB;
Dan Gohman8e5f2c62008-07-07 23:14:23 +00007252 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB);
7253 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB);
7254 F->insert(MBBIter, newMBB);
7255 F->insert(MBBIter, nextMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007256
Mon P Wang63307c32008-05-05 19:05:59 +00007257 // Move all successors to thisMBB to nextMBB
7258 nextMBB->transferSuccessors(thisMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007259
Mon P Wang63307c32008-05-05 19:05:59 +00007260 // Update thisMBB to fall through to newMBB
7261 thisMBB->addSuccessor(newMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007262
Mon P Wang63307c32008-05-05 19:05:59 +00007263 // newMBB jumps to itself and fall through to nextMBB
7264 newMBB->addSuccessor(nextMBB);
7265 newMBB->addSuccessor(newMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007266
Mon P Wang63307c32008-05-05 19:05:59 +00007267 // Insert instructions into newMBB based on incoming instruction
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007268 assert(bInstr->getNumOperands() < X86AddrNumOperands + 4 &&
Bill Wendling51b16f42009-05-30 01:09:53 +00007269 "unexpected number of operands");
Dale Johannesene4d209d2009-02-03 20:21:25 +00007270 DebugLoc dl = bInstr->getDebugLoc();
Mon P Wang63307c32008-05-05 19:05:59 +00007271 MachineOperand& destOper = bInstr->getOperand(0);
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007272 MachineOperand* argOpers[2 + X86AddrNumOperands];
Mon P Wang63307c32008-05-05 19:05:59 +00007273 int numArgs = bInstr->getNumOperands() - 1;
7274 for (int i=0; i < numArgs; ++i)
7275 argOpers[i] = &bInstr->getOperand(i+1);
7276
7277 // x86 address has 4 operands: base, index, scale, and displacement
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007278 int lastAddrIndx = X86AddrNumOperands - 1; // [0,3]
7279 int valArgIndx = lastAddrIndx + 1;
Scott Michelfdc40a02009-02-17 22:15:04 +00007280
Dale Johannesen140be2d2008-08-19 18:47:28 +00007281 unsigned t1 = F->getRegInfo().createVirtualRegister(RC);
Dale Johannesene4d209d2009-02-03 20:21:25 +00007282 MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(LoadOpc), t1);
Mon P Wang63307c32008-05-05 19:05:59 +00007283 for (int i=0; i <= lastAddrIndx; ++i)
7284 (*MIB).addOperand(*argOpers[i]);
Andrew Lenharth507a58a2008-06-14 05:48:15 +00007285
Dale Johannesen140be2d2008-08-19 18:47:28 +00007286 unsigned tt = F->getRegInfo().createVirtualRegister(RC);
Andrew Lenharth507a58a2008-06-14 05:48:15 +00007287 if (invSrc) {
Dale Johannesene4d209d2009-02-03 20:21:25 +00007288 MIB = BuildMI(newMBB, dl, TII->get(notOpc), tt).addReg(t1);
Andrew Lenharth507a58a2008-06-14 05:48:15 +00007289 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007290 else
Andrew Lenharth507a58a2008-06-14 05:48:15 +00007291 tt = t1;
7292
Dale Johannesen140be2d2008-08-19 18:47:28 +00007293 unsigned t2 = F->getRegInfo().createVirtualRegister(RC);
Dan Gohmand735b802008-10-03 15:45:36 +00007294 assert((argOpers[valArgIndx]->isReg() ||
7295 argOpers[valArgIndx]->isImm()) &&
Dan Gohman014278e2008-09-13 17:58:21 +00007296 "invalid operand");
Dan Gohmand735b802008-10-03 15:45:36 +00007297 if (argOpers[valArgIndx]->isReg())
Dale Johannesene4d209d2009-02-03 20:21:25 +00007298 MIB = BuildMI(newMBB, dl, TII->get(regOpc), t2);
Mon P Wang63307c32008-05-05 19:05:59 +00007299 else
Dale Johannesene4d209d2009-02-03 20:21:25 +00007300 MIB = BuildMI(newMBB, dl, TII->get(immOpc), t2);
Andrew Lenharth507a58a2008-06-14 05:48:15 +00007301 MIB.addReg(tt);
Mon P Wang63307c32008-05-05 19:05:59 +00007302 (*MIB).addOperand(*argOpers[valArgIndx]);
Andrew Lenharth507a58a2008-06-14 05:48:15 +00007303
Dale Johannesene4d209d2009-02-03 20:21:25 +00007304 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), EAXreg);
Mon P Wangab3e7472008-05-05 22:56:23 +00007305 MIB.addReg(t1);
Scott Michelfdc40a02009-02-17 22:15:04 +00007306
Dale Johannesene4d209d2009-02-03 20:21:25 +00007307 MIB = BuildMI(newMBB, dl, TII->get(CXchgOpc));
Mon P Wang63307c32008-05-05 19:05:59 +00007308 for (int i=0; i <= lastAddrIndx; ++i)
7309 (*MIB).addOperand(*argOpers[i]);
7310 MIB.addReg(t2);
Mon P Wangf5952662008-07-17 04:54:06 +00007311 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand");
7312 (*MIB).addMemOperand(*F, *bInstr->memoperands_begin());
7313
Dale Johannesene4d209d2009-02-03 20:21:25 +00007314 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), destOper.getReg());
Dale Johannesen140be2d2008-08-19 18:47:28 +00007315 MIB.addReg(EAXreg);
Scott Michelfdc40a02009-02-17 22:15:04 +00007316
Mon P Wang63307c32008-05-05 19:05:59 +00007317 // insert branch
Dale Johannesene4d209d2009-02-03 20:21:25 +00007318 BuildMI(newMBB, dl, TII->get(X86::JNE)).addMBB(newMBB);
Mon P Wang63307c32008-05-05 19:05:59 +00007319
Dan Gohman8e5f2c62008-07-07 23:14:23 +00007320 F->DeleteMachineInstr(bInstr); // The pseudo instruction is gone now.
Mon P Wang63307c32008-05-05 19:05:59 +00007321 return nextMBB;
7322}
7323
Dale Johannesen1b54c7f2008-10-03 19:41:08 +00007324// private utility function: 64 bit atomics on 32 bit host.
Mon P Wang63307c32008-05-05 19:05:59 +00007325MachineBasicBlock *
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007326X86TargetLowering::EmitAtomicBit6432WithCustomInserter(MachineInstr *bInstr,
7327 MachineBasicBlock *MBB,
7328 unsigned regOpcL,
7329 unsigned regOpcH,
7330 unsigned immOpcL,
7331 unsigned immOpcH,
Dan Gohman1fdbc1d2009-02-07 16:15:20 +00007332 bool invSrc) const {
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007333 // For the atomic bitwise operator, we generate
7334 // thisMBB (instructions are in pairs, except cmpxchg8b)
7335 // ld t1,t2 = [bitinstr.addr]
7336 // newMBB:
7337 // out1, out2 = phi (thisMBB, t1/t2) (newMBB, t3/t4)
7338 // op t5, t6 <- out1, out2, [bitinstr.val]
Dale Johannesen880ae362008-10-03 22:25:52 +00007339 // (for SWAP, substitute: mov t5, t6 <- [bitinstr.val])
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007340 // mov ECX, EBX <- t5, t6
7341 // mov EAX, EDX <- t1, t2
7342 // cmpxchg8b [bitinstr.addr] [EAX, EDX, EBX, ECX implicit]
7343 // mov t3, t4 <- EAX, EDX
7344 // bz newMBB
7345 // result in out1, out2
7346 // fallthrough -->nextMBB
7347
7348 const TargetRegisterClass *RC = X86::GR32RegisterClass;
7349 const unsigned LoadOpc = X86::MOV32rm;
7350 const unsigned copyOpc = X86::MOV32rr;
7351 const unsigned NotOpc = X86::NOT32r;
7352 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
7353 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
7354 MachineFunction::iterator MBBIter = MBB;
7355 ++MBBIter;
Scott Michelfdc40a02009-02-17 22:15:04 +00007356
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007357 /// First build the CFG
7358 MachineFunction *F = MBB->getParent();
7359 MachineBasicBlock *thisMBB = MBB;
7360 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB);
7361 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB);
7362 F->insert(MBBIter, newMBB);
7363 F->insert(MBBIter, nextMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007364
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007365 // Move all successors to thisMBB to nextMBB
7366 nextMBB->transferSuccessors(thisMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007367
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007368 // Update thisMBB to fall through to newMBB
7369 thisMBB->addSuccessor(newMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007370
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007371 // newMBB jumps to itself and fall through to nextMBB
7372 newMBB->addSuccessor(nextMBB);
7373 newMBB->addSuccessor(newMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007374
Dale Johannesene4d209d2009-02-03 20:21:25 +00007375 DebugLoc dl = bInstr->getDebugLoc();
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007376 // Insert instructions into newMBB based on incoming instruction
7377 // There are 8 "real" operands plus 9 implicit def/uses, ignored here.
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007378 assert(bInstr->getNumOperands() < X86AddrNumOperands + 14 &&
Bill Wendling51b16f42009-05-30 01:09:53 +00007379 "unexpected number of operands");
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007380 MachineOperand& dest1Oper = bInstr->getOperand(0);
7381 MachineOperand& dest2Oper = bInstr->getOperand(1);
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007382 MachineOperand* argOpers[2 + X86AddrNumOperands];
7383 for (int i=0; i < 2 + X86AddrNumOperands; ++i)
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007384 argOpers[i] = &bInstr->getOperand(i+2);
7385
7386 // x86 address has 4 operands: base, index, scale, and displacement
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007387 int lastAddrIndx = X86AddrNumOperands - 1; // [0,3]
Scott Michelfdc40a02009-02-17 22:15:04 +00007388
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007389 unsigned t1 = F->getRegInfo().createVirtualRegister(RC);
Dale Johannesene4d209d2009-02-03 20:21:25 +00007390 MachineInstrBuilder MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t1);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007391 for (int i=0; i <= lastAddrIndx; ++i)
7392 (*MIB).addOperand(*argOpers[i]);
7393 unsigned t2 = F->getRegInfo().createVirtualRegister(RC);
Dale Johannesene4d209d2009-02-03 20:21:25 +00007394 MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t2);
Dale Johannesen880ae362008-10-03 22:25:52 +00007395 // add 4 to displacement.
Rafael Espindola094fad32009-04-08 21:14:34 +00007396 for (int i=0; i <= lastAddrIndx-2; ++i)
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007397 (*MIB).addOperand(*argOpers[i]);
Dale Johannesen880ae362008-10-03 22:25:52 +00007398 MachineOperand newOp3 = *(argOpers[3]);
7399 if (newOp3.isImm())
7400 newOp3.setImm(newOp3.getImm()+4);
7401 else
7402 newOp3.setOffset(newOp3.getOffset()+4);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007403 (*MIB).addOperand(newOp3);
Rafael Espindola094fad32009-04-08 21:14:34 +00007404 (*MIB).addOperand(*argOpers[lastAddrIndx]);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007405
7406 // t3/4 are defined later, at the bottom of the loop
7407 unsigned t3 = F->getRegInfo().createVirtualRegister(RC);
7408 unsigned t4 = F->getRegInfo().createVirtualRegister(RC);
Dale Johannesene4d209d2009-02-03 20:21:25 +00007409 BuildMI(newMBB, dl, TII->get(X86::PHI), dest1Oper.getReg())
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007410 .addReg(t1).addMBB(thisMBB).addReg(t3).addMBB(newMBB);
Dale Johannesene4d209d2009-02-03 20:21:25 +00007411 BuildMI(newMBB, dl, TII->get(X86::PHI), dest2Oper.getReg())
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007412 .addReg(t2).addMBB(thisMBB).addReg(t4).addMBB(newMBB);
7413
7414 unsigned tt1 = F->getRegInfo().createVirtualRegister(RC);
7415 unsigned tt2 = F->getRegInfo().createVirtualRegister(RC);
Scott Michelfdc40a02009-02-17 22:15:04 +00007416 if (invSrc) {
Dale Johannesene4d209d2009-02-03 20:21:25 +00007417 MIB = BuildMI(newMBB, dl, TII->get(NotOpc), tt1).addReg(t1);
7418 MIB = BuildMI(newMBB, dl, TII->get(NotOpc), tt2).addReg(t2);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007419 } else {
7420 tt1 = t1;
7421 tt2 = t2;
7422 }
7423
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007424 int valArgIndx = lastAddrIndx + 1;
7425 assert((argOpers[valArgIndx]->isReg() ||
Bill Wendling51b16f42009-05-30 01:09:53 +00007426 argOpers[valArgIndx]->isImm()) &&
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007427 "invalid operand");
7428 unsigned t5 = F->getRegInfo().createVirtualRegister(RC);
7429 unsigned t6 = F->getRegInfo().createVirtualRegister(RC);
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007430 if (argOpers[valArgIndx]->isReg())
Dale Johannesene4d209d2009-02-03 20:21:25 +00007431 MIB = BuildMI(newMBB, dl, TII->get(regOpcL), t5);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007432 else
Dale Johannesene4d209d2009-02-03 20:21:25 +00007433 MIB = BuildMI(newMBB, dl, TII->get(immOpcL), t5);
Dale Johannesen880ae362008-10-03 22:25:52 +00007434 if (regOpcL != X86::MOV32rr)
7435 MIB.addReg(tt1);
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007436 (*MIB).addOperand(*argOpers[valArgIndx]);
7437 assert(argOpers[valArgIndx + 1]->isReg() ==
Bill Wendling51b16f42009-05-30 01:09:53 +00007438 argOpers[valArgIndx]->isReg());
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007439 assert(argOpers[valArgIndx + 1]->isImm() ==
Bill Wendling51b16f42009-05-30 01:09:53 +00007440 argOpers[valArgIndx]->isImm());
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007441 if (argOpers[valArgIndx + 1]->isReg())
Dale Johannesene4d209d2009-02-03 20:21:25 +00007442 MIB = BuildMI(newMBB, dl, TII->get(regOpcH), t6);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007443 else
Dale Johannesene4d209d2009-02-03 20:21:25 +00007444 MIB = BuildMI(newMBB, dl, TII->get(immOpcH), t6);
Dale Johannesen880ae362008-10-03 22:25:52 +00007445 if (regOpcH != X86::MOV32rr)
7446 MIB.addReg(tt2);
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007447 (*MIB).addOperand(*argOpers[valArgIndx + 1]);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007448
Dale Johannesene4d209d2009-02-03 20:21:25 +00007449 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::EAX);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007450 MIB.addReg(t1);
Dale Johannesene4d209d2009-02-03 20:21:25 +00007451 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::EDX);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007452 MIB.addReg(t2);
7453
Dale Johannesene4d209d2009-02-03 20:21:25 +00007454 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::EBX);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007455 MIB.addReg(t5);
Dale Johannesene4d209d2009-02-03 20:21:25 +00007456 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::ECX);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007457 MIB.addReg(t6);
Scott Michelfdc40a02009-02-17 22:15:04 +00007458
Dale Johannesene4d209d2009-02-03 20:21:25 +00007459 MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG8B));
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007460 for (int i=0; i <= lastAddrIndx; ++i)
7461 (*MIB).addOperand(*argOpers[i]);
7462
7463 assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand");
7464 (*MIB).addMemOperand(*F, *bInstr->memoperands_begin());
7465
Dale Johannesene4d209d2009-02-03 20:21:25 +00007466 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), t3);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007467 MIB.addReg(X86::EAX);
Dale Johannesene4d209d2009-02-03 20:21:25 +00007468 MIB = BuildMI(newMBB, dl, TII->get(copyOpc), t4);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007469 MIB.addReg(X86::EDX);
Scott Michelfdc40a02009-02-17 22:15:04 +00007470
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007471 // insert branch
Dale Johannesene4d209d2009-02-03 20:21:25 +00007472 BuildMI(newMBB, dl, TII->get(X86::JNE)).addMBB(newMBB);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007473
7474 F->DeleteMachineInstr(bInstr); // The pseudo instruction is gone now.
7475 return nextMBB;
7476}
7477
7478// private utility function
7479MachineBasicBlock *
Mon P Wang63307c32008-05-05 19:05:59 +00007480X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr,
7481 MachineBasicBlock *MBB,
Dan Gohman1fdbc1d2009-02-07 16:15:20 +00007482 unsigned cmovOpc) const {
Mon P Wang63307c32008-05-05 19:05:59 +00007483 // For the atomic min/max operator, we generate
7484 // thisMBB:
7485 // newMBB:
Mon P Wangab3e7472008-05-05 22:56:23 +00007486 // ld t1 = [min/max.addr]
Scott Michelfdc40a02009-02-17 22:15:04 +00007487 // mov t2 = [min/max.val]
Mon P Wang63307c32008-05-05 19:05:59 +00007488 // cmp t1, t2
7489 // cmov[cond] t2 = t1
Mon P Wangab3e7472008-05-05 22:56:23 +00007490 // mov EAX = t1
Mon P Wang63307c32008-05-05 19:05:59 +00007491 // lcs dest = [bitinstr.addr], t2 [EAX is implicit]
7492 // bz newMBB
7493 // fallthrough -->nextMBB
7494 //
7495 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
7496 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
Dan Gohman8e5f2c62008-07-07 23:14:23 +00007497 MachineFunction::iterator MBBIter = MBB;
Mon P Wang63307c32008-05-05 19:05:59 +00007498 ++MBBIter;
Scott Michelfdc40a02009-02-17 22:15:04 +00007499
Mon P Wang63307c32008-05-05 19:05:59 +00007500 /// First build the CFG
7501 MachineFunction *F = MBB->getParent();
7502 MachineBasicBlock *thisMBB = MBB;
Dan Gohman8e5f2c62008-07-07 23:14:23 +00007503 MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB);
7504 MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB);
7505 F->insert(MBBIter, newMBB);
7506 F->insert(MBBIter, nextMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007507
Mon P Wang63307c32008-05-05 19:05:59 +00007508 // Move all successors to thisMBB to nextMBB
7509 nextMBB->transferSuccessors(thisMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007510
Mon P Wang63307c32008-05-05 19:05:59 +00007511 // Update thisMBB to fall through to newMBB
7512 thisMBB->addSuccessor(newMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007513
Mon P Wang63307c32008-05-05 19:05:59 +00007514 // newMBB jumps to newMBB and fall through to nextMBB
7515 newMBB->addSuccessor(nextMBB);
7516 newMBB->addSuccessor(newMBB);
Scott Michelfdc40a02009-02-17 22:15:04 +00007517
Dale Johannesene4d209d2009-02-03 20:21:25 +00007518 DebugLoc dl = mInstr->getDebugLoc();
Mon P Wang63307c32008-05-05 19:05:59 +00007519 // Insert instructions into newMBB based on incoming instruction
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007520 assert(mInstr->getNumOperands() < X86AddrNumOperands + 4 &&
Bill Wendling51b16f42009-05-30 01:09:53 +00007521 "unexpected number of operands");
Mon P Wang63307c32008-05-05 19:05:59 +00007522 MachineOperand& destOper = mInstr->getOperand(0);
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007523 MachineOperand* argOpers[2 + X86AddrNumOperands];
Mon P Wang63307c32008-05-05 19:05:59 +00007524 int numArgs = mInstr->getNumOperands() - 1;
7525 for (int i=0; i < numArgs; ++i)
7526 argOpers[i] = &mInstr->getOperand(i+1);
Scott Michelfdc40a02009-02-17 22:15:04 +00007527
Mon P Wang63307c32008-05-05 19:05:59 +00007528 // x86 address has 4 operands: base, index, scale, and displacement
Rafael Espindolaa82dfca2009-03-27 15:26:30 +00007529 int lastAddrIndx = X86AddrNumOperands - 1; // [0,3]
7530 int valArgIndx = lastAddrIndx + 1;
Scott Michelfdc40a02009-02-17 22:15:04 +00007531
Mon P Wangab3e7472008-05-05 22:56:23 +00007532 unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass);
Dale Johannesene4d209d2009-02-03 20:21:25 +00007533 MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rm), t1);
Mon P Wang63307c32008-05-05 19:05:59 +00007534 for (int i=0; i <= lastAddrIndx; ++i)
7535 (*MIB).addOperand(*argOpers[i]);
Mon P Wangab3e7472008-05-05 22:56:23 +00007536
Mon P Wang63307c32008-05-05 19:05:59 +00007537 // We only support register and immediate values
Dan Gohmand735b802008-10-03 15:45:36 +00007538 assert((argOpers[valArgIndx]->isReg() ||
7539 argOpers[valArgIndx]->isImm()) &&
Dan Gohman014278e2008-09-13 17:58:21 +00007540 "invalid operand");
Scott Michelfdc40a02009-02-17 22:15:04 +00007541
7542 unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass);
Dan Gohmand735b802008-10-03 15:45:36 +00007543 if (argOpers[valArgIndx]->isReg())
Dale Johannesene4d209d2009-02-03 20:21:25 +00007544 MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), t2);
Scott Michelfdc40a02009-02-17 22:15:04 +00007545 else
Dale Johannesene4d209d2009-02-03 20:21:25 +00007546 MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), t2);
Mon P Wang63307c32008-05-05 19:05:59 +00007547 (*MIB).addOperand(*argOpers[valArgIndx]);
7548
Dale Johannesene4d209d2009-02-03 20:21:25 +00007549 MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), X86::EAX);
Mon P Wangab3e7472008-05-05 22:56:23 +00007550 MIB.addReg(t1);
7551
Dale Johannesene4d209d2009-02-03 20:21:25 +00007552 MIB = BuildMI(newMBB, dl, TII->get(X86::CMP32rr));
Mon P Wang63307c32008-05-05 19:05:59 +00007553 MIB.addReg(t1);
7554 MIB.addReg(t2);
7555
7556 // Generate movc
7557 unsigned t3 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass);
Dale Johannesene4d209d2009-02-03 20:21:25 +00007558 MIB = BuildMI(newMBB, dl, TII->get(cmovOpc),t3);
Mon P Wang63307c32008-05-05 19:05:59 +00007559 MIB.addReg(t2);
7560 MIB.addReg(t1);
7561
7562 // Cmp and exchange if none has modified the memory location
Dale Johannesene4d209d2009-02-03 20:21:25 +00007563 MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG32));
Mon P Wang63307c32008-05-05 19:05:59 +00007564 for (int i=0; i <= lastAddrIndx; ++i)
7565 (*MIB).addOperand(*argOpers[i]);
7566 MIB.addReg(t3);
Mon P Wangf5952662008-07-17 04:54:06 +00007567 assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand");
7568 (*MIB).addMemOperand(*F, *mInstr->memoperands_begin());
Scott Michelfdc40a02009-02-17 22:15:04 +00007569
Dale Johannesene4d209d2009-02-03 20:21:25 +00007570 MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), destOper.getReg());
Mon P Wang63307c32008-05-05 19:05:59 +00007571 MIB.addReg(X86::EAX);
Scott Michelfdc40a02009-02-17 22:15:04 +00007572
Mon P Wang63307c32008-05-05 19:05:59 +00007573 // insert branch
Dale Johannesene4d209d2009-02-03 20:21:25 +00007574 BuildMI(newMBB, dl, TII->get(X86::JNE)).addMBB(newMBB);
Mon P Wang63307c32008-05-05 19:05:59 +00007575
Dan Gohman8e5f2c62008-07-07 23:14:23 +00007576 F->DeleteMachineInstr(mInstr); // The pseudo instruction is gone now.
Mon P Wang63307c32008-05-05 19:05:59 +00007577 return nextMBB;
7578}
7579
7580
Evan Cheng60c07e12006-07-05 22:17:51 +00007581MachineBasicBlock *
Evan Chengff9b3732008-01-30 18:18:23 +00007582X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
Dan Gohman1fdbc1d2009-02-07 16:15:20 +00007583 MachineBasicBlock *BB) const {
Dale Johannesene4d209d2009-02-03 20:21:25 +00007584 DebugLoc dl = MI->getDebugLoc();
Evan Chengc0f64ff2006-11-27 23:37:22 +00007585 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
Evan Cheng60c07e12006-07-05 22:17:51 +00007586 switch (MI->getOpcode()) {
7587 default: assert(false && "Unexpected instr type to insert");
Mon P Wang9e5ecb82008-12-12 01:25:51 +00007588 case X86::CMOV_V1I64:
Evan Cheng60c07e12006-07-05 22:17:51 +00007589 case X86::CMOV_FR32:
7590 case X86::CMOV_FR64:
7591 case X86::CMOV_V4F32:
7592 case X86::CMOV_V2F64:
Evan Chenge5f62042007-09-29 00:00:36 +00007593 case X86::CMOV_V2I64: {
Evan Cheng60c07e12006-07-05 22:17:51 +00007594 // To "insert" a SELECT_CC instruction, we actually have to insert the
7595 // diamond control-flow pattern. The incoming instruction knows the
7596 // destination vreg to set, the condition code register to branch on, the
7597 // true/false values to select between, and a branch opcode to use.
7598 const BasicBlock *LLVM_BB = BB->getBasicBlock();
Dan Gohman8e5f2c62008-07-07 23:14:23 +00007599 MachineFunction::iterator It = BB;
Evan Cheng60c07e12006-07-05 22:17:51 +00007600 ++It;
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00007601
Evan Cheng60c07e12006-07-05 22:17:51 +00007602 // thisMBB:
7603 // ...
7604 // TrueVal = ...
7605 // cmpTY ccX, r1, r2
7606 // bCC copy1MBB
7607 // fallthrough --> copy0MBB
7608 MachineBasicBlock *thisMBB = BB;
Dan Gohman8e5f2c62008-07-07 23:14:23 +00007609 MachineFunction *F = BB->getParent();
7610 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
7611 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00007612 unsigned Opc =
Chris Lattner7fbe9722006-10-20 17:42:20 +00007613 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm());
Dale Johannesene4d209d2009-02-03 20:21:25 +00007614 BuildMI(BB, dl, TII->get(Opc)).addMBB(sinkMBB);
Dan Gohman8e5f2c62008-07-07 23:14:23 +00007615 F->insert(It, copy0MBB);
7616 F->insert(It, sinkMBB);
Mon P Wang63307c32008-05-05 19:05:59 +00007617 // Update machine-CFG edges by transferring all successors of the current
Evan Cheng60c07e12006-07-05 22:17:51 +00007618 // block to the new block which will contain the Phi node for the select.
Mon P Wang63307c32008-05-05 19:05:59 +00007619 sinkMBB->transferSuccessors(BB);
7620
7621 // Add the true and fallthrough blocks as its successors.
Evan Cheng60c07e12006-07-05 22:17:51 +00007622 BB->addSuccessor(copy0MBB);
7623 BB->addSuccessor(sinkMBB);
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00007624
Evan Cheng60c07e12006-07-05 22:17:51 +00007625 // copy0MBB:
7626 // %FalseValue = ...
7627 // # fallthrough to sinkMBB
7628 BB = copy0MBB;
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00007629
Evan Cheng60c07e12006-07-05 22:17:51 +00007630 // Update machine-CFG edges
7631 BB->addSuccessor(sinkMBB);
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00007632
Evan Cheng60c07e12006-07-05 22:17:51 +00007633 // sinkMBB:
7634 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
7635 // ...
7636 BB = sinkMBB;
Dale Johannesene4d209d2009-02-03 20:21:25 +00007637 BuildMI(BB, dl, TII->get(X86::PHI), MI->getOperand(0).getReg())
Evan Cheng60c07e12006-07-05 22:17:51 +00007638 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
7639 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
7640
Dan Gohman8e5f2c62008-07-07 23:14:23 +00007641 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now.
Evan Cheng60c07e12006-07-05 22:17:51 +00007642 return BB;
7643 }
7644
Dale Johannesen849f2142007-07-03 00:53:03 +00007645 case X86::FP32_TO_INT16_IN_MEM:
7646 case X86::FP32_TO_INT32_IN_MEM:
7647 case X86::FP32_TO_INT64_IN_MEM:
7648 case X86::FP64_TO_INT16_IN_MEM:
7649 case X86::FP64_TO_INT32_IN_MEM:
Dale Johannesena996d522007-08-07 01:17:37 +00007650 case X86::FP64_TO_INT64_IN_MEM:
7651 case X86::FP80_TO_INT16_IN_MEM:
7652 case X86::FP80_TO_INT32_IN_MEM:
7653 case X86::FP80_TO_INT64_IN_MEM: {
Evan Cheng60c07e12006-07-05 22:17:51 +00007654 // Change the floating point control register to use "round towards zero"
7655 // mode when truncating to an integer value.
7656 MachineFunction *F = BB->getParent();
7657 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2);
Dale Johannesene4d209d2009-02-03 20:21:25 +00007658 addFrameReference(BuildMI(BB, dl, TII->get(X86::FNSTCW16m)), CWFrameIdx);
Evan Cheng60c07e12006-07-05 22:17:51 +00007659
7660 // Load the old value of the high byte of the control word...
7661 unsigned OldCW =
Chris Lattner84bc5422007-12-31 04:13:23 +00007662 F->getRegInfo().createVirtualRegister(X86::GR16RegisterClass);
Scott Michelfdc40a02009-02-17 22:15:04 +00007663 addFrameReference(BuildMI(BB, dl, TII->get(X86::MOV16rm), OldCW),
Dale Johannesene4d209d2009-02-03 20:21:25 +00007664 CWFrameIdx);
Evan Cheng60c07e12006-07-05 22:17:51 +00007665
7666 // Set the high part to be round to zero...
Dale Johannesene4d209d2009-02-03 20:21:25 +00007667 addFrameReference(BuildMI(BB, dl, TII->get(X86::MOV16mi)), CWFrameIdx)
Evan Chengc0f64ff2006-11-27 23:37:22 +00007668 .addImm(0xC7F);
Evan Cheng60c07e12006-07-05 22:17:51 +00007669
7670 // Reload the modified control word now...
Dale Johannesene4d209d2009-02-03 20:21:25 +00007671 addFrameReference(BuildMI(BB, dl, TII->get(X86::FLDCW16m)), CWFrameIdx);
Evan Cheng60c07e12006-07-05 22:17:51 +00007672
7673 // Restore the memory image of control word to original value
Dale Johannesene4d209d2009-02-03 20:21:25 +00007674 addFrameReference(BuildMI(BB, dl, TII->get(X86::MOV16mr)), CWFrameIdx)
Evan Chengc0f64ff2006-11-27 23:37:22 +00007675 .addReg(OldCW);
Evan Cheng60c07e12006-07-05 22:17:51 +00007676
7677 // Get the X86 opcode to use.
7678 unsigned Opc;
7679 switch (MI->getOpcode()) {
Torok Edwinc23197a2009-07-14 16:55:14 +00007680 default: llvm_unreachable("illegal opcode!");
Dale Johannesene377d4d2007-07-04 21:07:47 +00007681 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break;
7682 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break;
7683 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break;
7684 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break;
7685 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break;
7686 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break;
Dale Johannesena996d522007-08-07 01:17:37 +00007687 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break;
7688 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break;
7689 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break;
Evan Cheng60c07e12006-07-05 22:17:51 +00007690 }
7691
7692 X86AddressMode AM;
7693 MachineOperand &Op = MI->getOperand(0);
Dan Gohmand735b802008-10-03 15:45:36 +00007694 if (Op.isReg()) {
Evan Cheng60c07e12006-07-05 22:17:51 +00007695 AM.BaseType = X86AddressMode::RegBase;
7696 AM.Base.Reg = Op.getReg();
7697 } else {
7698 AM.BaseType = X86AddressMode::FrameIndexBase;
Chris Lattner8aa797a2007-12-30 23:10:15 +00007699 AM.Base.FrameIndex = Op.getIndex();
Evan Cheng60c07e12006-07-05 22:17:51 +00007700 }
7701 Op = MI->getOperand(1);
Dan Gohmand735b802008-10-03 15:45:36 +00007702 if (Op.isImm())
Chris Lattner7fbe9722006-10-20 17:42:20 +00007703 AM.Scale = Op.getImm();
Evan Cheng60c07e12006-07-05 22:17:51 +00007704 Op = MI->getOperand(2);
Dan Gohmand735b802008-10-03 15:45:36 +00007705 if (Op.isImm())
Chris Lattner7fbe9722006-10-20 17:42:20 +00007706 AM.IndexReg = Op.getImm();
Evan Cheng60c07e12006-07-05 22:17:51 +00007707 Op = MI->getOperand(3);
Dan Gohmand735b802008-10-03 15:45:36 +00007708 if (Op.isGlobal()) {
Evan Cheng60c07e12006-07-05 22:17:51 +00007709 AM.GV = Op.getGlobal();
7710 } else {
Chris Lattner7fbe9722006-10-20 17:42:20 +00007711 AM.Disp = Op.getImm();
Evan Cheng60c07e12006-07-05 22:17:51 +00007712 }
Dale Johannesene4d209d2009-02-03 20:21:25 +00007713 addFullAddress(BuildMI(BB, dl, TII->get(Opc)), AM)
Rafael Espindola8ef2b892009-04-08 08:09:33 +00007714 .addReg(MI->getOperand(X86AddrNumOperands).getReg());
Evan Cheng60c07e12006-07-05 22:17:51 +00007715
7716 // Reload the original control word now.
Dale Johannesene4d209d2009-02-03 20:21:25 +00007717 addFrameReference(BuildMI(BB, dl, TII->get(X86::FLDCW16m)), CWFrameIdx);
Evan Cheng60c07e12006-07-05 22:17:51 +00007718
Dan Gohman8e5f2c62008-07-07 23:14:23 +00007719 F->DeleteMachineInstr(MI); // The pseudo instruction is gone now.
Evan Cheng60c07e12006-07-05 22:17:51 +00007720 return BB;
7721 }
Mon P Wang63307c32008-05-05 19:05:59 +00007722 case X86::ATOMAND32:
7723 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr,
Scott Michelfdc40a02009-02-17 22:15:04 +00007724 X86::AND32ri, X86::MOV32rm,
Dale Johannesen140be2d2008-08-19 18:47:28 +00007725 X86::LCMPXCHG32, X86::MOV32rr,
7726 X86::NOT32r, X86::EAX,
7727 X86::GR32RegisterClass);
Mon P Wang63307c32008-05-05 19:05:59 +00007728 case X86::ATOMOR32:
Scott Michelfdc40a02009-02-17 22:15:04 +00007729 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr,
7730 X86::OR32ri, X86::MOV32rm,
Dale Johannesen140be2d2008-08-19 18:47:28 +00007731 X86::LCMPXCHG32, X86::MOV32rr,
7732 X86::NOT32r, X86::EAX,
7733 X86::GR32RegisterClass);
Mon P Wang63307c32008-05-05 19:05:59 +00007734 case X86::ATOMXOR32:
7735 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr,
Scott Michelfdc40a02009-02-17 22:15:04 +00007736 X86::XOR32ri, X86::MOV32rm,
Dale Johannesen140be2d2008-08-19 18:47:28 +00007737 X86::LCMPXCHG32, X86::MOV32rr,
7738 X86::NOT32r, X86::EAX,
7739 X86::GR32RegisterClass);
Andrew Lenharth507a58a2008-06-14 05:48:15 +00007740 case X86::ATOMNAND32:
7741 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr,
Dale Johannesen140be2d2008-08-19 18:47:28 +00007742 X86::AND32ri, X86::MOV32rm,
7743 X86::LCMPXCHG32, X86::MOV32rr,
7744 X86::NOT32r, X86::EAX,
7745 X86::GR32RegisterClass, true);
Mon P Wang63307c32008-05-05 19:05:59 +00007746 case X86::ATOMMIN32:
7747 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr);
7748 case X86::ATOMMAX32:
7749 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr);
7750 case X86::ATOMUMIN32:
7751 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr);
7752 case X86::ATOMUMAX32:
7753 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr);
Dale Johannesen140be2d2008-08-19 18:47:28 +00007754
7755 case X86::ATOMAND16:
7756 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr,
7757 X86::AND16ri, X86::MOV16rm,
7758 X86::LCMPXCHG16, X86::MOV16rr,
7759 X86::NOT16r, X86::AX,
7760 X86::GR16RegisterClass);
7761 case X86::ATOMOR16:
Scott Michelfdc40a02009-02-17 22:15:04 +00007762 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR16rr,
Dale Johannesen140be2d2008-08-19 18:47:28 +00007763 X86::OR16ri, X86::MOV16rm,
7764 X86::LCMPXCHG16, X86::MOV16rr,
7765 X86::NOT16r, X86::AX,
7766 X86::GR16RegisterClass);
7767 case X86::ATOMXOR16:
7768 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR16rr,
7769 X86::XOR16ri, X86::MOV16rm,
7770 X86::LCMPXCHG16, X86::MOV16rr,
7771 X86::NOT16r, X86::AX,
7772 X86::GR16RegisterClass);
7773 case X86::ATOMNAND16:
7774 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr,
7775 X86::AND16ri, X86::MOV16rm,
7776 X86::LCMPXCHG16, X86::MOV16rr,
7777 X86::NOT16r, X86::AX,
7778 X86::GR16RegisterClass, true);
7779 case X86::ATOMMIN16:
7780 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL16rr);
7781 case X86::ATOMMAX16:
7782 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG16rr);
7783 case X86::ATOMUMIN16:
7784 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB16rr);
7785 case X86::ATOMUMAX16:
7786 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA16rr);
7787
7788 case X86::ATOMAND8:
7789 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr,
7790 X86::AND8ri, X86::MOV8rm,
7791 X86::LCMPXCHG8, X86::MOV8rr,
7792 X86::NOT8r, X86::AL,
7793 X86::GR8RegisterClass);
7794 case X86::ATOMOR8:
Scott Michelfdc40a02009-02-17 22:15:04 +00007795 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR8rr,
Dale Johannesen140be2d2008-08-19 18:47:28 +00007796 X86::OR8ri, X86::MOV8rm,
7797 X86::LCMPXCHG8, X86::MOV8rr,
7798 X86::NOT8r, X86::AL,
7799 X86::GR8RegisterClass);
7800 case X86::ATOMXOR8:
7801 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR8rr,
7802 X86::XOR8ri, X86::MOV8rm,
7803 X86::LCMPXCHG8, X86::MOV8rr,
7804 X86::NOT8r, X86::AL,
7805 X86::GR8RegisterClass);
7806 case X86::ATOMNAND8:
7807 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr,
7808 X86::AND8ri, X86::MOV8rm,
7809 X86::LCMPXCHG8, X86::MOV8rr,
7810 X86::NOT8r, X86::AL,
7811 X86::GR8RegisterClass, true);
7812 // FIXME: There are no CMOV8 instructions; MIN/MAX need some other way.
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007813 // This group is for 64-bit host.
Dale Johannesena99e3842008-08-20 00:48:50 +00007814 case X86::ATOMAND64:
7815 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr,
Scott Michelfdc40a02009-02-17 22:15:04 +00007816 X86::AND64ri32, X86::MOV64rm,
Dale Johannesena99e3842008-08-20 00:48:50 +00007817 X86::LCMPXCHG64, X86::MOV64rr,
7818 X86::NOT64r, X86::RAX,
7819 X86::GR64RegisterClass);
7820 case X86::ATOMOR64:
Scott Michelfdc40a02009-02-17 22:15:04 +00007821 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR64rr,
7822 X86::OR64ri32, X86::MOV64rm,
Dale Johannesena99e3842008-08-20 00:48:50 +00007823 X86::LCMPXCHG64, X86::MOV64rr,
7824 X86::NOT64r, X86::RAX,
7825 X86::GR64RegisterClass);
7826 case X86::ATOMXOR64:
7827 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR64rr,
Scott Michelfdc40a02009-02-17 22:15:04 +00007828 X86::XOR64ri32, X86::MOV64rm,
Dale Johannesena99e3842008-08-20 00:48:50 +00007829 X86::LCMPXCHG64, X86::MOV64rr,
7830 X86::NOT64r, X86::RAX,
7831 X86::GR64RegisterClass);
7832 case X86::ATOMNAND64:
7833 return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr,
7834 X86::AND64ri32, X86::MOV64rm,
7835 X86::LCMPXCHG64, X86::MOV64rr,
7836 X86::NOT64r, X86::RAX,
7837 X86::GR64RegisterClass, true);
7838 case X86::ATOMMIN64:
7839 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL64rr);
7840 case X86::ATOMMAX64:
7841 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG64rr);
7842 case X86::ATOMUMIN64:
7843 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB64rr);
7844 case X86::ATOMUMAX64:
7845 return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA64rr);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007846
7847 // This group does 64-bit operations on a 32-bit host.
7848 case X86::ATOMAND6432:
Scott Michelfdc40a02009-02-17 22:15:04 +00007849 return EmitAtomicBit6432WithCustomInserter(MI, BB,
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007850 X86::AND32rr, X86::AND32rr,
7851 X86::AND32ri, X86::AND32ri,
7852 false);
7853 case X86::ATOMOR6432:
Scott Michelfdc40a02009-02-17 22:15:04 +00007854 return EmitAtomicBit6432WithCustomInserter(MI, BB,
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007855 X86::OR32rr, X86::OR32rr,
7856 X86::OR32ri, X86::OR32ri,
7857 false);
7858 case X86::ATOMXOR6432:
Scott Michelfdc40a02009-02-17 22:15:04 +00007859 return EmitAtomicBit6432WithCustomInserter(MI, BB,
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007860 X86::XOR32rr, X86::XOR32rr,
7861 X86::XOR32ri, X86::XOR32ri,
7862 false);
7863 case X86::ATOMNAND6432:
Scott Michelfdc40a02009-02-17 22:15:04 +00007864 return EmitAtomicBit6432WithCustomInserter(MI, BB,
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007865 X86::AND32rr, X86::AND32rr,
7866 X86::AND32ri, X86::AND32ri,
7867 true);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007868 case X86::ATOMADD6432:
Scott Michelfdc40a02009-02-17 22:15:04 +00007869 return EmitAtomicBit6432WithCustomInserter(MI, BB,
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007870 X86::ADD32rr, X86::ADC32rr,
7871 X86::ADD32ri, X86::ADC32ri,
7872 false);
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007873 case X86::ATOMSUB6432:
Scott Michelfdc40a02009-02-17 22:15:04 +00007874 return EmitAtomicBit6432WithCustomInserter(MI, BB,
Dale Johannesen48c1bc22008-10-02 18:53:47 +00007875 X86::SUB32rr, X86::SBB32rr,
7876 X86::SUB32ri, X86::SBB32ri,
7877 false);
Dale Johannesen880ae362008-10-03 22:25:52 +00007878 case X86::ATOMSWAP6432:
Scott Michelfdc40a02009-02-17 22:15:04 +00007879 return EmitAtomicBit6432WithCustomInserter(MI, BB,
Dale Johannesen880ae362008-10-03 22:25:52 +00007880 X86::MOV32rr, X86::MOV32rr,
7881 X86::MOV32ri, X86::MOV32ri,
7882 false);
Evan Cheng60c07e12006-07-05 22:17:51 +00007883 }
7884}
7885
7886//===----------------------------------------------------------------------===//
7887// X86 Optimization Hooks
7888//===----------------------------------------------------------------------===//
7889
Dan Gohman475871a2008-07-27 21:46:04 +00007890void X86TargetLowering::computeMaskedBitsForTargetNode(const SDValue Op,
Dan Gohman977a76f2008-02-13 22:28:48 +00007891 const APInt &Mask,
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00007892 APInt &KnownZero,
7893 APInt &KnownOne,
Dan Gohmanea859be2007-06-22 14:59:07 +00007894 const SelectionDAG &DAG,
Nate Begeman368e18d2006-02-16 21:11:51 +00007895 unsigned Depth) const {
Evan Cheng3a03ebb2005-12-21 23:05:39 +00007896 unsigned Opc = Op.getOpcode();
Evan Cheng865f0602006-04-05 06:11:20 +00007897 assert((Opc >= ISD::BUILTIN_OP_END ||
7898 Opc == ISD::INTRINSIC_WO_CHAIN ||
7899 Opc == ISD::INTRINSIC_W_CHAIN ||
7900 Opc == ISD::INTRINSIC_VOID) &&
7901 "Should use MaskedValueIsZero if you don't know whether Op"
7902 " is a target node!");
Evan Cheng3a03ebb2005-12-21 23:05:39 +00007903
Dan Gohmanf4f92f52008-02-13 23:07:24 +00007904 KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0); // Don't know anything.
Evan Cheng3a03ebb2005-12-21 23:05:39 +00007905 switch (Opc) {
Evan Cheng865f0602006-04-05 06:11:20 +00007906 default: break;
Evan Cheng97d0e0e2009-02-02 09:15:04 +00007907 case X86ISD::ADD:
7908 case X86ISD::SUB:
7909 case X86ISD::SMUL:
7910 case X86ISD::UMUL:
Dan Gohman076aee32009-03-04 19:44:21 +00007911 case X86ISD::INC:
7912 case X86ISD::DEC:
Evan Cheng97d0e0e2009-02-02 09:15:04 +00007913 // These nodes' second result is a boolean.
7914 if (Op.getResNo() == 0)
7915 break;
7916 // Fallthrough
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00007917 case X86ISD::SETCC:
Dan Gohmanfd29e0e2008-02-13 00:35:47 +00007918 KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(),
7919 Mask.getBitWidth() - 1);
Nate Begeman368e18d2006-02-16 21:11:51 +00007920 break;
Evan Cheng3a03ebb2005-12-21 23:05:39 +00007921 }
Evan Cheng3a03ebb2005-12-21 23:05:39 +00007922}
Chris Lattner259e97c2006-01-31 19:43:35 +00007923
Evan Cheng206ee9d2006-07-07 08:33:52 +00007924/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the
Evan Chengad4196b2008-05-12 19:56:52 +00007925/// node is a GlobalAddress + offset.
7926bool X86TargetLowering::isGAPlusOffset(SDNode *N,
7927 GlobalValue* &GA, int64_t &Offset) const{
7928 if (N->getOpcode() == X86ISD::Wrapper) {
7929 if (isa<GlobalAddressSDNode>(N->getOperand(0))) {
Evan Cheng206ee9d2006-07-07 08:33:52 +00007930 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal();
Dan Gohman6520e202008-10-18 02:06:02 +00007931 Offset = cast<GlobalAddressSDNode>(N->getOperand(0))->getOffset();
Evan Cheng206ee9d2006-07-07 08:33:52 +00007932 return true;
7933 }
Evan Cheng206ee9d2006-07-07 08:33:52 +00007934 }
Evan Chengad4196b2008-05-12 19:56:52 +00007935 return TargetLowering::isGAPlusOffset(N, GA, Offset);
Evan Cheng206ee9d2006-07-07 08:33:52 +00007936}
7937
Evan Chengad4196b2008-05-12 19:56:52 +00007938static bool isBaseAlignmentOfN(unsigned N, SDNode *Base,
7939 const TargetLowering &TLI) {
Evan Cheng206ee9d2006-07-07 08:33:52 +00007940 GlobalValue *GV;
Nick Lewycky916a9f02008-02-02 08:29:58 +00007941 int64_t Offset = 0;
Evan Chengad4196b2008-05-12 19:56:52 +00007942 if (TLI.isGAPlusOffset(Base, GV, Offset))
Evan Cheng7e2ff772008-05-08 00:57:18 +00007943 return (GV->getAlignment() >= N && (Offset % N) == 0);
Chris Lattnerba96fbc2008-01-26 20:07:42 +00007944 // DAG combine handles the stack object case.
Evan Cheng206ee9d2006-07-07 08:33:52 +00007945 return false;
7946}
7947
Nate Begeman9008ca62009-04-27 18:41:29 +00007948static bool EltsFromConsecutiveLoads(ShuffleVectorSDNode *N, unsigned NumElems,
Owen Andersone50ed302009-08-10 22:56:29 +00007949 EVT EVT, LoadSDNode *&LDBase,
Eli Friedman7a5e5552009-06-07 06:52:44 +00007950 unsigned &LastLoadedElt,
Evan Chengad4196b2008-05-12 19:56:52 +00007951 SelectionDAG &DAG, MachineFrameInfo *MFI,
7952 const TargetLowering &TLI) {
Eli Friedman7a5e5552009-06-07 06:52:44 +00007953 LDBase = NULL;
Anton Korobeynikovb51b6cf2009-06-09 23:00:39 +00007954 LastLoadedElt = -1U;
Evan Cheng7e2ff772008-05-08 00:57:18 +00007955 for (unsigned i = 0; i < NumElems; ++i) {
Nate Begeman9008ca62009-04-27 18:41:29 +00007956 if (N->getMaskElt(i) < 0) {
Eli Friedman7a5e5552009-06-07 06:52:44 +00007957 if (!LDBase)
Evan Cheng7e2ff772008-05-08 00:57:18 +00007958 return false;
7959 continue;
7960 }
7961
Dan Gohman475871a2008-07-27 21:46:04 +00007962 SDValue Elt = DAG.getShuffleScalarElt(N, i);
Gabor Greifba36cb52008-08-28 21:40:38 +00007963 if (!Elt.getNode() ||
7964 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode())))
Evan Cheng7e2ff772008-05-08 00:57:18 +00007965 return false;
Eli Friedman7a5e5552009-06-07 06:52:44 +00007966 if (!LDBase) {
7967 if (Elt.getNode()->getOpcode() == ISD::UNDEF)
Evan Cheng50d9e722008-05-10 06:46:49 +00007968 return false;
Eli Friedman7a5e5552009-06-07 06:52:44 +00007969 LDBase = cast<LoadSDNode>(Elt.getNode());
7970 LastLoadedElt = i;
Evan Cheng7e2ff772008-05-08 00:57:18 +00007971 continue;
7972 }
7973 if (Elt.getOpcode() == ISD::UNDEF)
7974 continue;
7975
Nate Begemanabc01992009-06-05 21:37:30 +00007976 LoadSDNode *LD = cast<LoadSDNode>(Elt);
Nate Begemanabc01992009-06-05 21:37:30 +00007977 if (!TLI.isConsecutiveLoad(LD, LDBase, EVT.getSizeInBits()/8, i, MFI))
Evan Cheng7e2ff772008-05-08 00:57:18 +00007978 return false;
Eli Friedman7a5e5552009-06-07 06:52:44 +00007979 LastLoadedElt = i;
Evan Cheng7e2ff772008-05-08 00:57:18 +00007980 }
7981 return true;
7982}
Evan Cheng206ee9d2006-07-07 08:33:52 +00007983
7984/// PerformShuffleCombine - Combine a vector_shuffle that is equal to
7985/// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load
7986/// if the load addresses are consecutive, non-overlapping, and in the right
Mon P Wang1e955802009-04-03 02:43:30 +00007987/// order. In the case of v2i64, it will see if it can rewrite the
7988/// shuffle to be an appropriate build vector so it can take advantage of
7989// performBuildVectorCombine.
Dan Gohman475871a2008-07-27 21:46:04 +00007990static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG,
Nate Begeman9008ca62009-04-27 18:41:29 +00007991 const TargetLowering &TLI) {
Dale Johannesene4d209d2009-02-03 20:21:25 +00007992 DebugLoc dl = N->getDebugLoc();
Owen Andersone50ed302009-08-10 22:56:29 +00007993 EVT VT = N->getValueType(0);
7994 EVT EVT = VT.getVectorElementType();
Nate Begeman9008ca62009-04-27 18:41:29 +00007995 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
7996 unsigned NumElems = VT.getVectorNumElements();
Mon P Wang1e955802009-04-03 02:43:30 +00007997
Eli Friedman7a5e5552009-06-07 06:52:44 +00007998 if (VT.getSizeInBits() != 128)
7999 return SDValue();
8000
Mon P Wang1e955802009-04-03 02:43:30 +00008001 // Try to combine a vector_shuffle into a 128-bit load.
8002 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
Eli Friedman7a5e5552009-06-07 06:52:44 +00008003 LoadSDNode *LD = NULL;
8004 unsigned LastLoadedElt;
8005 if (!EltsFromConsecutiveLoads(SVN, NumElems, EVT, LD, LastLoadedElt, DAG,
8006 MFI, TLI))
Dan Gohman475871a2008-07-27 21:46:04 +00008007 return SDValue();
Evan Cheng206ee9d2006-07-07 08:33:52 +00008008
Eli Friedman7a5e5552009-06-07 06:52:44 +00008009 if (LastLoadedElt == NumElems - 1) {
8010 if (isBaseAlignmentOfN(16, LD->getBasePtr().getNode(), TLI))
8011 return DAG.getLoad(VT, dl, LD->getChain(), LD->getBasePtr(),
8012 LD->getSrcValue(), LD->getSrcValueOffset(),
8013 LD->isVolatile());
Dale Johannesene4d209d2009-02-03 20:21:25 +00008014 return DAG.getLoad(VT, dl, LD->getChain(), LD->getBasePtr(),
Scott Michelfdc40a02009-02-17 22:15:04 +00008015 LD->getSrcValue(), LD->getSrcValueOffset(),
Eli Friedman7a5e5552009-06-07 06:52:44 +00008016 LD->isVolatile(), LD->getAlignment());
8017 } else if (NumElems == 4 && LastLoadedElt == 1) {
Owen Anderson825b72b2009-08-11 20:47:22 +00008018 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other);
Nate Begemanabc01992009-06-05 21:37:30 +00008019 SDValue Ops[] = { LD->getChain(), LD->getBasePtr() };
8020 SDValue ResNode = DAG.getNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, 2);
Nate Begemanabc01992009-06-05 21:37:30 +00008021 return DAG.getNode(ISD::BIT_CONVERT, dl, VT, ResNode);
8022 }
8023 return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00008024}
Evan Chengd880b972008-05-09 21:53:03 +00008025
Chris Lattner83e6c992006-10-04 06:57:07 +00008026/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes.
Dan Gohman475871a2008-07-27 21:46:04 +00008027static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
Chris Lattner47b4ce82009-03-11 05:48:52 +00008028 const X86Subtarget *Subtarget) {
8029 DebugLoc DL = N->getDebugLoc();
Dan Gohman475871a2008-07-27 21:46:04 +00008030 SDValue Cond = N->getOperand(0);
Chris Lattner47b4ce82009-03-11 05:48:52 +00008031 // Get the LHS/RHS of the select.
8032 SDValue LHS = N->getOperand(1);
8033 SDValue RHS = N->getOperand(2);
8034
Chris Lattner83e6c992006-10-04 06:57:07 +00008035 // If we have SSE[12] support, try to form min/max nodes.
8036 if (Subtarget->hasSSE2() &&
Owen Anderson825b72b2009-08-11 20:47:22 +00008037 (LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64) &&
Chris Lattner47b4ce82009-03-11 05:48:52 +00008038 Cond.getOpcode() == ISD::SETCC) {
8039 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00008040
Chris Lattner47b4ce82009-03-11 05:48:52 +00008041 unsigned Opcode = 0;
8042 if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) {
8043 switch (CC) {
8044 default: break;
8045 case ISD::SETOLE: // (X <= Y) ? X : Y -> min
8046 case ISD::SETULE:
8047 case ISD::SETLE:
8048 if (!UnsafeFPMath) break;
8049 // FALL THROUGH.
8050 case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min
8051 case ISD::SETLT:
8052 Opcode = X86ISD::FMIN;
8053 break;
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00008054
Chris Lattner47b4ce82009-03-11 05:48:52 +00008055 case ISD::SETOGT: // (X > Y) ? X : Y -> max
8056 case ISD::SETUGT:
8057 case ISD::SETGT:
8058 if (!UnsafeFPMath) break;
8059 // FALL THROUGH.
8060 case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max
8061 case ISD::SETGE:
8062 Opcode = X86ISD::FMAX;
8063 break;
Chris Lattner83e6c992006-10-04 06:57:07 +00008064 }
Chris Lattner47b4ce82009-03-11 05:48:52 +00008065 } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) {
8066 switch (CC) {
8067 default: break;
8068 case ISD::SETOGT: // (X > Y) ? Y : X -> min
8069 case ISD::SETUGT:
8070 case ISD::SETGT:
8071 if (!UnsafeFPMath) break;
8072 // FALL THROUGH.
8073 case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min
8074 case ISD::SETGE:
8075 Opcode = X86ISD::FMIN;
8076 break;
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00008077
Chris Lattner47b4ce82009-03-11 05:48:52 +00008078 case ISD::SETOLE: // (X <= Y) ? Y : X -> max
8079 case ISD::SETULE:
8080 case ISD::SETLE:
8081 if (!UnsafeFPMath) break;
8082 // FALL THROUGH.
8083 case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max
8084 case ISD::SETLT:
8085 Opcode = X86ISD::FMAX;
8086 break;
8087 }
Chris Lattner83e6c992006-10-04 06:57:07 +00008088 }
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00008089
Chris Lattner47b4ce82009-03-11 05:48:52 +00008090 if (Opcode)
8091 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS);
Chris Lattner83e6c992006-10-04 06:57:07 +00008092 }
Chris Lattner47b4ce82009-03-11 05:48:52 +00008093
Chris Lattnerd1980a52009-03-12 06:52:53 +00008094 // If this is a select between two integer constants, try to do some
8095 // optimizations.
Chris Lattnercee56e72009-03-13 05:53:31 +00008096 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(LHS)) {
8097 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(RHS))
Chris Lattnerd1980a52009-03-12 06:52:53 +00008098 // Don't do this for crazy integer types.
8099 if (DAG.getTargetLoweringInfo().isTypeLegal(LHS.getValueType())) {
8100 // If this is efficiently invertible, canonicalize the LHSC/RHSC values
Chris Lattnercee56e72009-03-13 05:53:31 +00008101 // so that TrueC (the true value) is larger than FalseC.
Chris Lattnerd1980a52009-03-12 06:52:53 +00008102 bool NeedsCondInvert = false;
8103
Chris Lattnercee56e72009-03-13 05:53:31 +00008104 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue()) &&
Chris Lattnerd1980a52009-03-12 06:52:53 +00008105 // Efficiently invertible.
8106 (Cond.getOpcode() == ISD::SETCC || // setcc -> invertible.
8107 (Cond.getOpcode() == ISD::XOR && // xor(X, C) -> invertible.
8108 isa<ConstantSDNode>(Cond.getOperand(1))))) {
8109 NeedsCondInvert = true;
Chris Lattnercee56e72009-03-13 05:53:31 +00008110 std::swap(TrueC, FalseC);
Chris Lattnerd1980a52009-03-12 06:52:53 +00008111 }
8112
8113 // Optimize C ? 8 : 0 -> zext(C) << 3. Likewise for any pow2/0.
Chris Lattnercee56e72009-03-13 05:53:31 +00008114 if (FalseC->getAPIntValue() == 0 &&
8115 TrueC->getAPIntValue().isPowerOf2()) {
Chris Lattnerd1980a52009-03-12 06:52:53 +00008116 if (NeedsCondInvert) // Invert the condition if needed.
8117 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
8118 DAG.getConstant(1, Cond.getValueType()));
8119
8120 // Zero extend the condition if needed.
8121 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, LHS.getValueType(), Cond);
8122
Chris Lattnercee56e72009-03-13 05:53:31 +00008123 unsigned ShAmt = TrueC->getAPIntValue().logBase2();
Chris Lattnerd1980a52009-03-12 06:52:53 +00008124 return DAG.getNode(ISD::SHL, DL, LHS.getValueType(), Cond,
Owen Anderson825b72b2009-08-11 20:47:22 +00008125 DAG.getConstant(ShAmt, MVT::i8));
Chris Lattnerd1980a52009-03-12 06:52:53 +00008126 }
Chris Lattner97a29a52009-03-13 05:22:11 +00008127
8128 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst.
Chris Lattnercee56e72009-03-13 05:53:31 +00008129 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
Chris Lattner97a29a52009-03-13 05:22:11 +00008130 if (NeedsCondInvert) // Invert the condition if needed.
8131 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
8132 DAG.getConstant(1, Cond.getValueType()));
8133
8134 // Zero extend the condition if needed.
Chris Lattnercee56e72009-03-13 05:53:31 +00008135 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
8136 FalseC->getValueType(0), Cond);
Chris Lattner97a29a52009-03-13 05:22:11 +00008137 return DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
Chris Lattnercee56e72009-03-13 05:53:31 +00008138 SDValue(FalseC, 0));
Chris Lattner97a29a52009-03-13 05:22:11 +00008139 }
Chris Lattnercee56e72009-03-13 05:53:31 +00008140
8141 // Optimize cases that will turn into an LEA instruction. This requires
8142 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
Owen Anderson825b72b2009-08-11 20:47:22 +00008143 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
Chris Lattnercee56e72009-03-13 05:53:31 +00008144 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue();
Owen Anderson825b72b2009-08-11 20:47:22 +00008145 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff;
Chris Lattnercee56e72009-03-13 05:53:31 +00008146
8147 bool isFastMultiplier = false;
8148 if (Diff < 10) {
8149 switch ((unsigned char)Diff) {
8150 default: break;
8151 case 1: // result = add base, cond
8152 case 2: // result = lea base( , cond*2)
8153 case 3: // result = lea base(cond, cond*2)
8154 case 4: // result = lea base( , cond*4)
8155 case 5: // result = lea base(cond, cond*4)
8156 case 8: // result = lea base( , cond*8)
8157 case 9: // result = lea base(cond, cond*8)
8158 isFastMultiplier = true;
8159 break;
8160 }
8161 }
8162
8163 if (isFastMultiplier) {
8164 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
8165 if (NeedsCondInvert) // Invert the condition if needed.
8166 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
8167 DAG.getConstant(1, Cond.getValueType()));
8168
8169 // Zero extend the condition if needed.
8170 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
8171 Cond);
8172 // Scale the condition by the difference.
8173 if (Diff != 1)
8174 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
8175 DAG.getConstant(Diff, Cond.getValueType()));
8176
8177 // Add the base if non-zero.
8178 if (FalseC->getAPIntValue() != 0)
8179 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
8180 SDValue(FalseC, 0));
8181 return Cond;
8182 }
8183 }
Chris Lattnerd1980a52009-03-12 06:52:53 +00008184 }
8185 }
8186
Dan Gohman475871a2008-07-27 21:46:04 +00008187 return SDValue();
Chris Lattner83e6c992006-10-04 06:57:07 +00008188}
8189
Chris Lattnerd1980a52009-03-12 06:52:53 +00008190/// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
8191static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
8192 TargetLowering::DAGCombinerInfo &DCI) {
8193 DebugLoc DL = N->getDebugLoc();
8194
8195 // If the flag operand isn't dead, don't touch this CMOV.
8196 if (N->getNumValues() == 2 && !SDValue(N, 1).use_empty())
8197 return SDValue();
8198
8199 // If this is a select between two integer constants, try to do some
8200 // optimizations. Note that the operands are ordered the opposite of SELECT
8201 // operands.
8202 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
8203 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
8204 // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is
8205 // larger than FalseC (the false value).
8206 X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
8207
8208 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) {
8209 CC = X86::GetOppositeBranchCondition(CC);
8210 std::swap(TrueC, FalseC);
8211 }
8212
8213 // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3. Likewise for any pow2/0.
Chris Lattnercee56e72009-03-13 05:53:31 +00008214 // This is efficient for any integer data type (including i8/i16) and
8215 // shift amount.
Chris Lattnerd1980a52009-03-12 06:52:53 +00008216 if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) {
8217 SDValue Cond = N->getOperand(3);
Owen Anderson825b72b2009-08-11 20:47:22 +00008218 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
8219 DAG.getConstant(CC, MVT::i8), Cond);
Chris Lattnerd1980a52009-03-12 06:52:53 +00008220
8221 // Zero extend the condition if needed.
8222 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
8223
8224 unsigned ShAmt = TrueC->getAPIntValue().logBase2();
8225 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
Owen Anderson825b72b2009-08-11 20:47:22 +00008226 DAG.getConstant(ShAmt, MVT::i8));
Chris Lattnerd1980a52009-03-12 06:52:53 +00008227 if (N->getNumValues() == 2) // Dead flag value?
8228 return DCI.CombineTo(N, Cond, SDValue());
8229 return Cond;
8230 }
Chris Lattnercee56e72009-03-13 05:53:31 +00008231
8232 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. This is efficient
8233 // for any integer data type, including i8/i16.
Chris Lattner97a29a52009-03-13 05:22:11 +00008234 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
8235 SDValue Cond = N->getOperand(3);
Owen Anderson825b72b2009-08-11 20:47:22 +00008236 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
8237 DAG.getConstant(CC, MVT::i8), Cond);
Chris Lattner97a29a52009-03-13 05:22:11 +00008238
8239 // Zero extend the condition if needed.
Chris Lattnercee56e72009-03-13 05:53:31 +00008240 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
8241 FalseC->getValueType(0), Cond);
Chris Lattner97a29a52009-03-13 05:22:11 +00008242 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
8243 SDValue(FalseC, 0));
Chris Lattnercee56e72009-03-13 05:53:31 +00008244
Chris Lattner97a29a52009-03-13 05:22:11 +00008245 if (N->getNumValues() == 2) // Dead flag value?
8246 return DCI.CombineTo(N, Cond, SDValue());
8247 return Cond;
8248 }
Chris Lattnercee56e72009-03-13 05:53:31 +00008249
8250 // Optimize cases that will turn into an LEA instruction. This requires
8251 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
Owen Anderson825b72b2009-08-11 20:47:22 +00008252 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
Chris Lattnercee56e72009-03-13 05:53:31 +00008253 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue();
Owen Anderson825b72b2009-08-11 20:47:22 +00008254 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff;
Chris Lattnercee56e72009-03-13 05:53:31 +00008255
8256 bool isFastMultiplier = false;
8257 if (Diff < 10) {
8258 switch ((unsigned char)Diff) {
8259 default: break;
8260 case 1: // result = add base, cond
8261 case 2: // result = lea base( , cond*2)
8262 case 3: // result = lea base(cond, cond*2)
8263 case 4: // result = lea base( , cond*4)
8264 case 5: // result = lea base(cond, cond*4)
8265 case 8: // result = lea base( , cond*8)
8266 case 9: // result = lea base(cond, cond*8)
8267 isFastMultiplier = true;
8268 break;
8269 }
8270 }
8271
8272 if (isFastMultiplier) {
8273 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
8274 SDValue Cond = N->getOperand(3);
Owen Anderson825b72b2009-08-11 20:47:22 +00008275 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
8276 DAG.getConstant(CC, MVT::i8), Cond);
Chris Lattnercee56e72009-03-13 05:53:31 +00008277 // Zero extend the condition if needed.
8278 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
8279 Cond);
8280 // Scale the condition by the difference.
8281 if (Diff != 1)
8282 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
8283 DAG.getConstant(Diff, Cond.getValueType()));
8284
8285 // Add the base if non-zero.
8286 if (FalseC->getAPIntValue() != 0)
8287 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
8288 SDValue(FalseC, 0));
8289 if (N->getNumValues() == 2) // Dead flag value?
8290 return DCI.CombineTo(N, Cond, SDValue());
8291 return Cond;
8292 }
8293 }
Chris Lattnerd1980a52009-03-12 06:52:53 +00008294 }
8295 }
8296 return SDValue();
8297}
8298
8299
Evan Cheng0b0cd912009-03-28 05:57:29 +00008300/// PerformMulCombine - Optimize a single multiply with constant into two
8301/// in order to implement it with two cheaper instructions, e.g.
8302/// LEA + SHL, LEA + LEA.
8303static SDValue PerformMulCombine(SDNode *N, SelectionDAG &DAG,
8304 TargetLowering::DAGCombinerInfo &DCI) {
8305 if (DAG.getMachineFunction().
8306 getFunction()->hasFnAttr(Attribute::OptimizeForSize))
8307 return SDValue();
8308
8309 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
8310 return SDValue();
8311
Owen Andersone50ed302009-08-10 22:56:29 +00008312 EVT VT = N->getValueType(0);
Owen Anderson825b72b2009-08-11 20:47:22 +00008313 if (VT != MVT::i64)
Evan Cheng0b0cd912009-03-28 05:57:29 +00008314 return SDValue();
8315
8316 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
8317 if (!C)
8318 return SDValue();
8319 uint64_t MulAmt = C->getZExtValue();
8320 if (isPowerOf2_64(MulAmt) || MulAmt == 3 || MulAmt == 5 || MulAmt == 9)
8321 return SDValue();
8322
8323 uint64_t MulAmt1 = 0;
8324 uint64_t MulAmt2 = 0;
8325 if ((MulAmt % 9) == 0) {
8326 MulAmt1 = 9;
8327 MulAmt2 = MulAmt / 9;
8328 } else if ((MulAmt % 5) == 0) {
8329 MulAmt1 = 5;
8330 MulAmt2 = MulAmt / 5;
8331 } else if ((MulAmt % 3) == 0) {
8332 MulAmt1 = 3;
8333 MulAmt2 = MulAmt / 3;
8334 }
8335 if (MulAmt2 &&
8336 (isPowerOf2_64(MulAmt2) || MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)){
8337 DebugLoc DL = N->getDebugLoc();
8338
8339 if (isPowerOf2_64(MulAmt2) &&
8340 !(N->hasOneUse() && N->use_begin()->getOpcode() == ISD::ADD))
8341 // If second multiplifer is pow2, issue it first. We want the multiply by
8342 // 3, 5, or 9 to be folded into the addressing mode unless the lone use
8343 // is an add.
8344 std::swap(MulAmt1, MulAmt2);
8345
8346 SDValue NewMul;
8347 if (isPowerOf2_64(MulAmt1))
8348 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
Owen Anderson825b72b2009-08-11 20:47:22 +00008349 DAG.getConstant(Log2_64(MulAmt1), MVT::i8));
Evan Cheng0b0cd912009-03-28 05:57:29 +00008350 else
Evan Cheng73f24c92009-03-30 21:36:47 +00008351 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
Evan Cheng0b0cd912009-03-28 05:57:29 +00008352 DAG.getConstant(MulAmt1, VT));
8353
8354 if (isPowerOf2_64(MulAmt2))
8355 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
Owen Anderson825b72b2009-08-11 20:47:22 +00008356 DAG.getConstant(Log2_64(MulAmt2), MVT::i8));
Evan Cheng0b0cd912009-03-28 05:57:29 +00008357 else
Evan Cheng73f24c92009-03-30 21:36:47 +00008358 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
Evan Cheng0b0cd912009-03-28 05:57:29 +00008359 DAG.getConstant(MulAmt2, VT));
8360
8361 // Do not add new nodes to DAG combiner worklist.
8362 DCI.CombineTo(N, NewMul, false);
8363 }
8364 return SDValue();
8365}
8366
8367
Nate Begeman740ab032009-01-26 00:52:55 +00008368/// PerformShiftCombine - Transforms vector shift nodes to use vector shifts
8369/// when possible.
8370static SDValue PerformShiftCombine(SDNode* N, SelectionDAG &DAG,
8371 const X86Subtarget *Subtarget) {
8372 // On X86 with SSE2 support, we can transform this to a vector shift if
8373 // all elements are shifted by the same amount. We can't do this in legalize
8374 // because the a constant vector is typically transformed to a constant pool
8375 // so we have no knowledge of the shift amount.
Nate Begemanc2fd67f2009-01-26 03:15:31 +00008376 if (!Subtarget->hasSSE2())
8377 return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00008378
Owen Andersone50ed302009-08-10 22:56:29 +00008379 EVT VT = N->getValueType(0);
Owen Anderson825b72b2009-08-11 20:47:22 +00008380 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16)
Nate Begemanc2fd67f2009-01-26 03:15:31 +00008381 return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00008382
Mon P Wang3becd092009-01-28 08:12:05 +00008383 SDValue ShAmtOp = N->getOperand(1);
Owen Andersone50ed302009-08-10 22:56:29 +00008384 EVT EltVT = VT.getVectorElementType();
Chris Lattner47b4ce82009-03-11 05:48:52 +00008385 DebugLoc DL = N->getDebugLoc();
Mon P Wang3becd092009-01-28 08:12:05 +00008386 SDValue BaseShAmt;
8387 if (ShAmtOp.getOpcode() == ISD::BUILD_VECTOR) {
8388 unsigned NumElts = VT.getVectorNumElements();
8389 unsigned i = 0;
8390 for (; i != NumElts; ++i) {
8391 SDValue Arg = ShAmtOp.getOperand(i);
8392 if (Arg.getOpcode() == ISD::UNDEF) continue;
8393 BaseShAmt = Arg;
8394 break;
8395 }
8396 for (; i != NumElts; ++i) {
8397 SDValue Arg = ShAmtOp.getOperand(i);
8398 if (Arg.getOpcode() == ISD::UNDEF) continue;
8399 if (Arg != BaseShAmt) {
8400 return SDValue();
8401 }
8402 }
8403 } else if (ShAmtOp.getOpcode() == ISD::VECTOR_SHUFFLE &&
Nate Begeman9008ca62009-04-27 18:41:29 +00008404 cast<ShuffleVectorSDNode>(ShAmtOp)->isSplat()) {
8405 BaseShAmt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, ShAmtOp,
8406 DAG.getIntPtrConstant(0));
Mon P Wang3becd092009-01-28 08:12:05 +00008407 } else
Nate Begemanc2fd67f2009-01-26 03:15:31 +00008408 return SDValue();
Nate Begeman740ab032009-01-26 00:52:55 +00008409
Owen Anderson825b72b2009-08-11 20:47:22 +00008410 if (EltVT.bitsGT(MVT::i32))
8411 BaseShAmt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, BaseShAmt);
8412 else if (EltVT.bitsLT(MVT::i32))
8413 BaseShAmt = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, BaseShAmt);
Nate Begeman740ab032009-01-26 00:52:55 +00008414
Nate Begemanc2fd67f2009-01-26 03:15:31 +00008415 // The shift amount is identical so we can do a vector shift.
8416 SDValue ValOp = N->getOperand(0);
8417 switch (N->getOpcode()) {
8418 default:
Torok Edwinc23197a2009-07-14 16:55:14 +00008419 llvm_unreachable("Unknown shift opcode!");
Nate Begemanc2fd67f2009-01-26 03:15:31 +00008420 break;
8421 case ISD::SHL:
Owen Anderson825b72b2009-08-11 20:47:22 +00008422 if (VT == MVT::v2i64)
Chris Lattner47b4ce82009-03-11 05:48:52 +00008423 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00008424 DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32),
Nate Begeman740ab032009-01-26 00:52:55 +00008425 ValOp, BaseShAmt);
Owen Anderson825b72b2009-08-11 20:47:22 +00008426 if (VT == MVT::v4i32)
Chris Lattner47b4ce82009-03-11 05:48:52 +00008427 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00008428 DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32),
Nate Begeman740ab032009-01-26 00:52:55 +00008429 ValOp, BaseShAmt);
Owen Anderson825b72b2009-08-11 20:47:22 +00008430 if (VT == MVT::v8i16)
Chris Lattner47b4ce82009-03-11 05:48:52 +00008431 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00008432 DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32),
Nate Begeman740ab032009-01-26 00:52:55 +00008433 ValOp, BaseShAmt);
Nate Begemanc2fd67f2009-01-26 03:15:31 +00008434 break;
8435 case ISD::SRA:
Owen Anderson825b72b2009-08-11 20:47:22 +00008436 if (VT == MVT::v4i32)
Chris Lattner47b4ce82009-03-11 05:48:52 +00008437 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00008438 DAG.getConstant(Intrinsic::x86_sse2_psrai_d, MVT::i32),
Nate Begeman740ab032009-01-26 00:52:55 +00008439 ValOp, BaseShAmt);
Owen Anderson825b72b2009-08-11 20:47:22 +00008440 if (VT == MVT::v8i16)
Chris Lattner47b4ce82009-03-11 05:48:52 +00008441 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00008442 DAG.getConstant(Intrinsic::x86_sse2_psrai_w, MVT::i32),
Nate Begeman740ab032009-01-26 00:52:55 +00008443 ValOp, BaseShAmt);
Nate Begemanc2fd67f2009-01-26 03:15:31 +00008444 break;
8445 case ISD::SRL:
Owen Anderson825b72b2009-08-11 20:47:22 +00008446 if (VT == MVT::v2i64)
Chris Lattner47b4ce82009-03-11 05:48:52 +00008447 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00008448 DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32),
Nate Begeman740ab032009-01-26 00:52:55 +00008449 ValOp, BaseShAmt);
Owen Anderson825b72b2009-08-11 20:47:22 +00008450 if (VT == MVT::v4i32)
Chris Lattner47b4ce82009-03-11 05:48:52 +00008451 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00008452 DAG.getConstant(Intrinsic::x86_sse2_psrli_d, MVT::i32),
Nate Begeman740ab032009-01-26 00:52:55 +00008453 ValOp, BaseShAmt);
Owen Anderson825b72b2009-08-11 20:47:22 +00008454 if (VT == MVT::v8i16)
Chris Lattner47b4ce82009-03-11 05:48:52 +00008455 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
Owen Anderson825b72b2009-08-11 20:47:22 +00008456 DAG.getConstant(Intrinsic::x86_sse2_psrli_w, MVT::i32),
Nate Begeman740ab032009-01-26 00:52:55 +00008457 ValOp, BaseShAmt);
Nate Begemanc2fd67f2009-01-26 03:15:31 +00008458 break;
Nate Begeman740ab032009-01-26 00:52:55 +00008459 }
8460 return SDValue();
8461}
8462
Chris Lattner149a4e52008-02-22 02:09:43 +00008463/// PerformSTORECombine - Do target-specific dag combines on STORE nodes.
Dan Gohman475871a2008-07-27 21:46:04 +00008464static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
Evan Cheng536e6672009-03-12 05:59:15 +00008465 const X86Subtarget *Subtarget) {
Chris Lattner149a4e52008-02-22 02:09:43 +00008466 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering
8467 // the FP state in cases where an emms may be missing.
Dale Johannesen079f2a62008-02-25 19:20:14 +00008468 // A preferable solution to the general problem is to figure out the right
8469 // places to insert EMMS. This qualifies as a quick hack.
Evan Cheng536e6672009-03-12 05:59:15 +00008470
8471 // Similarly, turn load->store of i64 into double load/stores in 32-bit mode.
Evan Cheng7e2ff772008-05-08 00:57:18 +00008472 StoreSDNode *St = cast<StoreSDNode>(N);
Owen Andersone50ed302009-08-10 22:56:29 +00008473 EVT VT = St->getValue().getValueType();
Evan Cheng536e6672009-03-12 05:59:15 +00008474 if (VT.getSizeInBits() != 64)
8475 return SDValue();
8476
Devang Patel578efa92009-06-05 21:57:13 +00008477 const Function *F = DAG.getMachineFunction().getFunction();
8478 bool NoImplicitFloatOps = F->hasFnAttr(Attribute::NoImplicitFloat);
8479 bool F64IsLegal = !UseSoftFloat && !NoImplicitFloatOps
8480 && Subtarget->hasSSE2();
Evan Cheng536e6672009-03-12 05:59:15 +00008481 if ((VT.isVector() ||
Owen Anderson825b72b2009-08-11 20:47:22 +00008482 (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) &&
Dale Johannesen079f2a62008-02-25 19:20:14 +00008483 isa<LoadSDNode>(St->getValue()) &&
8484 !cast<LoadSDNode>(St->getValue())->isVolatile() &&
8485 St->getChain().hasOneUse() && !St->isVolatile()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00008486 SDNode* LdVal = St->getValue().getNode();
Dale Johannesen079f2a62008-02-25 19:20:14 +00008487 LoadSDNode *Ld = 0;
8488 int TokenFactorIndex = -1;
Dan Gohman475871a2008-07-27 21:46:04 +00008489 SmallVector<SDValue, 8> Ops;
Gabor Greifba36cb52008-08-28 21:40:38 +00008490 SDNode* ChainVal = St->getChain().getNode();
Dale Johannesen079f2a62008-02-25 19:20:14 +00008491 // Must be a store of a load. We currently handle two cases: the load
8492 // is a direct child, and it's under an intervening TokenFactor. It is
8493 // possible to dig deeper under nested TokenFactors.
Dale Johannesen14e2ea92008-02-25 22:29:22 +00008494 if (ChainVal == LdVal)
Dale Johannesen079f2a62008-02-25 19:20:14 +00008495 Ld = cast<LoadSDNode>(St->getChain());
8496 else if (St->getValue().hasOneUse() &&
8497 ChainVal->getOpcode() == ISD::TokenFactor) {
8498 for (unsigned i=0, e = ChainVal->getNumOperands(); i != e; ++i) {
Gabor Greifba36cb52008-08-28 21:40:38 +00008499 if (ChainVal->getOperand(i).getNode() == LdVal) {
Dale Johannesen079f2a62008-02-25 19:20:14 +00008500 TokenFactorIndex = i;
8501 Ld = cast<LoadSDNode>(St->getValue());
8502 } else
8503 Ops.push_back(ChainVal->getOperand(i));
8504 }
8505 }
Dale Johannesen079f2a62008-02-25 19:20:14 +00008506
Evan Cheng536e6672009-03-12 05:59:15 +00008507 if (!Ld || !ISD::isNormalLoad(Ld))
8508 return SDValue();
Dale Johannesen079f2a62008-02-25 19:20:14 +00008509
Evan Cheng536e6672009-03-12 05:59:15 +00008510 // If this is not the MMX case, i.e. we are just turning i64 load/store
8511 // into f64 load/store, avoid the transformation if there are multiple
8512 // uses of the loaded value.
8513 if (!VT.isVector() && !Ld->hasNUsesOfValue(1, 0))
8514 return SDValue();
Dale Johannesen079f2a62008-02-25 19:20:14 +00008515
Evan Cheng536e6672009-03-12 05:59:15 +00008516 DebugLoc LdDL = Ld->getDebugLoc();
8517 DebugLoc StDL = N->getDebugLoc();
8518 // If we are a 64-bit capable x86, lower to a single movq load/store pair.
8519 // Otherwise, if it's legal to use f64 SSE instructions, use f64 load/store
8520 // pair instead.
8521 if (Subtarget->is64Bit() || F64IsLegal) {
Owen Anderson825b72b2009-08-11 20:47:22 +00008522 EVT LdVT = Subtarget->is64Bit() ? MVT::i64 : MVT::f64;
Evan Cheng536e6672009-03-12 05:59:15 +00008523 SDValue NewLd = DAG.getLoad(LdVT, LdDL, Ld->getChain(),
8524 Ld->getBasePtr(), Ld->getSrcValue(),
8525 Ld->getSrcValueOffset(), Ld->isVolatile(),
8526 Ld->getAlignment());
8527 SDValue NewChain = NewLd.getValue(1);
Dale Johannesen079f2a62008-02-25 19:20:14 +00008528 if (TokenFactorIndex != -1) {
Evan Cheng536e6672009-03-12 05:59:15 +00008529 Ops.push_back(NewChain);
Owen Anderson825b72b2009-08-11 20:47:22 +00008530 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0],
Dale Johannesen079f2a62008-02-25 19:20:14 +00008531 Ops.size());
8532 }
Evan Cheng536e6672009-03-12 05:59:15 +00008533 return DAG.getStore(NewChain, StDL, NewLd, St->getBasePtr(),
Chris Lattner149a4e52008-02-22 02:09:43 +00008534 St->getSrcValue(), St->getSrcValueOffset(),
8535 St->isVolatile(), St->getAlignment());
8536 }
Evan Cheng536e6672009-03-12 05:59:15 +00008537
8538 // Otherwise, lower to two pairs of 32-bit loads / stores.
8539 SDValue LoAddr = Ld->getBasePtr();
Owen Anderson825b72b2009-08-11 20:47:22 +00008540 SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr,
8541 DAG.getConstant(4, MVT::i32));
Evan Cheng536e6672009-03-12 05:59:15 +00008542
Owen Anderson825b72b2009-08-11 20:47:22 +00008543 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr,
Evan Cheng536e6672009-03-12 05:59:15 +00008544 Ld->getSrcValue(), Ld->getSrcValueOffset(),
8545 Ld->isVolatile(), Ld->getAlignment());
Owen Anderson825b72b2009-08-11 20:47:22 +00008546 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr,
Evan Cheng536e6672009-03-12 05:59:15 +00008547 Ld->getSrcValue(), Ld->getSrcValueOffset()+4,
8548 Ld->isVolatile(),
8549 MinAlign(Ld->getAlignment(), 4));
8550
8551 SDValue NewChain = LoLd.getValue(1);
8552 if (TokenFactorIndex != -1) {
8553 Ops.push_back(LoLd);
8554 Ops.push_back(HiLd);
Owen Anderson825b72b2009-08-11 20:47:22 +00008555 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0],
Evan Cheng536e6672009-03-12 05:59:15 +00008556 Ops.size());
8557 }
8558
8559 LoAddr = St->getBasePtr();
Owen Anderson825b72b2009-08-11 20:47:22 +00008560 HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr,
8561 DAG.getConstant(4, MVT::i32));
Evan Cheng536e6672009-03-12 05:59:15 +00008562
8563 SDValue LoSt = DAG.getStore(NewChain, StDL, LoLd, LoAddr,
8564 St->getSrcValue(), St->getSrcValueOffset(),
8565 St->isVolatile(), St->getAlignment());
8566 SDValue HiSt = DAG.getStore(NewChain, StDL, HiLd, HiAddr,
8567 St->getSrcValue(),
8568 St->getSrcValueOffset() + 4,
8569 St->isVolatile(),
8570 MinAlign(St->getAlignment(), 4));
Owen Anderson825b72b2009-08-11 20:47:22 +00008571 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt);
Chris Lattner149a4e52008-02-22 02:09:43 +00008572 }
Dan Gohman475871a2008-07-27 21:46:04 +00008573 return SDValue();
Chris Lattner149a4e52008-02-22 02:09:43 +00008574}
8575
Chris Lattner6cf73262008-01-25 06:14:17 +00008576/// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and
8577/// X86ISD::FXOR nodes.
Dan Gohman475871a2008-07-27 21:46:04 +00008578static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) {
Chris Lattner6cf73262008-01-25 06:14:17 +00008579 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
8580 // F[X]OR(0.0, x) -> x
8581 // F[X]OR(x, 0.0) -> x
Chris Lattneraf723b92008-01-25 05:46:26 +00008582 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
8583 if (C->getValueAPF().isPosZero())
8584 return N->getOperand(1);
8585 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
8586 if (C->getValueAPF().isPosZero())
8587 return N->getOperand(0);
Dan Gohman475871a2008-07-27 21:46:04 +00008588 return SDValue();
Chris Lattneraf723b92008-01-25 05:46:26 +00008589}
8590
8591/// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes.
Dan Gohman475871a2008-07-27 21:46:04 +00008592static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) {
Chris Lattneraf723b92008-01-25 05:46:26 +00008593 // FAND(0.0, x) -> 0.0
8594 // FAND(x, 0.0) -> 0.0
8595 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
8596 if (C->getValueAPF().isPosZero())
8597 return N->getOperand(0);
8598 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
8599 if (C->getValueAPF().isPosZero())
8600 return N->getOperand(1);
Dan Gohman475871a2008-07-27 21:46:04 +00008601 return SDValue();
Chris Lattneraf723b92008-01-25 05:46:26 +00008602}
8603
Dan Gohmane5af2d32009-01-29 01:59:02 +00008604static SDValue PerformBTCombine(SDNode *N,
8605 SelectionDAG &DAG,
8606 TargetLowering::DAGCombinerInfo &DCI) {
8607 // BT ignores high bits in the bit index operand.
8608 SDValue Op1 = N->getOperand(1);
8609 if (Op1.hasOneUse()) {
8610 unsigned BitWidth = Op1.getValueSizeInBits();
8611 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth));
8612 APInt KnownZero, KnownOne;
8613 TargetLowering::TargetLoweringOpt TLO(DAG);
8614 TargetLowering &TLI = DAG.getTargetLoweringInfo();
8615 if (TLO.ShrinkDemandedConstant(Op1, DemandedMask) ||
8616 TLI.SimplifyDemandedBits(Op1, DemandedMask, KnownZero, KnownOne, TLO))
8617 DCI.CommitTargetLoweringOpt(TLO);
8618 }
8619 return SDValue();
8620}
Chris Lattner83e6c992006-10-04 06:57:07 +00008621
Eli Friedman7a5e5552009-06-07 06:52:44 +00008622static SDValue PerformVZEXT_MOVLCombine(SDNode *N, SelectionDAG &DAG) {
8623 SDValue Op = N->getOperand(0);
8624 if (Op.getOpcode() == ISD::BIT_CONVERT)
8625 Op = Op.getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00008626 EVT VT = N->getValueType(0), OpVT = Op.getValueType();
Eli Friedman7a5e5552009-06-07 06:52:44 +00008627 if (Op.getOpcode() == X86ISD::VZEXT_LOAD &&
8628 VT.getVectorElementType().getSizeInBits() ==
8629 OpVT.getVectorElementType().getSizeInBits()) {
8630 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), VT, Op);
8631 }
8632 return SDValue();
8633}
8634
Owen Anderson99177002009-06-29 18:04:45 +00008635// On X86 and X86-64, atomic operations are lowered to locked instructions.
8636// Locked instructions, in turn, have implicit fence semantics (all memory
8637// operations are flushed before issuing the locked instruction, and the
8638// are not buffered), so we can fold away the common pattern of
8639// fence-atomic-fence.
8640static SDValue PerformMEMBARRIERCombine(SDNode* N, SelectionDAG &DAG) {
8641 SDValue atomic = N->getOperand(0);
8642 switch (atomic.getOpcode()) {
8643 case ISD::ATOMIC_CMP_SWAP:
8644 case ISD::ATOMIC_SWAP:
8645 case ISD::ATOMIC_LOAD_ADD:
8646 case ISD::ATOMIC_LOAD_SUB:
8647 case ISD::ATOMIC_LOAD_AND:
8648 case ISD::ATOMIC_LOAD_OR:
8649 case ISD::ATOMIC_LOAD_XOR:
8650 case ISD::ATOMIC_LOAD_NAND:
8651 case ISD::ATOMIC_LOAD_MIN:
8652 case ISD::ATOMIC_LOAD_MAX:
8653 case ISD::ATOMIC_LOAD_UMIN:
8654 case ISD::ATOMIC_LOAD_UMAX:
8655 break;
8656 default:
8657 return SDValue();
8658 }
8659
8660 SDValue fence = atomic.getOperand(0);
8661 if (fence.getOpcode() != ISD::MEMBARRIER)
8662 return SDValue();
8663
8664 switch (atomic.getOpcode()) {
8665 case ISD::ATOMIC_CMP_SWAP:
8666 return DAG.UpdateNodeOperands(atomic, fence.getOperand(0),
8667 atomic.getOperand(1), atomic.getOperand(2),
8668 atomic.getOperand(3));
8669 case ISD::ATOMIC_SWAP:
8670 case ISD::ATOMIC_LOAD_ADD:
8671 case ISD::ATOMIC_LOAD_SUB:
8672 case ISD::ATOMIC_LOAD_AND:
8673 case ISD::ATOMIC_LOAD_OR:
8674 case ISD::ATOMIC_LOAD_XOR:
8675 case ISD::ATOMIC_LOAD_NAND:
8676 case ISD::ATOMIC_LOAD_MIN:
8677 case ISD::ATOMIC_LOAD_MAX:
8678 case ISD::ATOMIC_LOAD_UMIN:
8679 case ISD::ATOMIC_LOAD_UMAX:
8680 return DAG.UpdateNodeOperands(atomic, fence.getOperand(0),
8681 atomic.getOperand(1), atomic.getOperand(2));
8682 default:
8683 return SDValue();
8684 }
8685}
8686
Dan Gohman475871a2008-07-27 21:46:04 +00008687SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
Evan Cheng9dd93b32008-11-05 06:03:38 +00008688 DAGCombinerInfo &DCI) const {
Evan Cheng206ee9d2006-07-07 08:33:52 +00008689 SelectionDAG &DAG = DCI.DAG;
8690 switch (N->getOpcode()) {
8691 default: break;
Evan Chengad4196b2008-05-12 19:56:52 +00008692 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, *this);
Chris Lattneraf723b92008-01-25 05:46:26 +00008693 case ISD::SELECT: return PerformSELECTCombine(N, DAG, Subtarget);
Chris Lattnerd1980a52009-03-12 06:52:53 +00008694 case X86ISD::CMOV: return PerformCMOVCombine(N, DAG, DCI);
Evan Cheng0b0cd912009-03-28 05:57:29 +00008695 case ISD::MUL: return PerformMulCombine(N, DAG, DCI);
Nate Begeman740ab032009-01-26 00:52:55 +00008696 case ISD::SHL:
8697 case ISD::SRA:
8698 case ISD::SRL: return PerformShiftCombine(N, DAG, Subtarget);
Evan Cheng7e2ff772008-05-08 00:57:18 +00008699 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget);
Chris Lattner6cf73262008-01-25 06:14:17 +00008700 case X86ISD::FXOR:
Chris Lattneraf723b92008-01-25 05:46:26 +00008701 case X86ISD::FOR: return PerformFORCombine(N, DAG);
8702 case X86ISD::FAND: return PerformFANDCombine(N, DAG);
Dan Gohmane5af2d32009-01-29 01:59:02 +00008703 case X86ISD::BT: return PerformBTCombine(N, DAG, DCI);
Eli Friedman7a5e5552009-06-07 06:52:44 +00008704 case X86ISD::VZEXT_MOVL: return PerformVZEXT_MOVLCombine(N, DAG);
Owen Anderson99177002009-06-29 18:04:45 +00008705 case ISD::MEMBARRIER: return PerformMEMBARRIERCombine(N, DAG);
Evan Cheng206ee9d2006-07-07 08:33:52 +00008706 }
8707
Dan Gohman475871a2008-07-27 21:46:04 +00008708 return SDValue();
Evan Cheng206ee9d2006-07-07 08:33:52 +00008709}
8710
Evan Cheng60c07e12006-07-05 22:17:51 +00008711//===----------------------------------------------------------------------===//
8712// X86 Inline Assembly Support
8713//===----------------------------------------------------------------------===//
8714
Chris Lattnerb8105652009-07-20 17:51:36 +00008715static bool LowerToBSwap(CallInst *CI) {
8716 // FIXME: this should verify that we are targetting a 486 or better. If not,
8717 // we will turn this bswap into something that will be lowered to logical ops
8718 // instead of emitting the bswap asm. For now, we don't support 486 or lower
8719 // so don't worry about this.
8720
8721 // Verify this is a simple bswap.
8722 if (CI->getNumOperands() != 2 ||
8723 CI->getType() != CI->getOperand(1)->getType() ||
8724 !CI->getType()->isInteger())
8725 return false;
8726
8727 const IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
8728 if (!Ty || Ty->getBitWidth() % 16 != 0)
8729 return false;
8730
8731 // Okay, we can do this xform, do so now.
8732 const Type *Tys[] = { Ty };
8733 Module *M = CI->getParent()->getParent()->getParent();
8734 Constant *Int = Intrinsic::getDeclaration(M, Intrinsic::bswap, Tys, 1);
8735
8736 Value *Op = CI->getOperand(1);
8737 Op = CallInst::Create(Int, Op, CI->getName(), CI);
8738
8739 CI->replaceAllUsesWith(Op);
8740 CI->eraseFromParent();
8741 return true;
8742}
8743
8744bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
8745 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue());
8746 std::vector<InlineAsm::ConstraintInfo> Constraints = IA->ParseConstraints();
8747
8748 std::string AsmStr = IA->getAsmString();
8749
8750 // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a"
8751 std::vector<std::string> AsmPieces;
8752 SplitString(AsmStr, AsmPieces, "\n"); // ; as separator?
8753
8754 switch (AsmPieces.size()) {
8755 default: return false;
8756 case 1:
8757 AsmStr = AsmPieces[0];
8758 AsmPieces.clear();
8759 SplitString(AsmStr, AsmPieces, " \t"); // Split with whitespace.
8760
8761 // bswap $0
8762 if (AsmPieces.size() == 2 &&
8763 (AsmPieces[0] == "bswap" ||
8764 AsmPieces[0] == "bswapq" ||
8765 AsmPieces[0] == "bswapl") &&
8766 (AsmPieces[1] == "$0" ||
8767 AsmPieces[1] == "${0:q}")) {
8768 // No need to check constraints, nothing other than the equivalent of
8769 // "=r,0" would be valid here.
8770 return LowerToBSwap(CI);
8771 }
8772 // rorw $$8, ${0:w} --> llvm.bswap.i16
Owen Anderson1d0be152009-08-13 21:58:54 +00008773 if (CI->getType() == Type::getInt16Ty(CI->getContext()) &&
Chris Lattnerb8105652009-07-20 17:51:36 +00008774 AsmPieces.size() == 3 &&
8775 AsmPieces[0] == "rorw" &&
8776 AsmPieces[1] == "$$8," &&
8777 AsmPieces[2] == "${0:w}" &&
8778 IA->getConstraintString() == "=r,0,~{dirflag},~{fpsr},~{flags},~{cc}") {
8779 return LowerToBSwap(CI);
8780 }
8781 break;
8782 case 3:
Owen Anderson1d0be152009-08-13 21:58:54 +00008783 if (CI->getType() == Type::getInt64Ty(CI->getContext()) &&
8784 Constraints.size() >= 2 &&
Chris Lattnerb8105652009-07-20 17:51:36 +00008785 Constraints[0].Codes.size() == 1 && Constraints[0].Codes[0] == "A" &&
8786 Constraints[1].Codes.size() == 1 && Constraints[1].Codes[0] == "0") {
8787 // bswap %eax / bswap %edx / xchgl %eax, %edx -> llvm.bswap.i64
8788 std::vector<std::string> Words;
8789 SplitString(AsmPieces[0], Words, " \t");
8790 if (Words.size() == 2 && Words[0] == "bswap" && Words[1] == "%eax") {
8791 Words.clear();
8792 SplitString(AsmPieces[1], Words, " \t");
8793 if (Words.size() == 2 && Words[0] == "bswap" && Words[1] == "%edx") {
8794 Words.clear();
8795 SplitString(AsmPieces[2], Words, " \t,");
8796 if (Words.size() == 3 && Words[0] == "xchgl" && Words[1] == "%eax" &&
8797 Words[2] == "%edx") {
8798 return LowerToBSwap(CI);
8799 }
8800 }
8801 }
8802 }
8803 break;
8804 }
8805 return false;
8806}
8807
8808
8809
Chris Lattnerf4dff842006-07-11 02:54:03 +00008810/// getConstraintType - Given a constraint letter, return the type of
8811/// constraint it is for this target.
8812X86TargetLowering::ConstraintType
Chris Lattner4234f572007-03-25 02:14:49 +00008813X86TargetLowering::getConstraintType(const std::string &Constraint) const {
8814 if (Constraint.size() == 1) {
8815 switch (Constraint[0]) {
8816 case 'A':
Dale Johannesen330169f2008-11-13 21:52:36 +00008817 return C_Register;
Chris Lattnerfce84ac2008-03-11 19:06:29 +00008818 case 'f':
Chris Lattner4234f572007-03-25 02:14:49 +00008819 case 'r':
8820 case 'R':
8821 case 'l':
8822 case 'q':
8823 case 'Q':
8824 case 'x':
Dale Johannesen2ffbcac2008-04-01 00:57:48 +00008825 case 'y':
Chris Lattner4234f572007-03-25 02:14:49 +00008826 case 'Y':
8827 return C_RegisterClass;
Dale Johannesen78e3e522009-02-12 20:58:09 +00008828 case 'e':
8829 case 'Z':
8830 return C_Other;
Chris Lattner4234f572007-03-25 02:14:49 +00008831 default:
8832 break;
8833 }
Chris Lattnerf4dff842006-07-11 02:54:03 +00008834 }
Chris Lattner4234f572007-03-25 02:14:49 +00008835 return TargetLowering::getConstraintType(Constraint);
Chris Lattnerf4dff842006-07-11 02:54:03 +00008836}
8837
Dale Johannesenba2a0b92008-01-29 02:21:21 +00008838/// LowerXConstraint - try to replace an X constraint, which matches anything,
8839/// with another that has more specific requirements based on the type of the
8840/// corresponding operand.
Chris Lattner5e764232008-04-26 23:02:14 +00008841const char *X86TargetLowering::
Owen Andersone50ed302009-08-10 22:56:29 +00008842LowerXConstraint(EVT ConstraintVT) const {
Chris Lattner5e764232008-04-26 23:02:14 +00008843 // FP X constraints get lowered to SSE1/2 registers if available, otherwise
8844 // 'f' like normal targets.
Duncan Sands83ec4b62008-06-06 12:08:01 +00008845 if (ConstraintVT.isFloatingPoint()) {
Dale Johannesenba2a0b92008-01-29 02:21:21 +00008846 if (Subtarget->hasSSE2())
Chris Lattner5e764232008-04-26 23:02:14 +00008847 return "Y";
8848 if (Subtarget->hasSSE1())
8849 return "x";
8850 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008851
Chris Lattner5e764232008-04-26 23:02:14 +00008852 return TargetLowering::LowerXConstraint(ConstraintVT);
Dale Johannesenba2a0b92008-01-29 02:21:21 +00008853}
8854
Chris Lattner48884cd2007-08-25 00:47:38 +00008855/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
8856/// vector. If it is invalid, don't add anything to Ops.
Dan Gohman475871a2008-07-27 21:46:04 +00008857void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
Chris Lattner48884cd2007-08-25 00:47:38 +00008858 char Constraint,
Evan Chengda43bcf2008-09-24 00:05:32 +00008859 bool hasMemory,
Dan Gohman475871a2008-07-27 21:46:04 +00008860 std::vector<SDValue>&Ops,
Chris Lattner5e764232008-04-26 23:02:14 +00008861 SelectionDAG &DAG) const {
Dan Gohman475871a2008-07-27 21:46:04 +00008862 SDValue Result(0, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00008863
Chris Lattner22aaf1d2006-10-31 20:13:11 +00008864 switch (Constraint) {
8865 default: break;
Devang Patel84f7fd22007-03-17 00:13:28 +00008866 case 'I':
Chris Lattner188b9fe2007-03-25 01:57:35 +00008867 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00008868 if (C->getZExtValue() <= 31) {
8869 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
Chris Lattner48884cd2007-08-25 00:47:38 +00008870 break;
8871 }
Devang Patel84f7fd22007-03-17 00:13:28 +00008872 }
Chris Lattner48884cd2007-08-25 00:47:38 +00008873 return;
Evan Cheng364091e2008-09-22 23:57:37 +00008874 case 'J':
8875 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
Chris Lattner2e06dd22009-06-15 04:39:05 +00008876 if (C->getZExtValue() <= 63) {
Chris Lattnere4935152009-06-15 04:01:39 +00008877 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
8878 break;
8879 }
8880 }
8881 return;
8882 case 'K':
8883 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
Chris Lattner2e06dd22009-06-15 04:39:05 +00008884 if ((int8_t)C->getSExtValue() == C->getSExtValue()) {
Evan Cheng364091e2008-09-22 23:57:37 +00008885 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
8886 break;
8887 }
8888 }
8889 return;
Chris Lattner188b9fe2007-03-25 01:57:35 +00008890 case 'N':
8891 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00008892 if (C->getZExtValue() <= 255) {
8893 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
Chris Lattner48884cd2007-08-25 00:47:38 +00008894 break;
8895 }
Chris Lattner188b9fe2007-03-25 01:57:35 +00008896 }
Chris Lattner48884cd2007-08-25 00:47:38 +00008897 return;
Dale Johannesen78e3e522009-02-12 20:58:09 +00008898 case 'e': {
8899 // 32-bit signed value
8900 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
8901 const ConstantInt *CI = C->getConstantIntValue();
Owen Anderson1d0be152009-08-13 21:58:54 +00008902 if (CI->isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
8903 C->getSExtValue())) {
Dale Johannesen78e3e522009-02-12 20:58:09 +00008904 // Widen to 64 bits here to get it sign extended.
Owen Anderson825b72b2009-08-11 20:47:22 +00008905 Result = DAG.getTargetConstant(C->getSExtValue(), MVT::i64);
Dale Johannesen78e3e522009-02-12 20:58:09 +00008906 break;
8907 }
8908 // FIXME gcc accepts some relocatable values here too, but only in certain
8909 // memory models; it's complicated.
8910 }
8911 return;
8912 }
8913 case 'Z': {
8914 // 32-bit unsigned value
8915 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
8916 const ConstantInt *CI = C->getConstantIntValue();
Owen Anderson1d0be152009-08-13 21:58:54 +00008917 if (CI->isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
8918 C->getZExtValue())) {
Dale Johannesen78e3e522009-02-12 20:58:09 +00008919 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
8920 break;
8921 }
8922 }
8923 // FIXME gcc accepts some relocatable values here too, but only in certain
8924 // memory models; it's complicated.
8925 return;
8926 }
Chris Lattnerdc43a882007-05-03 16:52:29 +00008927 case 'i': {
Chris Lattner22aaf1d2006-10-31 20:13:11 +00008928 // Literal immediates are always ok.
Chris Lattner48884cd2007-08-25 00:47:38 +00008929 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) {
Dale Johannesen78e3e522009-02-12 20:58:09 +00008930 // Widen to 64 bits here to get it sign extended.
Owen Anderson825b72b2009-08-11 20:47:22 +00008931 Result = DAG.getTargetConstant(CST->getSExtValue(), MVT::i64);
Chris Lattner48884cd2007-08-25 00:47:38 +00008932 break;
8933 }
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00008934
Chris Lattnerdc43a882007-05-03 16:52:29 +00008935 // If we are in non-pic codegen mode, we allow the address of a global (with
8936 // an optional displacement) to be used with 'i'.
Chris Lattner49921962009-05-08 18:23:14 +00008937 GlobalAddressSDNode *GA = 0;
Chris Lattnerdc43a882007-05-03 16:52:29 +00008938 int64_t Offset = 0;
Scott Michelfdc40a02009-02-17 22:15:04 +00008939
Chris Lattner49921962009-05-08 18:23:14 +00008940 // Match either (GA), (GA+C), (GA+C1+C2), etc.
8941 while (1) {
8942 if ((GA = dyn_cast<GlobalAddressSDNode>(Op))) {
8943 Offset += GA->getOffset();
8944 break;
8945 } else if (Op.getOpcode() == ISD::ADD) {
8946 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
8947 Offset += C->getZExtValue();
8948 Op = Op.getOperand(0);
8949 continue;
8950 }
8951 } else if (Op.getOpcode() == ISD::SUB) {
8952 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
8953 Offset += -C->getZExtValue();
8954 Op = Op.getOperand(0);
8955 continue;
8956 }
Chris Lattnerdc43a882007-05-03 16:52:29 +00008957 }
Dale Johannesen76a1e2e2009-07-07 00:18:49 +00008958
Chris Lattner49921962009-05-08 18:23:14 +00008959 // Otherwise, this isn't something we can handle, reject it.
8960 return;
Chris Lattnerdc43a882007-05-03 16:52:29 +00008961 }
Chris Lattner3b6b36d2009-07-10 06:29:59 +00008962
Chris Lattner36c25012009-07-10 07:34:39 +00008963 GlobalValue *GV = GA->getGlobal();
Dale Johannesen76a1e2e2009-07-07 00:18:49 +00008964 // If we require an extra load to get this address, as in PIC mode, we
8965 // can't accept it.
Chris Lattner36c25012009-07-10 07:34:39 +00008966 if (isGlobalStubReference(Subtarget->ClassifyGlobalReference(GV,
8967 getTargetMachine())))
Dale Johannesen76a1e2e2009-07-07 00:18:49 +00008968 return;
Scott Michelfdc40a02009-02-17 22:15:04 +00008969
Dale Johannesen60b3ba02009-07-21 00:12:29 +00008970 if (hasMemory)
8971 Op = LowerGlobalAddress(GV, Op.getDebugLoc(), Offset, DAG);
8972 else
8973 Op = DAG.getTargetGlobalAddress(GV, GA->getValueType(0), Offset);
Chris Lattner49921962009-05-08 18:23:14 +00008974 Result = Op;
8975 break;
Chris Lattner22aaf1d2006-10-31 20:13:11 +00008976 }
Chris Lattnerdc43a882007-05-03 16:52:29 +00008977 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008978
Gabor Greifba36cb52008-08-28 21:40:38 +00008979 if (Result.getNode()) {
Chris Lattner48884cd2007-08-25 00:47:38 +00008980 Ops.push_back(Result);
8981 return;
8982 }
Evan Chengda43bcf2008-09-24 00:05:32 +00008983 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, hasMemory,
8984 Ops, DAG);
Chris Lattner22aaf1d2006-10-31 20:13:11 +00008985}
8986
Chris Lattner259e97c2006-01-31 19:43:35 +00008987std::vector<unsigned> X86TargetLowering::
Chris Lattner1efa40f2006-02-22 00:56:39 +00008988getRegClassForInlineAsmConstraint(const std::string &Constraint,
Owen Andersone50ed302009-08-10 22:56:29 +00008989 EVT VT) const {
Chris Lattner259e97c2006-01-31 19:43:35 +00008990 if (Constraint.size() == 1) {
8991 // FIXME: not handling fp-stack yet!
Chris Lattner259e97c2006-01-31 19:43:35 +00008992 switch (Constraint[0]) { // GCC X86 Constraint Letters
Chris Lattnerf4dff842006-07-11 02:54:03 +00008993 default: break; // Unknown constraint letter
Evan Cheng47e9fab2009-07-17 22:13:25 +00008994 case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
8995 if (Subtarget->is64Bit()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00008996 if (VT == MVT::i32)
Evan Cheng47e9fab2009-07-17 22:13:25 +00008997 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX,
8998 X86::ESI, X86::EDI, X86::R8D, X86::R9D,
8999 X86::R10D,X86::R11D,X86::R12D,
9000 X86::R13D,X86::R14D,X86::R15D,
9001 X86::EBP, X86::ESP, 0);
Owen Anderson825b72b2009-08-11 20:47:22 +00009002 else if (VT == MVT::i16)
Evan Cheng47e9fab2009-07-17 22:13:25 +00009003 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX,
9004 X86::SI, X86::DI, X86::R8W,X86::R9W,
9005 X86::R10W,X86::R11W,X86::R12W,
9006 X86::R13W,X86::R14W,X86::R15W,
9007 X86::BP, X86::SP, 0);
Owen Anderson825b72b2009-08-11 20:47:22 +00009008 else if (VT == MVT::i8)
Evan Cheng47e9fab2009-07-17 22:13:25 +00009009 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL,
9010 X86::SIL, X86::DIL, X86::R8B,X86::R9B,
9011 X86::R10B,X86::R11B,X86::R12B,
9012 X86::R13B,X86::R14B,X86::R15B,
9013 X86::BPL, X86::SPL, 0);
9014
Owen Anderson825b72b2009-08-11 20:47:22 +00009015 else if (VT == MVT::i64)
Evan Cheng47e9fab2009-07-17 22:13:25 +00009016 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX,
9017 X86::RSI, X86::RDI, X86::R8, X86::R9,
9018 X86::R10, X86::R11, X86::R12,
9019 X86::R13, X86::R14, X86::R15,
9020 X86::RBP, X86::RSP, 0);
9021
9022 break;
9023 }
9024 // 32-bit fallthrough
Chris Lattner259e97c2006-01-31 19:43:35 +00009025 case 'Q': // Q_REGS
Owen Anderson825b72b2009-08-11 20:47:22 +00009026 if (VT == MVT::i32)
Chris Lattner80a7ecc2006-05-06 00:29:37 +00009027 return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0);
Owen Anderson825b72b2009-08-11 20:47:22 +00009028 else if (VT == MVT::i16)
Chris Lattner80a7ecc2006-05-06 00:29:37 +00009029 return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0);
Owen Anderson825b72b2009-08-11 20:47:22 +00009030 else if (VT == MVT::i8)
Evan Cheng12914382007-08-13 23:27:11 +00009031 return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0);
Owen Anderson825b72b2009-08-11 20:47:22 +00009032 else if (VT == MVT::i64)
Chris Lattner03e6c702007-11-04 06:51:12 +00009033 return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, 0);
9034 break;
Chris Lattner259e97c2006-01-31 19:43:35 +00009035 }
9036 }
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00009037
Chris Lattner1efa40f2006-02-22 00:56:39 +00009038 return std::vector<unsigned>();
Chris Lattner259e97c2006-01-31 19:43:35 +00009039}
Chris Lattnerf76d1802006-07-31 23:26:50 +00009040
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00009041std::pair<unsigned, const TargetRegisterClass*>
Chris Lattnerf76d1802006-07-31 23:26:50 +00009042X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
Owen Andersone50ed302009-08-10 22:56:29 +00009043 EVT VT) const {
Chris Lattnerad043e82007-04-09 05:11:28 +00009044 // First, see if this is a constraint that directly corresponds to an LLVM
9045 // register class.
9046 if (Constraint.size() == 1) {
9047 // GCC Constraint Letters
9048 switch (Constraint[0]) {
9049 default: break;
Chris Lattner0f65cad2007-04-09 05:49:22 +00009050 case 'r': // GENERAL_REGS
9051 case 'R': // LEGACY_REGS
9052 case 'l': // INDEX_REGS
Owen Anderson825b72b2009-08-11 20:47:22 +00009053 if (VT == MVT::i8)
Chris Lattner0f65cad2007-04-09 05:49:22 +00009054 return std::make_pair(0U, X86::GR8RegisterClass);
Owen Anderson825b72b2009-08-11 20:47:22 +00009055 if (VT == MVT::i16)
Chris Lattner1fa71982008-10-17 18:15:05 +00009056 return std::make_pair(0U, X86::GR16RegisterClass);
Owen Anderson825b72b2009-08-11 20:47:22 +00009057 if (VT == MVT::i32 || !Subtarget->is64Bit())
Scott Michelfdc40a02009-02-17 22:15:04 +00009058 return std::make_pair(0U, X86::GR32RegisterClass);
Chris Lattner1fa71982008-10-17 18:15:05 +00009059 return std::make_pair(0U, X86::GR64RegisterClass);
Chris Lattnerfce84ac2008-03-11 19:06:29 +00009060 case 'f': // FP Stack registers.
9061 // If SSE is enabled for this VT, use f80 to ensure the isel moves the
9062 // value to the correct fpstack register class.
Owen Anderson825b72b2009-08-11 20:47:22 +00009063 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))
Chris Lattnerfce84ac2008-03-11 19:06:29 +00009064 return std::make_pair(0U, X86::RFP32RegisterClass);
Owen Anderson825b72b2009-08-11 20:47:22 +00009065 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT))
Chris Lattnerfce84ac2008-03-11 19:06:29 +00009066 return std::make_pair(0U, X86::RFP64RegisterClass);
9067 return std::make_pair(0U, X86::RFP80RegisterClass);
Chris Lattner6c284d72007-04-12 04:14:49 +00009068 case 'y': // MMX_REGS if MMX allowed.
9069 if (!Subtarget->hasMMX()) break;
9070 return std::make_pair(0U, X86::VR64RegisterClass);
Chris Lattner0f65cad2007-04-09 05:49:22 +00009071 case 'Y': // SSE_REGS if SSE2 allowed
9072 if (!Subtarget->hasSSE2()) break;
9073 // FALL THROUGH.
9074 case 'x': // SSE_REGS if SSE1 allowed
9075 if (!Subtarget->hasSSE1()) break;
Duncan Sands83ec4b62008-06-06 12:08:01 +00009076
Owen Anderson825b72b2009-08-11 20:47:22 +00009077 switch (VT.getSimpleVT().SimpleTy) {
Chris Lattner0f65cad2007-04-09 05:49:22 +00009078 default: break;
9079 // Scalar SSE types.
Owen Anderson825b72b2009-08-11 20:47:22 +00009080 case MVT::f32:
9081 case MVT::i32:
Chris Lattnerad043e82007-04-09 05:11:28 +00009082 return std::make_pair(0U, X86::FR32RegisterClass);
Owen Anderson825b72b2009-08-11 20:47:22 +00009083 case MVT::f64:
9084 case MVT::i64:
Chris Lattnerad043e82007-04-09 05:11:28 +00009085 return std::make_pair(0U, X86::FR64RegisterClass);
Chris Lattner0f65cad2007-04-09 05:49:22 +00009086 // Vector types.
Owen Anderson825b72b2009-08-11 20:47:22 +00009087 case MVT::v16i8:
9088 case MVT::v8i16:
9089 case MVT::v4i32:
9090 case MVT::v2i64:
9091 case MVT::v4f32:
9092 case MVT::v2f64:
Chris Lattner0f65cad2007-04-09 05:49:22 +00009093 return std::make_pair(0U, X86::VR128RegisterClass);
9094 }
Chris Lattnerad043e82007-04-09 05:11:28 +00009095 break;
9096 }
9097 }
Scott Michelfdc40a02009-02-17 22:15:04 +00009098
Chris Lattnerf76d1802006-07-31 23:26:50 +00009099 // Use the default implementation in TargetLowering to convert the register
9100 // constraint into a member of a register class.
9101 std::pair<unsigned, const TargetRegisterClass*> Res;
9102 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
Chris Lattner1a60aa72006-10-31 19:42:44 +00009103
9104 // Not found as a standard register?
9105 if (Res.second == 0) {
9106 // GCC calls "st(0)" just plain "st".
9107 if (StringsEqualNoCase("{st}", Constraint)) {
9108 Res.first = X86::ST0;
Chris Lattner9b4baf12007-09-24 05:27:37 +00009109 Res.second = X86::RFP80RegisterClass;
Chris Lattner1a60aa72006-10-31 19:42:44 +00009110 }
Dale Johannesen330169f2008-11-13 21:52:36 +00009111 // 'A' means EAX + EDX.
9112 if (Constraint == "A") {
9113 Res.first = X86::EAX;
Dan Gohman68a31c22009-07-30 17:02:08 +00009114 Res.second = X86::GR32_ADRegisterClass;
Dale Johannesen330169f2008-11-13 21:52:36 +00009115 }
Chris Lattner1a60aa72006-10-31 19:42:44 +00009116 return Res;
9117 }
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00009118
Chris Lattnerf76d1802006-07-31 23:26:50 +00009119 // Otherwise, check to see if this is a register class of the wrong value
9120 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to
9121 // turn into {ax},{dx}.
9122 if (Res.second->hasType(VT))
9123 return Res; // Correct type already, nothing to do.
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00009124
Chris Lattnerf76d1802006-07-31 23:26:50 +00009125 // All of the single-register GCC register classes map their values onto
9126 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we
9127 // really want an 8-bit or 32-bit register, map to the appropriate register
9128 // class and return the appropriate register.
Chris Lattner6ba50a92008-08-26 06:19:02 +00009129 if (Res.second == X86::GR16RegisterClass) {
Owen Anderson825b72b2009-08-11 20:47:22 +00009130 if (VT == MVT::i8) {
Chris Lattner6ba50a92008-08-26 06:19:02 +00009131 unsigned DestReg = 0;
9132 switch (Res.first) {
9133 default: break;
9134 case X86::AX: DestReg = X86::AL; break;
9135 case X86::DX: DestReg = X86::DL; break;
9136 case X86::CX: DestReg = X86::CL; break;
9137 case X86::BX: DestReg = X86::BL; break;
9138 }
9139 if (DestReg) {
9140 Res.first = DestReg;
Duncan Sands005e7982009-04-21 09:44:39 +00009141 Res.second = X86::GR8RegisterClass;
Chris Lattner6ba50a92008-08-26 06:19:02 +00009142 }
Owen Anderson825b72b2009-08-11 20:47:22 +00009143 } else if (VT == MVT::i32) {
Chris Lattner6ba50a92008-08-26 06:19:02 +00009144 unsigned DestReg = 0;
9145 switch (Res.first) {
9146 default: break;
9147 case X86::AX: DestReg = X86::EAX; break;
9148 case X86::DX: DestReg = X86::EDX; break;
9149 case X86::CX: DestReg = X86::ECX; break;
9150 case X86::BX: DestReg = X86::EBX; break;
9151 case X86::SI: DestReg = X86::ESI; break;
9152 case X86::DI: DestReg = X86::EDI; break;
9153 case X86::BP: DestReg = X86::EBP; break;
9154 case X86::SP: DestReg = X86::ESP; break;
9155 }
9156 if (DestReg) {
9157 Res.first = DestReg;
Duncan Sands005e7982009-04-21 09:44:39 +00009158 Res.second = X86::GR32RegisterClass;
Chris Lattner6ba50a92008-08-26 06:19:02 +00009159 }
Owen Anderson825b72b2009-08-11 20:47:22 +00009160 } else if (VT == MVT::i64) {
Chris Lattner6ba50a92008-08-26 06:19:02 +00009161 unsigned DestReg = 0;
9162 switch (Res.first) {
9163 default: break;
9164 case X86::AX: DestReg = X86::RAX; break;
9165 case X86::DX: DestReg = X86::RDX; break;
9166 case X86::CX: DestReg = X86::RCX; break;
9167 case X86::BX: DestReg = X86::RBX; break;
9168 case X86::SI: DestReg = X86::RSI; break;
9169 case X86::DI: DestReg = X86::RDI; break;
9170 case X86::BP: DestReg = X86::RBP; break;
9171 case X86::SP: DestReg = X86::RSP; break;
9172 }
9173 if (DestReg) {
9174 Res.first = DestReg;
Duncan Sands005e7982009-04-21 09:44:39 +00009175 Res.second = X86::GR64RegisterClass;
Chris Lattner6ba50a92008-08-26 06:19:02 +00009176 }
Chris Lattnerf76d1802006-07-31 23:26:50 +00009177 }
Chris Lattner6ba50a92008-08-26 06:19:02 +00009178 } else if (Res.second == X86::FR32RegisterClass ||
9179 Res.second == X86::FR64RegisterClass ||
9180 Res.second == X86::VR128RegisterClass) {
9181 // Handle references to XMM physical registers that got mapped into the
9182 // wrong class. This can happen with constraints like {xmm0} where the
9183 // target independent register mapper will just pick the first match it can
9184 // find, ignoring the required type.
Owen Anderson825b72b2009-08-11 20:47:22 +00009185 if (VT == MVT::f32)
Chris Lattner6ba50a92008-08-26 06:19:02 +00009186 Res.second = X86::FR32RegisterClass;
Owen Anderson825b72b2009-08-11 20:47:22 +00009187 else if (VT == MVT::f64)
Chris Lattner6ba50a92008-08-26 06:19:02 +00009188 Res.second = X86::FR64RegisterClass;
9189 else if (X86::VR128RegisterClass->hasType(VT))
9190 Res.second = X86::VR128RegisterClass;
Chris Lattnerf76d1802006-07-31 23:26:50 +00009191 }
Anton Korobeynikov12c49af2006-11-21 00:01:06 +00009192
Chris Lattnerf76d1802006-07-31 23:26:50 +00009193 return Res;
9194}
Mon P Wang0c397192008-10-30 08:01:45 +00009195
9196//===----------------------------------------------------------------------===//
9197// X86 Widen vector type
9198//===----------------------------------------------------------------------===//
9199
9200/// getWidenVectorType: given a vector type, returns the type to widen
9201/// to (e.g., v7i8 to v8i8). If the vector type is legal, it returns itself.
Owen Anderson825b72b2009-08-11 20:47:22 +00009202/// If there is no vector type that we want to widen to, returns MVT::Other
Mon P Wangf007a8b2008-11-06 05:31:54 +00009203/// When and where to widen is target dependent based on the cost of
Mon P Wang0c397192008-10-30 08:01:45 +00009204/// scalarizing vs using the wider vector type.
9205
Owen Andersone50ed302009-08-10 22:56:29 +00009206EVT X86TargetLowering::getWidenVectorType(EVT VT) const {
Mon P Wang0c397192008-10-30 08:01:45 +00009207 assert(VT.isVector());
9208 if (isTypeLegal(VT))
9209 return VT;
Scott Michelfdc40a02009-02-17 22:15:04 +00009210
Mon P Wang0c397192008-10-30 08:01:45 +00009211 // TODO: In computeRegisterProperty, we can compute the list of legal vector
9212 // type based on element type. This would speed up our search (though
9213 // it may not be worth it since the size of the list is relatively
9214 // small).
Owen Andersone50ed302009-08-10 22:56:29 +00009215 EVT EltVT = VT.getVectorElementType();
Mon P Wang0c397192008-10-30 08:01:45 +00009216 unsigned NElts = VT.getVectorNumElements();
Scott Michelfdc40a02009-02-17 22:15:04 +00009217
Mon P Wang0c397192008-10-30 08:01:45 +00009218 // On X86, it make sense to widen any vector wider than 1
9219 if (NElts <= 1)
Owen Anderson825b72b2009-08-11 20:47:22 +00009220 return MVT::Other;
Scott Michelfdc40a02009-02-17 22:15:04 +00009221
Owen Anderson825b72b2009-08-11 20:47:22 +00009222 for (unsigned nVT = MVT::FIRST_VECTOR_VALUETYPE;
9223 nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) {
9224 EVT SVT = (MVT::SimpleValueType)nVT;
Scott Michelfdc40a02009-02-17 22:15:04 +00009225
9226 if (isTypeLegal(SVT) &&
9227 SVT.getVectorElementType() == EltVT &&
Mon P Wang0c397192008-10-30 08:01:45 +00009228 SVT.getVectorNumElements() > NElts)
9229 return SVT;
9230 }
Owen Anderson825b72b2009-08-11 20:47:22 +00009231 return MVT::Other;
Mon P Wang0c397192008-10-30 08:01:45 +00009232}