Chris Lattner | c6d0567 | 2006-05-23 23:20:42 +0000 | [diff] [blame] | 1 | //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file was developed by Chris Lattner and is distributed under |
| 6 | // the University of Illinois Open Source License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file defines the interfaces that X86 uses to lower LLVM code into a |
| 11 | // selection DAG. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "X86.h" |
Evan Cheng | 0cc3945 | 2006-01-16 21:21:29 +0000 | [diff] [blame] | 16 | #include "X86InstrBuilder.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 17 | #include "X86ISelLowering.h" |
Evan Cheng | e8bd0a3 | 2006-06-06 23:30:24 +0000 | [diff] [blame] | 18 | #include "X86MachineFunctionInfo.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 19 | #include "X86TargetMachine.h" |
| 20 | #include "llvm/CallingConv.h" |
Evan Cheng | 223547a | 2006-01-31 22:28:30 +0000 | [diff] [blame] | 21 | #include "llvm/Constants.h" |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 22 | #include "llvm/DerivedTypes.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 23 | #include "llvm/Function.h" |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 24 | #include "llvm/Intrinsics.h" |
Evan Cheng | 30b37b5 | 2006-03-13 23:18:16 +0000 | [diff] [blame] | 25 | #include "llvm/ADT/VectorExtras.h" |
| 26 | #include "llvm/Analysis/ScalarEvolutionExpressions.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 27 | #include "llvm/CodeGen/MachineFrameInfo.h" |
Evan Cheng | 4a46080 | 2006-01-11 00:33:36 +0000 | [diff] [blame] | 28 | #include "llvm/CodeGen/MachineFunction.h" |
| 29 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 30 | #include "llvm/CodeGen/SelectionDAG.h" |
| 31 | #include "llvm/CodeGen/SSARegMap.h" |
Evan Cheng | ef6ffb1 | 2006-01-31 03:14:29 +0000 | [diff] [blame] | 32 | #include "llvm/Support/MathExtras.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 33 | #include "llvm/Target/TargetOptions.h" |
Evan Cheng | e111303 | 2006-10-04 18:33:38 +0000 | [diff] [blame] | 34 | #include "llvm/Support/CommandLine.h" |
Chris Lattner | 1a60aa7 | 2006-10-31 19:42:44 +0000 | [diff] [blame] | 35 | #include "llvm/ADT/StringExtras.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 36 | using namespace llvm; |
| 37 | |
| 38 | // FIXME: temporary. |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 39 | static cl::opt<bool> EnableFastCC("enable-x86-fastcc", cl::Hidden, |
| 40 | cl::desc("Enable fastcc on X86")); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 41 | X86TargetLowering::X86TargetLowering(TargetMachine &TM) |
| 42 | : TargetLowering(TM) { |
Evan Cheng | 559806f | 2006-01-27 08:10:46 +0000 | [diff] [blame] | 43 | Subtarget = &TM.getSubtarget<X86Subtarget>(); |
| 44 | X86ScalarSSE = Subtarget->hasSSE2(); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 45 | X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP; |
Evan Cheng | 559806f | 2006-01-27 08:10:46 +0000 | [diff] [blame] | 46 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 47 | // Set up the TargetLowering object. |
| 48 | |
| 49 | // X86 is weird, it always uses i8 for shift amounts and setcc results. |
| 50 | setShiftAmountType(MVT::i8); |
| 51 | setSetCCResultType(MVT::i8); |
| 52 | setSetCCResultContents(ZeroOrOneSetCCResult); |
Evan Cheng | 0b2afbd | 2006-01-25 09:15:17 +0000 | [diff] [blame] | 53 | setSchedulingPreference(SchedulingForRegPressure); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 54 | setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0 |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 55 | setStackPointerRegisterToSaveRestore(X86StackPtr); |
Evan Cheng | 714554d | 2006-03-16 21:47:42 +0000 | [diff] [blame] | 56 | |
Anton Korobeynikov | d27a258 | 2006-12-10 23:12:42 +0000 | [diff] [blame^] | 57 | if (Subtarget->isTargetDarwin()) { |
Evan Cheng | df57fa0 | 2006-03-17 20:31:41 +0000 | [diff] [blame] | 58 | // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. |
Anton Korobeynikov | d27a258 | 2006-12-10 23:12:42 +0000 | [diff] [blame^] | 59 | setUseUnderscoreSetJmp(false); |
| 60 | setUseUnderscoreLongJmp(false); |
| 61 | } else if (Subtarget->isTargetCygwin()) { |
| 62 | // MS runtime is weird: it exports _setjmp, but longjmp! |
| 63 | setUseUnderscoreSetJmp(true); |
| 64 | setUseUnderscoreLongJmp(false); |
| 65 | } else { |
| 66 | setUseUnderscoreSetJmp(true); |
| 67 | setUseUnderscoreLongJmp(true); |
| 68 | } |
| 69 | |
Evan Cheng | 714554d | 2006-03-16 21:47:42 +0000 | [diff] [blame] | 70 | // Add legal addressing mode scale values. |
| 71 | addLegalAddressScale(8); |
| 72 | addLegalAddressScale(4); |
| 73 | addLegalAddressScale(2); |
| 74 | // Enter the ones which require both scale + index last. These are more |
| 75 | // expensive. |
| 76 | addLegalAddressScale(9); |
| 77 | addLegalAddressScale(5); |
| 78 | addLegalAddressScale(3); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 79 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 80 | // Set up the register classes. |
Evan Cheng | 069287d | 2006-05-16 07:21:53 +0000 | [diff] [blame] | 81 | addRegisterClass(MVT::i8, X86::GR8RegisterClass); |
| 82 | addRegisterClass(MVT::i16, X86::GR16RegisterClass); |
| 83 | addRegisterClass(MVT::i32, X86::GR32RegisterClass); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 84 | if (Subtarget->is64Bit()) |
| 85 | addRegisterClass(MVT::i64, X86::GR64RegisterClass); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 86 | |
Evan Cheng | c548428 | 2006-10-04 00:56:09 +0000 | [diff] [blame] | 87 | setLoadXAction(ISD::SEXTLOAD, MVT::i1, Expand); |
| 88 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 89 | // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this |
| 90 | // operation. |
| 91 | setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); |
| 92 | setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); |
| 93 | setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); |
Evan Cheng | 6892f28 | 2006-01-17 02:32:49 +0000 | [diff] [blame] | 94 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 95 | if (Subtarget->is64Bit()) { |
| 96 | setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand); |
Evan Cheng | 6892f28 | 2006-01-17 02:32:49 +0000 | [diff] [blame] | 97 | setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 98 | } else { |
| 99 | if (X86ScalarSSE) |
| 100 | // If SSE i64 SINT_TO_FP is not available, expand i32 UINT_TO_FP. |
| 101 | setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand); |
| 102 | else |
| 103 | setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); |
| 104 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 105 | |
| 106 | // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have |
| 107 | // this operation. |
| 108 | setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); |
| 109 | setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 110 | // SSE has no i16 to fp conversion, only i32 |
Evan Cheng | 02568ff | 2006-01-30 22:13:22 +0000 | [diff] [blame] | 111 | if (X86ScalarSSE) |
Evan Cheng | 02568ff | 2006-01-30 22:13:22 +0000 | [diff] [blame] | 112 | setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 113 | else { |
| 114 | setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); |
| 115 | setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); |
| 116 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 117 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 118 | if (!Subtarget->is64Bit()) { |
| 119 | // Custom lower SINT_TO_FP and FP_TO_SINT from/to i64 in 32-bit mode. |
| 120 | setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); |
| 121 | setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); |
| 122 | } |
Evan Cheng | 6dab053 | 2006-01-30 08:02:57 +0000 | [diff] [blame] | 123 | |
Evan Cheng | 02568ff | 2006-01-30 22:13:22 +0000 | [diff] [blame] | 124 | // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have |
| 125 | // this operation. |
| 126 | setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); |
| 127 | setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); |
| 128 | |
| 129 | if (X86ScalarSSE) { |
| 130 | setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); |
| 131 | } else { |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 132 | setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); |
Evan Cheng | 02568ff | 2006-01-30 22:13:22 +0000 | [diff] [blame] | 133 | setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 134 | } |
| 135 | |
| 136 | // Handle FP_TO_UINT by promoting the destination to a larger signed |
| 137 | // conversion. |
| 138 | setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); |
| 139 | setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); |
| 140 | setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); |
| 141 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 142 | if (Subtarget->is64Bit()) { |
| 143 | setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 144 | setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 145 | } else { |
| 146 | if (X86ScalarSSE && !Subtarget->hasSSE3()) |
| 147 | // Expand FP_TO_UINT into a select. |
| 148 | // FIXME: We would like to use a Custom expander here eventually to do |
| 149 | // the optimal thing for SSE vs. the default expansion in the legalizer. |
| 150 | setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); |
| 151 | else |
| 152 | // With SSE3 we can use fisttpll to convert to a signed i64. |
| 153 | setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); |
| 154 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 155 | |
Chris Lattner | 399610a | 2006-12-05 18:22:22 +0000 | [diff] [blame] | 156 | // TODO: when we have SSE, these could be more efficient, by using movd/movq. |
Chris Lattner | f3597a1 | 2006-12-05 18:45:06 +0000 | [diff] [blame] | 157 | if (!X86ScalarSSE) { |
| 158 | setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand); |
| 159 | setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand); |
| 160 | } |
Chris Lattner | 399610a | 2006-12-05 18:22:22 +0000 | [diff] [blame] | 161 | if (Subtarget->is64Bit()) { |
| 162 | setOperationAction(ISD::BIT_CONVERT , MVT::f64 , Expand); |
| 163 | setOperationAction(ISD::BIT_CONVERT , MVT::i64 , Expand); |
| 164 | } |
Chris Lattner | 21f6685 | 2005-12-23 05:15:23 +0000 | [diff] [blame] | 165 | |
Evan Cheng | c35497f | 2006-10-30 08:02:39 +0000 | [diff] [blame] | 166 | setOperationAction(ISD::BR_JT , MVT::Other, Expand); |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 167 | setOperationAction(ISD::BRCOND , MVT::Other, Custom); |
Nate Begeman | 750ac1b | 2006-02-01 07:19:44 +0000 | [diff] [blame] | 168 | setOperationAction(ISD::BR_CC , MVT::Other, Expand); |
| 169 | setOperationAction(ISD::SELECT_CC , MVT::Other, Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 170 | setOperationAction(ISD::MEMMOVE , MVT::Other, Expand); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 171 | if (Subtarget->is64Bit()) |
| 172 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 173 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Expand); |
Chris Lattner | e80242a | 2005-12-07 17:59:14 +0000 | [diff] [blame] | 174 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 175 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); |
| 176 | setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 177 | setOperationAction(ISD::FREM , MVT::f64 , Expand); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 178 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 179 | setOperationAction(ISD::CTPOP , MVT::i8 , Expand); |
| 180 | setOperationAction(ISD::CTTZ , MVT::i8 , Expand); |
| 181 | setOperationAction(ISD::CTLZ , MVT::i8 , Expand); |
| 182 | setOperationAction(ISD::CTPOP , MVT::i16 , Expand); |
| 183 | setOperationAction(ISD::CTTZ , MVT::i16 , Expand); |
| 184 | setOperationAction(ISD::CTLZ , MVT::i16 , Expand); |
| 185 | setOperationAction(ISD::CTPOP , MVT::i32 , Expand); |
| 186 | setOperationAction(ISD::CTTZ , MVT::i32 , Expand); |
| 187 | setOperationAction(ISD::CTLZ , MVT::i32 , Expand); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 188 | if (Subtarget->is64Bit()) { |
| 189 | setOperationAction(ISD::CTPOP , MVT::i64 , Expand); |
| 190 | setOperationAction(ISD::CTTZ , MVT::i64 , Expand); |
| 191 | setOperationAction(ISD::CTLZ , MVT::i64 , Expand); |
| 192 | } |
| 193 | |
Andrew Lenharth | b873ff3 | 2005-11-20 21:41:10 +0000 | [diff] [blame] | 194 | setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); |
Nate Begeman | d88fc03 | 2006-01-14 03:14:10 +0000 | [diff] [blame] | 195 | setOperationAction(ISD::BSWAP , MVT::i16 , Expand); |
Nate Begeman | 35ef913 | 2006-01-11 21:21:00 +0000 | [diff] [blame] | 196 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 197 | // These should be promoted to a larger select which is supported. |
| 198 | setOperationAction(ISD::SELECT , MVT::i1 , Promote); |
| 199 | setOperationAction(ISD::SELECT , MVT::i8 , Promote); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 200 | // X86 wants to expand cmov itself. |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 201 | setOperationAction(ISD::SELECT , MVT::i16 , Custom); |
| 202 | setOperationAction(ISD::SELECT , MVT::i32 , Custom); |
| 203 | setOperationAction(ISD::SELECT , MVT::f32 , Custom); |
| 204 | setOperationAction(ISD::SELECT , MVT::f64 , Custom); |
| 205 | setOperationAction(ISD::SETCC , MVT::i8 , Custom); |
| 206 | setOperationAction(ISD::SETCC , MVT::i16 , Custom); |
| 207 | setOperationAction(ISD::SETCC , MVT::i32 , Custom); |
| 208 | setOperationAction(ISD::SETCC , MVT::f32 , Custom); |
| 209 | setOperationAction(ISD::SETCC , MVT::f64 , Custom); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 210 | if (Subtarget->is64Bit()) { |
| 211 | setOperationAction(ISD::SELECT , MVT::i64 , Custom); |
| 212 | setOperationAction(ISD::SETCC , MVT::i64 , Custom); |
| 213 | } |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 214 | // X86 ret instruction may pop stack. |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 215 | setOperationAction(ISD::RET , MVT::Other, Custom); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 216 | // Darwin ABI issue. |
Evan Cheng | 7ccced6 | 2006-02-18 00:15:05 +0000 | [diff] [blame] | 217 | setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); |
Nate Begeman | 37efe67 | 2006-04-22 18:53:45 +0000 | [diff] [blame] | 218 | setOperationAction(ISD::JumpTable , MVT::i32 , Custom); |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 219 | setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); |
Evan Cheng | 020d2e8 | 2006-02-23 20:41:18 +0000 | [diff] [blame] | 220 | setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 221 | if (Subtarget->is64Bit()) { |
| 222 | setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); |
| 223 | setOperationAction(ISD::JumpTable , MVT::i64 , Custom); |
| 224 | setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); |
| 225 | setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); |
| 226 | } |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 227 | // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 228 | setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); |
| 229 | setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); |
| 230 | setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 231 | // X86 wants to expand memset / memcpy itself. |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 232 | setOperationAction(ISD::MEMSET , MVT::Other, Custom); |
| 233 | setOperationAction(ISD::MEMCPY , MVT::Other, Custom); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 234 | |
Chris Lattner | f73bae1 | 2005-11-29 06:16:21 +0000 | [diff] [blame] | 235 | // We don't have line number support yet. |
| 236 | setOperationAction(ISD::LOCATION, MVT::Other, Expand); |
Jim Laskey | e0bce71 | 2006-01-05 01:47:43 +0000 | [diff] [blame] | 237 | setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand); |
Evan Cheng | 3c992d2 | 2006-03-07 02:02:57 +0000 | [diff] [blame] | 238 | // FIXME - use subtarget debug flags |
Anton Korobeynikov | ab4022f | 2006-10-31 08:31:24 +0000 | [diff] [blame] | 239 | if (!Subtarget->isTargetDarwin() && |
| 240 | !Subtarget->isTargetELF() && |
| 241 | !Subtarget->isTargetCygwin()) |
Evan Cheng | 3c992d2 | 2006-03-07 02:02:57 +0000 | [diff] [blame] | 242 | setOperationAction(ISD::DEBUG_LABEL, MVT::Other, Expand); |
Chris Lattner | f73bae1 | 2005-11-29 06:16:21 +0000 | [diff] [blame] | 243 | |
Nate Begeman | acc398c | 2006-01-25 18:21:52 +0000 | [diff] [blame] | 244 | // VASTART needs to be custom lowered to use the VarArgsFrameIndex |
| 245 | setOperationAction(ISD::VASTART , MVT::Other, Custom); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 246 | |
Nate Begeman | acc398c | 2006-01-25 18:21:52 +0000 | [diff] [blame] | 247 | // Use the default implementation. |
| 248 | setOperationAction(ISD::VAARG , MVT::Other, Expand); |
| 249 | setOperationAction(ISD::VACOPY , MVT::Other, Expand); |
| 250 | setOperationAction(ISD::VAEND , MVT::Other, Expand); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 251 | setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); |
Chris Lattner | e112552 | 2006-01-15 09:00:21 +0000 | [diff] [blame] | 252 | setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 253 | if (Subtarget->is64Bit()) |
| 254 | setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); |
Chris Lattner | e112552 | 2006-01-15 09:00:21 +0000 | [diff] [blame] | 255 | setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Expand); |
Chris Lattner | b99329e | 2006-01-13 02:42:53 +0000 | [diff] [blame] | 256 | |
Chris Lattner | 9601a86 | 2006-03-05 05:08:37 +0000 | [diff] [blame] | 257 | setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); |
| 258 | setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); |
| 259 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 260 | if (X86ScalarSSE) { |
| 261 | // Set up the FP register classes. |
Evan Cheng | 5ee4ccc | 2006-01-12 08:27:59 +0000 | [diff] [blame] | 262 | addRegisterClass(MVT::f32, X86::FR32RegisterClass); |
| 263 | addRegisterClass(MVT::f64, X86::FR64RegisterClass); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 264 | |
Evan Cheng | 223547a | 2006-01-31 22:28:30 +0000 | [diff] [blame] | 265 | // Use ANDPD to simulate FABS. |
| 266 | setOperationAction(ISD::FABS , MVT::f64, Custom); |
| 267 | setOperationAction(ISD::FABS , MVT::f32, Custom); |
| 268 | |
| 269 | // Use XORP to simulate FNEG. |
| 270 | setOperationAction(ISD::FNEG , MVT::f64, Custom); |
| 271 | setOperationAction(ISD::FNEG , MVT::f32, Custom); |
| 272 | |
Evan Cheng | d25e9e8 | 2006-02-02 00:28:23 +0000 | [diff] [blame] | 273 | // We don't support sin/cos/fmod |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 274 | setOperationAction(ISD::FSIN , MVT::f64, Expand); |
| 275 | setOperationAction(ISD::FCOS , MVT::f64, Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 276 | setOperationAction(ISD::FREM , MVT::f64, Expand); |
| 277 | setOperationAction(ISD::FSIN , MVT::f32, Expand); |
| 278 | setOperationAction(ISD::FCOS , MVT::f32, Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 279 | setOperationAction(ISD::FREM , MVT::f32, Expand); |
| 280 | |
Chris Lattner | a54aa94 | 2006-01-29 06:26:08 +0000 | [diff] [blame] | 281 | // Expand FP immediates into loads from the stack, except for the special |
| 282 | // cases we handle. |
| 283 | setOperationAction(ISD::ConstantFP, MVT::f64, Expand); |
| 284 | setOperationAction(ISD::ConstantFP, MVT::f32, Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 285 | addLegalFPImmediate(+0.0); // xorps / xorpd |
| 286 | } else { |
| 287 | // Set up the FP register classes. |
| 288 | addRegisterClass(MVT::f64, X86::RFPRegisterClass); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 289 | |
Chris Lattner | 44d9b9b | 2006-01-29 06:44:22 +0000 | [diff] [blame] | 290 | setOperationAction(ISD::UNDEF, MVT::f64, Expand); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 291 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 292 | if (!UnsafeFPMath) { |
| 293 | setOperationAction(ISD::FSIN , MVT::f64 , Expand); |
| 294 | setOperationAction(ISD::FCOS , MVT::f64 , Expand); |
| 295 | } |
| 296 | |
Chris Lattner | a54aa94 | 2006-01-29 06:26:08 +0000 | [diff] [blame] | 297 | setOperationAction(ISD::ConstantFP, MVT::f64, Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 298 | addLegalFPImmediate(+0.0); // FLD0 |
| 299 | addLegalFPImmediate(+1.0); // FLD1 |
| 300 | addLegalFPImmediate(-0.0); // FLD0/FCHS |
| 301 | addLegalFPImmediate(-1.0); // FLD1/FCHS |
| 302 | } |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 303 | |
Evan Cheng | d30bf01 | 2006-03-01 01:11:20 +0000 | [diff] [blame] | 304 | // First set operation action for all vector types to expand. Then we |
| 305 | // will selectively turn on ones that can be effectively codegen'd. |
| 306 | for (unsigned VT = (unsigned)MVT::Vector + 1; |
| 307 | VT != (unsigned)MVT::LAST_VALUETYPE; VT++) { |
| 308 | setOperationAction(ISD::ADD , (MVT::ValueType)VT, Expand); |
| 309 | setOperationAction(ISD::SUB , (MVT::ValueType)VT, Expand); |
Evan Cheng | 6bdb3f6 | 2006-10-27 18:49:08 +0000 | [diff] [blame] | 310 | setOperationAction(ISD::FADD, (MVT::ValueType)VT, Expand); |
| 311 | setOperationAction(ISD::FSUB, (MVT::ValueType)VT, Expand); |
Evan Cheng | d30bf01 | 2006-03-01 01:11:20 +0000 | [diff] [blame] | 312 | setOperationAction(ISD::MUL , (MVT::ValueType)VT, Expand); |
Evan Cheng | 6bdb3f6 | 2006-10-27 18:49:08 +0000 | [diff] [blame] | 313 | setOperationAction(ISD::FMUL, (MVT::ValueType)VT, Expand); |
| 314 | setOperationAction(ISD::SDIV, (MVT::ValueType)VT, Expand); |
| 315 | setOperationAction(ISD::UDIV, (MVT::ValueType)VT, Expand); |
| 316 | setOperationAction(ISD::FDIV, (MVT::ValueType)VT, Expand); |
| 317 | setOperationAction(ISD::SREM, (MVT::ValueType)VT, Expand); |
| 318 | setOperationAction(ISD::UREM, (MVT::ValueType)VT, Expand); |
Evan Cheng | d30bf01 | 2006-03-01 01:11:20 +0000 | [diff] [blame] | 319 | setOperationAction(ISD::LOAD, (MVT::ValueType)VT, Expand); |
Evan Cheng | b067a1e | 2006-03-31 19:22:53 +0000 | [diff] [blame] | 320 | setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::ValueType)VT, Expand); |
Chris Lattner | 9b3bd46 | 2006-03-21 20:51:05 +0000 | [diff] [blame] | 321 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::ValueType)VT, Expand); |
Evan Cheng | b067a1e | 2006-03-31 19:22:53 +0000 | [diff] [blame] | 322 | setOperationAction(ISD::INSERT_VECTOR_ELT, (MVT::ValueType)VT, Expand); |
Evan Cheng | d30bf01 | 2006-03-01 01:11:20 +0000 | [diff] [blame] | 323 | } |
| 324 | |
Evan Cheng | a88973f | 2006-03-22 19:22:18 +0000 | [diff] [blame] | 325 | if (Subtarget->hasMMX()) { |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 326 | addRegisterClass(MVT::v8i8, X86::VR64RegisterClass); |
| 327 | addRegisterClass(MVT::v4i16, X86::VR64RegisterClass); |
| 328 | addRegisterClass(MVT::v2i32, X86::VR64RegisterClass); |
| 329 | |
Evan Cheng | d30bf01 | 2006-03-01 01:11:20 +0000 | [diff] [blame] | 330 | // FIXME: add MMX packed arithmetics |
Evan Cheng | 48090aa | 2006-03-21 23:01:21 +0000 | [diff] [blame] | 331 | setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Expand); |
| 332 | setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Expand); |
| 333 | setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Expand); |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 334 | } |
| 335 | |
Evan Cheng | a88973f | 2006-03-22 19:22:18 +0000 | [diff] [blame] | 336 | if (Subtarget->hasSSE1()) { |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 337 | addRegisterClass(MVT::v4f32, X86::VR128RegisterClass); |
| 338 | |
Evan Cheng | 6bdb3f6 | 2006-10-27 18:49:08 +0000 | [diff] [blame] | 339 | setOperationAction(ISD::FADD, MVT::v4f32, Legal); |
| 340 | setOperationAction(ISD::FSUB, MVT::v4f32, Legal); |
| 341 | setOperationAction(ISD::FMUL, MVT::v4f32, Legal); |
| 342 | setOperationAction(ISD::FDIV, MVT::v4f32, Legal); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 343 | setOperationAction(ISD::LOAD, MVT::v4f32, Legal); |
| 344 | setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); |
| 345 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); |
Evan Cheng | 11e15b3 | 2006-04-03 20:53:28 +0000 | [diff] [blame] | 346 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 347 | setOperationAction(ISD::SELECT, MVT::v4f32, Custom); |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 348 | } |
| 349 | |
Evan Cheng | a88973f | 2006-03-22 19:22:18 +0000 | [diff] [blame] | 350 | if (Subtarget->hasSSE2()) { |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 351 | addRegisterClass(MVT::v2f64, X86::VR128RegisterClass); |
| 352 | addRegisterClass(MVT::v16i8, X86::VR128RegisterClass); |
| 353 | addRegisterClass(MVT::v8i16, X86::VR128RegisterClass); |
| 354 | addRegisterClass(MVT::v4i32, X86::VR128RegisterClass); |
| 355 | addRegisterClass(MVT::v2i64, X86::VR128RegisterClass); |
| 356 | |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 357 | setOperationAction(ISD::ADD, MVT::v16i8, Legal); |
| 358 | setOperationAction(ISD::ADD, MVT::v8i16, Legal); |
| 359 | setOperationAction(ISD::ADD, MVT::v4i32, Legal); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 360 | setOperationAction(ISD::SUB, MVT::v16i8, Legal); |
| 361 | setOperationAction(ISD::SUB, MVT::v8i16, Legal); |
| 362 | setOperationAction(ISD::SUB, MVT::v4i32, Legal); |
Evan Cheng | f998984 | 2006-04-13 05:10:25 +0000 | [diff] [blame] | 363 | setOperationAction(ISD::MUL, MVT::v8i16, Legal); |
Evan Cheng | 6bdb3f6 | 2006-10-27 18:49:08 +0000 | [diff] [blame] | 364 | setOperationAction(ISD::FADD, MVT::v2f64, Legal); |
| 365 | setOperationAction(ISD::FSUB, MVT::v2f64, Legal); |
| 366 | setOperationAction(ISD::FMUL, MVT::v2f64, Legal); |
| 367 | setOperationAction(ISD::FDIV, MVT::v2f64, Legal); |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 368 | |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 369 | setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); |
| 370 | setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); |
Evan Cheng | b067a1e | 2006-03-31 19:22:53 +0000 | [diff] [blame] | 371 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); |
Evan Cheng | 5edb8d2 | 2006-04-17 22:04:06 +0000 | [diff] [blame] | 372 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); |
| 373 | // Implement v4f32 insert_vector_elt in terms of SSE2 v8i16 ones. |
| 374 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 375 | |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 376 | // Custom lower build_vector, vector_shuffle, and extract_vector_elt. |
| 377 | for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) { |
| 378 | setOperationAction(ISD::BUILD_VECTOR, (MVT::ValueType)VT, Custom); |
| 379 | setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::ValueType)VT, Custom); |
| 380 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::ValueType)VT, Custom); |
| 381 | } |
| 382 | setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); |
| 383 | setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); |
| 384 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); |
| 385 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); |
| 386 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); |
| 387 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); |
| 388 | |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 389 | // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 390 | for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) { |
| 391 | setOperationAction(ISD::AND, (MVT::ValueType)VT, Promote); |
| 392 | AddPromotedToType (ISD::AND, (MVT::ValueType)VT, MVT::v2i64); |
| 393 | setOperationAction(ISD::OR, (MVT::ValueType)VT, Promote); |
| 394 | AddPromotedToType (ISD::OR, (MVT::ValueType)VT, MVT::v2i64); |
| 395 | setOperationAction(ISD::XOR, (MVT::ValueType)VT, Promote); |
| 396 | AddPromotedToType (ISD::XOR, (MVT::ValueType)VT, MVT::v2i64); |
Evan Cheng | 91b740d | 2006-04-12 17:12:36 +0000 | [diff] [blame] | 397 | setOperationAction(ISD::LOAD, (MVT::ValueType)VT, Promote); |
| 398 | AddPromotedToType (ISD::LOAD, (MVT::ValueType)VT, MVT::v2i64); |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 399 | setOperationAction(ISD::SELECT, (MVT::ValueType)VT, Promote); |
| 400 | AddPromotedToType (ISD::SELECT, (MVT::ValueType)VT, MVT::v2i64); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 401 | } |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 402 | |
| 403 | // Custom lower v2i64 and v2f64 selects. |
| 404 | setOperationAction(ISD::LOAD, MVT::v2f64, Legal); |
Evan Cheng | 91b740d | 2006-04-12 17:12:36 +0000 | [diff] [blame] | 405 | setOperationAction(ISD::LOAD, MVT::v2i64, Legal); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 406 | setOperationAction(ISD::SELECT, MVT::v2f64, Custom); |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 407 | setOperationAction(ISD::SELECT, MVT::v2i64, Custom); |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 408 | } |
| 409 | |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 410 | // We want to custom lower some of our intrinsics. |
| 411 | setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); |
| 412 | |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 413 | // We have target-specific dag combine patterns for the following nodes: |
| 414 | setTargetDAGCombine(ISD::VECTOR_SHUFFLE); |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 415 | setTargetDAGCombine(ISD::SELECT); |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 416 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 417 | computeRegisterProperties(); |
| 418 | |
Evan Cheng | 87ed716 | 2006-02-14 08:25:08 +0000 | [diff] [blame] | 419 | // FIXME: These should be based on subtarget info. Plus, the values should |
| 420 | // be smaller when we are in optimizing for size mode. |
Evan Cheng | a03a5dc | 2006-02-14 08:38:30 +0000 | [diff] [blame] | 421 | maxStoresPerMemset = 16; // For %llvm.memset -> sequence of stores |
| 422 | maxStoresPerMemcpy = 16; // For %llvm.memcpy -> sequence of stores |
| 423 | maxStoresPerMemmove = 16; // For %llvm.memmove -> sequence of stores |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 424 | allowUnalignedMemoryAccesses = true; // x86 supports it! |
| 425 | } |
| 426 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 427 | //===----------------------------------------------------------------------===// |
| 428 | // C Calling Convention implementation |
| 429 | //===----------------------------------------------------------------------===// |
| 430 | |
Evan Cheng | 85e3800 | 2006-04-27 05:35:28 +0000 | [diff] [blame] | 431 | /// AddLiveIn - This helper function adds the specified physical register to the |
| 432 | /// MachineFunction as a live in value. It also creates a corresponding virtual |
| 433 | /// register for it. |
| 434 | static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg, |
| 435 | TargetRegisterClass *RC) { |
| 436 | assert(RC->contains(PReg) && "Not the correct regclass!"); |
| 437 | unsigned VReg = MF.getSSARegMap()->createVirtualRegister(RC); |
| 438 | MF.addLiveIn(PReg, VReg); |
| 439 | return VReg; |
| 440 | } |
| 441 | |
Evan Cheng | 2fdd95e | 2006-04-27 08:31:10 +0000 | [diff] [blame] | 442 | /// HowToPassCCCArgument - Returns how an formal argument of the specified type |
| 443 | /// should be passed. If it is through stack, returns the size of the stack |
Evan Cheng | f9ff7c5 | 2006-05-26 18:25:43 +0000 | [diff] [blame] | 444 | /// slot; if it is through XMM register, returns the number of XMM registers |
Evan Cheng | 2fdd95e | 2006-04-27 08:31:10 +0000 | [diff] [blame] | 445 | /// are needed. |
| 446 | static void |
| 447 | HowToPassCCCArgument(MVT::ValueType ObjectVT, unsigned NumXMMRegs, |
| 448 | unsigned &ObjSize, unsigned &ObjXMMRegs) { |
Evan Cheng | 2675534 | 2006-06-01 05:53:27 +0000 | [diff] [blame] | 449 | ObjXMMRegs = 0; |
Evan Cheng | cc1fc22 | 2006-05-25 23:31:23 +0000 | [diff] [blame] | 450 | |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 451 | switch (ObjectVT) { |
| 452 | default: assert(0 && "Unhandled argument type!"); |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 453 | case MVT::i8: ObjSize = 1; break; |
| 454 | case MVT::i16: ObjSize = 2; break; |
| 455 | case MVT::i32: ObjSize = 4; break; |
| 456 | case MVT::i64: ObjSize = 8; break; |
| 457 | case MVT::f32: ObjSize = 4; break; |
| 458 | case MVT::f64: ObjSize = 8; break; |
Evan Cheng | 2fdd95e | 2006-04-27 08:31:10 +0000 | [diff] [blame] | 459 | case MVT::v16i8: |
| 460 | case MVT::v8i16: |
| 461 | case MVT::v4i32: |
| 462 | case MVT::v2i64: |
| 463 | case MVT::v4f32: |
| 464 | case MVT::v2f64: |
Evan Cheng | 1d6a9b3 | 2006-05-26 19:22:06 +0000 | [diff] [blame] | 465 | if (NumXMMRegs < 4) |
Evan Cheng | 2fdd95e | 2006-04-27 08:31:10 +0000 | [diff] [blame] | 466 | ObjXMMRegs = 1; |
| 467 | else |
| 468 | ObjSize = 16; |
| 469 | break; |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 470 | } |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 471 | } |
| 472 | |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 473 | SDOperand X86TargetLowering::LowerCCCArguments(SDOperand Op, SelectionDAG &DAG) { |
| 474 | unsigned NumArgs = Op.Val->getNumValues() - 1; |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 475 | MachineFunction &MF = DAG.getMachineFunction(); |
| 476 | MachineFrameInfo *MFI = MF.getFrameInfo(); |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 477 | SDOperand Root = Op.getOperand(0); |
| 478 | std::vector<SDOperand> ArgValues; |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 479 | |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 480 | // Add DAG nodes to load the arguments... On entry to a function on the X86, |
| 481 | // the stack frame looks like this: |
| 482 | // |
| 483 | // [ESP] -- return address |
| 484 | // [ESP + 4] -- first argument (leftmost lexically) |
Evan Cheng | f9d62dc | 2006-05-26 18:37:16 +0000 | [diff] [blame] | 485 | // [ESP + 8] -- second argument, if first argument is <= 4 bytes in size |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 486 | // ... |
| 487 | // |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 488 | unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot |
Evan Cheng | 2fdd95e | 2006-04-27 08:31:10 +0000 | [diff] [blame] | 489 | unsigned NumXMMRegs = 0; // XMM regs used for parameter passing. |
Evan Cheng | 1d6a9b3 | 2006-05-26 19:22:06 +0000 | [diff] [blame] | 490 | static const unsigned XMMArgRegs[] = { |
| 491 | X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 |
| 492 | }; |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 493 | for (unsigned i = 0; i < NumArgs; ++i) { |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 494 | MVT::ValueType ObjectVT = Op.getValue(i).getValueType(); |
| 495 | unsigned ArgIncrement = 4; |
| 496 | unsigned ObjSize = 0; |
| 497 | unsigned ObjXMMRegs = 0; |
| 498 | HowToPassCCCArgument(ObjectVT, NumXMMRegs, ObjSize, ObjXMMRegs); |
Evan Cheng | 052fb51 | 2006-05-26 18:39:59 +0000 | [diff] [blame] | 499 | if (ObjSize > 4) |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 500 | ArgIncrement = ObjSize; |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 501 | |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 502 | SDOperand ArgValue; |
| 503 | if (ObjXMMRegs) { |
| 504 | // Passed in a XMM register. |
| 505 | unsigned Reg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 506 | X86::VR128RegisterClass); |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 507 | ArgValue= DAG.getCopyFromReg(Root, Reg, ObjectVT); |
| 508 | ArgValues.push_back(ArgValue); |
| 509 | NumXMMRegs += ObjXMMRegs; |
| 510 | } else { |
Evan Cheng | 3fddf24 | 2006-05-26 20:37:47 +0000 | [diff] [blame] | 511 | // XMM arguments have to be aligned on 16-byte boundary. |
| 512 | if (ObjSize == 16) |
| 513 | ArgOffset = ((ArgOffset + 15) / 16) * 16; |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 514 | // Create the frame index object for this incoming parameter... |
| 515 | int FI = MFI->CreateFixedObject(ObjSize, ArgOffset); |
| 516 | SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy()); |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 517 | ArgValue = DAG.getLoad(Op.Val->getValueType(i), Root, FIN, NULL, 0); |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 518 | ArgValues.push_back(ArgValue); |
| 519 | ArgOffset += ArgIncrement; // Move on to the next argument... |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 520 | } |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 521 | } |
| 522 | |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 523 | ArgValues.push_back(Root); |
| 524 | |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 525 | // If the function takes variable number of arguments, make a frame index for |
| 526 | // the start of the first vararg value... for expansion of llvm.va_start. |
Evan Cheng | 4db3af3 | 2006-05-23 21:08:24 +0000 | [diff] [blame] | 527 | bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; |
| 528 | if (isVarArg) |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 529 | VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 530 | RegSaveFrameIndex = 0xAAAAAAA; // X86-64 only. |
| 531 | ReturnAddrIndex = 0; // No return address slot generated yet. |
| 532 | BytesToPopOnReturn = 0; // Callee pops nothing. |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 533 | BytesCallerReserves = ArgOffset; |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 534 | |
Anton Korobeynikov | df78611 | 2006-11-10 00:48:11 +0000 | [diff] [blame] | 535 | // If this is a struct return on, the callee pops the hidden struct |
| 536 | // pointer. This is common for Darwin/X86, Linux & Mingw32 targets. |
| 537 | if (MF.getFunction()->getCallingConv() == CallingConv::CSRet) |
Chris Lattner | 2d29709 | 2006-05-23 18:50:38 +0000 | [diff] [blame] | 538 | BytesToPopOnReturn = 4; |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 539 | |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 540 | // Return the new list of results. |
| 541 | std::vector<MVT::ValueType> RetVTs(Op.Val->value_begin(), |
| 542 | Op.Val->value_end()); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 543 | return DAG.getNode(ISD::MERGE_VALUES, RetVTs, &ArgValues[0],ArgValues.size()); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 544 | } |
| 545 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 546 | |
| 547 | SDOperand X86TargetLowering::LowerCCCCallTo(SDOperand Op, SelectionDAG &DAG) { |
| 548 | SDOperand Chain = Op.getOperand(0); |
| 549 | unsigned CallingConv= cast<ConstantSDNode>(Op.getOperand(1))->getValue(); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 550 | bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0; |
| 551 | SDOperand Callee = Op.getOperand(4); |
| 552 | MVT::ValueType RetVT= Op.Val->getValueType(0); |
| 553 | unsigned NumOps = (Op.getNumOperands() - 5) / 2; |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 554 | |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 555 | // Keep track of the number of XMM regs passed so far. |
| 556 | unsigned NumXMMRegs = 0; |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 557 | static const unsigned XMMArgRegs[] = { |
Evan Cheng | 1d6a9b3 | 2006-05-26 19:22:06 +0000 | [diff] [blame] | 558 | X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 559 | }; |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 560 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 561 | // Count how many bytes are to be pushed on the stack. |
| 562 | unsigned NumBytes = 0; |
| 563 | for (unsigned i = 0; i != NumOps; ++i) { |
| 564 | SDOperand Arg = Op.getOperand(5+2*i); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 565 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 566 | switch (Arg.getValueType()) { |
| 567 | default: assert(0 && "Unexpected ValueType for argument!"); |
| 568 | case MVT::i8: |
| 569 | case MVT::i16: |
| 570 | case MVT::i32: |
| 571 | case MVT::f32: |
| 572 | NumBytes += 4; |
| 573 | break; |
| 574 | case MVT::i64: |
| 575 | case MVT::f64: |
| 576 | NumBytes += 8; |
| 577 | break; |
| 578 | case MVT::v16i8: |
| 579 | case MVT::v8i16: |
| 580 | case MVT::v4i32: |
| 581 | case MVT::v2i64: |
| 582 | case MVT::v4f32: |
Evan Cheng | 25e71d1 | 2006-05-25 22:38:31 +0000 | [diff] [blame] | 583 | case MVT::v2f64: |
Evan Cheng | 1d6a9b3 | 2006-05-26 19:22:06 +0000 | [diff] [blame] | 584 | if (NumXMMRegs < 4) |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 585 | ++NumXMMRegs; |
Evan Cheng | 3fddf24 | 2006-05-26 20:37:47 +0000 | [diff] [blame] | 586 | else { |
| 587 | // XMM arguments have to be aligned on 16-byte boundary. |
| 588 | NumBytes = ((NumBytes + 15) / 16) * 16; |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 589 | NumBytes += 16; |
Evan Cheng | 3fddf24 | 2006-05-26 20:37:47 +0000 | [diff] [blame] | 590 | } |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 591 | break; |
| 592 | } |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 593 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 594 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 595 | Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy())); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 596 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 597 | // Arguments go on the stack in reverse order, as specified by the ABI. |
| 598 | unsigned ArgOffset = 0; |
| 599 | NumXMMRegs = 0; |
| 600 | std::vector<std::pair<unsigned, SDOperand> > RegsToPass; |
| 601 | std::vector<SDOperand> MemOpChains; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 602 | SDOperand StackPtr = DAG.getRegister(X86StackPtr, getPointerTy()); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 603 | for (unsigned i = 0; i != NumOps; ++i) { |
| 604 | SDOperand Arg = Op.getOperand(5+2*i); |
| 605 | |
| 606 | switch (Arg.getValueType()) { |
| 607 | default: assert(0 && "Unexpected ValueType for argument!"); |
| 608 | case MVT::i8: |
Evan Cheng | 6b5783d | 2006-05-25 18:56:34 +0000 | [diff] [blame] | 609 | case MVT::i16: { |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 610 | // Promote the integer to 32 bits. If the input type is signed use a |
| 611 | // sign extend, otherwise use a zero extend. |
| 612 | unsigned ExtOp = |
| 613 | dyn_cast<ConstantSDNode>(Op.getOperand(5+2*i+1))->getValue() ? |
| 614 | ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
| 615 | Arg = DAG.getNode(ExtOp, MVT::i32, Arg); |
Evan Cheng | 6b5783d | 2006-05-25 18:56:34 +0000 | [diff] [blame] | 616 | } |
| 617 | // Fallthrough |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 618 | |
| 619 | case MVT::i32: |
| 620 | case MVT::f32: { |
| 621 | SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); |
| 622 | PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 623 | MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0)); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 624 | ArgOffset += 4; |
| 625 | break; |
| 626 | } |
| 627 | case MVT::i64: |
| 628 | case MVT::f64: { |
| 629 | SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); |
| 630 | PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 631 | MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0)); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 632 | ArgOffset += 8; |
| 633 | break; |
| 634 | } |
| 635 | case MVT::v16i8: |
| 636 | case MVT::v8i16: |
| 637 | case MVT::v4i32: |
| 638 | case MVT::v2i64: |
| 639 | case MVT::v4f32: |
Evan Cheng | 25e71d1 | 2006-05-25 22:38:31 +0000 | [diff] [blame] | 640 | case MVT::v2f64: |
Evan Cheng | 1d6a9b3 | 2006-05-26 19:22:06 +0000 | [diff] [blame] | 641 | if (NumXMMRegs < 4) { |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 642 | RegsToPass.push_back(std::make_pair(XMMArgRegs[NumXMMRegs], Arg)); |
| 643 | NumXMMRegs++; |
| 644 | } else { |
Evan Cheng | 3fddf24 | 2006-05-26 20:37:47 +0000 | [diff] [blame] | 645 | // XMM arguments have to be aligned on 16-byte boundary. |
| 646 | ArgOffset = ((ArgOffset + 15) / 16) * 16; |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 647 | SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 648 | PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 649 | MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0)); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 650 | ArgOffset += 16; |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 651 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 652 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 653 | } |
| 654 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 655 | if (!MemOpChains.empty()) |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 656 | Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, |
| 657 | &MemOpChains[0], MemOpChains.size()); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 658 | |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 659 | // Build a sequence of copy-to-reg nodes chained together with token chain |
| 660 | // and flag operands which copy the outgoing args into registers. |
| 661 | SDOperand InFlag; |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 662 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { |
| 663 | Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, |
| 664 | InFlag); |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 665 | InFlag = Chain.getValue(1); |
| 666 | } |
| 667 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 668 | // If the callee is a GlobalAddress node (quite common, every direct call is) |
| 669 | // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. |
Anton Korobeynikov | a598685 | 2006-11-20 10:46:14 +0000 | [diff] [blame] | 670 | if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { |
| 671 | // We should use extra load for direct calls to dllimported functions |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 672 | if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(), true)) |
Anton Korobeynikov | a598685 | 2006-11-20 10:46:14 +0000 | [diff] [blame] | 673 | Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); |
| 674 | } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 675 | Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); |
| 676 | |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 677 | std::vector<MVT::ValueType> NodeTys; |
| 678 | NodeTys.push_back(MVT::Other); // Returns a chain |
| 679 | NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. |
| 680 | std::vector<SDOperand> Ops; |
| 681 | Ops.push_back(Chain); |
| 682 | Ops.push_back(Callee); |
Evan Cheng | b69d113 | 2006-06-14 18:17:40 +0000 | [diff] [blame] | 683 | |
| 684 | // Add argument registers to the end of the list so that they are known live |
| 685 | // into the call. |
| 686 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 687 | Ops.push_back(DAG.getRegister(RegsToPass[i].first, |
Evan Cheng | b69d113 | 2006-06-14 18:17:40 +0000 | [diff] [blame] | 688 | RegsToPass[i].second.getValueType())); |
| 689 | |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 690 | if (InFlag.Val) |
| 691 | Ops.push_back(InFlag); |
Evan Cheng | d90eb7f | 2006-01-05 00:27:02 +0000 | [diff] [blame] | 692 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 693 | Chain = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL, |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 694 | NodeTys, &Ops[0], Ops.size()); |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 695 | InFlag = Chain.getValue(1); |
Evan Cheng | d90eb7f | 2006-01-05 00:27:02 +0000 | [diff] [blame] | 696 | |
Chris Lattner | 2d29709 | 2006-05-23 18:50:38 +0000 | [diff] [blame] | 697 | // Create the CALLSEQ_END node. |
| 698 | unsigned NumBytesForCalleeToPush = 0; |
| 699 | |
Anton Korobeynikov | df78611 | 2006-11-10 00:48:11 +0000 | [diff] [blame] | 700 | // If this is is a call to a struct-return function, the callee |
Chris Lattner | 2d29709 | 2006-05-23 18:50:38 +0000 | [diff] [blame] | 701 | // pops the hidden struct pointer, so we have to push it back. |
Anton Korobeynikov | df78611 | 2006-11-10 00:48:11 +0000 | [diff] [blame] | 702 | // This is common for Darwin/X86, Linux & Mingw32 targets. |
| 703 | if (CallingConv == CallingConv::CSRet) |
Chris Lattner | 2d29709 | 2006-05-23 18:50:38 +0000 | [diff] [blame] | 704 | NumBytesForCalleeToPush = 4; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 705 | |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 706 | NodeTys.clear(); |
| 707 | NodeTys.push_back(MVT::Other); // Returns a chain |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 708 | if (RetVT != MVT::Other) |
| 709 | NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 710 | Ops.clear(); |
| 711 | Ops.push_back(Chain); |
| 712 | Ops.push_back(DAG.getConstant(NumBytes, getPointerTy())); |
Chris Lattner | 2d29709 | 2006-05-23 18:50:38 +0000 | [diff] [blame] | 713 | Ops.push_back(DAG.getConstant(NumBytesForCalleeToPush, getPointerTy())); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 714 | Ops.push_back(InFlag); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 715 | Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size()); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 716 | if (RetVT != MVT::Other) |
| 717 | InFlag = Chain.getValue(1); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 718 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 719 | std::vector<SDOperand> ResultVals; |
| 720 | NodeTys.clear(); |
| 721 | switch (RetVT) { |
| 722 | default: assert(0 && "Unknown value type to return!"); |
| 723 | case MVT::Other: break; |
| 724 | case MVT::i8: |
| 725 | Chain = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag).getValue(1); |
| 726 | ResultVals.push_back(Chain.getValue(0)); |
| 727 | NodeTys.push_back(MVT::i8); |
| 728 | break; |
| 729 | case MVT::i16: |
| 730 | Chain = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag).getValue(1); |
| 731 | ResultVals.push_back(Chain.getValue(0)); |
| 732 | NodeTys.push_back(MVT::i16); |
| 733 | break; |
| 734 | case MVT::i32: |
| 735 | if (Op.Val->getValueType(1) == MVT::i32) { |
| 736 | Chain = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag).getValue(1); |
| 737 | ResultVals.push_back(Chain.getValue(0)); |
| 738 | Chain = DAG.getCopyFromReg(Chain, X86::EDX, MVT::i32, |
| 739 | Chain.getValue(2)).getValue(1); |
| 740 | ResultVals.push_back(Chain.getValue(0)); |
| 741 | NodeTys.push_back(MVT::i32); |
| 742 | } else { |
| 743 | Chain = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag).getValue(1); |
| 744 | ResultVals.push_back(Chain.getValue(0)); |
Evan Cheng | d90eb7f | 2006-01-05 00:27:02 +0000 | [diff] [blame] | 745 | } |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 746 | NodeTys.push_back(MVT::i32); |
| 747 | break; |
| 748 | case MVT::v16i8: |
| 749 | case MVT::v8i16: |
| 750 | case MVT::v4i32: |
| 751 | case MVT::v2i64: |
| 752 | case MVT::v4f32: |
| 753 | case MVT::v2f64: |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 754 | Chain = DAG.getCopyFromReg(Chain, X86::XMM0, RetVT, InFlag).getValue(1); |
| 755 | ResultVals.push_back(Chain.getValue(0)); |
| 756 | NodeTys.push_back(RetVT); |
| 757 | break; |
| 758 | case MVT::f32: |
| 759 | case MVT::f64: { |
| 760 | std::vector<MVT::ValueType> Tys; |
| 761 | Tys.push_back(MVT::f64); |
| 762 | Tys.push_back(MVT::Other); |
| 763 | Tys.push_back(MVT::Flag); |
| 764 | std::vector<SDOperand> Ops; |
| 765 | Ops.push_back(Chain); |
| 766 | Ops.push_back(InFlag); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 767 | SDOperand RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 768 | &Ops[0], Ops.size()); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 769 | Chain = RetVal.getValue(1); |
| 770 | InFlag = RetVal.getValue(2); |
| 771 | if (X86ScalarSSE) { |
| 772 | // FIXME: Currently the FST is flagged to the FP_GET_RESULT. This |
| 773 | // shouldn't be necessary except that RFP cannot be live across |
| 774 | // multiple blocks. When stackifier is fixed, they can be uncoupled. |
| 775 | MachineFunction &MF = DAG.getMachineFunction(); |
| 776 | int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); |
| 777 | SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
| 778 | Tys.clear(); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 779 | Tys.push_back(MVT::Other); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 780 | Ops.clear(); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 781 | Ops.push_back(Chain); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 782 | Ops.push_back(RetVal); |
| 783 | Ops.push_back(StackSlot); |
| 784 | Ops.push_back(DAG.getValueType(RetVT)); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 785 | Ops.push_back(InFlag); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 786 | Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 787 | RetVal = DAG.getLoad(RetVT, Chain, StackSlot, NULL, 0); |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 788 | Chain = RetVal.getValue(1); |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 789 | } |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 790 | |
| 791 | if (RetVT == MVT::f32 && !X86ScalarSSE) |
| 792 | // FIXME: we would really like to remember that this FP_ROUND |
| 793 | // operation is okay to eliminate if we allow excess FP precision. |
| 794 | RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal); |
| 795 | ResultVals.push_back(RetVal); |
| 796 | NodeTys.push_back(RetVT); |
| 797 | break; |
| 798 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 799 | } |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 800 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 801 | // If the function returns void, just return the chain. |
| 802 | if (ResultVals.empty()) |
| 803 | return Chain; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 804 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 805 | // Otherwise, merge everything together with a MERGE_VALUES node. |
| 806 | NodeTys.push_back(MVT::Other); |
| 807 | ResultVals.push_back(Chain); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 808 | SDOperand Res = DAG.getNode(ISD::MERGE_VALUES, NodeTys, |
| 809 | &ResultVals[0], ResultVals.size()); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 810 | return Res.getValue(Op.ResNo); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 811 | } |
| 812 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 813 | |
| 814 | //===----------------------------------------------------------------------===// |
| 815 | // X86-64 C Calling Convention implementation |
| 816 | //===----------------------------------------------------------------------===// |
| 817 | |
| 818 | /// HowToPassX86_64CCCArgument - Returns how an formal argument of the specified |
| 819 | /// type should be passed. If it is through stack, returns the size of the stack |
| 820 | /// slot; if it is through integer or XMM register, returns the number of |
| 821 | /// integer or XMM registers are needed. |
| 822 | static void |
| 823 | HowToPassX86_64CCCArgument(MVT::ValueType ObjectVT, |
| 824 | unsigned NumIntRegs, unsigned NumXMMRegs, |
| 825 | unsigned &ObjSize, unsigned &ObjIntRegs, |
| 826 | unsigned &ObjXMMRegs) { |
| 827 | ObjSize = 0; |
| 828 | ObjIntRegs = 0; |
| 829 | ObjXMMRegs = 0; |
| 830 | |
| 831 | switch (ObjectVT) { |
| 832 | default: assert(0 && "Unhandled argument type!"); |
| 833 | case MVT::i8: |
| 834 | case MVT::i16: |
| 835 | case MVT::i32: |
| 836 | case MVT::i64: |
| 837 | if (NumIntRegs < 6) |
| 838 | ObjIntRegs = 1; |
| 839 | else { |
| 840 | switch (ObjectVT) { |
| 841 | default: break; |
| 842 | case MVT::i8: ObjSize = 1; break; |
| 843 | case MVT::i16: ObjSize = 2; break; |
| 844 | case MVT::i32: ObjSize = 4; break; |
| 845 | case MVT::i64: ObjSize = 8; break; |
| 846 | } |
| 847 | } |
| 848 | break; |
| 849 | case MVT::f32: |
| 850 | case MVT::f64: |
| 851 | case MVT::v16i8: |
| 852 | case MVT::v8i16: |
| 853 | case MVT::v4i32: |
| 854 | case MVT::v2i64: |
| 855 | case MVT::v4f32: |
| 856 | case MVT::v2f64: |
| 857 | if (NumXMMRegs < 8) |
| 858 | ObjXMMRegs = 1; |
| 859 | else { |
| 860 | switch (ObjectVT) { |
| 861 | default: break; |
| 862 | case MVT::f32: ObjSize = 4; break; |
| 863 | case MVT::f64: ObjSize = 8; break; |
| 864 | case MVT::v16i8: |
| 865 | case MVT::v8i16: |
| 866 | case MVT::v4i32: |
| 867 | case MVT::v2i64: |
| 868 | case MVT::v4f32: |
| 869 | case MVT::v2f64: ObjSize = 16; break; |
| 870 | } |
| 871 | break; |
| 872 | } |
| 873 | } |
| 874 | } |
| 875 | |
| 876 | SDOperand |
| 877 | X86TargetLowering::LowerX86_64CCCArguments(SDOperand Op, SelectionDAG &DAG) { |
| 878 | unsigned NumArgs = Op.Val->getNumValues() - 1; |
| 879 | MachineFunction &MF = DAG.getMachineFunction(); |
| 880 | MachineFrameInfo *MFI = MF.getFrameInfo(); |
| 881 | SDOperand Root = Op.getOperand(0); |
| 882 | bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; |
| 883 | std::vector<SDOperand> ArgValues; |
| 884 | |
| 885 | // Add DAG nodes to load the arguments... On entry to a function on the X86, |
| 886 | // the stack frame looks like this: |
| 887 | // |
| 888 | // [RSP] -- return address |
| 889 | // [RSP + 8] -- first nonreg argument (leftmost lexically) |
| 890 | // [RSP +16] -- second nonreg argument, if 1st argument is <= 8 bytes in size |
| 891 | // ... |
| 892 | // |
| 893 | unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot |
| 894 | unsigned NumIntRegs = 0; // Int regs used for parameter passing. |
| 895 | unsigned NumXMMRegs = 0; // XMM regs used for parameter passing. |
| 896 | |
| 897 | static const unsigned GPR8ArgRegs[] = { |
| 898 | X86::DIL, X86::SIL, X86::DL, X86::CL, X86::R8B, X86::R9B |
| 899 | }; |
| 900 | static const unsigned GPR16ArgRegs[] = { |
| 901 | X86::DI, X86::SI, X86::DX, X86::CX, X86::R8W, X86::R9W |
| 902 | }; |
| 903 | static const unsigned GPR32ArgRegs[] = { |
| 904 | X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D |
| 905 | }; |
| 906 | static const unsigned GPR64ArgRegs[] = { |
| 907 | X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 |
| 908 | }; |
| 909 | static const unsigned XMMArgRegs[] = { |
| 910 | X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, |
| 911 | X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 |
| 912 | }; |
| 913 | |
| 914 | for (unsigned i = 0; i < NumArgs; ++i) { |
| 915 | MVT::ValueType ObjectVT = Op.getValue(i).getValueType(); |
| 916 | unsigned ArgIncrement = 8; |
| 917 | unsigned ObjSize = 0; |
| 918 | unsigned ObjIntRegs = 0; |
| 919 | unsigned ObjXMMRegs = 0; |
| 920 | |
| 921 | // FIXME: __int128 and long double support? |
| 922 | HowToPassX86_64CCCArgument(ObjectVT, NumIntRegs, NumXMMRegs, |
| 923 | ObjSize, ObjIntRegs, ObjXMMRegs); |
| 924 | if (ObjSize > 8) |
| 925 | ArgIncrement = ObjSize; |
| 926 | |
| 927 | unsigned Reg = 0; |
| 928 | SDOperand ArgValue; |
| 929 | if (ObjIntRegs || ObjXMMRegs) { |
| 930 | switch (ObjectVT) { |
| 931 | default: assert(0 && "Unhandled argument type!"); |
| 932 | case MVT::i8: |
| 933 | case MVT::i16: |
| 934 | case MVT::i32: |
| 935 | case MVT::i64: { |
| 936 | TargetRegisterClass *RC = NULL; |
| 937 | switch (ObjectVT) { |
| 938 | default: break; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 939 | case MVT::i8: |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 940 | RC = X86::GR8RegisterClass; |
| 941 | Reg = GPR8ArgRegs[NumIntRegs]; |
| 942 | break; |
| 943 | case MVT::i16: |
| 944 | RC = X86::GR16RegisterClass; |
| 945 | Reg = GPR16ArgRegs[NumIntRegs]; |
| 946 | break; |
| 947 | case MVT::i32: |
| 948 | RC = X86::GR32RegisterClass; |
| 949 | Reg = GPR32ArgRegs[NumIntRegs]; |
| 950 | break; |
| 951 | case MVT::i64: |
| 952 | RC = X86::GR64RegisterClass; |
| 953 | Reg = GPR64ArgRegs[NumIntRegs]; |
| 954 | break; |
| 955 | } |
| 956 | Reg = AddLiveIn(MF, Reg, RC); |
| 957 | ArgValue = DAG.getCopyFromReg(Root, Reg, ObjectVT); |
| 958 | break; |
| 959 | } |
| 960 | case MVT::f32: |
| 961 | case MVT::f64: |
| 962 | case MVT::v16i8: |
| 963 | case MVT::v8i16: |
| 964 | case MVT::v4i32: |
| 965 | case MVT::v2i64: |
| 966 | case MVT::v4f32: |
| 967 | case MVT::v2f64: { |
| 968 | TargetRegisterClass *RC= (ObjectVT == MVT::f32) ? |
| 969 | X86::FR32RegisterClass : ((ObjectVT == MVT::f64) ? |
| 970 | X86::FR64RegisterClass : X86::VR128RegisterClass); |
| 971 | Reg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], RC); |
| 972 | ArgValue = DAG.getCopyFromReg(Root, Reg, ObjectVT); |
| 973 | break; |
| 974 | } |
| 975 | } |
| 976 | NumIntRegs += ObjIntRegs; |
| 977 | NumXMMRegs += ObjXMMRegs; |
| 978 | } else if (ObjSize) { |
| 979 | // XMM arguments have to be aligned on 16-byte boundary. |
| 980 | if (ObjSize == 16) |
| 981 | ArgOffset = ((ArgOffset + 15) / 16) * 16; |
| 982 | // Create the SelectionDAG nodes corresponding to a load from this |
| 983 | // parameter. |
| 984 | int FI = MFI->CreateFixedObject(ObjSize, ArgOffset); |
| 985 | SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy()); |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 986 | ArgValue = DAG.getLoad(Op.Val->getValueType(i), Root, FIN, NULL, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 987 | ArgOffset += ArgIncrement; // Move on to the next argument. |
| 988 | } |
| 989 | |
| 990 | ArgValues.push_back(ArgValue); |
| 991 | } |
| 992 | |
| 993 | // If the function takes variable number of arguments, make a frame index for |
| 994 | // the start of the first vararg value... for expansion of llvm.va_start. |
| 995 | if (isVarArg) { |
| 996 | // For X86-64, if there are vararg parameters that are passed via |
| 997 | // registers, then we must store them to their spots on the stack so they |
| 998 | // may be loaded by deferencing the result of va_next. |
| 999 | VarArgsGPOffset = NumIntRegs * 8; |
| 1000 | VarArgsFPOffset = 6 * 8 + NumXMMRegs * 16; |
| 1001 | VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset); |
| 1002 | RegSaveFrameIndex = MFI->CreateStackObject(6 * 8 + 8 * 16, 16); |
| 1003 | |
| 1004 | // Store the integer parameter registers. |
| 1005 | std::vector<SDOperand> MemOps; |
| 1006 | SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); |
| 1007 | SDOperand FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, |
| 1008 | DAG.getConstant(VarArgsGPOffset, getPointerTy())); |
| 1009 | for (; NumIntRegs != 6; ++NumIntRegs) { |
| 1010 | unsigned VReg = AddLiveIn(MF, GPR64ArgRegs[NumIntRegs], |
| 1011 | X86::GR64RegisterClass); |
| 1012 | SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::i64); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 1013 | SDOperand Store = DAG.getStore(Val.getValue(1), Val, FIN, NULL, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1014 | MemOps.push_back(Store); |
| 1015 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, |
| 1016 | DAG.getConstant(8, getPointerTy())); |
| 1017 | } |
| 1018 | |
| 1019 | // Now store the XMM (fp + vector) parameter registers. |
| 1020 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, |
| 1021 | DAG.getConstant(VarArgsFPOffset, getPointerTy())); |
| 1022 | for (; NumXMMRegs != 8; ++NumXMMRegs) { |
| 1023 | unsigned VReg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], |
| 1024 | X86::VR128RegisterClass); |
| 1025 | SDOperand Val = DAG.getCopyFromReg(Root, VReg, MVT::v4f32); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 1026 | SDOperand Store = DAG.getStore(Val.getValue(1), Val, FIN, NULL, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1027 | MemOps.push_back(Store); |
| 1028 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, |
| 1029 | DAG.getConstant(16, getPointerTy())); |
| 1030 | } |
| 1031 | if (!MemOps.empty()) |
| 1032 | Root = DAG.getNode(ISD::TokenFactor, MVT::Other, |
| 1033 | &MemOps[0], MemOps.size()); |
| 1034 | } |
| 1035 | |
| 1036 | ArgValues.push_back(Root); |
| 1037 | |
| 1038 | ReturnAddrIndex = 0; // No return address slot generated yet. |
| 1039 | BytesToPopOnReturn = 0; // Callee pops nothing. |
| 1040 | BytesCallerReserves = ArgOffset; |
| 1041 | |
| 1042 | // Return the new list of results. |
| 1043 | std::vector<MVT::ValueType> RetVTs(Op.Val->value_begin(), |
| 1044 | Op.Val->value_end()); |
| 1045 | return DAG.getNode(ISD::MERGE_VALUES, RetVTs, &ArgValues[0],ArgValues.size()); |
| 1046 | } |
| 1047 | |
| 1048 | SDOperand |
| 1049 | X86TargetLowering::LowerX86_64CCCCallTo(SDOperand Op, SelectionDAG &DAG) { |
| 1050 | SDOperand Chain = Op.getOperand(0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1051 | bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; |
| 1052 | bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0; |
| 1053 | SDOperand Callee = Op.getOperand(4); |
| 1054 | MVT::ValueType RetVT= Op.Val->getValueType(0); |
| 1055 | unsigned NumOps = (Op.getNumOperands() - 5) / 2; |
| 1056 | |
| 1057 | // Count how many bytes are to be pushed on the stack. |
| 1058 | unsigned NumBytes = 0; |
| 1059 | unsigned NumIntRegs = 0; // Int regs used for parameter passing. |
| 1060 | unsigned NumXMMRegs = 0; // XMM regs used for parameter passing. |
| 1061 | |
| 1062 | static const unsigned GPR8ArgRegs[] = { |
| 1063 | X86::DIL, X86::SIL, X86::DL, X86::CL, X86::R8B, X86::R9B |
| 1064 | }; |
| 1065 | static const unsigned GPR16ArgRegs[] = { |
| 1066 | X86::DI, X86::SI, X86::DX, X86::CX, X86::R8W, X86::R9W |
| 1067 | }; |
| 1068 | static const unsigned GPR32ArgRegs[] = { |
| 1069 | X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D |
| 1070 | }; |
| 1071 | static const unsigned GPR64ArgRegs[] = { |
| 1072 | X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 |
| 1073 | }; |
| 1074 | static const unsigned XMMArgRegs[] = { |
| 1075 | X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, |
| 1076 | X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 |
| 1077 | }; |
| 1078 | |
| 1079 | for (unsigned i = 0; i != NumOps; ++i) { |
| 1080 | SDOperand Arg = Op.getOperand(5+2*i); |
| 1081 | MVT::ValueType ArgVT = Arg.getValueType(); |
| 1082 | |
| 1083 | switch (ArgVT) { |
| 1084 | default: assert(0 && "Unknown value type!"); |
| 1085 | case MVT::i8: |
| 1086 | case MVT::i16: |
| 1087 | case MVT::i32: |
| 1088 | case MVT::i64: |
| 1089 | if (NumIntRegs < 6) |
| 1090 | ++NumIntRegs; |
| 1091 | else |
| 1092 | NumBytes += 8; |
| 1093 | break; |
| 1094 | case MVT::f32: |
| 1095 | case MVT::f64: |
| 1096 | case MVT::v16i8: |
| 1097 | case MVT::v8i16: |
| 1098 | case MVT::v4i32: |
| 1099 | case MVT::v2i64: |
| 1100 | case MVT::v4f32: |
| 1101 | case MVT::v2f64: |
| 1102 | if (NumXMMRegs < 8) |
| 1103 | NumXMMRegs++; |
| 1104 | else if (ArgVT == MVT::f32 || ArgVT == MVT::f64) |
| 1105 | NumBytes += 8; |
| 1106 | else { |
| 1107 | // XMM arguments have to be aligned on 16-byte boundary. |
| 1108 | NumBytes = ((NumBytes + 15) / 16) * 16; |
| 1109 | NumBytes += 16; |
| 1110 | } |
| 1111 | break; |
| 1112 | } |
| 1113 | } |
| 1114 | |
| 1115 | Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy())); |
| 1116 | |
| 1117 | // Arguments go on the stack in reverse order, as specified by the ABI. |
| 1118 | unsigned ArgOffset = 0; |
| 1119 | NumIntRegs = 0; |
| 1120 | NumXMMRegs = 0; |
| 1121 | std::vector<std::pair<unsigned, SDOperand> > RegsToPass; |
| 1122 | std::vector<SDOperand> MemOpChains; |
| 1123 | SDOperand StackPtr = DAG.getRegister(X86StackPtr, getPointerTy()); |
| 1124 | for (unsigned i = 0; i != NumOps; ++i) { |
| 1125 | SDOperand Arg = Op.getOperand(5+2*i); |
| 1126 | MVT::ValueType ArgVT = Arg.getValueType(); |
| 1127 | |
| 1128 | switch (ArgVT) { |
| 1129 | default: assert(0 && "Unexpected ValueType for argument!"); |
| 1130 | case MVT::i8: |
| 1131 | case MVT::i16: |
| 1132 | case MVT::i32: |
| 1133 | case MVT::i64: |
| 1134 | if (NumIntRegs < 6) { |
| 1135 | unsigned Reg = 0; |
| 1136 | switch (ArgVT) { |
| 1137 | default: break; |
| 1138 | case MVT::i8: Reg = GPR8ArgRegs[NumIntRegs]; break; |
| 1139 | case MVT::i16: Reg = GPR16ArgRegs[NumIntRegs]; break; |
| 1140 | case MVT::i32: Reg = GPR32ArgRegs[NumIntRegs]; break; |
| 1141 | case MVT::i64: Reg = GPR64ArgRegs[NumIntRegs]; break; |
| 1142 | } |
| 1143 | RegsToPass.push_back(std::make_pair(Reg, Arg)); |
| 1144 | ++NumIntRegs; |
| 1145 | } else { |
| 1146 | SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); |
| 1147 | PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 1148 | MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0)); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1149 | ArgOffset += 8; |
| 1150 | } |
| 1151 | break; |
| 1152 | case MVT::f32: |
| 1153 | case MVT::f64: |
| 1154 | case MVT::v16i8: |
| 1155 | case MVT::v8i16: |
| 1156 | case MVT::v4i32: |
| 1157 | case MVT::v2i64: |
| 1158 | case MVT::v4f32: |
| 1159 | case MVT::v2f64: |
| 1160 | if (NumXMMRegs < 8) { |
| 1161 | RegsToPass.push_back(std::make_pair(XMMArgRegs[NumXMMRegs], Arg)); |
| 1162 | NumXMMRegs++; |
| 1163 | } else { |
| 1164 | if (ArgVT != MVT::f32 && ArgVT != MVT::f64) { |
| 1165 | // XMM arguments have to be aligned on 16-byte boundary. |
| 1166 | ArgOffset = ((ArgOffset + 15) / 16) * 16; |
| 1167 | } |
| 1168 | SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); |
| 1169 | PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 1170 | MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0)); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1171 | if (ArgVT == MVT::f32 || ArgVT == MVT::f64) |
| 1172 | ArgOffset += 8; |
| 1173 | else |
| 1174 | ArgOffset += 16; |
| 1175 | } |
| 1176 | } |
| 1177 | } |
| 1178 | |
| 1179 | if (!MemOpChains.empty()) |
| 1180 | Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, |
| 1181 | &MemOpChains[0], MemOpChains.size()); |
| 1182 | |
| 1183 | // Build a sequence of copy-to-reg nodes chained together with token chain |
| 1184 | // and flag operands which copy the outgoing args into registers. |
| 1185 | SDOperand InFlag; |
| 1186 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { |
| 1187 | Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, |
| 1188 | InFlag); |
| 1189 | InFlag = Chain.getValue(1); |
| 1190 | } |
| 1191 | |
| 1192 | if (isVarArg) { |
| 1193 | // From AMD64 ABI document: |
| 1194 | // For calls that may call functions that use varargs or stdargs |
| 1195 | // (prototype-less calls or calls to functions containing ellipsis (...) in |
| 1196 | // the declaration) %al is used as hidden argument to specify the number |
| 1197 | // of SSE registers used. The contents of %al do not need to match exactly |
| 1198 | // the number of registers, but must be an ubound on the number of SSE |
| 1199 | // registers used and is in the range 0 - 8 inclusive. |
| 1200 | Chain = DAG.getCopyToReg(Chain, X86::AL, |
| 1201 | DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); |
| 1202 | InFlag = Chain.getValue(1); |
| 1203 | } |
| 1204 | |
| 1205 | // If the callee is a GlobalAddress node (quite common, every direct call is) |
| 1206 | // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. |
Anton Korobeynikov | a598685 | 2006-11-20 10:46:14 +0000 | [diff] [blame] | 1207 | if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { |
| 1208 | // We should use extra load for direct calls to dllimported functions |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1209 | if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(), true)) |
Anton Korobeynikov | a598685 | 2006-11-20 10:46:14 +0000 | [diff] [blame] | 1210 | Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); |
| 1211 | } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1212 | Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); |
| 1213 | |
| 1214 | std::vector<MVT::ValueType> NodeTys; |
| 1215 | NodeTys.push_back(MVT::Other); // Returns a chain |
| 1216 | NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. |
| 1217 | std::vector<SDOperand> Ops; |
| 1218 | Ops.push_back(Chain); |
| 1219 | Ops.push_back(Callee); |
| 1220 | |
| 1221 | // Add argument registers to the end of the list so that they are known live |
| 1222 | // into the call. |
| 1223 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1224 | Ops.push_back(DAG.getRegister(RegsToPass[i].first, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1225 | RegsToPass[i].second.getValueType())); |
| 1226 | |
| 1227 | if (InFlag.Val) |
| 1228 | Ops.push_back(InFlag); |
| 1229 | |
| 1230 | // FIXME: Do not generate X86ISD::TAILCALL for now. |
| 1231 | Chain = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL, |
| 1232 | NodeTys, &Ops[0], Ops.size()); |
| 1233 | InFlag = Chain.getValue(1); |
| 1234 | |
| 1235 | NodeTys.clear(); |
| 1236 | NodeTys.push_back(MVT::Other); // Returns a chain |
| 1237 | if (RetVT != MVT::Other) |
| 1238 | NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. |
| 1239 | Ops.clear(); |
| 1240 | Ops.push_back(Chain); |
| 1241 | Ops.push_back(DAG.getConstant(NumBytes, getPointerTy())); |
| 1242 | Ops.push_back(DAG.getConstant(0, getPointerTy())); |
| 1243 | Ops.push_back(InFlag); |
| 1244 | Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size()); |
| 1245 | if (RetVT != MVT::Other) |
| 1246 | InFlag = Chain.getValue(1); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1247 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1248 | std::vector<SDOperand> ResultVals; |
| 1249 | NodeTys.clear(); |
| 1250 | switch (RetVT) { |
| 1251 | default: assert(0 && "Unknown value type to return!"); |
| 1252 | case MVT::Other: break; |
| 1253 | case MVT::i8: |
| 1254 | Chain = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag).getValue(1); |
| 1255 | ResultVals.push_back(Chain.getValue(0)); |
| 1256 | NodeTys.push_back(MVT::i8); |
| 1257 | break; |
| 1258 | case MVT::i16: |
| 1259 | Chain = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag).getValue(1); |
| 1260 | ResultVals.push_back(Chain.getValue(0)); |
| 1261 | NodeTys.push_back(MVT::i16); |
| 1262 | break; |
| 1263 | case MVT::i32: |
| 1264 | Chain = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag).getValue(1); |
| 1265 | ResultVals.push_back(Chain.getValue(0)); |
| 1266 | NodeTys.push_back(MVT::i32); |
| 1267 | break; |
| 1268 | case MVT::i64: |
| 1269 | if (Op.Val->getValueType(1) == MVT::i64) { |
| 1270 | // FIXME: __int128 support? |
| 1271 | Chain = DAG.getCopyFromReg(Chain, X86::RAX, MVT::i64, InFlag).getValue(1); |
| 1272 | ResultVals.push_back(Chain.getValue(0)); |
| 1273 | Chain = DAG.getCopyFromReg(Chain, X86::RDX, MVT::i64, |
| 1274 | Chain.getValue(2)).getValue(1); |
| 1275 | ResultVals.push_back(Chain.getValue(0)); |
| 1276 | NodeTys.push_back(MVT::i64); |
| 1277 | } else { |
| 1278 | Chain = DAG.getCopyFromReg(Chain, X86::RAX, MVT::i64, InFlag).getValue(1); |
| 1279 | ResultVals.push_back(Chain.getValue(0)); |
| 1280 | } |
| 1281 | NodeTys.push_back(MVT::i64); |
| 1282 | break; |
| 1283 | case MVT::f32: |
| 1284 | case MVT::f64: |
| 1285 | case MVT::v16i8: |
| 1286 | case MVT::v8i16: |
| 1287 | case MVT::v4i32: |
| 1288 | case MVT::v2i64: |
| 1289 | case MVT::v4f32: |
| 1290 | case MVT::v2f64: |
| 1291 | // FIXME: long double support? |
| 1292 | Chain = DAG.getCopyFromReg(Chain, X86::XMM0, RetVT, InFlag).getValue(1); |
| 1293 | ResultVals.push_back(Chain.getValue(0)); |
| 1294 | NodeTys.push_back(RetVT); |
| 1295 | break; |
| 1296 | } |
| 1297 | |
| 1298 | // If the function returns void, just return the chain. |
| 1299 | if (ResultVals.empty()) |
| 1300 | return Chain; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1301 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1302 | // Otherwise, merge everything together with a MERGE_VALUES node. |
| 1303 | NodeTys.push_back(MVT::Other); |
| 1304 | ResultVals.push_back(Chain); |
| 1305 | SDOperand Res = DAG.getNode(ISD::MERGE_VALUES, NodeTys, |
| 1306 | &ResultVals[0], ResultVals.size()); |
| 1307 | return Res.getValue(Op.ResNo); |
| 1308 | } |
| 1309 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1310 | //===----------------------------------------------------------------------===// |
| 1311 | // Fast Calling Convention implementation |
| 1312 | //===----------------------------------------------------------------------===// |
| 1313 | // |
| 1314 | // The X86 'fast' calling convention passes up to two integer arguments in |
| 1315 | // registers (an appropriate portion of EAX/EDX), passes arguments in C order, |
| 1316 | // and requires that the callee pop its arguments off the stack (allowing proper |
| 1317 | // tail calls), and has the same return value conventions as C calling convs. |
| 1318 | // |
| 1319 | // This calling convention always arranges for the callee pop value to be 8n+4 |
| 1320 | // bytes, which is needed for tail recursion elimination and stack alignment |
| 1321 | // reasons. |
| 1322 | // |
| 1323 | // Note that this can be enhanced in the future to pass fp vals in registers |
| 1324 | // (when we have a global fp allocator) and do other tricks. |
| 1325 | // |
| 1326 | |
Evan Cheng | 2fdd95e | 2006-04-27 08:31:10 +0000 | [diff] [blame] | 1327 | /// HowToPassFastCCArgument - Returns how an formal argument of the specified |
| 1328 | /// type should be passed. If it is through stack, returns the size of the stack |
Evan Cheng | f9ff7c5 | 2006-05-26 18:25:43 +0000 | [diff] [blame] | 1329 | /// slot; if it is through integer or XMM register, returns the number of |
Evan Cheng | 2fdd95e | 2006-04-27 08:31:10 +0000 | [diff] [blame] | 1330 | /// integer or XMM registers are needed. |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1331 | static void |
Evan Cheng | 2fdd95e | 2006-04-27 08:31:10 +0000 | [diff] [blame] | 1332 | HowToPassFastCCArgument(MVT::ValueType ObjectVT, |
| 1333 | unsigned NumIntRegs, unsigned NumXMMRegs, |
| 1334 | unsigned &ObjSize, unsigned &ObjIntRegs, |
| 1335 | unsigned &ObjXMMRegs) { |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1336 | ObjSize = 0; |
Evan Cheng | 2675534 | 2006-06-01 05:53:27 +0000 | [diff] [blame] | 1337 | ObjIntRegs = 0; |
| 1338 | ObjXMMRegs = 0; |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1339 | |
| 1340 | switch (ObjectVT) { |
| 1341 | default: assert(0 && "Unhandled argument type!"); |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1342 | case MVT::i8: |
Evan Cheng | da08d2c | 2006-06-24 08:36:10 +0000 | [diff] [blame] | 1343 | #if FASTCC_NUM_INT_ARGS_INREGS > 0 |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1344 | if (NumIntRegs < FASTCC_NUM_INT_ARGS_INREGS) |
Evan Cheng | 85e3800 | 2006-04-27 05:35:28 +0000 | [diff] [blame] | 1345 | ObjIntRegs = 1; |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1346 | else |
Evan Cheng | da08d2c | 2006-06-24 08:36:10 +0000 | [diff] [blame] | 1347 | #endif |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1348 | ObjSize = 1; |
| 1349 | break; |
| 1350 | case MVT::i16: |
Evan Cheng | da08d2c | 2006-06-24 08:36:10 +0000 | [diff] [blame] | 1351 | #if FASTCC_NUM_INT_ARGS_INREGS > 0 |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1352 | if (NumIntRegs < FASTCC_NUM_INT_ARGS_INREGS) |
Evan Cheng | 85e3800 | 2006-04-27 05:35:28 +0000 | [diff] [blame] | 1353 | ObjIntRegs = 1; |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1354 | else |
Evan Cheng | da08d2c | 2006-06-24 08:36:10 +0000 | [diff] [blame] | 1355 | #endif |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1356 | ObjSize = 2; |
| 1357 | break; |
| 1358 | case MVT::i32: |
Evan Cheng | da08d2c | 2006-06-24 08:36:10 +0000 | [diff] [blame] | 1359 | #if FASTCC_NUM_INT_ARGS_INREGS > 0 |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1360 | if (NumIntRegs < FASTCC_NUM_INT_ARGS_INREGS) |
Evan Cheng | 85e3800 | 2006-04-27 05:35:28 +0000 | [diff] [blame] | 1361 | ObjIntRegs = 1; |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1362 | else |
Evan Cheng | da08d2c | 2006-06-24 08:36:10 +0000 | [diff] [blame] | 1363 | #endif |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1364 | ObjSize = 4; |
| 1365 | break; |
| 1366 | case MVT::i64: |
Evan Cheng | da08d2c | 2006-06-24 08:36:10 +0000 | [diff] [blame] | 1367 | #if FASTCC_NUM_INT_ARGS_INREGS > 0 |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1368 | if (NumIntRegs+2 <= FASTCC_NUM_INT_ARGS_INREGS) { |
Evan Cheng | 85e3800 | 2006-04-27 05:35:28 +0000 | [diff] [blame] | 1369 | ObjIntRegs = 2; |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1370 | } else if (NumIntRegs+1 <= FASTCC_NUM_INT_ARGS_INREGS) { |
Evan Cheng | 85e3800 | 2006-04-27 05:35:28 +0000 | [diff] [blame] | 1371 | ObjIntRegs = 1; |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1372 | ObjSize = 4; |
| 1373 | } else |
Evan Cheng | da08d2c | 2006-06-24 08:36:10 +0000 | [diff] [blame] | 1374 | #endif |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1375 | ObjSize = 8; |
| 1376 | case MVT::f32: |
| 1377 | ObjSize = 4; |
| 1378 | break; |
| 1379 | case MVT::f64: |
| 1380 | ObjSize = 8; |
| 1381 | break; |
Evan Cheng | 2fdd95e | 2006-04-27 08:31:10 +0000 | [diff] [blame] | 1382 | case MVT::v16i8: |
| 1383 | case MVT::v8i16: |
| 1384 | case MVT::v4i32: |
| 1385 | case MVT::v2i64: |
| 1386 | case MVT::v4f32: |
| 1387 | case MVT::v2f64: |
Evan Cheng | 1d6a9b3 | 2006-05-26 19:22:06 +0000 | [diff] [blame] | 1388 | if (NumXMMRegs < 4) |
Evan Cheng | 2fdd95e | 2006-04-27 08:31:10 +0000 | [diff] [blame] | 1389 | ObjXMMRegs = 1; |
| 1390 | else |
| 1391 | ObjSize = 16; |
| 1392 | break; |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1393 | } |
| 1394 | } |
| 1395 | |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1396 | SDOperand |
| 1397 | X86TargetLowering::LowerFastCCArguments(SDOperand Op, SelectionDAG &DAG) { |
| 1398 | unsigned NumArgs = Op.Val->getNumValues()-1; |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1399 | MachineFunction &MF = DAG.getMachineFunction(); |
| 1400 | MachineFrameInfo *MFI = MF.getFrameInfo(); |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1401 | SDOperand Root = Op.getOperand(0); |
| 1402 | std::vector<SDOperand> ArgValues; |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1403 | |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1404 | // Add DAG nodes to load the arguments... On entry to a function the stack |
| 1405 | // frame looks like this: |
| 1406 | // |
| 1407 | // [ESP] -- return address |
| 1408 | // [ESP + 4] -- first nonreg argument (leftmost lexically) |
Evan Cheng | f9d62dc | 2006-05-26 18:37:16 +0000 | [diff] [blame] | 1409 | // [ESP + 8] -- second nonreg argument, if 1st argument is <= 4 bytes in size |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1410 | // ... |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1411 | unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot |
| 1412 | |
| 1413 | // Keep track of the number of integer regs passed so far. This can be either |
| 1414 | // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both |
| 1415 | // used). |
| 1416 | unsigned NumIntRegs = 0; |
Evan Cheng | 2fdd95e | 2006-04-27 08:31:10 +0000 | [diff] [blame] | 1417 | unsigned NumXMMRegs = 0; // XMM regs used for parameter passing. |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1418 | |
| 1419 | static const unsigned XMMArgRegs[] = { |
Evan Cheng | 1d6a9b3 | 2006-05-26 19:22:06 +0000 | [diff] [blame] | 1420 | X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1421 | }; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1422 | |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 1423 | for (unsigned i = 0; i < NumArgs; ++i) { |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1424 | MVT::ValueType ObjectVT = Op.getValue(i).getValueType(); |
| 1425 | unsigned ArgIncrement = 4; |
| 1426 | unsigned ObjSize = 0; |
| 1427 | unsigned ObjIntRegs = 0; |
| 1428 | unsigned ObjXMMRegs = 0; |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1429 | |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1430 | HowToPassFastCCArgument(ObjectVT, NumIntRegs, NumXMMRegs, |
| 1431 | ObjSize, ObjIntRegs, ObjXMMRegs); |
Evan Cheng | 052fb51 | 2006-05-26 18:39:59 +0000 | [diff] [blame] | 1432 | if (ObjSize > 4) |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1433 | ArgIncrement = ObjSize; |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1434 | |
Evan Cheng | 04b2562 | 2006-06-01 00:30:39 +0000 | [diff] [blame] | 1435 | unsigned Reg = 0; |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1436 | SDOperand ArgValue; |
| 1437 | if (ObjIntRegs || ObjXMMRegs) { |
| 1438 | switch (ObjectVT) { |
| 1439 | default: assert(0 && "Unhandled argument type!"); |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1440 | case MVT::i8: |
| 1441 | Reg = AddLiveIn(MF, NumIntRegs ? X86::DL : X86::AL, |
| 1442 | X86::GR8RegisterClass); |
| 1443 | ArgValue = DAG.getCopyFromReg(Root, Reg, MVT::i8); |
| 1444 | break; |
| 1445 | case MVT::i16: |
| 1446 | Reg = AddLiveIn(MF, NumIntRegs ? X86::DX : X86::AX, |
| 1447 | X86::GR16RegisterClass); |
| 1448 | ArgValue = DAG.getCopyFromReg(Root, Reg, MVT::i16); |
| 1449 | break; |
| 1450 | case MVT::i32: |
| 1451 | Reg = AddLiveIn(MF, NumIntRegs ? X86::EDX : X86::EAX, |
| 1452 | X86::GR32RegisterClass); |
| 1453 | ArgValue = DAG.getCopyFromReg(Root, Reg, MVT::i32); |
| 1454 | break; |
| 1455 | case MVT::i64: |
| 1456 | Reg = AddLiveIn(MF, NumIntRegs ? X86::EDX : X86::EAX, |
| 1457 | X86::GR32RegisterClass); |
| 1458 | ArgValue = DAG.getCopyFromReg(Root, Reg, MVT::i32); |
| 1459 | if (ObjIntRegs == 2) { |
| 1460 | Reg = AddLiveIn(MF, X86::EDX, X86::GR32RegisterClass); |
| 1461 | SDOperand ArgValue2 = DAG.getCopyFromReg(Root, Reg, MVT::i32); |
| 1462 | ArgValue= DAG.getNode(ISD::BUILD_PAIR, MVT::i64, ArgValue, ArgValue2); |
Evan Cheng | 85e3800 | 2006-04-27 05:35:28 +0000 | [diff] [blame] | 1463 | } |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1464 | break; |
| 1465 | case MVT::v16i8: |
| 1466 | case MVT::v8i16: |
| 1467 | case MVT::v4i32: |
| 1468 | case MVT::v2i64: |
| 1469 | case MVT::v4f32: |
| 1470 | case MVT::v2f64: |
| 1471 | Reg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], X86::VR128RegisterClass); |
| 1472 | ArgValue = DAG.getCopyFromReg(Root, Reg, ObjectVT); |
| 1473 | break; |
Evan Cheng | eda65fa | 2006-04-27 01:32:22 +0000 | [diff] [blame] | 1474 | } |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1475 | NumIntRegs += ObjIntRegs; |
| 1476 | NumXMMRegs += ObjXMMRegs; |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1477 | } |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1478 | |
| 1479 | if (ObjSize) { |
Evan Cheng | 3fddf24 | 2006-05-26 20:37:47 +0000 | [diff] [blame] | 1480 | // XMM arguments have to be aligned on 16-byte boundary. |
| 1481 | if (ObjSize == 16) |
| 1482 | ArgOffset = ((ArgOffset + 15) / 16) * 16; |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1483 | // Create the SelectionDAG nodes corresponding to a load from this |
| 1484 | // parameter. |
| 1485 | int FI = MFI->CreateFixedObject(ObjSize, ArgOffset); |
| 1486 | SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy()); |
| 1487 | if (ObjectVT == MVT::i64 && ObjIntRegs) { |
| 1488 | SDOperand ArgValue2 = DAG.getLoad(Op.Val->getValueType(i), Root, FIN, |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 1489 | NULL, 0); |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1490 | ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, ArgValue, ArgValue2); |
| 1491 | } else |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 1492 | ArgValue = DAG.getLoad(Op.Val->getValueType(i), Root, FIN, NULL, 0); |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1493 | ArgOffset += ArgIncrement; // Move on to the next argument. |
| 1494 | } |
| 1495 | |
| 1496 | ArgValues.push_back(ArgValue); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1497 | } |
| 1498 | |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1499 | ArgValues.push_back(Root); |
| 1500 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1501 | // Make sure the instruction takes 8n+4 bytes to make sure the start of the |
| 1502 | // arguments and the arguments after the retaddr has been pushed are aligned. |
| 1503 | if ((ArgOffset & 7) == 0) |
| 1504 | ArgOffset += 4; |
| 1505 | |
| 1506 | VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1507 | RegSaveFrameIndex = 0xAAAAAAA; // X86-64 only. |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1508 | ReturnAddrIndex = 0; // No return address slot generated yet. |
| 1509 | BytesToPopOnReturn = ArgOffset; // Callee pops all stack arguments. |
| 1510 | BytesCallerReserves = 0; |
| 1511 | |
| 1512 | // Finally, inform the code generator which regs we return values in. |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1513 | switch (getValueType(MF.getFunction()->getReturnType())) { |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1514 | default: assert(0 && "Unknown type!"); |
| 1515 | case MVT::isVoid: break; |
Chris Lattner | 13bf6c1 | 2006-10-03 17:18:42 +0000 | [diff] [blame] | 1516 | case MVT::i1: |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1517 | case MVT::i8: |
| 1518 | case MVT::i16: |
| 1519 | case MVT::i32: |
| 1520 | MF.addLiveOut(X86::EAX); |
| 1521 | break; |
| 1522 | case MVT::i64: |
| 1523 | MF.addLiveOut(X86::EAX); |
| 1524 | MF.addLiveOut(X86::EDX); |
| 1525 | break; |
| 1526 | case MVT::f32: |
| 1527 | case MVT::f64: |
| 1528 | MF.addLiveOut(X86::ST0); |
| 1529 | break; |
Evan Cheng | 6b5783d | 2006-05-25 18:56:34 +0000 | [diff] [blame] | 1530 | case MVT::v16i8: |
| 1531 | case MVT::v8i16: |
| 1532 | case MVT::v4i32: |
| 1533 | case MVT::v2i64: |
| 1534 | case MVT::v4f32: |
| 1535 | case MVT::v2f64: |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 1536 | MF.addLiveOut(X86::XMM0); |
| 1537 | break; |
| 1538 | } |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 1539 | |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1540 | // Return the new list of results. |
| 1541 | std::vector<MVT::ValueType> RetVTs(Op.Val->value_begin(), |
| 1542 | Op.Val->value_end()); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 1543 | return DAG.getNode(ISD::MERGE_VALUES, RetVTs, &ArgValues[0],ArgValues.size()); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1544 | } |
| 1545 | |
Chris Lattner | e87e115 | 2006-09-26 03:57:53 +0000 | [diff] [blame] | 1546 | SDOperand X86TargetLowering::LowerFastCCCallTo(SDOperand Op, SelectionDAG &DAG, |
| 1547 | bool isFastCall) { |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1548 | SDOperand Chain = Op.getOperand(0); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1549 | bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0; |
| 1550 | SDOperand Callee = Op.getOperand(4); |
| 1551 | MVT::ValueType RetVT= Op.Val->getValueType(0); |
| 1552 | unsigned NumOps = (Op.getNumOperands() - 5) / 2; |
| 1553 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1554 | // Count how many bytes are to be pushed on the stack. |
| 1555 | unsigned NumBytes = 0; |
| 1556 | |
| 1557 | // Keep track of the number of integer regs passed so far. This can be either |
| 1558 | // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both |
| 1559 | // used). |
| 1560 | unsigned NumIntRegs = 0; |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1561 | unsigned NumXMMRegs = 0; // XMM regs used for parameter passing. |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1562 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1563 | static const unsigned GPRArgRegs[][2] = { |
| 1564 | { X86::AL, X86::DL }, |
| 1565 | { X86::AX, X86::DX }, |
| 1566 | { X86::EAX, X86::EDX } |
| 1567 | }; |
Reid Spencer | 3ed469c | 2006-11-02 20:25:50 +0000 | [diff] [blame] | 1568 | #if 0 |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1569 | static const unsigned FastCallGPRArgRegs[][2] = { |
| 1570 | { X86::CL, X86::DL }, |
| 1571 | { X86::CX, X86::DX }, |
| 1572 | { X86::ECX, X86::EDX } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1573 | }; |
Reid Spencer | 3ed469c | 2006-11-02 20:25:50 +0000 | [diff] [blame] | 1574 | #endif |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1575 | static const unsigned XMMArgRegs[] = { |
Evan Cheng | 1d6a9b3 | 2006-05-26 19:22:06 +0000 | [diff] [blame] | 1576 | X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1577 | }; |
| 1578 | |
| 1579 | for (unsigned i = 0; i != NumOps; ++i) { |
| 1580 | SDOperand Arg = Op.getOperand(5+2*i); |
| 1581 | |
| 1582 | switch (Arg.getValueType()) { |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1583 | default: assert(0 && "Unknown value type!"); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1584 | case MVT::i8: |
| 1585 | case MVT::i16: |
Nick Lewycky | 70084fd | 2006-09-21 02:08:31 +0000 | [diff] [blame] | 1586 | case MVT::i32: { |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1587 | unsigned MaxNumIntRegs = (isFastCall ? 2 : FASTCC_NUM_INT_ARGS_INREGS); |
| 1588 | if (NumIntRegs < MaxNumIntRegs) { |
| 1589 | ++NumIntRegs; |
| 1590 | break; |
| 1591 | } |
Nick Lewycky | 70084fd | 2006-09-21 02:08:31 +0000 | [diff] [blame] | 1592 | } // Fall through |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1593 | case MVT::f32: |
| 1594 | NumBytes += 4; |
| 1595 | break; |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1596 | case MVT::f64: |
| 1597 | NumBytes += 8; |
| 1598 | break; |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1599 | case MVT::v16i8: |
| 1600 | case MVT::v8i16: |
| 1601 | case MVT::v4i32: |
| 1602 | case MVT::v2i64: |
| 1603 | case MVT::v4f32: |
Evan Cheng | 6b5783d | 2006-05-25 18:56:34 +0000 | [diff] [blame] | 1604 | case MVT::v2f64: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1605 | if (isFastCall) { |
| 1606 | assert(0 && "Unknown value type!"); |
| 1607 | } else { |
| 1608 | if (NumXMMRegs < 4) |
| 1609 | NumXMMRegs++; |
| 1610 | else { |
| 1611 | // XMM arguments have to be aligned on 16-byte boundary. |
| 1612 | NumBytes = ((NumBytes + 15) / 16) * 16; |
| 1613 | NumBytes += 16; |
| 1614 | } |
| 1615 | } |
| 1616 | break; |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1617 | } |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1618 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1619 | |
| 1620 | // Make sure the instruction takes 8n+4 bytes to make sure the start of the |
| 1621 | // arguments and the arguments after the retaddr has been pushed are aligned. |
| 1622 | if ((NumBytes & 7) == 0) |
| 1623 | NumBytes += 4; |
| 1624 | |
Chris Lattner | 94dd292 | 2006-02-13 09:00:43 +0000 | [diff] [blame] | 1625 | Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy())); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1626 | |
| 1627 | // Arguments go on the stack in reverse order, as specified by the ABI. |
| 1628 | unsigned ArgOffset = 0; |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1629 | NumIntRegs = 0; |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1630 | std::vector<std::pair<unsigned, SDOperand> > RegsToPass; |
| 1631 | std::vector<SDOperand> MemOpChains; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1632 | SDOperand StackPtr = DAG.getRegister(X86StackPtr, getPointerTy()); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1633 | for (unsigned i = 0; i != NumOps; ++i) { |
| 1634 | SDOperand Arg = Op.getOperand(5+2*i); |
| 1635 | |
| 1636 | switch (Arg.getValueType()) { |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1637 | default: assert(0 && "Unexpected ValueType for argument!"); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1638 | case MVT::i8: |
| 1639 | case MVT::i16: |
Nick Lewycky | 70084fd | 2006-09-21 02:08:31 +0000 | [diff] [blame] | 1640 | case MVT::i32: { |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1641 | unsigned MaxNumIntRegs = (isFastCall ? 2 : FASTCC_NUM_INT_ARGS_INREGS); |
| 1642 | if (NumIntRegs < MaxNumIntRegs) { |
| 1643 | RegsToPass.push_back( |
| 1644 | std::make_pair(GPRArgRegs[Arg.getValueType()-MVT::i8][NumIntRegs], |
| 1645 | Arg)); |
| 1646 | ++NumIntRegs; |
| 1647 | break; |
| 1648 | } |
Nick Lewycky | 70084fd | 2006-09-21 02:08:31 +0000 | [diff] [blame] | 1649 | } // Fall through |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1650 | case MVT::f32: { |
| 1651 | SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1652 | PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 1653 | MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0)); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1654 | ArgOffset += 4; |
| 1655 | break; |
| 1656 | } |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1657 | case MVT::f64: { |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1658 | SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1659 | PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 1660 | MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0)); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1661 | ArgOffset += 8; |
| 1662 | break; |
| 1663 | } |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1664 | case MVT::v16i8: |
| 1665 | case MVT::v8i16: |
| 1666 | case MVT::v4i32: |
| 1667 | case MVT::v2i64: |
| 1668 | case MVT::v4f32: |
Evan Cheng | 6b5783d | 2006-05-25 18:56:34 +0000 | [diff] [blame] | 1669 | case MVT::v2f64: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1670 | if (isFastCall) { |
| 1671 | assert(0 && "Unexpected ValueType for argument!"); |
| 1672 | } else { |
| 1673 | if (NumXMMRegs < 4) { |
| 1674 | RegsToPass.push_back(std::make_pair(XMMArgRegs[NumXMMRegs], Arg)); |
| 1675 | NumXMMRegs++; |
| 1676 | } else { |
| 1677 | // XMM arguments have to be aligned on 16-byte boundary. |
| 1678 | ArgOffset = ((ArgOffset + 15) / 16) * 16; |
| 1679 | SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); |
| 1680 | PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 1681 | MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0)); |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1682 | ArgOffset += 16; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1683 | } |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1684 | } |
| 1685 | break; |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1686 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1687 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1688 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1689 | if (!MemOpChains.empty()) |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 1690 | Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, |
| 1691 | &MemOpChains[0], MemOpChains.size()); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1692 | |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1693 | // Build a sequence of copy-to-reg nodes chained together with token chain |
| 1694 | // and flag operands which copy the outgoing args into registers. |
| 1695 | SDOperand InFlag; |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1696 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { |
| 1697 | Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, |
| 1698 | InFlag); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1699 | InFlag = Chain.getValue(1); |
| 1700 | } |
| 1701 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1702 | // If the callee is a GlobalAddress node (quite common, every direct call is) |
| 1703 | // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. |
Anton Korobeynikov | a598685 | 2006-11-20 10:46:14 +0000 | [diff] [blame] | 1704 | if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { |
| 1705 | // We should use extra load for direct calls to dllimported functions |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1706 | if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(), true)) |
Anton Korobeynikov | a598685 | 2006-11-20 10:46:14 +0000 | [diff] [blame] | 1707 | Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); |
| 1708 | } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1709 | Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); |
| 1710 | |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1711 | std::vector<MVT::ValueType> NodeTys; |
| 1712 | NodeTys.push_back(MVT::Other); // Returns a chain |
| 1713 | NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. |
| 1714 | std::vector<SDOperand> Ops; |
| 1715 | Ops.push_back(Chain); |
| 1716 | Ops.push_back(Callee); |
Evan Cheng | b69d113 | 2006-06-14 18:17:40 +0000 | [diff] [blame] | 1717 | |
| 1718 | // Add argument registers to the end of the list so that they are known live |
| 1719 | // into the call. |
| 1720 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1721 | Ops.push_back(DAG.getRegister(RegsToPass[i].first, |
Evan Cheng | b69d113 | 2006-06-14 18:17:40 +0000 | [diff] [blame] | 1722 | RegsToPass[i].second.getValueType())); |
| 1723 | |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1724 | if (InFlag.Val) |
| 1725 | Ops.push_back(InFlag); |
| 1726 | |
| 1727 | // FIXME: Do not generate X86ISD::TAILCALL for now. |
Chris Lattner | 8c0c10c | 2006-05-16 06:45:34 +0000 | [diff] [blame] | 1728 | Chain = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL, |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 1729 | NodeTys, &Ops[0], Ops.size()); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1730 | InFlag = Chain.getValue(1); |
| 1731 | |
| 1732 | NodeTys.clear(); |
| 1733 | NodeTys.push_back(MVT::Other); // Returns a chain |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1734 | if (RetVT != MVT::Other) |
| 1735 | NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1736 | Ops.clear(); |
| 1737 | Ops.push_back(Chain); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1738 | Ops.push_back(DAG.getConstant(NumBytes, getPointerTy())); |
| 1739 | Ops.push_back(DAG.getConstant(NumBytes, getPointerTy())); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1740 | Ops.push_back(InFlag); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 1741 | Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size()); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1742 | if (RetVT != MVT::Other) |
| 1743 | InFlag = Chain.getValue(1); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1744 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1745 | std::vector<SDOperand> ResultVals; |
| 1746 | NodeTys.clear(); |
| 1747 | switch (RetVT) { |
| 1748 | default: assert(0 && "Unknown value type to return!"); |
| 1749 | case MVT::Other: break; |
| 1750 | case MVT::i8: |
| 1751 | Chain = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag).getValue(1); |
| 1752 | ResultVals.push_back(Chain.getValue(0)); |
| 1753 | NodeTys.push_back(MVT::i8); |
| 1754 | break; |
| 1755 | case MVT::i16: |
| 1756 | Chain = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag).getValue(1); |
| 1757 | ResultVals.push_back(Chain.getValue(0)); |
| 1758 | NodeTys.push_back(MVT::i16); |
| 1759 | break; |
| 1760 | case MVT::i32: |
| 1761 | if (Op.Val->getValueType(1) == MVT::i32) { |
| 1762 | Chain = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag).getValue(1); |
| 1763 | ResultVals.push_back(Chain.getValue(0)); |
| 1764 | Chain = DAG.getCopyFromReg(Chain, X86::EDX, MVT::i32, |
| 1765 | Chain.getValue(2)).getValue(1); |
| 1766 | ResultVals.push_back(Chain.getValue(0)); |
| 1767 | NodeTys.push_back(MVT::i32); |
| 1768 | } else { |
| 1769 | Chain = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag).getValue(1); |
| 1770 | ResultVals.push_back(Chain.getValue(0)); |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 1771 | } |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1772 | NodeTys.push_back(MVT::i32); |
| 1773 | break; |
| 1774 | case MVT::v16i8: |
| 1775 | case MVT::v8i16: |
| 1776 | case MVT::v4i32: |
| 1777 | case MVT::v2i64: |
| 1778 | case MVT::v4f32: |
| 1779 | case MVT::v2f64: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1780 | if (isFastCall) { |
| 1781 | assert(0 && "Unknown value type to return!"); |
| 1782 | } else { |
| 1783 | Chain = DAG.getCopyFromReg(Chain, X86::XMM0, RetVT, InFlag).getValue(1); |
| 1784 | ResultVals.push_back(Chain.getValue(0)); |
| 1785 | NodeTys.push_back(RetVT); |
| 1786 | } |
| 1787 | break; |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1788 | case MVT::f32: |
| 1789 | case MVT::f64: { |
| 1790 | std::vector<MVT::ValueType> Tys; |
| 1791 | Tys.push_back(MVT::f64); |
| 1792 | Tys.push_back(MVT::Other); |
| 1793 | Tys.push_back(MVT::Flag); |
| 1794 | std::vector<SDOperand> Ops; |
| 1795 | Ops.push_back(Chain); |
| 1796 | Ops.push_back(InFlag); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 1797 | SDOperand RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, |
| 1798 | &Ops[0], Ops.size()); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1799 | Chain = RetVal.getValue(1); |
| 1800 | InFlag = RetVal.getValue(2); |
| 1801 | if (X86ScalarSSE) { |
| 1802 | // FIXME: Currently the FST is flagged to the FP_GET_RESULT. This |
| 1803 | // shouldn't be necessary except that RFP cannot be live across |
| 1804 | // multiple blocks. When stackifier is fixed, they can be uncoupled. |
| 1805 | MachineFunction &MF = DAG.getMachineFunction(); |
| 1806 | int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); |
| 1807 | SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
| 1808 | Tys.clear(); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1809 | Tys.push_back(MVT::Other); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1810 | Ops.clear(); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1811 | Ops.push_back(Chain); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1812 | Ops.push_back(RetVal); |
| 1813 | Ops.push_back(StackSlot); |
| 1814 | Ops.push_back(DAG.getValueType(RetVT)); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1815 | Ops.push_back(InFlag); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 1816 | Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 1817 | RetVal = DAG.getLoad(RetVT, Chain, StackSlot, NULL, 0); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1818 | Chain = RetVal.getValue(1); |
| 1819 | } |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 1820 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1821 | if (RetVT == MVT::f32 && !X86ScalarSSE) |
| 1822 | // FIXME: we would really like to remember that this FP_ROUND |
| 1823 | // operation is okay to eliminate if we allow excess FP precision. |
| 1824 | RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal); |
| 1825 | ResultVals.push_back(RetVal); |
| 1826 | NodeTys.push_back(RetVT); |
| 1827 | break; |
| 1828 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1829 | } |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1830 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1831 | |
| 1832 | // If the function returns void, just return the chain. |
| 1833 | if (ResultVals.empty()) |
| 1834 | return Chain; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1835 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1836 | // Otherwise, merge everything together with a MERGE_VALUES node. |
| 1837 | NodeTys.push_back(MVT::Other); |
| 1838 | ResultVals.push_back(Chain); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 1839 | SDOperand Res = DAG.getNode(ISD::MERGE_VALUES, NodeTys, |
| 1840 | &ResultVals[0], ResultVals.size()); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1841 | return Res.getValue(Op.ResNo); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1842 | } |
| 1843 | |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1844 | //===----------------------------------------------------------------------===// |
| 1845 | // StdCall Calling Convention implementation |
| 1846 | //===----------------------------------------------------------------------===// |
| 1847 | // StdCall calling convention seems to be standard for many Windows' API |
| 1848 | // routines and around. It differs from C calling convention just a little: |
| 1849 | // callee should clean up the stack, not caller. Symbols should be also |
| 1850 | // decorated in some fancy way :) It doesn't support any vector arguments. |
| 1851 | |
| 1852 | /// HowToPassStdCallCCArgument - Returns how an formal argument of the specified |
| 1853 | /// type should be passed. Returns the size of the stack slot |
| 1854 | static void |
| 1855 | HowToPassStdCallCCArgument(MVT::ValueType ObjectVT, unsigned &ObjSize) { |
| 1856 | switch (ObjectVT) { |
| 1857 | default: assert(0 && "Unhandled argument type!"); |
| 1858 | case MVT::i8: ObjSize = 1; break; |
| 1859 | case MVT::i16: ObjSize = 2; break; |
| 1860 | case MVT::i32: ObjSize = 4; break; |
| 1861 | case MVT::i64: ObjSize = 8; break; |
| 1862 | case MVT::f32: ObjSize = 4; break; |
| 1863 | case MVT::f64: ObjSize = 8; break; |
| 1864 | } |
| 1865 | } |
| 1866 | |
| 1867 | SDOperand X86TargetLowering::LowerStdCallCCArguments(SDOperand Op, |
| 1868 | SelectionDAG &DAG) { |
| 1869 | unsigned NumArgs = Op.Val->getNumValues() - 1; |
| 1870 | MachineFunction &MF = DAG.getMachineFunction(); |
| 1871 | MachineFrameInfo *MFI = MF.getFrameInfo(); |
| 1872 | SDOperand Root = Op.getOperand(0); |
| 1873 | std::vector<SDOperand> ArgValues; |
| 1874 | |
| 1875 | // Add DAG nodes to load the arguments... On entry to a function on the X86, |
| 1876 | // the stack frame looks like this: |
| 1877 | // |
| 1878 | // [ESP] -- return address |
| 1879 | // [ESP + 4] -- first argument (leftmost lexically) |
| 1880 | // [ESP + 8] -- second argument, if first argument is <= 4 bytes in size |
| 1881 | // ... |
| 1882 | // |
| 1883 | unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot |
| 1884 | for (unsigned i = 0; i < NumArgs; ++i) { |
| 1885 | MVT::ValueType ObjectVT = Op.getValue(i).getValueType(); |
| 1886 | unsigned ArgIncrement = 4; |
| 1887 | unsigned ObjSize = 0; |
| 1888 | HowToPassStdCallCCArgument(ObjectVT, ObjSize); |
| 1889 | if (ObjSize > 4) |
| 1890 | ArgIncrement = ObjSize; |
| 1891 | |
| 1892 | SDOperand ArgValue; |
| 1893 | // Create the frame index object for this incoming parameter... |
| 1894 | int FI = MFI->CreateFixedObject(ObjSize, ArgOffset); |
| 1895 | SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy()); |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 1896 | ArgValue = DAG.getLoad(Op.Val->getValueType(i), Root, FIN, NULL, 0); |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1897 | ArgValues.push_back(ArgValue); |
| 1898 | ArgOffset += ArgIncrement; // Move on to the next argument... |
| 1899 | } |
| 1900 | |
| 1901 | ArgValues.push_back(Root); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1902 | |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1903 | // If the function takes variable number of arguments, make a frame index for |
| 1904 | // the start of the first vararg value... for expansion of llvm.va_start. |
| 1905 | bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; |
| 1906 | if (isVarArg) { |
| 1907 | BytesToPopOnReturn = 0; // Callee pops nothing. |
| 1908 | BytesCallerReserves = ArgOffset; |
| 1909 | VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset); |
| 1910 | } else { |
| 1911 | BytesToPopOnReturn = ArgOffset; // Callee pops everything.. |
| 1912 | BytesCallerReserves = 0; |
| 1913 | } |
| 1914 | RegSaveFrameIndex = 0xAAAAAAA; // X86-64 only. |
| 1915 | ReturnAddrIndex = 0; // No return address slot generated yet. |
| 1916 | |
| 1917 | MF.getInfo<X86FunctionInfo>()->setBytesToPopOnReturn(BytesToPopOnReturn); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1918 | |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1919 | // Return the new list of results. |
| 1920 | std::vector<MVT::ValueType> RetVTs(Op.Val->value_begin(), |
| 1921 | Op.Val->value_end()); |
| 1922 | return DAG.getNode(ISD::MERGE_VALUES, RetVTs, &ArgValues[0],ArgValues.size()); |
| 1923 | } |
| 1924 | |
| 1925 | |
| 1926 | SDOperand X86TargetLowering::LowerStdCallCCCallTo(SDOperand Op, |
| 1927 | SelectionDAG &DAG) { |
| 1928 | SDOperand Chain = Op.getOperand(0); |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1929 | bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0; |
| 1930 | bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0; |
| 1931 | SDOperand Callee = Op.getOperand(4); |
| 1932 | MVT::ValueType RetVT= Op.Val->getValueType(0); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1933 | unsigned NumOps = (Op.getNumOperands() - 5) / 2; |
| 1934 | |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1935 | // Count how many bytes are to be pushed on the stack. |
| 1936 | unsigned NumBytes = 0; |
| 1937 | for (unsigned i = 0; i != NumOps; ++i) { |
| 1938 | SDOperand Arg = Op.getOperand(5+2*i); |
| 1939 | |
| 1940 | switch (Arg.getValueType()) { |
| 1941 | default: assert(0 && "Unexpected ValueType for argument!"); |
| 1942 | case MVT::i8: |
| 1943 | case MVT::i16: |
| 1944 | case MVT::i32: |
| 1945 | case MVT::f32: |
| 1946 | NumBytes += 4; |
| 1947 | break; |
| 1948 | case MVT::i64: |
| 1949 | case MVT::f64: |
| 1950 | NumBytes += 8; |
| 1951 | break; |
| 1952 | } |
| 1953 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1954 | |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1955 | Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy())); |
| 1956 | |
| 1957 | // Arguments go on the stack in reverse order, as specified by the ABI. |
| 1958 | unsigned ArgOffset = 0; |
| 1959 | std::vector<SDOperand> MemOpChains; |
| 1960 | SDOperand StackPtr = DAG.getRegister(X86StackPtr, getPointerTy()); |
| 1961 | for (unsigned i = 0; i != NumOps; ++i) { |
| 1962 | SDOperand Arg = Op.getOperand(5+2*i); |
| 1963 | |
| 1964 | switch (Arg.getValueType()) { |
| 1965 | default: assert(0 && "Unexpected ValueType for argument!"); |
| 1966 | case MVT::i8: |
| 1967 | case MVT::i16: { |
| 1968 | // Promote the integer to 32 bits. If the input type is signed use a |
| 1969 | // sign extend, otherwise use a zero extend. |
| 1970 | unsigned ExtOp = |
| 1971 | dyn_cast<ConstantSDNode>(Op.getOperand(5+2*i+1))->getValue() ? |
| 1972 | ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
| 1973 | Arg = DAG.getNode(ExtOp, MVT::i32, Arg); |
| 1974 | } |
| 1975 | // Fallthrough |
| 1976 | |
| 1977 | case MVT::i32: |
| 1978 | case MVT::f32: { |
| 1979 | SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); |
| 1980 | PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 1981 | MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0)); |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1982 | ArgOffset += 4; |
| 1983 | break; |
| 1984 | } |
| 1985 | case MVT::i64: |
| 1986 | case MVT::f64: { |
| 1987 | SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy()); |
| 1988 | PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 1989 | MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0)); |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 1990 | ArgOffset += 8; |
| 1991 | break; |
| 1992 | } |
| 1993 | } |
| 1994 | } |
| 1995 | |
| 1996 | if (!MemOpChains.empty()) |
| 1997 | Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, |
| 1998 | &MemOpChains[0], MemOpChains.size()); |
| 1999 | |
| 2000 | // If the callee is a GlobalAddress node (quite common, every direct call is) |
| 2001 | // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. |
Anton Korobeynikov | a598685 | 2006-11-20 10:46:14 +0000 | [diff] [blame] | 2002 | if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { |
| 2003 | // We should use extra load for direct calls to dllimported functions |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 2004 | if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(), true)) |
Anton Korobeynikov | a598685 | 2006-11-20 10:46:14 +0000 | [diff] [blame] | 2005 | Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy()); |
| 2006 | } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 2007 | Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); |
| 2008 | |
| 2009 | std::vector<MVT::ValueType> NodeTys; |
| 2010 | NodeTys.push_back(MVT::Other); // Returns a chain |
| 2011 | NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. |
| 2012 | std::vector<SDOperand> Ops; |
| 2013 | Ops.push_back(Chain); |
| 2014 | Ops.push_back(Callee); |
| 2015 | |
| 2016 | Chain = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL, |
| 2017 | NodeTys, &Ops[0], Ops.size()); |
| 2018 | SDOperand InFlag = Chain.getValue(1); |
| 2019 | |
| 2020 | // Create the CALLSEQ_END node. |
| 2021 | unsigned NumBytesForCalleeToPush; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 2022 | |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 2023 | if (isVarArg) { |
| 2024 | NumBytesForCalleeToPush = 0; |
| 2025 | } else { |
| 2026 | NumBytesForCalleeToPush = NumBytes; |
| 2027 | } |
| 2028 | |
| 2029 | NodeTys.clear(); |
| 2030 | NodeTys.push_back(MVT::Other); // Returns a chain |
| 2031 | if (RetVT != MVT::Other) |
| 2032 | NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use. |
| 2033 | Ops.clear(); |
| 2034 | Ops.push_back(Chain); |
| 2035 | Ops.push_back(DAG.getConstant(NumBytes, getPointerTy())); |
| 2036 | Ops.push_back(DAG.getConstant(NumBytesForCalleeToPush, getPointerTy())); |
| 2037 | Ops.push_back(InFlag); |
| 2038 | Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size()); |
| 2039 | if (RetVT != MVT::Other) |
| 2040 | InFlag = Chain.getValue(1); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 2041 | |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 2042 | std::vector<SDOperand> ResultVals; |
| 2043 | NodeTys.clear(); |
| 2044 | switch (RetVT) { |
| 2045 | default: assert(0 && "Unknown value type to return!"); |
| 2046 | case MVT::Other: break; |
| 2047 | case MVT::i8: |
| 2048 | Chain = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag).getValue(1); |
| 2049 | ResultVals.push_back(Chain.getValue(0)); |
| 2050 | NodeTys.push_back(MVT::i8); |
| 2051 | break; |
| 2052 | case MVT::i16: |
| 2053 | Chain = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag).getValue(1); |
| 2054 | ResultVals.push_back(Chain.getValue(0)); |
| 2055 | NodeTys.push_back(MVT::i16); |
| 2056 | break; |
| 2057 | case MVT::i32: |
| 2058 | if (Op.Val->getValueType(1) == MVT::i32) { |
| 2059 | Chain = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag).getValue(1); |
| 2060 | ResultVals.push_back(Chain.getValue(0)); |
| 2061 | Chain = DAG.getCopyFromReg(Chain, X86::EDX, MVT::i32, |
| 2062 | Chain.getValue(2)).getValue(1); |
| 2063 | ResultVals.push_back(Chain.getValue(0)); |
| 2064 | NodeTys.push_back(MVT::i32); |
| 2065 | } else { |
| 2066 | Chain = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag).getValue(1); |
| 2067 | ResultVals.push_back(Chain.getValue(0)); |
| 2068 | } |
| 2069 | NodeTys.push_back(MVT::i32); |
| 2070 | break; |
| 2071 | case MVT::f32: |
| 2072 | case MVT::f64: { |
| 2073 | std::vector<MVT::ValueType> Tys; |
| 2074 | Tys.push_back(MVT::f64); |
| 2075 | Tys.push_back(MVT::Other); |
| 2076 | Tys.push_back(MVT::Flag); |
| 2077 | std::vector<SDOperand> Ops; |
| 2078 | Ops.push_back(Chain); |
| 2079 | Ops.push_back(InFlag); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 2080 | SDOperand RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 2081 | &Ops[0], Ops.size()); |
| 2082 | Chain = RetVal.getValue(1); |
| 2083 | InFlag = RetVal.getValue(2); |
| 2084 | if (X86ScalarSSE) { |
| 2085 | // FIXME: Currently the FST is flagged to the FP_GET_RESULT. This |
| 2086 | // shouldn't be necessary except that RFP cannot be live across |
| 2087 | // multiple blocks. When stackifier is fixed, they can be uncoupled. |
| 2088 | MachineFunction &MF = DAG.getMachineFunction(); |
| 2089 | int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); |
| 2090 | SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
| 2091 | Tys.clear(); |
| 2092 | Tys.push_back(MVT::Other); |
| 2093 | Ops.clear(); |
| 2094 | Ops.push_back(Chain); |
| 2095 | Ops.push_back(RetVal); |
| 2096 | Ops.push_back(StackSlot); |
| 2097 | Ops.push_back(DAG.getValueType(RetVT)); |
| 2098 | Ops.push_back(InFlag); |
| 2099 | Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 2100 | RetVal = DAG.getLoad(RetVT, Chain, StackSlot, NULL, 0); |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 2101 | Chain = RetVal.getValue(1); |
| 2102 | } |
| 2103 | |
| 2104 | if (RetVT == MVT::f32 && !X86ScalarSSE) |
| 2105 | // FIXME: we would really like to remember that this FP_ROUND |
| 2106 | // operation is okay to eliminate if we allow excess FP precision. |
| 2107 | RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal); |
| 2108 | ResultVals.push_back(RetVal); |
| 2109 | NodeTys.push_back(RetVT); |
| 2110 | break; |
| 2111 | } |
| 2112 | } |
| 2113 | |
| 2114 | // If the function returns void, just return the chain. |
| 2115 | if (ResultVals.empty()) |
| 2116 | return Chain; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 2117 | |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 2118 | // Otherwise, merge everything together with a MERGE_VALUES node. |
| 2119 | NodeTys.push_back(MVT::Other); |
| 2120 | ResultVals.push_back(Chain); |
| 2121 | SDOperand Res = DAG.getNode(ISD::MERGE_VALUES, NodeTys, |
| 2122 | &ResultVals[0], ResultVals.size()); |
| 2123 | return Res.getValue(Op.ResNo); |
| 2124 | } |
| 2125 | |
| 2126 | //===----------------------------------------------------------------------===// |
| 2127 | // FastCall Calling Convention implementation |
| 2128 | //===----------------------------------------------------------------------===// |
| 2129 | // |
| 2130 | // The X86 'fastcall' calling convention passes up to two integer arguments in |
| 2131 | // registers (an appropriate portion of ECX/EDX), passes arguments in C order, |
| 2132 | // and requires that the callee pop its arguments off the stack (allowing proper |
| 2133 | // tail calls), and has the same return value conventions as C calling convs. |
| 2134 | // |
| 2135 | // This calling convention always arranges for the callee pop value to be 8n+4 |
| 2136 | // bytes, which is needed for tail recursion elimination and stack alignment |
| 2137 | // reasons. |
| 2138 | // |
| 2139 | |
| 2140 | /// HowToPassFastCallCCArgument - Returns how an formal argument of the |
| 2141 | /// specified type should be passed. If it is through stack, returns the size of |
| 2142 | /// the stack slot; if it is through integer register, returns the number of |
| 2143 | /// integer registers are needed. |
| 2144 | static void |
| 2145 | HowToPassFastCallCCArgument(MVT::ValueType ObjectVT, |
| 2146 | unsigned NumIntRegs, |
| 2147 | unsigned &ObjSize, |
| 2148 | unsigned &ObjIntRegs) |
| 2149 | { |
| 2150 | ObjSize = 0; |
| 2151 | ObjIntRegs = 0; |
| 2152 | |
| 2153 | switch (ObjectVT) { |
| 2154 | default: assert(0 && "Unhandled argument type!"); |
| 2155 | case MVT::i8: |
| 2156 | if (NumIntRegs < 2) |
| 2157 | ObjIntRegs = 1; |
| 2158 | else |
| 2159 | ObjSize = 1; |
| 2160 | break; |
| 2161 | case MVT::i16: |
| 2162 | if (NumIntRegs < 2) |
| 2163 | ObjIntRegs = 1; |
| 2164 | else |
| 2165 | ObjSize = 2; |
| 2166 | break; |
| 2167 | case MVT::i32: |
| 2168 | if (NumIntRegs < 2) |
| 2169 | ObjIntRegs = 1; |
| 2170 | else |
| 2171 | ObjSize = 4; |
| 2172 | break; |
| 2173 | case MVT::i64: |
| 2174 | if (NumIntRegs+2 <= 2) { |
| 2175 | ObjIntRegs = 2; |
| 2176 | } else if (NumIntRegs+1 <= 2) { |
| 2177 | ObjIntRegs = 1; |
| 2178 | ObjSize = 4; |
| 2179 | } else |
| 2180 | ObjSize = 8; |
| 2181 | case MVT::f32: |
| 2182 | ObjSize = 4; |
| 2183 | break; |
| 2184 | case MVT::f64: |
| 2185 | ObjSize = 8; |
| 2186 | break; |
| 2187 | } |
| 2188 | } |
| 2189 | |
| 2190 | SDOperand |
| 2191 | X86TargetLowering::LowerFastCallCCArguments(SDOperand Op, SelectionDAG &DAG) { |
| 2192 | unsigned NumArgs = Op.Val->getNumValues()-1; |
| 2193 | MachineFunction &MF = DAG.getMachineFunction(); |
| 2194 | MachineFrameInfo *MFI = MF.getFrameInfo(); |
| 2195 | SDOperand Root = Op.getOperand(0); |
| 2196 | std::vector<SDOperand> ArgValues; |
| 2197 | |
| 2198 | // Add DAG nodes to load the arguments... On entry to a function the stack |
| 2199 | // frame looks like this: |
| 2200 | // |
| 2201 | // [ESP] -- return address |
| 2202 | // [ESP + 4] -- first nonreg argument (leftmost lexically) |
| 2203 | // [ESP + 8] -- second nonreg argument, if 1st argument is <= 4 bytes in size |
| 2204 | // ... |
| 2205 | unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot |
| 2206 | |
| 2207 | // Keep track of the number of integer regs passed so far. This can be either |
| 2208 | // 0 (neither ECX or EDX used), 1 (ECX is used) or 2 (ECX and EDX are both |
| 2209 | // used). |
| 2210 | unsigned NumIntRegs = 0; |
| 2211 | |
| 2212 | for (unsigned i = 0; i < NumArgs; ++i) { |
| 2213 | MVT::ValueType ObjectVT = Op.getValue(i).getValueType(); |
| 2214 | unsigned ArgIncrement = 4; |
| 2215 | unsigned ObjSize = 0; |
| 2216 | unsigned ObjIntRegs = 0; |
| 2217 | |
| 2218 | HowToPassFastCallCCArgument(ObjectVT, NumIntRegs, ObjSize, ObjIntRegs); |
| 2219 | if (ObjSize > 4) |
| 2220 | ArgIncrement = ObjSize; |
| 2221 | |
| 2222 | unsigned Reg = 0; |
| 2223 | SDOperand ArgValue; |
| 2224 | if (ObjIntRegs) { |
| 2225 | switch (ObjectVT) { |
| 2226 | default: assert(0 && "Unhandled argument type!"); |
| 2227 | case MVT::i8: |
| 2228 | Reg = AddLiveIn(MF, NumIntRegs ? X86::DL : X86::CL, |
| 2229 | X86::GR8RegisterClass); |
| 2230 | ArgValue = DAG.getCopyFromReg(Root, Reg, MVT::i8); |
| 2231 | break; |
| 2232 | case MVT::i16: |
| 2233 | Reg = AddLiveIn(MF, NumIntRegs ? X86::DX : X86::CX, |
| 2234 | X86::GR16RegisterClass); |
| 2235 | ArgValue = DAG.getCopyFromReg(Root, Reg, MVT::i16); |
| 2236 | break; |
| 2237 | case MVT::i32: |
| 2238 | Reg = AddLiveIn(MF, NumIntRegs ? X86::EDX : X86::ECX, |
| 2239 | X86::GR32RegisterClass); |
| 2240 | ArgValue = DAG.getCopyFromReg(Root, Reg, MVT::i32); |
| 2241 | break; |
| 2242 | case MVT::i64: |
| 2243 | Reg = AddLiveIn(MF, NumIntRegs ? X86::EDX : X86::ECX, |
| 2244 | X86::GR32RegisterClass); |
| 2245 | ArgValue = DAG.getCopyFromReg(Root, Reg, MVT::i32); |
| 2246 | if (ObjIntRegs == 2) { |
| 2247 | Reg = AddLiveIn(MF, X86::EDX, X86::GR32RegisterClass); |
| 2248 | SDOperand ArgValue2 = DAG.getCopyFromReg(Root, Reg, MVT::i32); |
| 2249 | ArgValue= DAG.getNode(ISD::BUILD_PAIR, MVT::i64, ArgValue, ArgValue2); |
| 2250 | } |
| 2251 | break; |
| 2252 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 2253 | |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 2254 | NumIntRegs += ObjIntRegs; |
| 2255 | } |
| 2256 | |
| 2257 | if (ObjSize) { |
| 2258 | // Create the SelectionDAG nodes corresponding to a load from this |
| 2259 | // parameter. |
| 2260 | int FI = MFI->CreateFixedObject(ObjSize, ArgOffset); |
| 2261 | SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy()); |
| 2262 | if (ObjectVT == MVT::i64 && ObjIntRegs) { |
| 2263 | SDOperand ArgValue2 = DAG.getLoad(Op.Val->getValueType(i), Root, FIN, |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 2264 | NULL, 0); |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 2265 | ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, ArgValue, ArgValue2); |
| 2266 | } else |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 2267 | ArgValue = DAG.getLoad(Op.Val->getValueType(i), Root, FIN, NULL, 0); |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 2268 | ArgOffset += ArgIncrement; // Move on to the next argument. |
| 2269 | } |
| 2270 | |
| 2271 | ArgValues.push_back(ArgValue); |
| 2272 | } |
| 2273 | |
| 2274 | ArgValues.push_back(Root); |
| 2275 | |
| 2276 | // Make sure the instruction takes 8n+4 bytes to make sure the start of the |
| 2277 | // arguments and the arguments after the retaddr has been pushed are aligned. |
| 2278 | if ((ArgOffset & 7) == 0) |
| 2279 | ArgOffset += 4; |
| 2280 | |
| 2281 | VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. |
| 2282 | RegSaveFrameIndex = 0xAAAAAAA; // X86-64 only. |
| 2283 | ReturnAddrIndex = 0; // No return address slot generated yet. |
| 2284 | BytesToPopOnReturn = ArgOffset; // Callee pops all stack arguments. |
| 2285 | BytesCallerReserves = 0; |
| 2286 | |
| 2287 | MF.getInfo<X86FunctionInfo>()->setBytesToPopOnReturn(BytesToPopOnReturn); |
| 2288 | |
| 2289 | // Finally, inform the code generator which regs we return values in. |
| 2290 | switch (getValueType(MF.getFunction()->getReturnType())) { |
| 2291 | default: assert(0 && "Unknown type!"); |
| 2292 | case MVT::isVoid: break; |
Chris Lattner | 13bf6c1 | 2006-10-03 17:18:42 +0000 | [diff] [blame] | 2293 | case MVT::i1: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 2294 | case MVT::i8: |
| 2295 | case MVT::i16: |
| 2296 | case MVT::i32: |
| 2297 | MF.addLiveOut(X86::ECX); |
| 2298 | break; |
| 2299 | case MVT::i64: |
| 2300 | MF.addLiveOut(X86::ECX); |
| 2301 | MF.addLiveOut(X86::EDX); |
| 2302 | break; |
| 2303 | case MVT::f32: |
| 2304 | case MVT::f64: |
| 2305 | MF.addLiveOut(X86::ST0); |
| 2306 | break; |
| 2307 | } |
| 2308 | |
| 2309 | // Return the new list of results. |
| 2310 | std::vector<MVT::ValueType> RetVTs(Op.Val->value_begin(), |
| 2311 | Op.Val->value_end()); |
| 2312 | return DAG.getNode(ISD::MERGE_VALUES, RetVTs, &ArgValues[0],ArgValues.size()); |
| 2313 | } |
| 2314 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 2315 | SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) { |
| 2316 | if (ReturnAddrIndex == 0) { |
| 2317 | // Set up a frame object for the return address. |
| 2318 | MachineFunction &MF = DAG.getMachineFunction(); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 2319 | if (Subtarget->is64Bit()) |
| 2320 | ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(8, -8); |
| 2321 | else |
| 2322 | ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 2323 | } |
| 2324 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 2325 | return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 2326 | } |
| 2327 | |
| 2328 | |
| 2329 | |
| 2330 | std::pair<SDOperand, SDOperand> X86TargetLowering:: |
| 2331 | LowerFrameReturnAddress(bool isFrameAddress, SDOperand Chain, unsigned Depth, |
| 2332 | SelectionDAG &DAG) { |
| 2333 | SDOperand Result; |
| 2334 | if (Depth) // Depths > 0 not supported yet! |
| 2335 | Result = DAG.getConstant(0, getPointerTy()); |
| 2336 | else { |
| 2337 | SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG); |
| 2338 | if (!isFrameAddress) |
| 2339 | // Just load the return address |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 2340 | Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), RetAddrFI, |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 2341 | NULL, 0); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 2342 | else |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 2343 | Result = DAG.getNode(ISD::SUB, getPointerTy(), RetAddrFI, |
| 2344 | DAG.getConstant(4, getPointerTy())); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 2345 | } |
| 2346 | return std::make_pair(Result, Chain); |
| 2347 | } |
| 2348 | |
Evan Cheng | 6dfa999 | 2006-01-30 23:41:35 +0000 | [diff] [blame] | 2349 | /// translateX86CC - do a one to one translation of a ISD::CondCode to the X86 |
| 2350 | /// specific condition code. It returns a false if it cannot do a direct |
Chris Lattner | f957051 | 2006-09-13 03:22:10 +0000 | [diff] [blame] | 2351 | /// translation. X86CC is the translated CondCode. LHS/RHS are modified as |
| 2352 | /// needed. |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 2353 | static bool translateX86CC(ISD::CondCode SetCCOpcode, bool isFP, |
Chris Lattner | f957051 | 2006-09-13 03:22:10 +0000 | [diff] [blame] | 2354 | unsigned &X86CC, SDOperand &LHS, SDOperand &RHS, |
| 2355 | SelectionDAG &DAG) { |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2356 | X86CC = X86::COND_INVALID; |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2357 | if (!isFP) { |
Chris Lattner | bfd68a7 | 2006-09-13 17:04:54 +0000 | [diff] [blame] | 2358 | if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { |
| 2359 | if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { |
| 2360 | // X > -1 -> X == 0, jump !sign. |
| 2361 | RHS = DAG.getConstant(0, RHS.getValueType()); |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2362 | X86CC = X86::COND_NS; |
Chris Lattner | bfd68a7 | 2006-09-13 17:04:54 +0000 | [diff] [blame] | 2363 | return true; |
| 2364 | } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { |
| 2365 | // X < 0 -> X == 0, jump on sign. |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2366 | X86CC = X86::COND_S; |
Chris Lattner | bfd68a7 | 2006-09-13 17:04:54 +0000 | [diff] [blame] | 2367 | return true; |
| 2368 | } |
Chris Lattner | f957051 | 2006-09-13 03:22:10 +0000 | [diff] [blame] | 2369 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 2370 | |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2371 | switch (SetCCOpcode) { |
| 2372 | default: break; |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2373 | case ISD::SETEQ: X86CC = X86::COND_E; break; |
| 2374 | case ISD::SETGT: X86CC = X86::COND_G; break; |
| 2375 | case ISD::SETGE: X86CC = X86::COND_GE; break; |
| 2376 | case ISD::SETLT: X86CC = X86::COND_L; break; |
| 2377 | case ISD::SETLE: X86CC = X86::COND_LE; break; |
| 2378 | case ISD::SETNE: X86CC = X86::COND_NE; break; |
| 2379 | case ISD::SETULT: X86CC = X86::COND_B; break; |
| 2380 | case ISD::SETUGT: X86CC = X86::COND_A; break; |
| 2381 | case ISD::SETULE: X86CC = X86::COND_BE; break; |
| 2382 | case ISD::SETUGE: X86CC = X86::COND_AE; break; |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2383 | } |
| 2384 | } else { |
| 2385 | // On a floating point condition, the flags are set as follows: |
| 2386 | // ZF PF CF op |
| 2387 | // 0 | 0 | 0 | X > Y |
| 2388 | // 0 | 0 | 1 | X < Y |
| 2389 | // 1 | 0 | 0 | X == Y |
| 2390 | // 1 | 1 | 1 | unordered |
Chris Lattner | f957051 | 2006-09-13 03:22:10 +0000 | [diff] [blame] | 2391 | bool Flip = false; |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2392 | switch (SetCCOpcode) { |
| 2393 | default: break; |
| 2394 | case ISD::SETUEQ: |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2395 | case ISD::SETEQ: X86CC = X86::COND_E; break; |
Evan Cheng | 5001ea1 | 2006-04-17 07:24:10 +0000 | [diff] [blame] | 2396 | case ISD::SETOLT: Flip = true; // Fallthrough |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2397 | case ISD::SETOGT: |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2398 | case ISD::SETGT: X86CC = X86::COND_A; break; |
Evan Cheng | 5001ea1 | 2006-04-17 07:24:10 +0000 | [diff] [blame] | 2399 | case ISD::SETOLE: Flip = true; // Fallthrough |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2400 | case ISD::SETOGE: |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2401 | case ISD::SETGE: X86CC = X86::COND_AE; break; |
Evan Cheng | 5001ea1 | 2006-04-17 07:24:10 +0000 | [diff] [blame] | 2402 | case ISD::SETUGT: Flip = true; // Fallthrough |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2403 | case ISD::SETULT: |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2404 | case ISD::SETLT: X86CC = X86::COND_B; break; |
Evan Cheng | 5001ea1 | 2006-04-17 07:24:10 +0000 | [diff] [blame] | 2405 | case ISD::SETUGE: Flip = true; // Fallthrough |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2406 | case ISD::SETULE: |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2407 | case ISD::SETLE: X86CC = X86::COND_BE; break; |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2408 | case ISD::SETONE: |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2409 | case ISD::SETNE: X86CC = X86::COND_NE; break; |
| 2410 | case ISD::SETUO: X86CC = X86::COND_P; break; |
| 2411 | case ISD::SETO: X86CC = X86::COND_NP; break; |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2412 | } |
Chris Lattner | f957051 | 2006-09-13 03:22:10 +0000 | [diff] [blame] | 2413 | if (Flip) |
| 2414 | std::swap(LHS, RHS); |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2415 | } |
Evan Cheng | 6dfa999 | 2006-01-30 23:41:35 +0000 | [diff] [blame] | 2416 | |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2417 | return X86CC != X86::COND_INVALID; |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2418 | } |
| 2419 | |
Evan Cheng | 4a46080 | 2006-01-11 00:33:36 +0000 | [diff] [blame] | 2420 | /// hasFPCMov - is there a floating point cmov for the specific X86 condition |
| 2421 | /// code. Current x86 isa includes the following FP cmov instructions: |
Evan Cheng | aaca22c | 2006-01-10 20:26:56 +0000 | [diff] [blame] | 2422 | /// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. |
Evan Cheng | 4a46080 | 2006-01-11 00:33:36 +0000 | [diff] [blame] | 2423 | static bool hasFPCMov(unsigned X86CC) { |
Evan Cheng | aaca22c | 2006-01-10 20:26:56 +0000 | [diff] [blame] | 2424 | switch (X86CC) { |
| 2425 | default: |
| 2426 | return false; |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2427 | case X86::COND_B: |
| 2428 | case X86::COND_BE: |
| 2429 | case X86::COND_E: |
| 2430 | case X86::COND_P: |
| 2431 | case X86::COND_A: |
| 2432 | case X86::COND_AE: |
| 2433 | case X86::COND_NE: |
| 2434 | case X86::COND_NP: |
Evan Cheng | aaca22c | 2006-01-10 20:26:56 +0000 | [diff] [blame] | 2435 | return true; |
| 2436 | } |
| 2437 | } |
| 2438 | |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2439 | /// isUndefOrInRange - Op is either an undef node or a ConstantSDNode. Return |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2440 | /// true if Op is undef or if its value falls within the specified range (L, H]. |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2441 | static bool isUndefOrInRange(SDOperand Op, unsigned Low, unsigned Hi) { |
| 2442 | if (Op.getOpcode() == ISD::UNDEF) |
| 2443 | return true; |
| 2444 | |
| 2445 | unsigned Val = cast<ConstantSDNode>(Op)->getValue(); |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2446 | return (Val >= Low && Val < Hi); |
| 2447 | } |
| 2448 | |
| 2449 | /// isUndefOrEqual - Op is either an undef node or a ConstantSDNode. Return |
| 2450 | /// true if Op is undef or if its value equal to the specified value. |
| 2451 | static bool isUndefOrEqual(SDOperand Op, unsigned Val) { |
| 2452 | if (Op.getOpcode() == ISD::UNDEF) |
| 2453 | return true; |
| 2454 | return cast<ConstantSDNode>(Op)->getValue() == Val; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2455 | } |
| 2456 | |
Evan Cheng | 0188ecb | 2006-03-22 18:59:22 +0000 | [diff] [blame] | 2457 | /// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2458 | /// specifies a shuffle of elements that is suitable for input to PSHUFD. |
| 2459 | bool X86::isPSHUFDMask(SDNode *N) { |
| 2460 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2461 | |
| 2462 | if (N->getNumOperands() != 4) |
| 2463 | return false; |
| 2464 | |
| 2465 | // Check if the value doesn't reference the second vector. |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2466 | for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2467 | SDOperand Arg = N->getOperand(i); |
| 2468 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2469 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
| 2470 | if (cast<ConstantSDNode>(Arg)->getValue() >= 4) |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2471 | return false; |
| 2472 | } |
| 2473 | |
| 2474 | return true; |
| 2475 | } |
| 2476 | |
| 2477 | /// isPSHUFHWMask - Return true if the specified VECTOR_SHUFFLE operand |
Evan Cheng | c21a053 | 2006-04-05 01:47:37 +0000 | [diff] [blame] | 2478 | /// specifies a shuffle of elements that is suitable for input to PSHUFHW. |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2479 | bool X86::isPSHUFHWMask(SDNode *N) { |
| 2480 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2481 | |
| 2482 | if (N->getNumOperands() != 8) |
| 2483 | return false; |
| 2484 | |
| 2485 | // Lower quadword copied in order. |
| 2486 | for (unsigned i = 0; i != 4; ++i) { |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2487 | SDOperand Arg = N->getOperand(i); |
| 2488 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2489 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
| 2490 | if (cast<ConstantSDNode>(Arg)->getValue() != i) |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2491 | return false; |
| 2492 | } |
| 2493 | |
| 2494 | // Upper quadword shuffled. |
| 2495 | for (unsigned i = 4; i != 8; ++i) { |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2496 | SDOperand Arg = N->getOperand(i); |
| 2497 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2498 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
| 2499 | unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2500 | if (Val < 4 || Val > 7) |
| 2501 | return false; |
| 2502 | } |
| 2503 | |
| 2504 | return true; |
| 2505 | } |
| 2506 | |
| 2507 | /// isPSHUFLWMask - Return true if the specified VECTOR_SHUFFLE operand |
Evan Cheng | c21a053 | 2006-04-05 01:47:37 +0000 | [diff] [blame] | 2508 | /// specifies a shuffle of elements that is suitable for input to PSHUFLW. |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2509 | bool X86::isPSHUFLWMask(SDNode *N) { |
| 2510 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2511 | |
| 2512 | if (N->getNumOperands() != 8) |
| 2513 | return false; |
| 2514 | |
| 2515 | // Upper quadword copied in order. |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2516 | for (unsigned i = 4; i != 8; ++i) |
| 2517 | if (!isUndefOrEqual(N->getOperand(i), i)) |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2518 | return false; |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2519 | |
| 2520 | // Lower quadword shuffled. |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2521 | for (unsigned i = 0; i != 4; ++i) |
| 2522 | if (!isUndefOrInRange(N->getOperand(i), 0, 4)) |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2523 | return false; |
Evan Cheng | 0188ecb | 2006-03-22 18:59:22 +0000 | [diff] [blame] | 2524 | |
| 2525 | return true; |
| 2526 | } |
| 2527 | |
Evan Cheng | 14aed5e | 2006-03-24 01:18:28 +0000 | [diff] [blame] | 2528 | /// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2529 | /// specifies a shuffle of elements that is suitable for input to SHUFP*. |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2530 | static bool isSHUFPMask(std::vector<SDOperand> &N) { |
| 2531 | unsigned NumElems = N.size(); |
| 2532 | if (NumElems != 2 && NumElems != 4) return false; |
Evan Cheng | 14aed5e | 2006-03-24 01:18:28 +0000 | [diff] [blame] | 2533 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2534 | unsigned Half = NumElems / 2; |
| 2535 | for (unsigned i = 0; i < Half; ++i) |
| 2536 | if (!isUndefOrInRange(N[i], 0, NumElems)) |
| 2537 | return false; |
| 2538 | for (unsigned i = Half; i < NumElems; ++i) |
| 2539 | if (!isUndefOrInRange(N[i], NumElems, NumElems*2)) |
| 2540 | return false; |
Evan Cheng | 14aed5e | 2006-03-24 01:18:28 +0000 | [diff] [blame] | 2541 | |
| 2542 | return true; |
| 2543 | } |
| 2544 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2545 | bool X86::isSHUFPMask(SDNode *N) { |
| 2546 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2547 | std::vector<SDOperand> Ops(N->op_begin(), N->op_end()); |
| 2548 | return ::isSHUFPMask(Ops); |
| 2549 | } |
| 2550 | |
| 2551 | /// isCommutedSHUFP - Returns true if the shuffle mask is except |
| 2552 | /// the reverse of what x86 shuffles want. x86 shuffles requires the lower |
| 2553 | /// half elements to come from vector 1 (which would equal the dest.) and |
| 2554 | /// the upper half to come from vector 2. |
| 2555 | static bool isCommutedSHUFP(std::vector<SDOperand> &Ops) { |
| 2556 | unsigned NumElems = Ops.size(); |
| 2557 | if (NumElems != 2 && NumElems != 4) return false; |
| 2558 | |
| 2559 | unsigned Half = NumElems / 2; |
| 2560 | for (unsigned i = 0; i < Half; ++i) |
| 2561 | if (!isUndefOrInRange(Ops[i], NumElems, NumElems*2)) |
| 2562 | return false; |
| 2563 | for (unsigned i = Half; i < NumElems; ++i) |
| 2564 | if (!isUndefOrInRange(Ops[i], 0, NumElems)) |
| 2565 | return false; |
| 2566 | return true; |
| 2567 | } |
| 2568 | |
| 2569 | static bool isCommutedSHUFP(SDNode *N) { |
| 2570 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2571 | std::vector<SDOperand> Ops(N->op_begin(), N->op_end()); |
| 2572 | return isCommutedSHUFP(Ops); |
| 2573 | } |
| 2574 | |
Evan Cheng | 2c0dbd0 | 2006-03-24 02:58:06 +0000 | [diff] [blame] | 2575 | /// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2576 | /// specifies a shuffle of elements that is suitable for input to MOVHLPS. |
| 2577 | bool X86::isMOVHLPSMask(SDNode *N) { |
| 2578 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2579 | |
Evan Cheng | 2064a2b | 2006-03-28 06:50:32 +0000 | [diff] [blame] | 2580 | if (N->getNumOperands() != 4) |
Evan Cheng | 2c0dbd0 | 2006-03-24 02:58:06 +0000 | [diff] [blame] | 2581 | return false; |
| 2582 | |
Evan Cheng | 2064a2b | 2006-03-28 06:50:32 +0000 | [diff] [blame] | 2583 | // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2584 | return isUndefOrEqual(N->getOperand(0), 6) && |
| 2585 | isUndefOrEqual(N->getOperand(1), 7) && |
| 2586 | isUndefOrEqual(N->getOperand(2), 2) && |
| 2587 | isUndefOrEqual(N->getOperand(3), 3); |
Evan Cheng | 2064a2b | 2006-03-28 06:50:32 +0000 | [diff] [blame] | 2588 | } |
| 2589 | |
Evan Cheng | 6e56e2c | 2006-11-07 22:14:24 +0000 | [diff] [blame] | 2590 | /// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form |
| 2591 | /// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, |
| 2592 | /// <2, 3, 2, 3> |
| 2593 | bool X86::isMOVHLPS_v_undef_Mask(SDNode *N) { |
| 2594 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2595 | |
| 2596 | if (N->getNumOperands() != 4) |
| 2597 | return false; |
| 2598 | |
| 2599 | // Expect bit0 == 2, bit1 == 3, bit2 == 2, bit3 == 3 |
| 2600 | return isUndefOrEqual(N->getOperand(0), 2) && |
| 2601 | isUndefOrEqual(N->getOperand(1), 3) && |
| 2602 | isUndefOrEqual(N->getOperand(2), 2) && |
| 2603 | isUndefOrEqual(N->getOperand(3), 3); |
| 2604 | } |
| 2605 | |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2606 | /// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2607 | /// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. |
| 2608 | bool X86::isMOVLPMask(SDNode *N) { |
| 2609 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2610 | |
| 2611 | unsigned NumElems = N->getNumOperands(); |
| 2612 | if (NumElems != 2 && NumElems != 4) |
| 2613 | return false; |
| 2614 | |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2615 | for (unsigned i = 0; i < NumElems/2; ++i) |
| 2616 | if (!isUndefOrEqual(N->getOperand(i), i + NumElems)) |
| 2617 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2618 | |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2619 | for (unsigned i = NumElems/2; i < NumElems; ++i) |
| 2620 | if (!isUndefOrEqual(N->getOperand(i), i)) |
| 2621 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2622 | |
| 2623 | return true; |
| 2624 | } |
| 2625 | |
| 2626 | /// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 2627 | /// specifies a shuffle of elements that is suitable for input to MOVHP{S|D} |
| 2628 | /// and MOVLHPS. |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2629 | bool X86::isMOVHPMask(SDNode *N) { |
| 2630 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2631 | |
| 2632 | unsigned NumElems = N->getNumOperands(); |
| 2633 | if (NumElems != 2 && NumElems != 4) |
| 2634 | return false; |
| 2635 | |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2636 | for (unsigned i = 0; i < NumElems/2; ++i) |
| 2637 | if (!isUndefOrEqual(N->getOperand(i), i)) |
| 2638 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2639 | |
| 2640 | for (unsigned i = 0; i < NumElems/2; ++i) { |
| 2641 | SDOperand Arg = N->getOperand(i + NumElems/2); |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2642 | if (!isUndefOrEqual(Arg, i + NumElems)) |
| 2643 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2644 | } |
| 2645 | |
| 2646 | return true; |
| 2647 | } |
| 2648 | |
Evan Cheng | 0038e59 | 2006-03-28 00:39:58 +0000 | [diff] [blame] | 2649 | /// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2650 | /// specifies a shuffle of elements that is suitable for input to UNPCKL. |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2651 | bool static isUNPCKLMask(std::vector<SDOperand> &N, bool V2IsSplat = false) { |
| 2652 | unsigned NumElems = N.size(); |
Evan Cheng | 0038e59 | 2006-03-28 00:39:58 +0000 | [diff] [blame] | 2653 | if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) |
| 2654 | return false; |
| 2655 | |
| 2656 | for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2657 | SDOperand BitI = N[i]; |
| 2658 | SDOperand BitI1 = N[i+1]; |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2659 | if (!isUndefOrEqual(BitI, j)) |
| 2660 | return false; |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2661 | if (V2IsSplat) { |
| 2662 | if (isUndefOrEqual(BitI1, NumElems)) |
| 2663 | return false; |
| 2664 | } else { |
| 2665 | if (!isUndefOrEqual(BitI1, j + NumElems)) |
| 2666 | return false; |
| 2667 | } |
Evan Cheng | 0038e59 | 2006-03-28 00:39:58 +0000 | [diff] [blame] | 2668 | } |
| 2669 | |
| 2670 | return true; |
| 2671 | } |
| 2672 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2673 | bool X86::isUNPCKLMask(SDNode *N, bool V2IsSplat) { |
| 2674 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2675 | std::vector<SDOperand> Ops(N->op_begin(), N->op_end()); |
| 2676 | return ::isUNPCKLMask(Ops, V2IsSplat); |
| 2677 | } |
| 2678 | |
Evan Cheng | 4fcb922 | 2006-03-28 02:43:26 +0000 | [diff] [blame] | 2679 | /// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2680 | /// specifies a shuffle of elements that is suitable for input to UNPCKH. |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2681 | bool static isUNPCKHMask(std::vector<SDOperand> &N, bool V2IsSplat = false) { |
| 2682 | unsigned NumElems = N.size(); |
Evan Cheng | 4fcb922 | 2006-03-28 02:43:26 +0000 | [diff] [blame] | 2683 | if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) |
| 2684 | return false; |
| 2685 | |
| 2686 | for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2687 | SDOperand BitI = N[i]; |
| 2688 | SDOperand BitI1 = N[i+1]; |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2689 | if (!isUndefOrEqual(BitI, j + NumElems/2)) |
| 2690 | return false; |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2691 | if (V2IsSplat) { |
| 2692 | if (isUndefOrEqual(BitI1, NumElems)) |
| 2693 | return false; |
| 2694 | } else { |
| 2695 | if (!isUndefOrEqual(BitI1, j + NumElems/2 + NumElems)) |
| 2696 | return false; |
| 2697 | } |
Evan Cheng | 4fcb922 | 2006-03-28 02:43:26 +0000 | [diff] [blame] | 2698 | } |
| 2699 | |
| 2700 | return true; |
| 2701 | } |
| 2702 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2703 | bool X86::isUNPCKHMask(SDNode *N, bool V2IsSplat) { |
| 2704 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2705 | std::vector<SDOperand> Ops(N->op_begin(), N->op_end()); |
| 2706 | return ::isUNPCKHMask(Ops, V2IsSplat); |
| 2707 | } |
| 2708 | |
Evan Cheng | 1d5a8cc | 2006-04-05 07:20:06 +0000 | [diff] [blame] | 2709 | /// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form |
| 2710 | /// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, |
| 2711 | /// <0, 0, 1, 1> |
| 2712 | bool X86::isUNPCKL_v_undef_Mask(SDNode *N) { |
| 2713 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2714 | |
| 2715 | unsigned NumElems = N->getNumOperands(); |
| 2716 | if (NumElems != 4 && NumElems != 8 && NumElems != 16) |
| 2717 | return false; |
| 2718 | |
| 2719 | for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) { |
| 2720 | SDOperand BitI = N->getOperand(i); |
| 2721 | SDOperand BitI1 = N->getOperand(i+1); |
| 2722 | |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2723 | if (!isUndefOrEqual(BitI, j)) |
| 2724 | return false; |
| 2725 | if (!isUndefOrEqual(BitI1, j)) |
| 2726 | return false; |
Evan Cheng | 1d5a8cc | 2006-04-05 07:20:06 +0000 | [diff] [blame] | 2727 | } |
| 2728 | |
| 2729 | return true; |
| 2730 | } |
| 2731 | |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 2732 | /// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2733 | /// specifies a shuffle of elements that is suitable for input to MOVSS, |
| 2734 | /// MOVSD, and MOVD, i.e. setting the lowest element. |
| 2735 | static bool isMOVLMask(std::vector<SDOperand> &N) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2736 | unsigned NumElems = N.size(); |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 2737 | if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) |
Evan Cheng | d6d1cbd | 2006-04-11 00:19:04 +0000 | [diff] [blame] | 2738 | return false; |
| 2739 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2740 | if (!isUndefOrEqual(N[0], NumElems)) |
Evan Cheng | d6d1cbd | 2006-04-11 00:19:04 +0000 | [diff] [blame] | 2741 | return false; |
| 2742 | |
| 2743 | for (unsigned i = 1; i < NumElems; ++i) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2744 | SDOperand Arg = N[i]; |
Evan Cheng | d6d1cbd | 2006-04-11 00:19:04 +0000 | [diff] [blame] | 2745 | if (!isUndefOrEqual(Arg, i)) |
| 2746 | return false; |
| 2747 | } |
| 2748 | |
| 2749 | return true; |
| 2750 | } |
Evan Cheng | 1d5a8cc | 2006-04-05 07:20:06 +0000 | [diff] [blame] | 2751 | |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 2752 | bool X86::isMOVLMask(SDNode *N) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2753 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2754 | std::vector<SDOperand> Ops(N->op_begin(), N->op_end()); |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 2755 | return ::isMOVLMask(Ops); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2756 | } |
| 2757 | |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 2758 | /// isCommutedMOVL - Returns true if the shuffle mask is except the reverse |
| 2759 | /// of what x86 movss want. X86 movs requires the lowest element to be lowest |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2760 | /// element of vector 2 and the other elements to come from vector 1 in order. |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2761 | static bool isCommutedMOVL(std::vector<SDOperand> &Ops, bool V2IsSplat = false, |
| 2762 | bool V2IsUndef = false) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2763 | unsigned NumElems = Ops.size(); |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 2764 | if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2765 | return false; |
| 2766 | |
| 2767 | if (!isUndefOrEqual(Ops[0], 0)) |
| 2768 | return false; |
| 2769 | |
| 2770 | for (unsigned i = 1; i < NumElems; ++i) { |
| 2771 | SDOperand Arg = Ops[i]; |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2772 | if (!(isUndefOrEqual(Arg, i+NumElems) || |
| 2773 | (V2IsUndef && isUndefOrInRange(Arg, NumElems, NumElems*2)) || |
| 2774 | (V2IsSplat && isUndefOrEqual(Arg, NumElems)))) |
| 2775 | return false; |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2776 | } |
| 2777 | |
| 2778 | return true; |
| 2779 | } |
| 2780 | |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2781 | static bool isCommutedMOVL(SDNode *N, bool V2IsSplat = false, |
| 2782 | bool V2IsUndef = false) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2783 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2784 | std::vector<SDOperand> Ops(N->op_begin(), N->op_end()); |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2785 | return isCommutedMOVL(Ops, V2IsSplat, V2IsUndef); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2786 | } |
| 2787 | |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2788 | /// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2789 | /// specifies a shuffle of elements that is suitable for input to MOVSHDUP. |
| 2790 | bool X86::isMOVSHDUPMask(SDNode *N) { |
| 2791 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2792 | |
| 2793 | if (N->getNumOperands() != 4) |
| 2794 | return false; |
| 2795 | |
| 2796 | // Expect 1, 1, 3, 3 |
| 2797 | for (unsigned i = 0; i < 2; ++i) { |
| 2798 | SDOperand Arg = N->getOperand(i); |
| 2799 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2800 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
| 2801 | unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); |
| 2802 | if (Val != 1) return false; |
| 2803 | } |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2804 | |
| 2805 | bool HasHi = false; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2806 | for (unsigned i = 2; i < 4; ++i) { |
| 2807 | SDOperand Arg = N->getOperand(i); |
| 2808 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2809 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
| 2810 | unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); |
| 2811 | if (Val != 3) return false; |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2812 | HasHi = true; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2813 | } |
Evan Cheng | 39fc145 | 2006-04-15 03:13:24 +0000 | [diff] [blame] | 2814 | |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2815 | // Don't use movshdup if it can be done with a shufps. |
| 2816 | return HasHi; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2817 | } |
| 2818 | |
| 2819 | /// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2820 | /// specifies a shuffle of elements that is suitable for input to MOVSLDUP. |
| 2821 | bool X86::isMOVSLDUPMask(SDNode *N) { |
| 2822 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2823 | |
| 2824 | if (N->getNumOperands() != 4) |
| 2825 | return false; |
| 2826 | |
| 2827 | // Expect 0, 0, 2, 2 |
| 2828 | for (unsigned i = 0; i < 2; ++i) { |
| 2829 | SDOperand Arg = N->getOperand(i); |
| 2830 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2831 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
| 2832 | unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); |
| 2833 | if (Val != 0) return false; |
| 2834 | } |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2835 | |
| 2836 | bool HasHi = false; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2837 | for (unsigned i = 2; i < 4; ++i) { |
| 2838 | SDOperand Arg = N->getOperand(i); |
| 2839 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2840 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
| 2841 | unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); |
| 2842 | if (Val != 2) return false; |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2843 | HasHi = true; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2844 | } |
Evan Cheng | 39fc145 | 2006-04-15 03:13:24 +0000 | [diff] [blame] | 2845 | |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2846 | // Don't use movshdup if it can be done with a shufps. |
| 2847 | return HasHi; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2848 | } |
| 2849 | |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2850 | /// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies |
| 2851 | /// a splat of a single element. |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 2852 | static bool isSplatMask(SDNode *N) { |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2853 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2854 | |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2855 | // This is a splat operation if each element of the permute is the same, and |
| 2856 | // if the value doesn't reference the second vector. |
Evan Cheng | 94fe5eb | 2006-04-19 23:28:59 +0000 | [diff] [blame] | 2857 | unsigned NumElems = N->getNumOperands(); |
| 2858 | SDOperand ElementBase; |
| 2859 | unsigned i = 0; |
| 2860 | for (; i != NumElems; ++i) { |
| 2861 | SDOperand Elt = N->getOperand(i); |
Reid Spencer | 3ed469c | 2006-11-02 20:25:50 +0000 | [diff] [blame] | 2862 | if (isa<ConstantSDNode>(Elt)) { |
Evan Cheng | 94fe5eb | 2006-04-19 23:28:59 +0000 | [diff] [blame] | 2863 | ElementBase = Elt; |
| 2864 | break; |
| 2865 | } |
| 2866 | } |
| 2867 | |
| 2868 | if (!ElementBase.Val) |
| 2869 | return false; |
| 2870 | |
| 2871 | for (; i != NumElems; ++i) { |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2872 | SDOperand Arg = N->getOperand(i); |
| 2873 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2874 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Evan Cheng | 94fe5eb | 2006-04-19 23:28:59 +0000 | [diff] [blame] | 2875 | if (Arg != ElementBase) return false; |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2876 | } |
| 2877 | |
| 2878 | // Make sure it is a splat of the first vector operand. |
Evan Cheng | 94fe5eb | 2006-04-19 23:28:59 +0000 | [diff] [blame] | 2879 | return cast<ConstantSDNode>(ElementBase)->getValue() < NumElems; |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2880 | } |
| 2881 | |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 2882 | /// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies |
| 2883 | /// a splat of a single element and it's a 2 or 4 element mask. |
| 2884 | bool X86::isSplatMask(SDNode *N) { |
| 2885 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2886 | |
Evan Cheng | 94fe5eb | 2006-04-19 23:28:59 +0000 | [diff] [blame] | 2887 | // We can only splat 64-bit, and 32-bit quantities with a single instruction. |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 2888 | if (N->getNumOperands() != 4 && N->getNumOperands() != 2) |
| 2889 | return false; |
| 2890 | return ::isSplatMask(N); |
| 2891 | } |
| 2892 | |
Evan Cheng | f686d9b | 2006-10-27 21:08:32 +0000 | [diff] [blame] | 2893 | /// isSplatLoMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2894 | /// specifies a splat of zero element. |
| 2895 | bool X86::isSplatLoMask(SDNode *N) { |
| 2896 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2897 | |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 2898 | for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) |
Evan Cheng | f686d9b | 2006-10-27 21:08:32 +0000 | [diff] [blame] | 2899 | if (!isUndefOrEqual(N->getOperand(i), 0)) |
| 2900 | return false; |
| 2901 | return true; |
| 2902 | } |
| 2903 | |
Evan Cheng | 63d3300 | 2006-03-22 08:01:21 +0000 | [diff] [blame] | 2904 | /// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle |
| 2905 | /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP* |
| 2906 | /// instructions. |
| 2907 | unsigned X86::getShuffleSHUFImmediate(SDNode *N) { |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2908 | unsigned NumOperands = N->getNumOperands(); |
| 2909 | unsigned Shift = (NumOperands == 4) ? 2 : 1; |
| 2910 | unsigned Mask = 0; |
Evan Cheng | 36b27f3 | 2006-03-28 23:41:33 +0000 | [diff] [blame] | 2911 | for (unsigned i = 0; i < NumOperands; ++i) { |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2912 | unsigned Val = 0; |
| 2913 | SDOperand Arg = N->getOperand(NumOperands-i-1); |
| 2914 | if (Arg.getOpcode() != ISD::UNDEF) |
| 2915 | Val = cast<ConstantSDNode>(Arg)->getValue(); |
Evan Cheng | 14aed5e | 2006-03-24 01:18:28 +0000 | [diff] [blame] | 2916 | if (Val >= NumOperands) Val -= NumOperands; |
Evan Cheng | 63d3300 | 2006-03-22 08:01:21 +0000 | [diff] [blame] | 2917 | Mask |= Val; |
Evan Cheng | 36b27f3 | 2006-03-28 23:41:33 +0000 | [diff] [blame] | 2918 | if (i != NumOperands - 1) |
| 2919 | Mask <<= Shift; |
| 2920 | } |
Evan Cheng | 63d3300 | 2006-03-22 08:01:21 +0000 | [diff] [blame] | 2921 | |
| 2922 | return Mask; |
| 2923 | } |
| 2924 | |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2925 | /// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle |
| 2926 | /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW |
| 2927 | /// instructions. |
| 2928 | unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) { |
| 2929 | unsigned Mask = 0; |
| 2930 | // 8 nodes, but we only care about the last 4. |
| 2931 | for (unsigned i = 7; i >= 4; --i) { |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2932 | unsigned Val = 0; |
| 2933 | SDOperand Arg = N->getOperand(i); |
| 2934 | if (Arg.getOpcode() != ISD::UNDEF) |
| 2935 | Val = cast<ConstantSDNode>(Arg)->getValue(); |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2936 | Mask |= (Val - 4); |
| 2937 | if (i != 4) |
| 2938 | Mask <<= 2; |
| 2939 | } |
| 2940 | |
| 2941 | return Mask; |
| 2942 | } |
| 2943 | |
| 2944 | /// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle |
| 2945 | /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW |
| 2946 | /// instructions. |
| 2947 | unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) { |
| 2948 | unsigned Mask = 0; |
| 2949 | // 8 nodes, but we only care about the first 4. |
| 2950 | for (int i = 3; i >= 0; --i) { |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2951 | unsigned Val = 0; |
| 2952 | SDOperand Arg = N->getOperand(i); |
| 2953 | if (Arg.getOpcode() != ISD::UNDEF) |
| 2954 | Val = cast<ConstantSDNode>(Arg)->getValue(); |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2955 | Mask |= Val; |
| 2956 | if (i != 0) |
| 2957 | Mask <<= 2; |
| 2958 | } |
| 2959 | |
| 2960 | return Mask; |
| 2961 | } |
| 2962 | |
Evan Cheng | c21a053 | 2006-04-05 01:47:37 +0000 | [diff] [blame] | 2963 | /// isPSHUFHW_PSHUFLWMask - true if the specified VECTOR_SHUFFLE operand |
| 2964 | /// specifies a 8 element shuffle that can be broken into a pair of |
| 2965 | /// PSHUFHW and PSHUFLW. |
| 2966 | static bool isPSHUFHW_PSHUFLWMask(SDNode *N) { |
| 2967 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2968 | |
| 2969 | if (N->getNumOperands() != 8) |
| 2970 | return false; |
| 2971 | |
| 2972 | // Lower quadword shuffled. |
| 2973 | for (unsigned i = 0; i != 4; ++i) { |
| 2974 | SDOperand Arg = N->getOperand(i); |
| 2975 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2976 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
| 2977 | unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); |
| 2978 | if (Val > 4) |
| 2979 | return false; |
| 2980 | } |
| 2981 | |
| 2982 | // Upper quadword shuffled. |
| 2983 | for (unsigned i = 4; i != 8; ++i) { |
| 2984 | SDOperand Arg = N->getOperand(i); |
| 2985 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2986 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
| 2987 | unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); |
| 2988 | if (Val < 4 || Val > 7) |
| 2989 | return false; |
| 2990 | } |
| 2991 | |
| 2992 | return true; |
| 2993 | } |
| 2994 | |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2995 | /// CommuteVectorShuffle - Swap vector_shuffle operandsas well as |
| 2996 | /// values in ther permute mask. |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 2997 | static SDOperand CommuteVectorShuffle(SDOperand Op, SDOperand &V1, |
| 2998 | SDOperand &V2, SDOperand &Mask, |
| 2999 | SelectionDAG &DAG) { |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 3000 | MVT::ValueType VT = Op.getValueType(); |
| 3001 | MVT::ValueType MaskVT = Mask.getValueType(); |
| 3002 | MVT::ValueType EltVT = MVT::getVectorBaseType(MaskVT); |
| 3003 | unsigned NumElems = Mask.getNumOperands(); |
| 3004 | std::vector<SDOperand> MaskVec; |
| 3005 | |
| 3006 | for (unsigned i = 0; i != NumElems; ++i) { |
| 3007 | SDOperand Arg = Mask.getOperand(i); |
Evan Cheng | 80d428c | 2006-04-19 22:48:17 +0000 | [diff] [blame] | 3008 | if (Arg.getOpcode() == ISD::UNDEF) { |
| 3009 | MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); |
| 3010 | continue; |
| 3011 | } |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 3012 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
| 3013 | unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); |
| 3014 | if (Val < NumElems) |
| 3015 | MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); |
| 3016 | else |
| 3017 | MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); |
| 3018 | } |
| 3019 | |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 3020 | std::swap(V1, V2); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 3021 | Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 3022 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 3023 | } |
| 3024 | |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 3025 | /// ShouldXformToMOVHLPS - Return true if the node should be transformed to |
| 3026 | /// match movhlps. The lower half elements should come from upper half of |
| 3027 | /// V1 (and in order), and the upper half elements should come from the upper |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 3028 | /// half of V2 (and in order). |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 3029 | static bool ShouldXformToMOVHLPS(SDNode *Mask) { |
| 3030 | unsigned NumElems = Mask->getNumOperands(); |
| 3031 | if (NumElems != 4) |
| 3032 | return false; |
| 3033 | for (unsigned i = 0, e = 2; i != e; ++i) |
| 3034 | if (!isUndefOrEqual(Mask->getOperand(i), i+2)) |
| 3035 | return false; |
| 3036 | for (unsigned i = 2; i != 4; ++i) |
| 3037 | if (!isUndefOrEqual(Mask->getOperand(i), i+4)) |
| 3038 | return false; |
| 3039 | return true; |
| 3040 | } |
| 3041 | |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 3042 | /// isScalarLoadToVector - Returns true if the node is a scalar load that |
| 3043 | /// is promoted to a vector. |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 3044 | static inline bool isScalarLoadToVector(SDNode *N) { |
| 3045 | if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) { |
| 3046 | N = N->getOperand(0).Val; |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 3047 | return ISD::isNON_EXTLoad(N); |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 3048 | } |
| 3049 | return false; |
| 3050 | } |
| 3051 | |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 3052 | /// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to |
| 3053 | /// match movlp{s|d}. The lower half elements should come from lower half of |
| 3054 | /// V1 (and in order), and the upper half elements should come from the upper |
| 3055 | /// half of V2 (and in order). And since V1 will become the source of the |
| 3056 | /// MOVLP, it must be either a vector load or a scalar load to vector. |
Evan Cheng | 23425f5 | 2006-10-09 21:39:25 +0000 | [diff] [blame] | 3057 | static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, SDNode *Mask) { |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 3058 | if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 3059 | return false; |
Evan Cheng | 23425f5 | 2006-10-09 21:39:25 +0000 | [diff] [blame] | 3060 | // Is V2 is a vector load, don't do this transformation. We will try to use |
| 3061 | // load folding shufps op. |
| 3062 | if (ISD::isNON_EXTLoad(V2)) |
| 3063 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 3064 | |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 3065 | unsigned NumElems = Mask->getNumOperands(); |
| 3066 | if (NumElems != 2 && NumElems != 4) |
| 3067 | return false; |
| 3068 | for (unsigned i = 0, e = NumElems/2; i != e; ++i) |
| 3069 | if (!isUndefOrEqual(Mask->getOperand(i), i)) |
| 3070 | return false; |
| 3071 | for (unsigned i = NumElems/2; i != NumElems; ++i) |
| 3072 | if (!isUndefOrEqual(Mask->getOperand(i), i+NumElems)) |
| 3073 | return false; |
| 3074 | return true; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 3075 | } |
| 3076 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 3077 | /// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are |
| 3078 | /// all the same. |
| 3079 | static bool isSplatVector(SDNode *N) { |
| 3080 | if (N->getOpcode() != ISD::BUILD_VECTOR) |
| 3081 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 3082 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 3083 | SDOperand SplatValue = N->getOperand(0); |
| 3084 | for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) |
| 3085 | if (N->getOperand(i) != SplatValue) |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 3086 | return false; |
| 3087 | return true; |
| 3088 | } |
| 3089 | |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 3090 | /// isUndefShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved |
| 3091 | /// to an undef. |
| 3092 | static bool isUndefShuffle(SDNode *N) { |
| 3093 | if (N->getOpcode() != ISD::BUILD_VECTOR) |
| 3094 | return false; |
| 3095 | |
| 3096 | SDOperand V1 = N->getOperand(0); |
| 3097 | SDOperand V2 = N->getOperand(1); |
| 3098 | SDOperand Mask = N->getOperand(2); |
| 3099 | unsigned NumElems = Mask.getNumOperands(); |
| 3100 | for (unsigned i = 0; i != NumElems; ++i) { |
| 3101 | SDOperand Arg = Mask.getOperand(i); |
| 3102 | if (Arg.getOpcode() != ISD::UNDEF) { |
| 3103 | unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); |
| 3104 | if (Val < NumElems && V1.getOpcode() != ISD::UNDEF) |
| 3105 | return false; |
| 3106 | else if (Val >= NumElems && V2.getOpcode() != ISD::UNDEF) |
| 3107 | return false; |
| 3108 | } |
| 3109 | } |
| 3110 | return true; |
| 3111 | } |
| 3112 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 3113 | /// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements |
| 3114 | /// that point to V2 points to its first element. |
| 3115 | static SDOperand NormalizeMask(SDOperand Mask, SelectionDAG &DAG) { |
| 3116 | assert(Mask.getOpcode() == ISD::BUILD_VECTOR); |
| 3117 | |
| 3118 | bool Changed = false; |
| 3119 | std::vector<SDOperand> MaskVec; |
| 3120 | unsigned NumElems = Mask.getNumOperands(); |
| 3121 | for (unsigned i = 0; i != NumElems; ++i) { |
| 3122 | SDOperand Arg = Mask.getOperand(i); |
| 3123 | if (Arg.getOpcode() != ISD::UNDEF) { |
| 3124 | unsigned Val = cast<ConstantSDNode>(Arg)->getValue(); |
| 3125 | if (Val > NumElems) { |
| 3126 | Arg = DAG.getConstant(NumElems, Arg.getValueType()); |
| 3127 | Changed = true; |
| 3128 | } |
| 3129 | } |
| 3130 | MaskVec.push_back(Arg); |
| 3131 | } |
| 3132 | |
| 3133 | if (Changed) |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 3134 | Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(), |
| 3135 | &MaskVec[0], MaskVec.size()); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 3136 | return Mask; |
| 3137 | } |
| 3138 | |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3139 | /// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd |
| 3140 | /// operation of specified width. |
| 3141 | static SDOperand getMOVLMask(unsigned NumElems, SelectionDAG &DAG) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 3142 | MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 3143 | MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT); |
| 3144 | |
| 3145 | std::vector<SDOperand> MaskVec; |
| 3146 | MaskVec.push_back(DAG.getConstant(NumElems, BaseVT)); |
| 3147 | for (unsigned i = 1; i != NumElems; ++i) |
| 3148 | MaskVec.push_back(DAG.getConstant(i, BaseVT)); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 3149 | return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 3150 | } |
| 3151 | |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 3152 | /// getUnpacklMask - Returns a vector_shuffle mask for an unpackl operation |
| 3153 | /// of specified width. |
| 3154 | static SDOperand getUnpacklMask(unsigned NumElems, SelectionDAG &DAG) { |
| 3155 | MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 3156 | MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT); |
| 3157 | std::vector<SDOperand> MaskVec; |
| 3158 | for (unsigned i = 0, e = NumElems/2; i != e; ++i) { |
| 3159 | MaskVec.push_back(DAG.getConstant(i, BaseVT)); |
| 3160 | MaskVec.push_back(DAG.getConstant(i + NumElems, BaseVT)); |
| 3161 | } |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 3162 | return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 3163 | } |
| 3164 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 3165 | /// getUnpackhMask - Returns a vector_shuffle mask for an unpackh operation |
| 3166 | /// of specified width. |
| 3167 | static SDOperand getUnpackhMask(unsigned NumElems, SelectionDAG &DAG) { |
| 3168 | MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 3169 | MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT); |
| 3170 | unsigned Half = NumElems/2; |
| 3171 | std::vector<SDOperand> MaskVec; |
| 3172 | for (unsigned i = 0; i != Half; ++i) { |
| 3173 | MaskVec.push_back(DAG.getConstant(i + Half, BaseVT)); |
| 3174 | MaskVec.push_back(DAG.getConstant(i + NumElems + Half, BaseVT)); |
| 3175 | } |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 3176 | return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 3177 | } |
| 3178 | |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3179 | /// getZeroVector - Returns a vector of specified type with all zero elements. |
| 3180 | /// |
| 3181 | static SDOperand getZeroVector(MVT::ValueType VT, SelectionDAG &DAG) { |
| 3182 | assert(MVT::isVector(VT) && "Expected a vector type"); |
| 3183 | unsigned NumElems = getVectorNumElements(VT); |
| 3184 | MVT::ValueType EVT = MVT::getVectorBaseType(VT); |
| 3185 | bool isFP = MVT::isFloatingPoint(EVT); |
| 3186 | SDOperand Zero = isFP ? DAG.getConstantFP(0.0, EVT) : DAG.getConstant(0, EVT); |
| 3187 | std::vector<SDOperand> ZeroVec(NumElems, Zero); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 3188 | return DAG.getNode(ISD::BUILD_VECTOR, VT, &ZeroVec[0], ZeroVec.size()); |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3189 | } |
| 3190 | |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 3191 | /// PromoteSplat - Promote a splat of v8i16 or v16i8 to v4i32. |
| 3192 | /// |
| 3193 | static SDOperand PromoteSplat(SDOperand Op, SelectionDAG &DAG) { |
| 3194 | SDOperand V1 = Op.getOperand(0); |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3195 | SDOperand Mask = Op.getOperand(2); |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 3196 | MVT::ValueType VT = Op.getValueType(); |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3197 | unsigned NumElems = Mask.getNumOperands(); |
| 3198 | Mask = getUnpacklMask(NumElems, DAG); |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 3199 | while (NumElems != 4) { |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3200 | V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask); |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 3201 | NumElems >>= 1; |
| 3202 | } |
| 3203 | V1 = DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, V1); |
| 3204 | |
| 3205 | MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4); |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3206 | Mask = getZeroVector(MaskVT, DAG); |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 3207 | SDOperand Shuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v4i32, V1, |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3208 | DAG.getNode(ISD::UNDEF, MVT::v4i32), Mask); |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 3209 | return DAG.getNode(ISD::BIT_CONVERT, VT, Shuffle); |
| 3210 | } |
| 3211 | |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3212 | /// isZeroNode - Returns true if Elt is a constant zero or a floating point |
| 3213 | /// constant +0.0. |
| 3214 | static inline bool isZeroNode(SDOperand Elt) { |
| 3215 | return ((isa<ConstantSDNode>(Elt) && |
| 3216 | cast<ConstantSDNode>(Elt)->getValue() == 0) || |
| 3217 | (isa<ConstantFPSDNode>(Elt) && |
| 3218 | cast<ConstantFPSDNode>(Elt)->isExactlyValue(0.0))); |
| 3219 | } |
| 3220 | |
Evan Cheng | ba05f72 | 2006-04-21 23:03:30 +0000 | [diff] [blame] | 3221 | /// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified |
| 3222 | /// vector and zero or undef vector. |
| 3223 | static SDOperand getShuffleVectorZeroOrUndef(SDOperand V2, MVT::ValueType VT, |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3224 | unsigned NumElems, unsigned Idx, |
Evan Cheng | ba05f72 | 2006-04-21 23:03:30 +0000 | [diff] [blame] | 3225 | bool isZero, SelectionDAG &DAG) { |
| 3226 | SDOperand V1 = isZero ? getZeroVector(VT, DAG) : DAG.getNode(ISD::UNDEF, VT); |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3227 | MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 3228 | MVT::ValueType EVT = MVT::getVectorBaseType(MaskVT); |
| 3229 | SDOperand Zero = DAG.getConstant(0, EVT); |
| 3230 | std::vector<SDOperand> MaskVec(NumElems, Zero); |
| 3231 | MaskVec[Idx] = DAG.getConstant(NumElems, EVT); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 3232 | SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3233 | &MaskVec[0], MaskVec.size()); |
Evan Cheng | ba05f72 | 2006-04-21 23:03:30 +0000 | [diff] [blame] | 3234 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3235 | } |
| 3236 | |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3237 | /// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. |
| 3238 | /// |
| 3239 | static SDOperand LowerBuildVectorv16i8(SDOperand Op, unsigned NonZeros, |
| 3240 | unsigned NumNonZero, unsigned NumZero, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3241 | SelectionDAG &DAG, TargetLowering &TLI) { |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3242 | if (NumNonZero > 8) |
| 3243 | return SDOperand(); |
| 3244 | |
| 3245 | SDOperand V(0, 0); |
| 3246 | bool First = true; |
| 3247 | for (unsigned i = 0; i < 16; ++i) { |
| 3248 | bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; |
| 3249 | if (ThisIsNonZero && First) { |
| 3250 | if (NumZero) |
| 3251 | V = getZeroVector(MVT::v8i16, DAG); |
| 3252 | else |
| 3253 | V = DAG.getNode(ISD::UNDEF, MVT::v8i16); |
| 3254 | First = false; |
| 3255 | } |
| 3256 | |
| 3257 | if ((i & 1) != 0) { |
| 3258 | SDOperand ThisElt(0, 0), LastElt(0, 0); |
| 3259 | bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; |
| 3260 | if (LastIsNonZero) { |
| 3261 | LastElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i-1)); |
| 3262 | } |
| 3263 | if (ThisIsNonZero) { |
| 3264 | ThisElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i)); |
| 3265 | ThisElt = DAG.getNode(ISD::SHL, MVT::i16, |
| 3266 | ThisElt, DAG.getConstant(8, MVT::i8)); |
| 3267 | if (LastIsNonZero) |
| 3268 | ThisElt = DAG.getNode(ISD::OR, MVT::i16, ThisElt, LastElt); |
| 3269 | } else |
| 3270 | ThisElt = LastElt; |
| 3271 | |
| 3272 | if (ThisElt.Val) |
| 3273 | V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, ThisElt, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3274 | DAG.getConstant(i/2, TLI.getPointerTy())); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3275 | } |
| 3276 | } |
| 3277 | |
| 3278 | return DAG.getNode(ISD::BIT_CONVERT, MVT::v16i8, V); |
| 3279 | } |
| 3280 | |
| 3281 | /// LowerBuildVectorv16i8 - Custom lower build_vector of v8i16. |
| 3282 | /// |
| 3283 | static SDOperand LowerBuildVectorv8i16(SDOperand Op, unsigned NonZeros, |
| 3284 | unsigned NumNonZero, unsigned NumZero, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3285 | SelectionDAG &DAG, TargetLowering &TLI) { |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3286 | if (NumNonZero > 4) |
| 3287 | return SDOperand(); |
| 3288 | |
| 3289 | SDOperand V(0, 0); |
| 3290 | bool First = true; |
| 3291 | for (unsigned i = 0; i < 8; ++i) { |
| 3292 | bool isNonZero = (NonZeros & (1 << i)) != 0; |
| 3293 | if (isNonZero) { |
| 3294 | if (First) { |
| 3295 | if (NumZero) |
| 3296 | V = getZeroVector(MVT::v8i16, DAG); |
| 3297 | else |
| 3298 | V = DAG.getNode(ISD::UNDEF, MVT::v8i16); |
| 3299 | First = false; |
| 3300 | } |
| 3301 | V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, Op.getOperand(i), |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3302 | DAG.getConstant(i, TLI.getPointerTy())); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3303 | } |
| 3304 | } |
| 3305 | |
| 3306 | return V; |
| 3307 | } |
| 3308 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3309 | SDOperand |
| 3310 | X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) { |
| 3311 | // All zero's are handled with pxor. |
| 3312 | if (ISD::isBuildVectorAllZeros(Op.Val)) |
| 3313 | return Op; |
| 3314 | |
| 3315 | // All one's are handled with pcmpeqd. |
| 3316 | if (ISD::isBuildVectorAllOnes(Op.Val)) |
| 3317 | return Op; |
| 3318 | |
| 3319 | MVT::ValueType VT = Op.getValueType(); |
| 3320 | MVT::ValueType EVT = MVT::getVectorBaseType(VT); |
| 3321 | unsigned EVTBits = MVT::getSizeInBits(EVT); |
| 3322 | |
| 3323 | unsigned NumElems = Op.getNumOperands(); |
| 3324 | unsigned NumZero = 0; |
| 3325 | unsigned NumNonZero = 0; |
| 3326 | unsigned NonZeros = 0; |
| 3327 | std::set<SDOperand> Values; |
| 3328 | for (unsigned i = 0; i < NumElems; ++i) { |
| 3329 | SDOperand Elt = Op.getOperand(i); |
| 3330 | if (Elt.getOpcode() != ISD::UNDEF) { |
| 3331 | Values.insert(Elt); |
| 3332 | if (isZeroNode(Elt)) |
| 3333 | NumZero++; |
| 3334 | else { |
| 3335 | NonZeros |= (1 << i); |
| 3336 | NumNonZero++; |
| 3337 | } |
| 3338 | } |
| 3339 | } |
| 3340 | |
| 3341 | if (NumNonZero == 0) |
| 3342 | // Must be a mix of zero and undef. Return a zero vector. |
| 3343 | return getZeroVector(VT, DAG); |
| 3344 | |
| 3345 | // Splat is obviously ok. Let legalizer expand it to a shuffle. |
| 3346 | if (Values.size() == 1) |
| 3347 | return SDOperand(); |
| 3348 | |
| 3349 | // Special case for single non-zero element. |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3350 | if (NumNonZero == 1) { |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3351 | unsigned Idx = CountTrailingZeros_32(NonZeros); |
| 3352 | SDOperand Item = Op.getOperand(Idx); |
| 3353 | Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); |
| 3354 | if (Idx == 0) |
| 3355 | // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. |
| 3356 | return getShuffleVectorZeroOrUndef(Item, VT, NumElems, Idx, |
| 3357 | NumZero > 0, DAG); |
| 3358 | |
| 3359 | if (EVTBits == 32) { |
| 3360 | // Turn it into a shuffle of zero and zero-extended scalar to vector. |
| 3361 | Item = getShuffleVectorZeroOrUndef(Item, VT, NumElems, 0, NumZero > 0, |
| 3362 | DAG); |
| 3363 | MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 3364 | MVT::ValueType MaskEVT = MVT::getVectorBaseType(MaskVT); |
| 3365 | std::vector<SDOperand> MaskVec; |
| 3366 | for (unsigned i = 0; i < NumElems; i++) |
| 3367 | MaskVec.push_back(DAG.getConstant((i == Idx) ? 0 : 1, MaskEVT)); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 3368 | SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3369 | &MaskVec[0], MaskVec.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3370 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Item, |
| 3371 | DAG.getNode(ISD::UNDEF, VT), Mask); |
| 3372 | } |
| 3373 | } |
| 3374 | |
Evan Cheng | e111303 | 2006-10-04 18:33:38 +0000 | [diff] [blame] | 3375 | // Let legalizer expand 2-wide build_vector's. |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3376 | if (EVTBits == 64) |
| 3377 | return SDOperand(); |
| 3378 | |
| 3379 | // If element VT is < 32 bits, convert it to inserts into a zero vector. |
| 3380 | if (EVTBits == 8) { |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3381 | SDOperand V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, |
| 3382 | *this); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3383 | if (V.Val) return V; |
| 3384 | } |
| 3385 | |
| 3386 | if (EVTBits == 16) { |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3387 | SDOperand V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, |
| 3388 | *this); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3389 | if (V.Val) return V; |
| 3390 | } |
| 3391 | |
| 3392 | // If element VT is == 32 bits, turn it into a number of shuffles. |
| 3393 | std::vector<SDOperand> V(NumElems); |
| 3394 | if (NumElems == 4 && NumZero > 0) { |
| 3395 | for (unsigned i = 0; i < 4; ++i) { |
| 3396 | bool isZero = !(NonZeros & (1 << i)); |
| 3397 | if (isZero) |
| 3398 | V[i] = getZeroVector(VT, DAG); |
| 3399 | else |
| 3400 | V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); |
| 3401 | } |
| 3402 | |
| 3403 | for (unsigned i = 0; i < 2; ++i) { |
| 3404 | switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { |
| 3405 | default: break; |
| 3406 | case 0: |
| 3407 | V[i] = V[i*2]; // Must be a zero vector. |
| 3408 | break; |
| 3409 | case 1: |
| 3410 | V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2+1], V[i*2], |
| 3411 | getMOVLMask(NumElems, DAG)); |
| 3412 | break; |
| 3413 | case 2: |
| 3414 | V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], |
| 3415 | getMOVLMask(NumElems, DAG)); |
| 3416 | break; |
| 3417 | case 3: |
| 3418 | V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], |
| 3419 | getUnpacklMask(NumElems, DAG)); |
| 3420 | break; |
| 3421 | } |
| 3422 | } |
| 3423 | |
Evan Cheng | 069287d | 2006-05-16 07:21:53 +0000 | [diff] [blame] | 3424 | // Take advantage of the fact GR32 to VR128 scalar_to_vector (i.e. movd) |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 3425 | // clears the upper bits. |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3426 | // FIXME: we can do the same for v4f32 case when we know both parts of |
| 3427 | // the lower half come from scalar_to_vector (loadf32). We should do |
| 3428 | // that in post legalizer dag combiner with target specific hooks. |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3429 | if (MVT::isInteger(EVT) && (NonZeros & (0x3 << 2)) == 0) |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3430 | return V[0]; |
| 3431 | MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 3432 | MVT::ValueType EVT = MVT::getVectorBaseType(MaskVT); |
| 3433 | std::vector<SDOperand> MaskVec; |
| 3434 | bool Reverse = (NonZeros & 0x3) == 2; |
| 3435 | for (unsigned i = 0; i < 2; ++i) |
| 3436 | if (Reverse) |
| 3437 | MaskVec.push_back(DAG.getConstant(1-i, EVT)); |
| 3438 | else |
| 3439 | MaskVec.push_back(DAG.getConstant(i, EVT)); |
| 3440 | Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2; |
| 3441 | for (unsigned i = 0; i < 2; ++i) |
| 3442 | if (Reverse) |
| 3443 | MaskVec.push_back(DAG.getConstant(1-i+NumElems, EVT)); |
| 3444 | else |
| 3445 | MaskVec.push_back(DAG.getConstant(i+NumElems, EVT)); |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3446 | SDOperand ShufMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3447 | &MaskVec[0], MaskVec.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3448 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[0], V[1], ShufMask); |
| 3449 | } |
| 3450 | |
| 3451 | if (Values.size() > 2) { |
| 3452 | // Expand into a number of unpckl*. |
| 3453 | // e.g. for v4f32 |
| 3454 | // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0> |
| 3455 | // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1> |
| 3456 | // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> |
| 3457 | SDOperand UnpckMask = getUnpacklMask(NumElems, DAG); |
| 3458 | for (unsigned i = 0; i < NumElems; ++i) |
| 3459 | V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); |
| 3460 | NumElems >>= 1; |
| 3461 | while (NumElems != 0) { |
| 3462 | for (unsigned i = 0; i < NumElems; ++i) |
| 3463 | V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i], V[i + NumElems], |
| 3464 | UnpckMask); |
| 3465 | NumElems >>= 1; |
| 3466 | } |
| 3467 | return V[0]; |
| 3468 | } |
| 3469 | |
| 3470 | return SDOperand(); |
| 3471 | } |
| 3472 | |
| 3473 | SDOperand |
| 3474 | X86TargetLowering::LowerVECTOR_SHUFFLE(SDOperand Op, SelectionDAG &DAG) { |
| 3475 | SDOperand V1 = Op.getOperand(0); |
| 3476 | SDOperand V2 = Op.getOperand(1); |
| 3477 | SDOperand PermMask = Op.getOperand(2); |
| 3478 | MVT::ValueType VT = Op.getValueType(); |
| 3479 | unsigned NumElems = PermMask.getNumOperands(); |
| 3480 | bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; |
| 3481 | bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; |
Evan Cheng | d9b8e40 | 2006-10-16 06:36:00 +0000 | [diff] [blame] | 3482 | bool V1IsSplat = false; |
| 3483 | bool V2IsSplat = false; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3484 | |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 3485 | if (isUndefShuffle(Op.Val)) |
| 3486 | return DAG.getNode(ISD::UNDEF, VT); |
| 3487 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3488 | if (isSplatMask(PermMask.Val)) { |
| 3489 | if (NumElems <= 4) return Op; |
| 3490 | // Promote it to a v4i32 splat. |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3491 | return PromoteSplat(Op, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3492 | } |
| 3493 | |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3494 | if (X86::isMOVLMask(PermMask.Val)) |
| 3495 | return (V1IsUndef) ? V2 : Op; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 3496 | |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3497 | if (X86::isMOVSHDUPMask(PermMask.Val) || |
| 3498 | X86::isMOVSLDUPMask(PermMask.Val) || |
| 3499 | X86::isMOVHLPSMask(PermMask.Val) || |
| 3500 | X86::isMOVHPMask(PermMask.Val) || |
| 3501 | X86::isMOVLPMask(PermMask.Val)) |
| 3502 | return Op; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3503 | |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3504 | if (ShouldXformToMOVHLPS(PermMask.Val) || |
| 3505 | ShouldXformToMOVLP(V1.Val, V2.Val, PermMask.Val)) |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 3506 | return CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3507 | |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 3508 | bool Commuted = false; |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3509 | V1IsSplat = isSplatVector(V1.Val); |
| 3510 | V2IsSplat = isSplatVector(V2.Val); |
| 3511 | if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) { |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 3512 | Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3513 | std::swap(V1IsSplat, V2IsSplat); |
| 3514 | std::swap(V1IsUndef, V2IsUndef); |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 3515 | Commuted = true; |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3516 | } |
| 3517 | |
| 3518 | if (isCommutedMOVL(PermMask.Val, V2IsSplat, V2IsUndef)) { |
| 3519 | if (V2IsUndef) return V1; |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 3520 | Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3521 | if (V2IsSplat) { |
| 3522 | // V2 is a splat, so the mask may be malformed. That is, it may point |
| 3523 | // to any V2 element. The instruction selectior won't like this. Get |
| 3524 | // a corrected mask and commute to form a proper MOVS{S|D}. |
| 3525 | SDOperand NewMask = getMOVLMask(NumElems, DAG); |
| 3526 | if (NewMask.Val != PermMask.Val) |
| 3527 | Op = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3528 | } |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3529 | return Op; |
Evan Cheng | d9b8e40 | 2006-10-16 06:36:00 +0000 | [diff] [blame] | 3530 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3531 | |
Evan Cheng | d9b8e40 | 2006-10-16 06:36:00 +0000 | [diff] [blame] | 3532 | if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) || |
| 3533 | X86::isUNPCKLMask(PermMask.Val) || |
| 3534 | X86::isUNPCKHMask(PermMask.Val)) |
| 3535 | return Op; |
Evan Cheng | e111303 | 2006-10-04 18:33:38 +0000 | [diff] [blame] | 3536 | |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3537 | if (V2IsSplat) { |
| 3538 | // Normalize mask so all entries that point to V2 points to its first |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 3539 | // element then try to match unpck{h|l} again. If match, return a |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 3540 | // new vector_shuffle with the corrected mask. |
| 3541 | SDOperand NewMask = NormalizeMask(PermMask, DAG); |
| 3542 | if (NewMask.Val != PermMask.Val) { |
| 3543 | if (X86::isUNPCKLMask(PermMask.Val, true)) { |
| 3544 | SDOperand NewMask = getUnpacklMask(NumElems, DAG); |
| 3545 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); |
| 3546 | } else if (X86::isUNPCKHMask(PermMask.Val, true)) { |
| 3547 | SDOperand NewMask = getUnpackhMask(NumElems, DAG); |
| 3548 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3549 | } |
| 3550 | } |
| 3551 | } |
| 3552 | |
| 3553 | // Normalize the node to match x86 shuffle ops if needed |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 3554 | if (V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(PermMask.Val)) |
| 3555 | Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); |
| 3556 | |
| 3557 | if (Commuted) { |
| 3558 | // Commute is back and try unpck* again. |
| 3559 | Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); |
| 3560 | if (X86::isUNPCKL_v_undef_Mask(PermMask.Val) || |
| 3561 | X86::isUNPCKLMask(PermMask.Val) || |
| 3562 | X86::isUNPCKHMask(PermMask.Val)) |
| 3563 | return Op; |
| 3564 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3565 | |
| 3566 | // If VT is integer, try PSHUF* first, then SHUFP*. |
| 3567 | if (MVT::isInteger(VT)) { |
| 3568 | if (X86::isPSHUFDMask(PermMask.Val) || |
| 3569 | X86::isPSHUFHWMask(PermMask.Val) || |
| 3570 | X86::isPSHUFLWMask(PermMask.Val)) { |
| 3571 | if (V2.getOpcode() != ISD::UNDEF) |
| 3572 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, |
| 3573 | DAG.getNode(ISD::UNDEF, V1.getValueType()),PermMask); |
| 3574 | return Op; |
| 3575 | } |
| 3576 | |
| 3577 | if (X86::isSHUFPMask(PermMask.Val)) |
| 3578 | return Op; |
| 3579 | |
| 3580 | // Handle v8i16 shuffle high / low shuffle node pair. |
| 3581 | if (VT == MVT::v8i16 && isPSHUFHW_PSHUFLWMask(PermMask.Val)) { |
| 3582 | MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 3583 | MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT); |
| 3584 | std::vector<SDOperand> MaskVec; |
| 3585 | for (unsigned i = 0; i != 4; ++i) |
| 3586 | MaskVec.push_back(PermMask.getOperand(i)); |
| 3587 | for (unsigned i = 4; i != 8; ++i) |
| 3588 | MaskVec.push_back(DAG.getConstant(i, BaseVT)); |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3589 | SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3590 | &MaskVec[0], MaskVec.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3591 | V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); |
| 3592 | MaskVec.clear(); |
| 3593 | for (unsigned i = 0; i != 4; ++i) |
| 3594 | MaskVec.push_back(DAG.getConstant(i, BaseVT)); |
| 3595 | for (unsigned i = 4; i != 8; ++i) |
| 3596 | MaskVec.push_back(PermMask.getOperand(i)); |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3597 | Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0],MaskVec.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3598 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); |
| 3599 | } |
| 3600 | } else { |
| 3601 | // Floating point cases in the other order. |
| 3602 | if (X86::isSHUFPMask(PermMask.Val)) |
| 3603 | return Op; |
| 3604 | if (X86::isPSHUFDMask(PermMask.Val) || |
| 3605 | X86::isPSHUFHWMask(PermMask.Val) || |
| 3606 | X86::isPSHUFLWMask(PermMask.Val)) { |
| 3607 | if (V2.getOpcode() != ISD::UNDEF) |
| 3608 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, |
| 3609 | DAG.getNode(ISD::UNDEF, V1.getValueType()),PermMask); |
| 3610 | return Op; |
| 3611 | } |
| 3612 | } |
| 3613 | |
| 3614 | if (NumElems == 4) { |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3615 | MVT::ValueType MaskVT = PermMask.getValueType(); |
| 3616 | MVT::ValueType MaskEVT = MVT::getVectorBaseType(MaskVT); |
Evan Cheng | 43f3bd3 | 2006-04-28 07:03:38 +0000 | [diff] [blame] | 3617 | std::vector<std::pair<int, int> > Locs; |
| 3618 | Locs.reserve(NumElems); |
| 3619 | std::vector<SDOperand> Mask1(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT)); |
| 3620 | std::vector<SDOperand> Mask2(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT)); |
| 3621 | unsigned NumHi = 0; |
| 3622 | unsigned NumLo = 0; |
| 3623 | // If no more than two elements come from either vector. This can be |
| 3624 | // implemented with two shuffles. First shuffle gather the elements. |
| 3625 | // The second shuffle, which takes the first shuffle as both of its |
| 3626 | // vector operands, put the elements into the right order. |
| 3627 | for (unsigned i = 0; i != NumElems; ++i) { |
| 3628 | SDOperand Elt = PermMask.getOperand(i); |
| 3629 | if (Elt.getOpcode() == ISD::UNDEF) { |
| 3630 | Locs[i] = std::make_pair(-1, -1); |
| 3631 | } else { |
| 3632 | unsigned Val = cast<ConstantSDNode>(Elt)->getValue(); |
| 3633 | if (Val < NumElems) { |
| 3634 | Locs[i] = std::make_pair(0, NumLo); |
| 3635 | Mask1[NumLo] = Elt; |
| 3636 | NumLo++; |
| 3637 | } else { |
| 3638 | Locs[i] = std::make_pair(1, NumHi); |
| 3639 | if (2+NumHi < NumElems) |
| 3640 | Mask1[2+NumHi] = Elt; |
| 3641 | NumHi++; |
| 3642 | } |
| 3643 | } |
| 3644 | } |
| 3645 | if (NumLo <= 2 && NumHi <= 2) { |
| 3646 | V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3647 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3648 | &Mask1[0], Mask1.size())); |
Evan Cheng | 43f3bd3 | 2006-04-28 07:03:38 +0000 | [diff] [blame] | 3649 | for (unsigned i = 0; i != NumElems; ++i) { |
| 3650 | if (Locs[i].first == -1) |
| 3651 | continue; |
| 3652 | else { |
| 3653 | unsigned Idx = (i < NumElems/2) ? 0 : NumElems; |
| 3654 | Idx += Locs[i].first * (NumElems/2) + Locs[i].second; |
| 3655 | Mask2[i] = DAG.getConstant(Idx, MaskEVT); |
| 3656 | } |
| 3657 | } |
| 3658 | |
| 3659 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3660 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3661 | &Mask2[0], Mask2.size())); |
Evan Cheng | 43f3bd3 | 2006-04-28 07:03:38 +0000 | [diff] [blame] | 3662 | } |
| 3663 | |
| 3664 | // Break it into (shuffle shuffle_hi, shuffle_lo). |
| 3665 | Locs.clear(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3666 | std::vector<SDOperand> LoMask(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT)); |
| 3667 | std::vector<SDOperand> HiMask(NumElems, DAG.getNode(ISD::UNDEF, MaskEVT)); |
| 3668 | std::vector<SDOperand> *MaskPtr = &LoMask; |
| 3669 | unsigned MaskIdx = 0; |
| 3670 | unsigned LoIdx = 0; |
| 3671 | unsigned HiIdx = NumElems/2; |
| 3672 | for (unsigned i = 0; i != NumElems; ++i) { |
| 3673 | if (i == NumElems/2) { |
| 3674 | MaskPtr = &HiMask; |
| 3675 | MaskIdx = 1; |
| 3676 | LoIdx = 0; |
| 3677 | HiIdx = NumElems/2; |
| 3678 | } |
| 3679 | SDOperand Elt = PermMask.getOperand(i); |
| 3680 | if (Elt.getOpcode() == ISD::UNDEF) { |
| 3681 | Locs[i] = std::make_pair(-1, -1); |
| 3682 | } else if (cast<ConstantSDNode>(Elt)->getValue() < NumElems) { |
| 3683 | Locs[i] = std::make_pair(MaskIdx, LoIdx); |
| 3684 | (*MaskPtr)[LoIdx] = Elt; |
| 3685 | LoIdx++; |
| 3686 | } else { |
| 3687 | Locs[i] = std::make_pair(MaskIdx, HiIdx); |
| 3688 | (*MaskPtr)[HiIdx] = Elt; |
| 3689 | HiIdx++; |
| 3690 | } |
| 3691 | } |
| 3692 | |
Chris Lattner | 8c0c10c | 2006-05-16 06:45:34 +0000 | [diff] [blame] | 3693 | SDOperand LoShuffle = |
| 3694 | DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3695 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3696 | &LoMask[0], LoMask.size())); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 3697 | SDOperand HiShuffle = |
Chris Lattner | 8c0c10c | 2006-05-16 06:45:34 +0000 | [diff] [blame] | 3698 | DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3699 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3700 | &HiMask[0], HiMask.size())); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3701 | std::vector<SDOperand> MaskOps; |
| 3702 | for (unsigned i = 0; i != NumElems; ++i) { |
| 3703 | if (Locs[i].first == -1) { |
| 3704 | MaskOps.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); |
| 3705 | } else { |
| 3706 | unsigned Idx = Locs[i].first * NumElems + Locs[i].second; |
| 3707 | MaskOps.push_back(DAG.getConstant(Idx, MaskEVT)); |
| 3708 | } |
| 3709 | } |
| 3710 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, LoShuffle, HiShuffle, |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3711 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3712 | &MaskOps[0], MaskOps.size())); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3713 | } |
| 3714 | |
| 3715 | return SDOperand(); |
| 3716 | } |
| 3717 | |
| 3718 | SDOperand |
| 3719 | X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) { |
| 3720 | if (!isa<ConstantSDNode>(Op.getOperand(1))) |
| 3721 | return SDOperand(); |
| 3722 | |
| 3723 | MVT::ValueType VT = Op.getValueType(); |
| 3724 | // TODO: handle v16i8. |
| 3725 | if (MVT::getSizeInBits(VT) == 16) { |
| 3726 | // Transform it so it match pextrw which produces a 32-bit result. |
| 3727 | MVT::ValueType EVT = (MVT::ValueType)(VT+1); |
| 3728 | SDOperand Extract = DAG.getNode(X86ISD::PEXTRW, EVT, |
| 3729 | Op.getOperand(0), Op.getOperand(1)); |
| 3730 | SDOperand Assert = DAG.getNode(ISD::AssertZext, EVT, Extract, |
| 3731 | DAG.getValueType(VT)); |
| 3732 | return DAG.getNode(ISD::TRUNCATE, VT, Assert); |
| 3733 | } else if (MVT::getSizeInBits(VT) == 32) { |
| 3734 | SDOperand Vec = Op.getOperand(0); |
| 3735 | unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); |
| 3736 | if (Idx == 0) |
| 3737 | return Op; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3738 | // SHUFPS the element to the lowest double word, then movss. |
| 3739 | MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3740 | std::vector<SDOperand> IdxVec; |
| 3741 | IdxVec.push_back(DAG.getConstant(Idx, MVT::getVectorBaseType(MaskVT))); |
| 3742 | IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT))); |
| 3743 | IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT))); |
| 3744 | IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT))); |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3745 | SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3746 | &IdxVec[0], IdxVec.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3747 | Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), |
Evan Cheng | 6e56e2c | 2006-11-07 22:14:24 +0000 | [diff] [blame] | 3748 | Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3749 | return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, |
Evan Cheng | 015188f | 2006-06-15 08:14:54 +0000 | [diff] [blame] | 3750 | DAG.getConstant(0, getPointerTy())); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3751 | } else if (MVT::getSizeInBits(VT) == 64) { |
| 3752 | SDOperand Vec = Op.getOperand(0); |
| 3753 | unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); |
| 3754 | if (Idx == 0) |
| 3755 | return Op; |
| 3756 | |
| 3757 | // UNPCKHPD the element to the lowest double word, then movsd. |
| 3758 | // Note if the lower 64 bits of the result of the UNPCKHPD is then stored |
| 3759 | // to a f64mem, the whole operation is folded into a single MOVHPDmr. |
| 3760 | MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4); |
| 3761 | std::vector<SDOperand> IdxVec; |
| 3762 | IdxVec.push_back(DAG.getConstant(1, MVT::getVectorBaseType(MaskVT))); |
| 3763 | IdxVec.push_back(DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(MaskVT))); |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3764 | SDOperand Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3765 | &IdxVec[0], IdxVec.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3766 | Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), |
| 3767 | Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); |
| 3768 | return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, |
Evan Cheng | 015188f | 2006-06-15 08:14:54 +0000 | [diff] [blame] | 3769 | DAG.getConstant(0, getPointerTy())); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3770 | } |
| 3771 | |
| 3772 | return SDOperand(); |
| 3773 | } |
| 3774 | |
| 3775 | SDOperand |
| 3776 | X86TargetLowering::LowerINSERT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) { |
Evan Cheng | 069287d | 2006-05-16 07:21:53 +0000 | [diff] [blame] | 3777 | // Transform it so it match pinsrw which expects a 16-bit value in a GR32 |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3778 | // as its second argument. |
| 3779 | MVT::ValueType VT = Op.getValueType(); |
| 3780 | MVT::ValueType BaseVT = MVT::getVectorBaseType(VT); |
| 3781 | SDOperand N0 = Op.getOperand(0); |
| 3782 | SDOperand N1 = Op.getOperand(1); |
| 3783 | SDOperand N2 = Op.getOperand(2); |
| 3784 | if (MVT::getSizeInBits(BaseVT) == 16) { |
| 3785 | if (N1.getValueType() != MVT::i32) |
| 3786 | N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); |
| 3787 | if (N2.getValueType() != MVT::i32) |
| 3788 | N2 = DAG.getConstant(cast<ConstantSDNode>(N2)->getValue(), MVT::i32); |
| 3789 | return DAG.getNode(X86ISD::PINSRW, VT, N0, N1, N2); |
| 3790 | } else if (MVT::getSizeInBits(BaseVT) == 32) { |
| 3791 | unsigned Idx = cast<ConstantSDNode>(N2)->getValue(); |
| 3792 | if (Idx == 0) { |
| 3793 | // Use a movss. |
| 3794 | N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, N1); |
| 3795 | MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(4); |
| 3796 | MVT::ValueType BaseVT = MVT::getVectorBaseType(MaskVT); |
| 3797 | std::vector<SDOperand> MaskVec; |
| 3798 | MaskVec.push_back(DAG.getConstant(4, BaseVT)); |
| 3799 | for (unsigned i = 1; i <= 3; ++i) |
| 3800 | MaskVec.push_back(DAG.getConstant(i, BaseVT)); |
| 3801 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, N0, N1, |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3802 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3803 | &MaskVec[0], MaskVec.size())); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3804 | } else { |
| 3805 | // Use two pinsrw instructions to insert a 32 bit value. |
| 3806 | Idx <<= 1; |
| 3807 | if (MVT::isFloatingPoint(N1.getValueType())) { |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 3808 | if (ISD::isNON_EXTLoad(N1.Val)) { |
Evan Cheng | 069287d | 2006-05-16 07:21:53 +0000 | [diff] [blame] | 3809 | // Just load directly from f32mem to GR32. |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 3810 | LoadSDNode *LD = cast<LoadSDNode>(N1); |
| 3811 | N1 = DAG.getLoad(MVT::i32, LD->getChain(), LD->getBasePtr(), |
| 3812 | LD->getSrcValue(), LD->getSrcValueOffset()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3813 | } else { |
| 3814 | N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v4f32, N1); |
| 3815 | N1 = DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, N1); |
| 3816 | N1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, N1, |
Evan Cheng | 015188f | 2006-06-15 08:14:54 +0000 | [diff] [blame] | 3817 | DAG.getConstant(0, getPointerTy())); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3818 | } |
| 3819 | } |
| 3820 | N0 = DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, N0); |
| 3821 | N0 = DAG.getNode(X86ISD::PINSRW, MVT::v8i16, N0, N1, |
Evan Cheng | 015188f | 2006-06-15 08:14:54 +0000 | [diff] [blame] | 3822 | DAG.getConstant(Idx, getPointerTy())); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3823 | N1 = DAG.getNode(ISD::SRL, MVT::i32, N1, DAG.getConstant(16, MVT::i8)); |
| 3824 | N0 = DAG.getNode(X86ISD::PINSRW, MVT::v8i16, N0, N1, |
Evan Cheng | 015188f | 2006-06-15 08:14:54 +0000 | [diff] [blame] | 3825 | DAG.getConstant(Idx+1, getPointerTy())); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3826 | return DAG.getNode(ISD::BIT_CONVERT, VT, N0); |
| 3827 | } |
| 3828 | } |
| 3829 | |
| 3830 | return SDOperand(); |
| 3831 | } |
| 3832 | |
| 3833 | SDOperand |
| 3834 | X86TargetLowering::LowerSCALAR_TO_VECTOR(SDOperand Op, SelectionDAG &DAG) { |
| 3835 | SDOperand AnyExt = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, Op.getOperand(0)); |
| 3836 | return DAG.getNode(X86ISD::S2VEC, Op.getValueType(), AnyExt); |
| 3837 | } |
| 3838 | |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 3839 | // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3840 | // their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is |
| 3841 | // one of the above mentioned nodes. It has to be wrapped because otherwise |
| 3842 | // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only |
| 3843 | // be used to form addressing mode. These wrapped nodes will be selected |
| 3844 | // into MOV32ri. |
| 3845 | SDOperand |
| 3846 | X86TargetLowering::LowerConstantPool(SDOperand Op, SelectionDAG &DAG) { |
| 3847 | ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); |
Evan Cheng | d0ff02c | 2006-11-29 23:19:46 +0000 | [diff] [blame] | 3848 | SDOperand Result = DAG.getTargetConstantPool(CP->getConstVal(), |
| 3849 | getPointerTy(), |
| 3850 | CP->getAlignment()); |
Evan Cheng | 19f2ffc | 2006-12-05 04:01:03 +0000 | [diff] [blame] | 3851 | Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3852 | if (Subtarget->isTargetDarwin()) { |
| 3853 | // With PIC, the address is actually $g + Offset. |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3854 | if (!Subtarget->is64Bit() && |
| 3855 | getTargetMachine().getRelocationModel() == Reloc::PIC_) |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3856 | Result = DAG.getNode(ISD::ADD, getPointerTy(), |
| 3857 | DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), Result); |
| 3858 | } |
| 3859 | |
| 3860 | return Result; |
| 3861 | } |
| 3862 | |
| 3863 | SDOperand |
| 3864 | X86TargetLowering::LowerGlobalAddress(SDOperand Op, SelectionDAG &DAG) { |
| 3865 | GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); |
Evan Cheng | d0ff02c | 2006-11-29 23:19:46 +0000 | [diff] [blame] | 3866 | SDOperand Result = DAG.getTargetGlobalAddress(GV, getPointerTy()); |
Evan Cheng | 19f2ffc | 2006-12-05 04:01:03 +0000 | [diff] [blame] | 3867 | Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3868 | if (Subtarget->isTargetDarwin()) { |
| 3869 | // With PIC, the address is actually $g + Offset. |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3870 | if (!Subtarget->is64Bit() && |
| 3871 | getTargetMachine().getRelocationModel() == Reloc::PIC_) |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3872 | Result = DAG.getNode(ISD::ADD, getPointerTy(), |
Chris Lattner | 8c0c10c | 2006-05-16 06:45:34 +0000 | [diff] [blame] | 3873 | DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), |
| 3874 | Result); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3875 | |
| 3876 | // For Darwin, external and weak symbols are indirect, so we want to load |
| 3877 | // the value at address GV, not the value of GV itself. This means that |
| 3878 | // the GlobalAddress must be in the base or index register of the address, |
| 3879 | // not the GV offset field. |
| 3880 | if (getTargetMachine().getRelocationModel() != Reloc::Static && |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 3881 | Subtarget->GVRequiresExtraLoad(GV, false)) |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 3882 | Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), Result, NULL, 0); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 3883 | } else if (Subtarget->GVRequiresExtraLoad(GV, false)) { |
| 3884 | Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), Result, NULL, 0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3885 | } |
| 3886 | |
| 3887 | return Result; |
| 3888 | } |
| 3889 | |
| 3890 | SDOperand |
| 3891 | X86TargetLowering::LowerExternalSymbol(SDOperand Op, SelectionDAG &DAG) { |
| 3892 | const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol(); |
Evan Cheng | d0ff02c | 2006-11-29 23:19:46 +0000 | [diff] [blame] | 3893 | SDOperand Result = DAG.getTargetExternalSymbol(Sym, getPointerTy()); |
Evan Cheng | 19f2ffc | 2006-12-05 04:01:03 +0000 | [diff] [blame] | 3894 | Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3895 | if (Subtarget->isTargetDarwin()) { |
| 3896 | // With PIC, the address is actually $g + Offset. |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3897 | if (!Subtarget->is64Bit() && |
| 3898 | getTargetMachine().getRelocationModel() == Reloc::PIC_) |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3899 | Result = DAG.getNode(ISD::ADD, getPointerTy(), |
Chris Lattner | 8c0c10c | 2006-05-16 06:45:34 +0000 | [diff] [blame] | 3900 | DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), |
| 3901 | Result); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3902 | } |
| 3903 | |
| 3904 | return Result; |
| 3905 | } |
| 3906 | |
| 3907 | SDOperand X86TargetLowering::LowerShift(SDOperand Op, SelectionDAG &DAG) { |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 3908 | assert(Op.getNumOperands() == 3 && Op.getValueType() == MVT::i32 && |
| 3909 | "Not an i64 shift!"); |
| 3910 | bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; |
| 3911 | SDOperand ShOpLo = Op.getOperand(0); |
| 3912 | SDOperand ShOpHi = Op.getOperand(1); |
| 3913 | SDOperand ShAmt = Op.getOperand(2); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 3914 | SDOperand Tmp1 = isSRA ? |
| 3915 | DAG.getNode(ISD::SRA, MVT::i32, ShOpHi, DAG.getConstant(31, MVT::i8)) : |
| 3916 | DAG.getConstant(0, MVT::i32); |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 3917 | |
| 3918 | SDOperand Tmp2, Tmp3; |
| 3919 | if (Op.getOpcode() == ISD::SHL_PARTS) { |
| 3920 | Tmp2 = DAG.getNode(X86ISD::SHLD, MVT::i32, ShOpHi, ShOpLo, ShAmt); |
| 3921 | Tmp3 = DAG.getNode(ISD::SHL, MVT::i32, ShOpLo, ShAmt); |
| 3922 | } else { |
| 3923 | Tmp2 = DAG.getNode(X86ISD::SHRD, MVT::i32, ShOpLo, ShOpHi, ShAmt); |
Evan Cheng | b7b5706 | 2006-01-19 01:46:14 +0000 | [diff] [blame] | 3924 | Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, MVT::i32, ShOpHi, ShAmt); |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 3925 | } |
| 3926 | |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 3927 | const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag); |
| 3928 | SDOperand AndNode = DAG.getNode(ISD::AND, MVT::i8, ShAmt, |
| 3929 | DAG.getConstant(32, MVT::i8)); |
| 3930 | SDOperand COps[]={DAG.getEntryNode(), AndNode, DAG.getConstant(0, MVT::i8)}; |
| 3931 | SDOperand InFlag = DAG.getNode(X86ISD::CMP, VTs, 2, COps, 3).getValue(1); |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 3932 | |
| 3933 | SDOperand Hi, Lo; |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 3934 | SDOperand CC = DAG.getConstant(X86::COND_NE, MVT::i8); |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 3935 | |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 3936 | VTs = DAG.getNodeValueTypes(MVT::i32, MVT::Flag); |
| 3937 | SmallVector<SDOperand, 4> Ops; |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 3938 | if (Op.getOpcode() == ISD::SHL_PARTS) { |
| 3939 | Ops.push_back(Tmp2); |
| 3940 | Ops.push_back(Tmp3); |
| 3941 | Ops.push_back(CC); |
| 3942 | Ops.push_back(InFlag); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 3943 | Hi = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 3944 | InFlag = Hi.getValue(1); |
| 3945 | |
| 3946 | Ops.clear(); |
| 3947 | Ops.push_back(Tmp3); |
| 3948 | Ops.push_back(Tmp1); |
| 3949 | Ops.push_back(CC); |
| 3950 | Ops.push_back(InFlag); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 3951 | Lo = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 3952 | } else { |
| 3953 | Ops.push_back(Tmp2); |
| 3954 | Ops.push_back(Tmp3); |
| 3955 | Ops.push_back(CC); |
Evan Cheng | 910cd3c | 2006-01-09 22:29:54 +0000 | [diff] [blame] | 3956 | Ops.push_back(InFlag); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 3957 | Lo = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 3958 | InFlag = Lo.getValue(1); |
| 3959 | |
| 3960 | Ops.clear(); |
| 3961 | Ops.push_back(Tmp3); |
| 3962 | Ops.push_back(Tmp1); |
| 3963 | Ops.push_back(CC); |
| 3964 | Ops.push_back(InFlag); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 3965 | Hi = DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 3966 | } |
| 3967 | |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 3968 | VTs = DAG.getNodeValueTypes(MVT::i32, MVT::i32); |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 3969 | Ops.clear(); |
| 3970 | Ops.push_back(Lo); |
| 3971 | Ops.push_back(Hi); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 3972 | return DAG.getNode(ISD::MERGE_VALUES, VTs, 2, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3973 | } |
Evan Cheng | a3195e8 | 2006-01-12 22:54:21 +0000 | [diff] [blame] | 3974 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3975 | SDOperand X86TargetLowering::LowerSINT_TO_FP(SDOperand Op, SelectionDAG &DAG) { |
| 3976 | assert(Op.getOperand(0).getValueType() <= MVT::i64 && |
| 3977 | Op.getOperand(0).getValueType() >= MVT::i16 && |
| 3978 | "Unknown SINT_TO_FP to lower!"); |
| 3979 | |
| 3980 | SDOperand Result; |
| 3981 | MVT::ValueType SrcVT = Op.getOperand(0).getValueType(); |
| 3982 | unsigned Size = MVT::getSizeInBits(SrcVT)/8; |
| 3983 | MachineFunction &MF = DAG.getMachineFunction(); |
| 3984 | int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); |
| 3985 | SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
Evan Cheng | 786225a | 2006-10-05 23:01:46 +0000 | [diff] [blame] | 3986 | SDOperand Chain = DAG.getStore(DAG.getEntryNode(), Op.getOperand(0), |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 3987 | StackSlot, NULL, 0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3988 | |
| 3989 | // Build the FILD |
| 3990 | std::vector<MVT::ValueType> Tys; |
| 3991 | Tys.push_back(MVT::f64); |
| 3992 | Tys.push_back(MVT::Other); |
| 3993 | if (X86ScalarSSE) Tys.push_back(MVT::Flag); |
| 3994 | std::vector<SDOperand> Ops; |
| 3995 | Ops.push_back(Chain); |
| 3996 | Ops.push_back(StackSlot); |
| 3997 | Ops.push_back(DAG.getValueType(SrcVT)); |
| 3998 | Result = DAG.getNode(X86ScalarSSE ? X86ISD::FILD_FLAG :X86ISD::FILD, |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 3999 | Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4000 | |
| 4001 | if (X86ScalarSSE) { |
| 4002 | Chain = Result.getValue(1); |
| 4003 | SDOperand InFlag = Result.getValue(2); |
| 4004 | |
| 4005 | // FIXME: Currently the FST is flagged to the FILD_FLAG. This |
| 4006 | // shouldn't be necessary except that RFP cannot be live across |
| 4007 | // multiple blocks. When stackifier is fixed, they can be uncoupled. |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4008 | MachineFunction &MF = DAG.getMachineFunction(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4009 | int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4010 | SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
Evan Cheng | a3195e8 | 2006-01-12 22:54:21 +0000 | [diff] [blame] | 4011 | std::vector<MVT::ValueType> Tys; |
Evan Cheng | 6dab053 | 2006-01-30 08:02:57 +0000 | [diff] [blame] | 4012 | Tys.push_back(MVT::Other); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4013 | std::vector<SDOperand> Ops; |
Evan Cheng | a3195e8 | 2006-01-12 22:54:21 +0000 | [diff] [blame] | 4014 | Ops.push_back(Chain); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4015 | Ops.push_back(Result); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4016 | Ops.push_back(StackSlot); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4017 | Ops.push_back(DAG.getValueType(Op.getValueType())); |
| 4018 | Ops.push_back(InFlag); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 4019 | Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 4020 | Result = DAG.getLoad(Op.getValueType(), Chain, StackSlot, NULL, 0); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4021 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4022 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4023 | return Result; |
| 4024 | } |
| 4025 | |
| 4026 | SDOperand X86TargetLowering::LowerFP_TO_SINT(SDOperand Op, SelectionDAG &DAG) { |
| 4027 | assert(Op.getValueType() <= MVT::i64 && Op.getValueType() >= MVT::i16 && |
| 4028 | "Unknown FP_TO_SINT to lower!"); |
| 4029 | // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary |
| 4030 | // stack slot. |
| 4031 | MachineFunction &MF = DAG.getMachineFunction(); |
| 4032 | unsigned MemSize = MVT::getSizeInBits(Op.getValueType())/8; |
| 4033 | int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); |
| 4034 | SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
| 4035 | |
| 4036 | unsigned Opc; |
| 4037 | switch (Op.getValueType()) { |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4038 | default: assert(0 && "Invalid FP_TO_SINT to lower!"); |
| 4039 | case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; |
| 4040 | case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; |
| 4041 | case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4042 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4043 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4044 | SDOperand Chain = DAG.getEntryNode(); |
| 4045 | SDOperand Value = Op.getOperand(0); |
| 4046 | if (X86ScalarSSE) { |
| 4047 | assert(Op.getValueType() == MVT::i64 && "Invalid FP_TO_SINT to lower!"); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4048 | Chain = DAG.getStore(Chain, Value, StackSlot, NULL, 0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4049 | std::vector<MVT::ValueType> Tys; |
| 4050 | Tys.push_back(MVT::f64); |
| 4051 | Tys.push_back(MVT::Other); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4052 | std::vector<SDOperand> Ops; |
Evan Cheng | 6dab053 | 2006-01-30 08:02:57 +0000 | [diff] [blame] | 4053 | Ops.push_back(Chain); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4054 | Ops.push_back(StackSlot); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4055 | Ops.push_back(DAG.getValueType(Op.getOperand(0).getValueType())); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 4056 | Value = DAG.getNode(X86ISD::FLD, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4057 | Chain = Value.getValue(1); |
| 4058 | SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); |
| 4059 | StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
| 4060 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4061 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4062 | // Build the FP_TO_INT*_IN_MEM |
| 4063 | std::vector<SDOperand> Ops; |
| 4064 | Ops.push_back(Chain); |
| 4065 | Ops.push_back(Value); |
| 4066 | Ops.push_back(StackSlot); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 4067 | SDOperand FIST = DAG.getNode(Opc, MVT::Other, &Ops[0], Ops.size()); |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 4068 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4069 | // Load the result. |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 4070 | return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4071 | } |
| 4072 | |
| 4073 | SDOperand X86TargetLowering::LowerFABS(SDOperand Op, SelectionDAG &DAG) { |
| 4074 | MVT::ValueType VT = Op.getValueType(); |
| 4075 | const Type *OpNTy = MVT::getTypeForValueType(VT); |
| 4076 | std::vector<Constant*> CV; |
| 4077 | if (VT == MVT::f64) { |
| 4078 | CV.push_back(ConstantFP::get(OpNTy, BitsToDouble(~(1ULL << 63)))); |
| 4079 | CV.push_back(ConstantFP::get(OpNTy, 0.0)); |
| 4080 | } else { |
| 4081 | CV.push_back(ConstantFP::get(OpNTy, BitsToFloat(~(1U << 31)))); |
| 4082 | CV.push_back(ConstantFP::get(OpNTy, 0.0)); |
| 4083 | CV.push_back(ConstantFP::get(OpNTy, 0.0)); |
| 4084 | CV.push_back(ConstantFP::get(OpNTy, 0.0)); |
| 4085 | } |
| 4086 | Constant *CS = ConstantStruct::get(CV); |
| 4087 | SDOperand CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4); |
Evan Cheng | 64a752f | 2006-08-11 09:08:15 +0000 | [diff] [blame] | 4088 | std::vector<MVT::ValueType> Tys; |
| 4089 | Tys.push_back(VT); |
| 4090 | Tys.push_back(MVT::Other); |
| 4091 | SmallVector<SDOperand, 3> Ops; |
| 4092 | Ops.push_back(DAG.getEntryNode()); |
| 4093 | Ops.push_back(CPIdx); |
| 4094 | Ops.push_back(DAG.getSrcValue(NULL)); |
| 4095 | SDOperand Mask = DAG.getNode(X86ISD::LOAD_PACK, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4096 | return DAG.getNode(X86ISD::FAND, VT, Op.getOperand(0), Mask); |
| 4097 | } |
| 4098 | |
| 4099 | SDOperand X86TargetLowering::LowerFNEG(SDOperand Op, SelectionDAG &DAG) { |
| 4100 | MVT::ValueType VT = Op.getValueType(); |
| 4101 | const Type *OpNTy = MVT::getTypeForValueType(VT); |
| 4102 | std::vector<Constant*> CV; |
| 4103 | if (VT == MVT::f64) { |
| 4104 | CV.push_back(ConstantFP::get(OpNTy, BitsToDouble(1ULL << 63))); |
| 4105 | CV.push_back(ConstantFP::get(OpNTy, 0.0)); |
| 4106 | } else { |
| 4107 | CV.push_back(ConstantFP::get(OpNTy, BitsToFloat(1U << 31))); |
| 4108 | CV.push_back(ConstantFP::get(OpNTy, 0.0)); |
| 4109 | CV.push_back(ConstantFP::get(OpNTy, 0.0)); |
| 4110 | CV.push_back(ConstantFP::get(OpNTy, 0.0)); |
| 4111 | } |
| 4112 | Constant *CS = ConstantStruct::get(CV); |
| 4113 | SDOperand CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4); |
Evan Cheng | 64a752f | 2006-08-11 09:08:15 +0000 | [diff] [blame] | 4114 | std::vector<MVT::ValueType> Tys; |
| 4115 | Tys.push_back(VT); |
| 4116 | Tys.push_back(MVT::Other); |
| 4117 | SmallVector<SDOperand, 3> Ops; |
| 4118 | Ops.push_back(DAG.getEntryNode()); |
| 4119 | Ops.push_back(CPIdx); |
| 4120 | Ops.push_back(DAG.getSrcValue(NULL)); |
| 4121 | SDOperand Mask = DAG.getNode(X86ISD::LOAD_PACK, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4122 | return DAG.getNode(X86ISD::FXOR, VT, Op.getOperand(0), Mask); |
| 4123 | } |
| 4124 | |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4125 | SDOperand X86TargetLowering::LowerSETCC(SDOperand Op, SelectionDAG &DAG, |
| 4126 | SDOperand Chain) { |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4127 | assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); |
| 4128 | SDOperand Cond; |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4129 | SDOperand Op0 = Op.getOperand(0); |
| 4130 | SDOperand Op1 = Op.getOperand(1); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4131 | SDOperand CC = Op.getOperand(2); |
| 4132 | ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); |
Evan Cheng | cf12ec4 | 2006-10-12 19:12:56 +0000 | [diff] [blame] | 4133 | const MVT::ValueType *VTs1 = DAG.getNodeValueTypes(MVT::Other, MVT::Flag); |
| 4134 | const MVT::ValueType *VTs2 = DAG.getNodeValueTypes(MVT::i8, MVT::Flag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4135 | bool isFP = MVT::isFloatingPoint(Op.getOperand(1).getValueType()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4136 | unsigned X86CC; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4137 | |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 4138 | if (translateX86CC(cast<CondCodeSDNode>(CC)->get(), isFP, X86CC, |
Chris Lattner | f957051 | 2006-09-13 03:22:10 +0000 | [diff] [blame] | 4139 | Op0, Op1, DAG)) { |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4140 | SDOperand Ops1[] = { Chain, Op0, Op1 }; |
Evan Cheng | cf12ec4 | 2006-10-12 19:12:56 +0000 | [diff] [blame] | 4141 | Cond = DAG.getNode(X86ISD::CMP, VTs1, 2, Ops1, 3).getValue(1); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4142 | SDOperand Ops2[] = { DAG.getConstant(X86CC, MVT::i8), Cond }; |
Evan Cheng | cf12ec4 | 2006-10-12 19:12:56 +0000 | [diff] [blame] | 4143 | return DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops2, 2); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4144 | } |
| 4145 | |
| 4146 | assert(isFP && "Illegal integer SetCC!"); |
| 4147 | |
| 4148 | SDOperand COps[] = { Chain, Op0, Op1 }; |
Evan Cheng | cf12ec4 | 2006-10-12 19:12:56 +0000 | [diff] [blame] | 4149 | Cond = DAG.getNode(X86ISD::CMP, VTs1, 2, COps, 3).getValue(1); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4150 | |
| 4151 | switch (SetCCOpcode) { |
| 4152 | default: assert(false && "Illegal floating point SetCC!"); |
| 4153 | case ISD::SETOEQ: { // !PF & ZF |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 4154 | SDOperand Ops1[] = { DAG.getConstant(X86::COND_NP, MVT::i8), Cond }; |
Evan Cheng | cf12ec4 | 2006-10-12 19:12:56 +0000 | [diff] [blame] | 4155 | SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops1, 2); |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 4156 | SDOperand Ops2[] = { DAG.getConstant(X86::COND_E, MVT::i8), |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4157 | Tmp1.getValue(1) }; |
Evan Cheng | cf12ec4 | 2006-10-12 19:12:56 +0000 | [diff] [blame] | 4158 | SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops2, 2); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4159 | return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2); |
| 4160 | } |
| 4161 | case ISD::SETUNE: { // PF | !ZF |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 4162 | SDOperand Ops1[] = { DAG.getConstant(X86::COND_P, MVT::i8), Cond }; |
Evan Cheng | cf12ec4 | 2006-10-12 19:12:56 +0000 | [diff] [blame] | 4163 | SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops1, 2); |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 4164 | SDOperand Ops2[] = { DAG.getConstant(X86::COND_NE, MVT::i8), |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4165 | Tmp1.getValue(1) }; |
Evan Cheng | cf12ec4 | 2006-10-12 19:12:56 +0000 | [diff] [blame] | 4166 | SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, VTs2, 2, Ops2, 2); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4167 | return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2); |
| 4168 | } |
Evan Cheng | d5781fc | 2005-12-21 20:21:51 +0000 | [diff] [blame] | 4169 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4170 | } |
Evan Cheng | 6dfa999 | 2006-01-30 23:41:35 +0000 | [diff] [blame] | 4171 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4172 | SDOperand X86TargetLowering::LowerSELECT(SDOperand Op, SelectionDAG &DAG) { |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4173 | bool addTest = true; |
| 4174 | SDOperand Chain = DAG.getEntryNode(); |
| 4175 | SDOperand Cond = Op.getOperand(0); |
| 4176 | SDOperand CC; |
| 4177 | const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag); |
Evan Cheng | 9bba894 | 2006-01-26 02:13:10 +0000 | [diff] [blame] | 4178 | |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4179 | if (Cond.getOpcode() == ISD::SETCC) |
| 4180 | Cond = LowerSETCC(Cond, DAG, Chain); |
| 4181 | |
| 4182 | if (Cond.getOpcode() == X86ISD::SETCC) { |
| 4183 | CC = Cond.getOperand(0); |
| 4184 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4185 | // If condition flag is set by a X86ISD::CMP, then make a copy of it |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4186 | // (since flag operand cannot be shared). Use it as the condition setting |
| 4187 | // operand in place of the X86ISD::SETCC. |
| 4188 | // If the X86ISD::SETCC has more than one use, then perhaps it's better |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4189 | // to use a test instead of duplicating the X86ISD::CMP (for register |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4190 | // pressure reason)? |
| 4191 | SDOperand Cmp = Cond.getOperand(1); |
| 4192 | unsigned Opc = Cmp.getOpcode(); |
| 4193 | bool IllegalFPCMov = !X86ScalarSSE && |
| 4194 | MVT::isFloatingPoint(Op.getValueType()) && |
| 4195 | !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended()); |
| 4196 | if ((Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI) && |
| 4197 | !IllegalFPCMov) { |
| 4198 | SDOperand Ops[] = { Chain, Cmp.getOperand(1), Cmp.getOperand(2) }; |
| 4199 | Cond = DAG.getNode(Opc, VTs, 2, Ops, 3); |
| 4200 | addTest = false; |
| 4201 | } |
| 4202 | } |
Evan Cheng | aaca22c | 2006-01-10 20:26:56 +0000 | [diff] [blame] | 4203 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4204 | if (addTest) { |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 4205 | CC = DAG.getConstant(X86::COND_NE, MVT::i8); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4206 | SDOperand Ops[] = { Chain, Cond, DAG.getConstant(0, MVT::i8) }; |
| 4207 | Cond = DAG.getNode(X86ISD::CMP, VTs, 2, Ops, 3); |
Evan Cheng | 7df96d6 | 2005-12-17 01:21:05 +0000 | [diff] [blame] | 4208 | } |
Evan Cheng | 6dfa999 | 2006-01-30 23:41:35 +0000 | [diff] [blame] | 4209 | |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4210 | VTs = DAG.getNodeValueTypes(Op.getValueType(), MVT::Flag); |
| 4211 | SmallVector<SDOperand, 4> Ops; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4212 | // X86ISD::CMOV means set the result (which is operand 1) to the RHS if |
| 4213 | // condition is true. |
| 4214 | Ops.push_back(Op.getOperand(2)); |
| 4215 | Ops.push_back(Op.getOperand(1)); |
| 4216 | Ops.push_back(CC); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4217 | Ops.push_back(Cond.getValue(1)); |
| 4218 | return DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4219 | } |
Evan Cheng | 9bba894 | 2006-01-26 02:13:10 +0000 | [diff] [blame] | 4220 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4221 | SDOperand X86TargetLowering::LowerBRCOND(SDOperand Op, SelectionDAG &DAG) { |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4222 | bool addTest = true; |
| 4223 | SDOperand Chain = Op.getOperand(0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4224 | SDOperand Cond = Op.getOperand(1); |
| 4225 | SDOperand Dest = Op.getOperand(2); |
| 4226 | SDOperand CC; |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4227 | const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag); |
| 4228 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4229 | if (Cond.getOpcode() == ISD::SETCC) |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4230 | Cond = LowerSETCC(Cond, DAG, Chain); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4231 | |
| 4232 | if (Cond.getOpcode() == X86ISD::SETCC) { |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4233 | CC = Cond.getOperand(0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4234 | |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4235 | // If condition flag is set by a X86ISD::CMP, then make a copy of it |
| 4236 | // (since flag operand cannot be shared). Use it as the condition setting |
| 4237 | // operand in place of the X86ISD::SETCC. |
| 4238 | // If the X86ISD::SETCC has more than one use, then perhaps it's better |
| 4239 | // to use a test instead of duplicating the X86ISD::CMP (for register |
| 4240 | // pressure reason)? |
| 4241 | SDOperand Cmp = Cond.getOperand(1); |
| 4242 | unsigned Opc = Cmp.getOpcode(); |
| 4243 | if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI) { |
| 4244 | SDOperand Ops[] = { Chain, Cmp.getOperand(1), Cmp.getOperand(2) }; |
| 4245 | Cond = DAG.getNode(Opc, VTs, 2, Ops, 3); |
| 4246 | addTest = false; |
| 4247 | } |
| 4248 | } |
Evan Cheng | 1bcee36 | 2006-01-13 01:03:02 +0000 | [diff] [blame] | 4249 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4250 | if (addTest) { |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 4251 | CC = DAG.getConstant(X86::COND_NE, MVT::i8); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4252 | SDOperand Ops[] = { Chain, Cond, DAG.getConstant(0, MVT::i8) }; |
| 4253 | Cond = DAG.getNode(X86ISD::CMP, VTs, 2, Ops, 3); |
Evan Cheng | 898101c | 2005-12-19 23:12:38 +0000 | [diff] [blame] | 4254 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4255 | return DAG.getNode(X86ISD::BRCOND, Op.getValueType(), |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4256 | Cond, Op.getOperand(2), CC, Cond.getValue(1)); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4257 | } |
Evan Cheng | 67f92a7 | 2006-01-11 22:15:48 +0000 | [diff] [blame] | 4258 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4259 | SDOperand X86TargetLowering::LowerJumpTable(SDOperand Op, SelectionDAG &DAG) { |
| 4260 | JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); |
Evan Cheng | d0ff02c | 2006-11-29 23:19:46 +0000 | [diff] [blame] | 4261 | SDOperand Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy()); |
Evan Cheng | 19f2ffc | 2006-12-05 04:01:03 +0000 | [diff] [blame] | 4262 | Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4263 | if (Subtarget->isTargetDarwin()) { |
| 4264 | // With PIC, the address is actually $g + Offset. |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4265 | if (!Subtarget->is64Bit() && |
| 4266 | getTargetMachine().getRelocationModel() == Reloc::PIC_) |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4267 | Result = DAG.getNode(ISD::ADD, getPointerTy(), |
Chris Lattner | 8c0c10c | 2006-05-16 06:45:34 +0000 | [diff] [blame] | 4268 | DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 4269 | Result); |
Evan Cheng | 67f92a7 | 2006-01-11 22:15:48 +0000 | [diff] [blame] | 4270 | } |
Evan Cheng | bbbb2fb | 2006-02-25 09:55:19 +0000 | [diff] [blame] | 4271 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4272 | return Result; |
| 4273 | } |
Evan Cheng | 7ccced6 | 2006-02-18 00:15:05 +0000 | [diff] [blame] | 4274 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 4275 | SDOperand X86TargetLowering::LowerCALL(SDOperand Op, SelectionDAG &DAG) { |
| 4276 | unsigned CallingConv= cast<ConstantSDNode>(Op.getOperand(1))->getValue(); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 4277 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4278 | if (Subtarget->is64Bit()) |
| 4279 | return LowerX86_64CCCCallTo(Op, DAG); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 4280 | else |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4281 | switch (CallingConv) { |
Chris Lattner | f38f543 | 2006-09-27 18:29:38 +0000 | [diff] [blame] | 4282 | default: |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 4283 | assert(0 && "Unsupported calling convention"); |
Chris Lattner | f38f543 | 2006-09-27 18:29:38 +0000 | [diff] [blame] | 4284 | case CallingConv::Fast: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4285 | if (EnableFastCC) { |
| 4286 | return LowerFastCCCallTo(Op, DAG, false); |
| 4287 | } |
| 4288 | // Falls through |
Chris Lattner | f38f543 | 2006-09-27 18:29:38 +0000 | [diff] [blame] | 4289 | case CallingConv::C: |
| 4290 | case CallingConv::CSRet: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4291 | return LowerCCCCallTo(Op, DAG); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 4292 | case CallingConv::X86_StdCall: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4293 | return LowerStdCallCCCallTo(Op, DAG); |
Chris Lattner | f38f543 | 2006-09-27 18:29:38 +0000 | [diff] [blame] | 4294 | case CallingConv::X86_FastCall: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4295 | return LowerFastCCCallTo(Op, DAG, true); |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4296 | } |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 4297 | } |
| 4298 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4299 | SDOperand X86TargetLowering::LowerRET(SDOperand Op, SelectionDAG &DAG) { |
| 4300 | SDOperand Copy; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 4301 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4302 | switch(Op.getNumOperands()) { |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4303 | default: |
| 4304 | assert(0 && "Do not know how to return this many arguments!"); |
| 4305 | abort(); |
Chris Lattner | b2be403 | 2006-04-17 20:32:50 +0000 | [diff] [blame] | 4306 | case 1: // ret void. |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4307 | return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, Op.getOperand(0), |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4308 | DAG.getConstant(getBytesToPopOnReturn(), MVT::i16)); |
Evan Cheng | 6848be1 | 2006-05-26 23:10:12 +0000 | [diff] [blame] | 4309 | case 3: { |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4310 | MVT::ValueType ArgVT = Op.getOperand(1).getValueType(); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 4311 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4312 | if (MVT::isVector(ArgVT) || |
| 4313 | (Subtarget->is64Bit() && MVT::isFloatingPoint(ArgVT))) { |
Chris Lattner | b2be403 | 2006-04-17 20:32:50 +0000 | [diff] [blame] | 4314 | // Integer or FP vector result -> XMM0. |
| 4315 | if (DAG.getMachineFunction().liveout_empty()) |
| 4316 | DAG.getMachineFunction().addLiveOut(X86::XMM0); |
| 4317 | Copy = DAG.getCopyToReg(Op.getOperand(0), X86::XMM0, Op.getOperand(1), |
| 4318 | SDOperand()); |
| 4319 | } else if (MVT::isInteger(ArgVT)) { |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4320 | // Integer result -> EAX / RAX. |
| 4321 | // The C calling convention guarantees the return value has been |
| 4322 | // promoted to at least MVT::i32. The X86-64 ABI doesn't require the |
| 4323 | // value to be promoted MVT::i64. So we don't have to extend it to |
| 4324 | // 64-bit. Return the value in EAX, but mark RAX as liveout. |
| 4325 | unsigned Reg = Subtarget->is64Bit() ? X86::RAX : X86::EAX; |
Chris Lattner | b2be403 | 2006-04-17 20:32:50 +0000 | [diff] [blame] | 4326 | if (DAG.getMachineFunction().liveout_empty()) |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4327 | DAG.getMachineFunction().addLiveOut(Reg); |
Chris Lattner | b2be403 | 2006-04-17 20:32:50 +0000 | [diff] [blame] | 4328 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4329 | Reg = (ArgVT == MVT::i64) ? X86::RAX : X86::EAX; |
| 4330 | Copy = DAG.getCopyToReg(Op.getOperand(0), Reg, Op.getOperand(1), |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4331 | SDOperand()); |
Chris Lattner | b2be403 | 2006-04-17 20:32:50 +0000 | [diff] [blame] | 4332 | } else if (!X86ScalarSSE) { |
| 4333 | // FP return with fp-stack value. |
| 4334 | if (DAG.getMachineFunction().liveout_empty()) |
| 4335 | DAG.getMachineFunction().addLiveOut(X86::ST0); |
| 4336 | |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4337 | std::vector<MVT::ValueType> Tys; |
| 4338 | Tys.push_back(MVT::Other); |
| 4339 | Tys.push_back(MVT::Flag); |
| 4340 | std::vector<SDOperand> Ops; |
| 4341 | Ops.push_back(Op.getOperand(0)); |
| 4342 | Ops.push_back(Op.getOperand(1)); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 4343 | Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, &Ops[0], Ops.size()); |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4344 | } else { |
Chris Lattner | b2be403 | 2006-04-17 20:32:50 +0000 | [diff] [blame] | 4345 | // FP return with ScalarSSE (return on fp-stack). |
| 4346 | if (DAG.getMachineFunction().liveout_empty()) |
| 4347 | DAG.getMachineFunction().addLiveOut(X86::ST0); |
| 4348 | |
Evan Cheng | 0d084c9 | 2006-02-01 00:20:21 +0000 | [diff] [blame] | 4349 | SDOperand MemLoc; |
| 4350 | SDOperand Chain = Op.getOperand(0); |
Evan Cheng | 0e8671b | 2006-01-31 23:19:54 +0000 | [diff] [blame] | 4351 | SDOperand Value = Op.getOperand(1); |
| 4352 | |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 4353 | if (ISD::isNON_EXTLoad(Value.Val) && |
Evan Cheng | 760df29 | 2006-02-01 01:19:32 +0000 | [diff] [blame] | 4354 | (Chain == Value.getValue(1) || Chain == Value.getOperand(0))) { |
Evan Cheng | 0e8671b | 2006-01-31 23:19:54 +0000 | [diff] [blame] | 4355 | Chain = Value.getOperand(0); |
| 4356 | MemLoc = Value.getOperand(1); |
| 4357 | } else { |
| 4358 | // Spill the value to memory and reload it into top of stack. |
| 4359 | unsigned Size = MVT::getSizeInBits(ArgVT)/8; |
| 4360 | MachineFunction &MF = DAG.getMachineFunction(); |
| 4361 | int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); |
| 4362 | MemLoc = DAG.getFrameIndex(SSFI, getPointerTy()); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4363 | Chain = DAG.getStore(Op.getOperand(0), Value, MemLoc, NULL, 0); |
Evan Cheng | 0e8671b | 2006-01-31 23:19:54 +0000 | [diff] [blame] | 4364 | } |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4365 | std::vector<MVT::ValueType> Tys; |
| 4366 | Tys.push_back(MVT::f64); |
| 4367 | Tys.push_back(MVT::Other); |
| 4368 | std::vector<SDOperand> Ops; |
| 4369 | Ops.push_back(Chain); |
Evan Cheng | 0e8671b | 2006-01-31 23:19:54 +0000 | [diff] [blame] | 4370 | Ops.push_back(MemLoc); |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4371 | Ops.push_back(DAG.getValueType(ArgVT)); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 4372 | Copy = DAG.getNode(X86ISD::FLD, Tys, &Ops[0], Ops.size()); |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4373 | Tys.clear(); |
| 4374 | Tys.push_back(MVT::Other); |
| 4375 | Tys.push_back(MVT::Flag); |
| 4376 | Ops.clear(); |
| 4377 | Ops.push_back(Copy.getValue(1)); |
| 4378 | Ops.push_back(Copy); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 4379 | Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, &Ops[0], Ops.size()); |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4380 | } |
| 4381 | break; |
| 4382 | } |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4383 | case 5: { |
| 4384 | unsigned Reg1 = Subtarget->is64Bit() ? X86::RAX : X86::EAX; |
| 4385 | unsigned Reg2 = Subtarget->is64Bit() ? X86::RDX : X86::EDX; |
Chris Lattner | b2be403 | 2006-04-17 20:32:50 +0000 | [diff] [blame] | 4386 | if (DAG.getMachineFunction().liveout_empty()) { |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4387 | DAG.getMachineFunction().addLiveOut(Reg1); |
| 4388 | DAG.getMachineFunction().addLiveOut(Reg2); |
Chris Lattner | b2be403 | 2006-04-17 20:32:50 +0000 | [diff] [blame] | 4389 | } |
| 4390 | |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 4391 | Copy = DAG.getCopyToReg(Op.getOperand(0), Reg2, Op.getOperand(3), |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4392 | SDOperand()); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4393 | Copy = DAG.getCopyToReg(Copy, Reg1, Op.getOperand(1), Copy.getValue(1)); |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4394 | break; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4395 | } |
Nate Begeman | ee62557 | 2006-01-27 21:09:22 +0000 | [diff] [blame] | 4396 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4397 | return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4398 | Copy, DAG.getConstant(getBytesToPopOnReturn(), MVT::i16), |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4399 | Copy.getValue(1)); |
| 4400 | } |
| 4401 | |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 4402 | SDOperand |
| 4403 | X86TargetLowering::LowerFORMAL_ARGUMENTS(SDOperand Op, SelectionDAG &DAG) { |
Evan Cheng | e8bd0a3 | 2006-06-06 23:30:24 +0000 | [diff] [blame] | 4404 | MachineFunction &MF = DAG.getMachineFunction(); |
| 4405 | const Function* Fn = MF.getFunction(); |
| 4406 | if (Fn->hasExternalLinkage() && |
Anton Korobeynikov | bcb9770 | 2006-09-17 20:25:45 +0000 | [diff] [blame] | 4407 | Subtarget->isTargetCygwin() && |
Evan Cheng | b12223e | 2006-06-09 06:24:42 +0000 | [diff] [blame] | 4408 | Fn->getName() == "main") |
Evan Cheng | e8bd0a3 | 2006-06-06 23:30:24 +0000 | [diff] [blame] | 4409 | MF.getInfo<X86FunctionInfo>()->setForceFramePointer(true); |
| 4410 | |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 4411 | unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getValue(); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4412 | if (Subtarget->is64Bit()) |
| 4413 | return LowerX86_64CCCArguments(Op, DAG); |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 4414 | else |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4415 | switch(CC) { |
Chris Lattner | f38f543 | 2006-09-27 18:29:38 +0000 | [diff] [blame] | 4416 | default: |
| 4417 | assert(0 && "Unsupported calling convention"); |
| 4418 | case CallingConv::Fast: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4419 | if (EnableFastCC) { |
| 4420 | return LowerFastCCArguments(Op, DAG); |
| 4421 | } |
| 4422 | // Falls through |
Chris Lattner | f38f543 | 2006-09-27 18:29:38 +0000 | [diff] [blame] | 4423 | case CallingConv::C: |
| 4424 | case CallingConv::CSRet: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4425 | return LowerCCCArguments(Op, DAG); |
Chris Lattner | f38f543 | 2006-09-27 18:29:38 +0000 | [diff] [blame] | 4426 | case CallingConv::X86_StdCall: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4427 | MF.getInfo<X86FunctionInfo>()->setDecorationStyle(StdCall); |
| 4428 | return LowerStdCallCCArguments(Op, DAG); |
Chris Lattner | f38f543 | 2006-09-27 18:29:38 +0000 | [diff] [blame] | 4429 | case CallingConv::X86_FastCall: |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4430 | MF.getInfo<X86FunctionInfo>()->setDecorationStyle(FastCall); |
| 4431 | return LowerFastCallCCArguments(Op, DAG); |
Anton Korobeynikov | f824868 | 2006-09-20 22:03:51 +0000 | [diff] [blame] | 4432 | } |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 4433 | } |
| 4434 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4435 | SDOperand X86TargetLowering::LowerMEMSET(SDOperand Op, SelectionDAG &DAG) { |
| 4436 | SDOperand InFlag(0, 0); |
| 4437 | SDOperand Chain = Op.getOperand(0); |
| 4438 | unsigned Align = |
| 4439 | (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue(); |
| 4440 | if (Align == 0) Align = 1; |
| 4441 | |
| 4442 | ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)); |
| 4443 | // If not DWORD aligned, call memset if size is less than the threshold. |
| 4444 | // It knows how to align to the right boundary first. |
| 4445 | if ((Align & 3) != 0 || |
| 4446 | (I && I->getValue() < Subtarget->getMinRepStrSizeThreshold())) { |
| 4447 | MVT::ValueType IntPtr = getPointerTy(); |
Owen Anderson | a69571c | 2006-05-03 01:29:57 +0000 | [diff] [blame] | 4448 | const Type *IntPtrTy = getTargetData()->getIntPtrType(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4449 | std::vector<std::pair<SDOperand, const Type*> > Args; |
| 4450 | Args.push_back(std::make_pair(Op.getOperand(1), IntPtrTy)); |
| 4451 | // Extend the ubyte argument to be an int value for the call. |
| 4452 | SDOperand Val = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Op.getOperand(2)); |
| 4453 | Args.push_back(std::make_pair(Val, IntPtrTy)); |
| 4454 | Args.push_back(std::make_pair(Op.getOperand(3), IntPtrTy)); |
| 4455 | std::pair<SDOperand,SDOperand> CallResult = |
| 4456 | LowerCallTo(Chain, Type::VoidTy, false, CallingConv::C, false, |
| 4457 | DAG.getExternalSymbol("memset", IntPtr), Args, DAG); |
| 4458 | return CallResult.second; |
Evan Cheng | 48090aa | 2006-03-21 23:01:21 +0000 | [diff] [blame] | 4459 | } |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 4460 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4461 | MVT::ValueType AVT; |
| 4462 | SDOperand Count; |
| 4463 | ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Op.getOperand(2)); |
| 4464 | unsigned BytesLeft = 0; |
| 4465 | bool TwoRepStos = false; |
| 4466 | if (ValC) { |
| 4467 | unsigned ValReg; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4468 | uint64_t Val = ValC->getValue() & 255; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 4469 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4470 | // If the value is a constant, then we can potentially use larger sets. |
| 4471 | switch (Align & 3) { |
| 4472 | case 2: // WORD aligned |
| 4473 | AVT = MVT::i16; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4474 | ValReg = X86::AX; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4475 | Val = (Val << 8) | Val; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4476 | break; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4477 | case 0: // DWORD aligned |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4478 | AVT = MVT::i32; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4479 | ValReg = X86::EAX; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4480 | Val = (Val << 8) | Val; |
| 4481 | Val = (Val << 16) | Val; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4482 | if (Subtarget->is64Bit() && ((Align & 0xF) == 0)) { // QWORD aligned |
| 4483 | AVT = MVT::i64; |
| 4484 | ValReg = X86::RAX; |
| 4485 | Val = (Val << 32) | Val; |
| 4486 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4487 | break; |
| 4488 | default: // Byte aligned |
| 4489 | AVT = MVT::i8; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4490 | ValReg = X86::AL; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4491 | Count = Op.getOperand(3); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4492 | break; |
Evan Cheng | 80d428c | 2006-04-19 22:48:17 +0000 | [diff] [blame] | 4493 | } |
| 4494 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4495 | if (AVT > MVT::i8) { |
| 4496 | if (I) { |
| 4497 | unsigned UBytes = MVT::getSizeInBits(AVT) / 8; |
| 4498 | Count = DAG.getConstant(I->getValue() / UBytes, getPointerTy()); |
| 4499 | BytesLeft = I->getValue() % UBytes; |
| 4500 | } else { |
| 4501 | assert(AVT >= MVT::i32 && |
| 4502 | "Do not use rep;stos if not at least DWORD aligned"); |
| 4503 | Count = DAG.getNode(ISD::SRL, Op.getOperand(3).getValueType(), |
| 4504 | Op.getOperand(3), DAG.getConstant(2, MVT::i8)); |
| 4505 | TwoRepStos = true; |
| 4506 | } |
| 4507 | } |
| 4508 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4509 | Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT), |
| 4510 | InFlag); |
| 4511 | InFlag = Chain.getValue(1); |
| 4512 | } else { |
| 4513 | AVT = MVT::i8; |
| 4514 | Count = Op.getOperand(3); |
| 4515 | Chain = DAG.getCopyToReg(Chain, X86::AL, Op.getOperand(2), InFlag); |
| 4516 | InFlag = Chain.getValue(1); |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 4517 | } |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 4518 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4519 | Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, |
| 4520 | Count, InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4521 | InFlag = Chain.getValue(1); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4522 | Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, |
| 4523 | Op.getOperand(1), InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4524 | InFlag = Chain.getValue(1); |
Evan Cheng | a0b3afb | 2006-03-27 07:00:16 +0000 | [diff] [blame] | 4525 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4526 | std::vector<MVT::ValueType> Tys; |
| 4527 | Tys.push_back(MVT::Other); |
| 4528 | Tys.push_back(MVT::Flag); |
| 4529 | std::vector<SDOperand> Ops; |
| 4530 | Ops.push_back(Chain); |
| 4531 | Ops.push_back(DAG.getValueType(AVT)); |
| 4532 | Ops.push_back(InFlag); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 4533 | Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 4534 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4535 | if (TwoRepStos) { |
| 4536 | InFlag = Chain.getValue(1); |
| 4537 | Count = Op.getOperand(3); |
| 4538 | MVT::ValueType CVT = Count.getValueType(); |
| 4539 | SDOperand Left = DAG.getNode(ISD::AND, CVT, Count, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4540 | DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT)); |
| 4541 | Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX, |
| 4542 | Left, InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4543 | InFlag = Chain.getValue(1); |
| 4544 | Tys.clear(); |
| 4545 | Tys.push_back(MVT::Other); |
| 4546 | Tys.push_back(MVT::Flag); |
| 4547 | Ops.clear(); |
| 4548 | Ops.push_back(Chain); |
| 4549 | Ops.push_back(DAG.getValueType(MVT::i8)); |
| 4550 | Ops.push_back(InFlag); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 4551 | Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4552 | } else if (BytesLeft) { |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4553 | // Issue stores for the last 1 - 7 bytes. |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4554 | SDOperand Value; |
| 4555 | unsigned Val = ValC->getValue() & 255; |
| 4556 | unsigned Offset = I->getValue() - BytesLeft; |
| 4557 | SDOperand DstAddr = Op.getOperand(1); |
| 4558 | MVT::ValueType AddrVT = DstAddr.getValueType(); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4559 | if (BytesLeft >= 4) { |
| 4560 | Val = (Val << 8) | Val; |
| 4561 | Val = (Val << 16) | Val; |
| 4562 | Value = DAG.getConstant(Val, MVT::i32); |
Evan Cheng | 786225a | 2006-10-05 23:01:46 +0000 | [diff] [blame] | 4563 | Chain = DAG.getStore(Chain, Value, |
| 4564 | DAG.getNode(ISD::ADD, AddrVT, DstAddr, |
| 4565 | DAG.getConstant(Offset, AddrVT)), |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4566 | NULL, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4567 | BytesLeft -= 4; |
| 4568 | Offset += 4; |
| 4569 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4570 | if (BytesLeft >= 2) { |
| 4571 | Value = DAG.getConstant((Val << 8) | Val, MVT::i16); |
Evan Cheng | 786225a | 2006-10-05 23:01:46 +0000 | [diff] [blame] | 4572 | Chain = DAG.getStore(Chain, Value, |
| 4573 | DAG.getNode(ISD::ADD, AddrVT, DstAddr, |
| 4574 | DAG.getConstant(Offset, AddrVT)), |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4575 | NULL, 0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4576 | BytesLeft -= 2; |
| 4577 | Offset += 2; |
Evan Cheng | 386031a | 2006-03-24 07:29:27 +0000 | [diff] [blame] | 4578 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4579 | if (BytesLeft == 1) { |
| 4580 | Value = DAG.getConstant(Val, MVT::i8); |
Evan Cheng | 786225a | 2006-10-05 23:01:46 +0000 | [diff] [blame] | 4581 | Chain = DAG.getStore(Chain, Value, |
| 4582 | DAG.getNode(ISD::ADD, AddrVT, DstAddr, |
| 4583 | DAG.getConstant(Offset, AddrVT)), |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4584 | NULL, 0); |
Evan Cheng | ba05f72 | 2006-04-21 23:03:30 +0000 | [diff] [blame] | 4585 | } |
Evan Cheng | 386031a | 2006-03-24 07:29:27 +0000 | [diff] [blame] | 4586 | } |
Evan Cheng | 11e15b3 | 2006-04-03 20:53:28 +0000 | [diff] [blame] | 4587 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4588 | return Chain; |
| 4589 | } |
Evan Cheng | 11e15b3 | 2006-04-03 20:53:28 +0000 | [diff] [blame] | 4590 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4591 | SDOperand X86TargetLowering::LowerMEMCPY(SDOperand Op, SelectionDAG &DAG) { |
| 4592 | SDOperand Chain = Op.getOperand(0); |
| 4593 | unsigned Align = |
| 4594 | (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue(); |
| 4595 | if (Align == 0) Align = 1; |
Evan Cheng | 11e15b3 | 2006-04-03 20:53:28 +0000 | [diff] [blame] | 4596 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4597 | ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)); |
| 4598 | // If not DWORD aligned, call memcpy if size is less than the threshold. |
| 4599 | // It knows how to align to the right boundary first. |
| 4600 | if ((Align & 3) != 0 || |
| 4601 | (I && I->getValue() < Subtarget->getMinRepStrSizeThreshold())) { |
| 4602 | MVT::ValueType IntPtr = getPointerTy(); |
Owen Anderson | a69571c | 2006-05-03 01:29:57 +0000 | [diff] [blame] | 4603 | const Type *IntPtrTy = getTargetData()->getIntPtrType(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4604 | std::vector<std::pair<SDOperand, const Type*> > Args; |
| 4605 | Args.push_back(std::make_pair(Op.getOperand(1), IntPtrTy)); |
| 4606 | Args.push_back(std::make_pair(Op.getOperand(2), IntPtrTy)); |
| 4607 | Args.push_back(std::make_pair(Op.getOperand(3), IntPtrTy)); |
| 4608 | std::pair<SDOperand,SDOperand> CallResult = |
| 4609 | LowerCallTo(Chain, Type::VoidTy, false, CallingConv::C, false, |
| 4610 | DAG.getExternalSymbol("memcpy", IntPtr), Args, DAG); |
| 4611 | return CallResult.second; |
Evan Cheng | b067a1e | 2006-03-31 19:22:53 +0000 | [diff] [blame] | 4612 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4613 | |
| 4614 | MVT::ValueType AVT; |
| 4615 | SDOperand Count; |
| 4616 | unsigned BytesLeft = 0; |
| 4617 | bool TwoRepMovs = false; |
| 4618 | switch (Align & 3) { |
| 4619 | case 2: // WORD aligned |
| 4620 | AVT = MVT::i16; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4621 | break; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4622 | case 0: // DWORD aligned |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4623 | AVT = MVT::i32; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4624 | if (Subtarget->is64Bit() && ((Align & 0xF) == 0)) // QWORD aligned |
| 4625 | AVT = MVT::i64; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4626 | break; |
| 4627 | default: // Byte aligned |
| 4628 | AVT = MVT::i8; |
| 4629 | Count = Op.getOperand(3); |
| 4630 | break; |
| 4631 | } |
| 4632 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4633 | if (AVT > MVT::i8) { |
| 4634 | if (I) { |
| 4635 | unsigned UBytes = MVT::getSizeInBits(AVT) / 8; |
| 4636 | Count = DAG.getConstant(I->getValue() / UBytes, getPointerTy()); |
| 4637 | BytesLeft = I->getValue() % UBytes; |
| 4638 | } else { |
| 4639 | assert(AVT >= MVT::i32 && |
| 4640 | "Do not use rep;movs if not at least DWORD aligned"); |
| 4641 | Count = DAG.getNode(ISD::SRL, Op.getOperand(3).getValueType(), |
| 4642 | Op.getOperand(3), DAG.getConstant(2, MVT::i8)); |
| 4643 | TwoRepMovs = true; |
| 4644 | } |
| 4645 | } |
| 4646 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4647 | SDOperand InFlag(0, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4648 | Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, |
| 4649 | Count, InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4650 | InFlag = Chain.getValue(1); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4651 | Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, |
| 4652 | Op.getOperand(1), InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4653 | InFlag = Chain.getValue(1); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4654 | Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RSI : X86::ESI, |
| 4655 | Op.getOperand(2), InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4656 | InFlag = Chain.getValue(1); |
| 4657 | |
| 4658 | std::vector<MVT::ValueType> Tys; |
| 4659 | Tys.push_back(MVT::Other); |
| 4660 | Tys.push_back(MVT::Flag); |
| 4661 | std::vector<SDOperand> Ops; |
| 4662 | Ops.push_back(Chain); |
| 4663 | Ops.push_back(DAG.getValueType(AVT)); |
| 4664 | Ops.push_back(InFlag); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 4665 | Chain = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4666 | |
| 4667 | if (TwoRepMovs) { |
| 4668 | InFlag = Chain.getValue(1); |
| 4669 | Count = Op.getOperand(3); |
| 4670 | MVT::ValueType CVT = Count.getValueType(); |
| 4671 | SDOperand Left = DAG.getNode(ISD::AND, CVT, Count, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4672 | DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT)); |
| 4673 | Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX, |
| 4674 | Left, InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4675 | InFlag = Chain.getValue(1); |
| 4676 | Tys.clear(); |
| 4677 | Tys.push_back(MVT::Other); |
| 4678 | Tys.push_back(MVT::Flag); |
| 4679 | Ops.clear(); |
| 4680 | Ops.push_back(Chain); |
| 4681 | Ops.push_back(DAG.getValueType(MVT::i8)); |
| 4682 | Ops.push_back(InFlag); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 4683 | Chain = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4684 | } else if (BytesLeft) { |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4685 | // Issue loads and stores for the last 1 - 7 bytes. |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4686 | unsigned Offset = I->getValue() - BytesLeft; |
| 4687 | SDOperand DstAddr = Op.getOperand(1); |
| 4688 | MVT::ValueType DstVT = DstAddr.getValueType(); |
| 4689 | SDOperand SrcAddr = Op.getOperand(2); |
| 4690 | MVT::ValueType SrcVT = SrcAddr.getValueType(); |
| 4691 | SDOperand Value; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4692 | if (BytesLeft >= 4) { |
| 4693 | Value = DAG.getLoad(MVT::i32, Chain, |
| 4694 | DAG.getNode(ISD::ADD, SrcVT, SrcAddr, |
| 4695 | DAG.getConstant(Offset, SrcVT)), |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 4696 | NULL, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4697 | Chain = Value.getValue(1); |
Evan Cheng | 786225a | 2006-10-05 23:01:46 +0000 | [diff] [blame] | 4698 | Chain = DAG.getStore(Chain, Value, |
| 4699 | DAG.getNode(ISD::ADD, DstVT, DstAddr, |
| 4700 | DAG.getConstant(Offset, DstVT)), |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4701 | NULL, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4702 | BytesLeft -= 4; |
| 4703 | Offset += 4; |
| 4704 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4705 | if (BytesLeft >= 2) { |
| 4706 | Value = DAG.getLoad(MVT::i16, Chain, |
| 4707 | DAG.getNode(ISD::ADD, SrcVT, SrcAddr, |
| 4708 | DAG.getConstant(Offset, SrcVT)), |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 4709 | NULL, 0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4710 | Chain = Value.getValue(1); |
Evan Cheng | 786225a | 2006-10-05 23:01:46 +0000 | [diff] [blame] | 4711 | Chain = DAG.getStore(Chain, Value, |
| 4712 | DAG.getNode(ISD::ADD, DstVT, DstAddr, |
| 4713 | DAG.getConstant(Offset, DstVT)), |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4714 | NULL, 0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4715 | BytesLeft -= 2; |
| 4716 | Offset += 2; |
Evan Cheng | b067a1e | 2006-03-31 19:22:53 +0000 | [diff] [blame] | 4717 | } |
| 4718 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4719 | if (BytesLeft == 1) { |
| 4720 | Value = DAG.getLoad(MVT::i8, Chain, |
| 4721 | DAG.getNode(ISD::ADD, SrcVT, SrcAddr, |
| 4722 | DAG.getConstant(Offset, SrcVT)), |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 4723 | NULL, 0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4724 | Chain = Value.getValue(1); |
Evan Cheng | 786225a | 2006-10-05 23:01:46 +0000 | [diff] [blame] | 4725 | Chain = DAG.getStore(Chain, Value, |
| 4726 | DAG.getNode(ISD::ADD, DstVT, DstAddr, |
| 4727 | DAG.getConstant(Offset, DstVT)), |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4728 | NULL, 0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4729 | } |
Evan Cheng | b067a1e | 2006-03-31 19:22:53 +0000 | [diff] [blame] | 4730 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4731 | |
| 4732 | return Chain; |
| 4733 | } |
| 4734 | |
| 4735 | SDOperand |
| 4736 | X86TargetLowering::LowerREADCYCLCECOUNTER(SDOperand Op, SelectionDAG &DAG) { |
| 4737 | std::vector<MVT::ValueType> Tys; |
| 4738 | Tys.push_back(MVT::Other); |
| 4739 | Tys.push_back(MVT::Flag); |
| 4740 | std::vector<SDOperand> Ops; |
| 4741 | Ops.push_back(Op.getOperand(0)); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 4742 | SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4743 | Ops.clear(); |
Evan Cheng | 3fa9dff | 2006-11-29 08:28:13 +0000 | [diff] [blame] | 4744 | if (Subtarget->is64Bit()) { |
| 4745 | SDOperand Copy1 = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1)); |
| 4746 | SDOperand Copy2 = DAG.getCopyFromReg(Copy1.getValue(1), X86::RDX, |
| 4747 | MVT::i64, Copy1.getValue(2)); |
| 4748 | SDOperand Tmp = DAG.getNode(ISD::SHL, MVT::i64, Copy2, |
| 4749 | DAG.getConstant(32, MVT::i8)); |
| 4750 | Ops.push_back(DAG.getNode(ISD::OR, MVT::i64, Copy1, Tmp)); |
| 4751 | Ops.push_back(Copy2.getValue(1)); |
| 4752 | Tys[0] = MVT::i64; |
| 4753 | Tys[1] = MVT::Other; |
| 4754 | } else { |
| 4755 | SDOperand Copy1 = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)); |
| 4756 | SDOperand Copy2 = DAG.getCopyFromReg(Copy1.getValue(1), X86::EDX, |
| 4757 | MVT::i32, Copy1.getValue(2)); |
| 4758 | Ops.push_back(Copy1); |
| 4759 | Ops.push_back(Copy2); |
| 4760 | Ops.push_back(Copy2.getValue(1)); |
| 4761 | Tys[0] = Tys[1] = MVT::i32; |
| 4762 | Tys.push_back(MVT::Other); |
| 4763 | } |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 4764 | return DAG.getNode(ISD::MERGE_VALUES, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4765 | } |
| 4766 | |
| 4767 | SDOperand X86TargetLowering::LowerVASTART(SDOperand Op, SelectionDAG &DAG) { |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4768 | SrcValueSDNode *SV = cast<SrcValueSDNode>(Op.getOperand(2)); |
| 4769 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4770 | if (!Subtarget->is64Bit()) { |
| 4771 | // vastart just stores the address of the VarArgsFrameIndex slot into the |
| 4772 | // memory location argument. |
| 4773 | SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4774 | return DAG.getStore(Op.getOperand(0), FR,Op.getOperand(1), SV->getValue(), |
| 4775 | SV->getOffset()); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4776 | } |
| 4777 | |
| 4778 | // __va_list_tag: |
| 4779 | // gp_offset (0 - 6 * 8) |
| 4780 | // fp_offset (48 - 48 + 8 * 16) |
| 4781 | // overflow_arg_area (point to parameters coming in memory). |
| 4782 | // reg_save_area |
| 4783 | std::vector<SDOperand> MemOps; |
| 4784 | SDOperand FIN = Op.getOperand(1); |
| 4785 | // Store gp_offset |
Evan Cheng | 786225a | 2006-10-05 23:01:46 +0000 | [diff] [blame] | 4786 | SDOperand Store = DAG.getStore(Op.getOperand(0), |
| 4787 | DAG.getConstant(VarArgsGPOffset, MVT::i32), |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4788 | FIN, SV->getValue(), SV->getOffset()); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4789 | MemOps.push_back(Store); |
| 4790 | |
| 4791 | // Store fp_offset |
| 4792 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, |
| 4793 | DAG.getConstant(4, getPointerTy())); |
Evan Cheng | 786225a | 2006-10-05 23:01:46 +0000 | [diff] [blame] | 4794 | Store = DAG.getStore(Op.getOperand(0), |
| 4795 | DAG.getConstant(VarArgsFPOffset, MVT::i32), |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4796 | FIN, SV->getValue(), SV->getOffset()); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4797 | MemOps.push_back(Store); |
| 4798 | |
| 4799 | // Store ptr to overflow_arg_area |
| 4800 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, |
| 4801 | DAG.getConstant(4, getPointerTy())); |
| 4802 | SDOperand OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4803 | Store = DAG.getStore(Op.getOperand(0), OVFIN, FIN, SV->getValue(), |
| 4804 | SV->getOffset()); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4805 | MemOps.push_back(Store); |
| 4806 | |
| 4807 | // Store ptr to reg_save_area. |
| 4808 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, |
| 4809 | DAG.getConstant(8, getPointerTy())); |
| 4810 | SDOperand RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 4811 | Store = DAG.getStore(Op.getOperand(0), RSFIN, FIN, SV->getValue(), |
| 4812 | SV->getOffset()); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 4813 | MemOps.push_back(Store); |
| 4814 | return DAG.getNode(ISD::TokenFactor, MVT::Other, &MemOps[0], MemOps.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4815 | } |
| 4816 | |
| 4817 | SDOperand |
| 4818 | X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDOperand Op, SelectionDAG &DAG) { |
| 4819 | unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getValue(); |
| 4820 | switch (IntNo) { |
| 4821 | default: return SDOperand(); // Don't custom lower most intrinsics. |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4822 | // Comparison intrinsics. |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4823 | case Intrinsic::x86_sse_comieq_ss: |
| 4824 | case Intrinsic::x86_sse_comilt_ss: |
| 4825 | case Intrinsic::x86_sse_comile_ss: |
| 4826 | case Intrinsic::x86_sse_comigt_ss: |
| 4827 | case Intrinsic::x86_sse_comige_ss: |
| 4828 | case Intrinsic::x86_sse_comineq_ss: |
| 4829 | case Intrinsic::x86_sse_ucomieq_ss: |
| 4830 | case Intrinsic::x86_sse_ucomilt_ss: |
| 4831 | case Intrinsic::x86_sse_ucomile_ss: |
| 4832 | case Intrinsic::x86_sse_ucomigt_ss: |
| 4833 | case Intrinsic::x86_sse_ucomige_ss: |
| 4834 | case Intrinsic::x86_sse_ucomineq_ss: |
| 4835 | case Intrinsic::x86_sse2_comieq_sd: |
| 4836 | case Intrinsic::x86_sse2_comilt_sd: |
| 4837 | case Intrinsic::x86_sse2_comile_sd: |
| 4838 | case Intrinsic::x86_sse2_comigt_sd: |
| 4839 | case Intrinsic::x86_sse2_comige_sd: |
| 4840 | case Intrinsic::x86_sse2_comineq_sd: |
| 4841 | case Intrinsic::x86_sse2_ucomieq_sd: |
| 4842 | case Intrinsic::x86_sse2_ucomilt_sd: |
| 4843 | case Intrinsic::x86_sse2_ucomile_sd: |
| 4844 | case Intrinsic::x86_sse2_ucomigt_sd: |
| 4845 | case Intrinsic::x86_sse2_ucomige_sd: |
| 4846 | case Intrinsic::x86_sse2_ucomineq_sd: { |
| 4847 | unsigned Opc = 0; |
| 4848 | ISD::CondCode CC = ISD::SETCC_INVALID; |
| 4849 | switch (IntNo) { |
| 4850 | default: break; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 4851 | case Intrinsic::x86_sse_comieq_ss: |
| 4852 | case Intrinsic::x86_sse2_comieq_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4853 | Opc = X86ISD::COMI; |
| 4854 | CC = ISD::SETEQ; |
| 4855 | break; |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4856 | case Intrinsic::x86_sse_comilt_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4857 | case Intrinsic::x86_sse2_comilt_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4858 | Opc = X86ISD::COMI; |
| 4859 | CC = ISD::SETLT; |
| 4860 | break; |
| 4861 | case Intrinsic::x86_sse_comile_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4862 | case Intrinsic::x86_sse2_comile_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4863 | Opc = X86ISD::COMI; |
| 4864 | CC = ISD::SETLE; |
| 4865 | break; |
| 4866 | case Intrinsic::x86_sse_comigt_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4867 | case Intrinsic::x86_sse2_comigt_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4868 | Opc = X86ISD::COMI; |
| 4869 | CC = ISD::SETGT; |
| 4870 | break; |
| 4871 | case Intrinsic::x86_sse_comige_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4872 | case Intrinsic::x86_sse2_comige_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4873 | Opc = X86ISD::COMI; |
| 4874 | CC = ISD::SETGE; |
| 4875 | break; |
| 4876 | case Intrinsic::x86_sse_comineq_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4877 | case Intrinsic::x86_sse2_comineq_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4878 | Opc = X86ISD::COMI; |
| 4879 | CC = ISD::SETNE; |
| 4880 | break; |
| 4881 | case Intrinsic::x86_sse_ucomieq_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4882 | case Intrinsic::x86_sse2_ucomieq_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4883 | Opc = X86ISD::UCOMI; |
| 4884 | CC = ISD::SETEQ; |
| 4885 | break; |
| 4886 | case Intrinsic::x86_sse_ucomilt_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4887 | case Intrinsic::x86_sse2_ucomilt_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4888 | Opc = X86ISD::UCOMI; |
| 4889 | CC = ISD::SETLT; |
| 4890 | break; |
| 4891 | case Intrinsic::x86_sse_ucomile_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4892 | case Intrinsic::x86_sse2_ucomile_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4893 | Opc = X86ISD::UCOMI; |
| 4894 | CC = ISD::SETLE; |
| 4895 | break; |
| 4896 | case Intrinsic::x86_sse_ucomigt_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4897 | case Intrinsic::x86_sse2_ucomigt_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4898 | Opc = X86ISD::UCOMI; |
| 4899 | CC = ISD::SETGT; |
| 4900 | break; |
| 4901 | case Intrinsic::x86_sse_ucomige_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4902 | case Intrinsic::x86_sse2_ucomige_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4903 | Opc = X86ISD::UCOMI; |
| 4904 | CC = ISD::SETGE; |
| 4905 | break; |
| 4906 | case Intrinsic::x86_sse_ucomineq_ss: |
| 4907 | case Intrinsic::x86_sse2_ucomineq_sd: |
| 4908 | Opc = X86ISD::UCOMI; |
| 4909 | CC = ISD::SETNE; |
| 4910 | break; |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4911 | } |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4912 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4913 | unsigned X86CC; |
Chris Lattner | f957051 | 2006-09-13 03:22:10 +0000 | [diff] [blame] | 4914 | SDOperand LHS = Op.getOperand(1); |
| 4915 | SDOperand RHS = Op.getOperand(2); |
| 4916 | translateX86CC(CC, true, X86CC, LHS, RHS, DAG); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4917 | |
| 4918 | const MVT::ValueType *VTs = DAG.getNodeValueTypes(MVT::Other, MVT::Flag); |
Chris Lattner | f957051 | 2006-09-13 03:22:10 +0000 | [diff] [blame] | 4919 | SDOperand Ops1[] = { DAG.getEntryNode(), LHS, RHS }; |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4920 | SDOperand Cond = DAG.getNode(Opc, VTs, 2, Ops1, 3); |
| 4921 | VTs = DAG.getNodeValueTypes(MVT::i8, MVT::Flag); |
| 4922 | SDOperand Ops2[] = { DAG.getConstant(X86CC, MVT::i8), Cond }; |
| 4923 | SDOperand SetCC = DAG.getNode(X86ISD::SETCC, VTs, 2, Ops2, 2); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4924 | return DAG.getNode(ISD::ANY_EXTEND, MVT::i32, SetCC); |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4925 | } |
Evan Cheng | 38bcbaf | 2005-12-23 07:31:11 +0000 | [diff] [blame] | 4926 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4927 | } |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 4928 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4929 | /// LowerOperation - Provide custom lowering hooks for some operations. |
| 4930 | /// |
| 4931 | SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { |
| 4932 | switch (Op.getOpcode()) { |
| 4933 | default: assert(0 && "Should not custom lower this!"); |
| 4934 | case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); |
| 4935 | case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); |
| 4936 | case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); |
| 4937 | case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); |
| 4938 | case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); |
| 4939 | case ISD::ConstantPool: return LowerConstantPool(Op, DAG); |
| 4940 | case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); |
| 4941 | case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); |
| 4942 | case ISD::SHL_PARTS: |
| 4943 | case ISD::SRA_PARTS: |
| 4944 | case ISD::SRL_PARTS: return LowerShift(Op, DAG); |
| 4945 | case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); |
| 4946 | case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); |
| 4947 | case ISD::FABS: return LowerFABS(Op, DAG); |
| 4948 | case ISD::FNEG: return LowerFNEG(Op, DAG); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 4949 | case ISD::SETCC: return LowerSETCC(Op, DAG, DAG.getEntryNode()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4950 | case ISD::SELECT: return LowerSELECT(Op, DAG); |
| 4951 | case ISD::BRCOND: return LowerBRCOND(Op, DAG); |
| 4952 | case ISD::JumpTable: return LowerJumpTable(Op, DAG); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 4953 | case ISD::CALL: return LowerCALL(Op, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4954 | case ISD::RET: return LowerRET(Op, DAG); |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 4955 | case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4956 | case ISD::MEMSET: return LowerMEMSET(Op, DAG); |
| 4957 | case ISD::MEMCPY: return LowerMEMCPY(Op, DAG); |
| 4958 | case ISD::READCYCLECOUNTER: return LowerREADCYCLCECOUNTER(Op, DAG); |
| 4959 | case ISD::VASTART: return LowerVASTART(Op, DAG); |
| 4960 | case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); |
| 4961 | } |
| 4962 | } |
| 4963 | |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 4964 | const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { |
| 4965 | switch (Opcode) { |
| 4966 | default: return NULL; |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 4967 | case X86ISD::SHLD: return "X86ISD::SHLD"; |
| 4968 | case X86ISD::SHRD: return "X86ISD::SHRD"; |
Evan Cheng | ef6ffb1 | 2006-01-31 03:14:29 +0000 | [diff] [blame] | 4969 | case X86ISD::FAND: return "X86ISD::FAND"; |
Evan Cheng | 223547a | 2006-01-31 22:28:30 +0000 | [diff] [blame] | 4970 | case X86ISD::FXOR: return "X86ISD::FXOR"; |
Evan Cheng | a3195e8 | 2006-01-12 22:54:21 +0000 | [diff] [blame] | 4971 | case X86ISD::FILD: return "X86ISD::FILD"; |
Evan Cheng | e3de85b | 2006-02-04 02:20:30 +0000 | [diff] [blame] | 4972 | case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG"; |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 4973 | case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; |
| 4974 | case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; |
| 4975 | case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; |
Evan Cheng | b077b84 | 2005-12-21 02:39:21 +0000 | [diff] [blame] | 4976 | case X86ISD::FLD: return "X86ISD::FLD"; |
Evan Cheng | d90eb7f | 2006-01-05 00:27:02 +0000 | [diff] [blame] | 4977 | case X86ISD::FST: return "X86ISD::FST"; |
| 4978 | case X86ISD::FP_GET_RESULT: return "X86ISD::FP_GET_RESULT"; |
Evan Cheng | b077b84 | 2005-12-21 02:39:21 +0000 | [diff] [blame] | 4979 | case X86ISD::FP_SET_RESULT: return "X86ISD::FP_SET_RESULT"; |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 4980 | case X86ISD::CALL: return "X86ISD::CALL"; |
| 4981 | case X86ISD::TAILCALL: return "X86ISD::TAILCALL"; |
| 4982 | case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; |
| 4983 | case X86ISD::CMP: return "X86ISD::CMP"; |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 4984 | case X86ISD::COMI: return "X86ISD::COMI"; |
| 4985 | case X86ISD::UCOMI: return "X86ISD::UCOMI"; |
Evan Cheng | d5781fc | 2005-12-21 20:21:51 +0000 | [diff] [blame] | 4986 | case X86ISD::SETCC: return "X86ISD::SETCC"; |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 4987 | case X86ISD::CMOV: return "X86ISD::CMOV"; |
| 4988 | case X86ISD::BRCOND: return "X86ISD::BRCOND"; |
Evan Cheng | b077b84 | 2005-12-21 02:39:21 +0000 | [diff] [blame] | 4989 | case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; |
Evan Cheng | 8df346b | 2006-03-04 01:12:00 +0000 | [diff] [blame] | 4990 | case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; |
| 4991 | case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; |
Evan Cheng | 223547a | 2006-01-31 22:28:30 +0000 | [diff] [blame] | 4992 | case X86ISD::LOAD_PACK: return "X86ISD::LOAD_PACK"; |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 4993 | case X86ISD::LOAD_UA: return "X86ISD::LOAD_UA"; |
Evan Cheng | 7ccced6 | 2006-02-18 00:15:05 +0000 | [diff] [blame] | 4994 | case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; |
Evan Cheng | 020d2e8 | 2006-02-23 20:41:18 +0000 | [diff] [blame] | 4995 | case X86ISD::Wrapper: return "X86ISD::Wrapper"; |
Evan Cheng | bc4832b | 2006-03-24 23:15:12 +0000 | [diff] [blame] | 4996 | case X86ISD::S2VEC: return "X86ISD::S2VEC"; |
Evan Cheng | b067a1e | 2006-03-31 19:22:53 +0000 | [diff] [blame] | 4997 | case X86ISD::PEXTRW: return "X86ISD::PEXTRW"; |
Evan Cheng | 653159f | 2006-03-31 21:55:24 +0000 | [diff] [blame] | 4998 | case X86ISD::PINSRW: return "X86ISD::PINSRW"; |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 4999 | case X86ISD::FMAX: return "X86ISD::FMAX"; |
| 5000 | case X86ISD::FMIN: return "X86ISD::FMIN"; |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 5001 | } |
| 5002 | } |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 5003 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5004 | /// isLegalAddressImmediate - Return true if the integer value or |
| 5005 | /// GlobalValue can be used as the offset of the target addressing mode. |
| 5006 | bool X86TargetLowering::isLegalAddressImmediate(int64_t V) const { |
| 5007 | // X86 allows a sign-extended 32-bit immediate field. |
| 5008 | return (V > -(1LL << 32) && V < (1LL << 32)-1); |
| 5009 | } |
| 5010 | |
| 5011 | bool X86TargetLowering::isLegalAddressImmediate(GlobalValue *GV) const { |
Evan Cheng | c8306bd | 2006-11-29 23:48:14 +0000 | [diff] [blame] | 5012 | // In 64-bit mode, GV is 64-bit so it won't fit in the 32-bit displacement |
| 5013 | // field unless we are in small code model. |
| 5014 | if (Subtarget->is64Bit() && |
| 5015 | getTargetMachine().getCodeModel() != CodeModel::Small) |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5016 | return false; |
Evan Cheng | c8306bd | 2006-11-29 23:48:14 +0000 | [diff] [blame] | 5017 | Reloc::Model RModel = getTargetMachine().getRelocationModel(); |
| 5018 | return (RModel == Reloc::Static) || |
| 5019 | !Subtarget->GVRequiresExtraLoad(GV, false); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5020 | } |
| 5021 | |
| 5022 | /// isShuffleMaskLegal - Targets can use this to indicate that they only |
| 5023 | /// support *some* VECTOR_SHUFFLE operations, those with specific masks. |
| 5024 | /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values |
| 5025 | /// are assumed to be legal. |
| 5026 | bool |
| 5027 | X86TargetLowering::isShuffleMaskLegal(SDOperand Mask, MVT::ValueType VT) const { |
| 5028 | // Only do shuffles on 128-bit vector types for now. |
| 5029 | if (MVT::getSizeInBits(VT) == 64) return false; |
| 5030 | return (Mask.Val->getNumOperands() <= 4 || |
| 5031 | isSplatMask(Mask.Val) || |
| 5032 | isPSHUFHW_PSHUFLWMask(Mask.Val) || |
| 5033 | X86::isUNPCKLMask(Mask.Val) || |
| 5034 | X86::isUNPCKL_v_undef_Mask(Mask.Val) || |
| 5035 | X86::isUNPCKHMask(Mask.Val)); |
| 5036 | } |
| 5037 | |
| 5038 | bool X86TargetLowering::isVectorClearMaskLegal(std::vector<SDOperand> &BVOps, |
| 5039 | MVT::ValueType EVT, |
| 5040 | SelectionDAG &DAG) const { |
| 5041 | unsigned NumElts = BVOps.size(); |
| 5042 | // Only do shuffles on 128-bit vector types for now. |
| 5043 | if (MVT::getSizeInBits(EVT) * NumElts == 64) return false; |
| 5044 | if (NumElts == 2) return true; |
| 5045 | if (NumElts == 4) { |
| 5046 | return (isMOVLMask(BVOps) || isCommutedMOVL(BVOps, true) || |
| 5047 | isSHUFPMask(BVOps) || isCommutedSHUFP(BVOps)); |
| 5048 | } |
| 5049 | return false; |
| 5050 | } |
| 5051 | |
| 5052 | //===----------------------------------------------------------------------===// |
| 5053 | // X86 Scheduler Hooks |
| 5054 | //===----------------------------------------------------------------------===// |
| 5055 | |
| 5056 | MachineBasicBlock * |
| 5057 | X86TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI, |
| 5058 | MachineBasicBlock *BB) { |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 5059 | const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5060 | switch (MI->getOpcode()) { |
| 5061 | default: assert(false && "Unexpected instr type to insert"); |
| 5062 | case X86::CMOV_FR32: |
| 5063 | case X86::CMOV_FR64: |
| 5064 | case X86::CMOV_V4F32: |
| 5065 | case X86::CMOV_V2F64: |
| 5066 | case X86::CMOV_V2I64: { |
| 5067 | // To "insert" a SELECT_CC instruction, we actually have to insert the |
| 5068 | // diamond control-flow pattern. The incoming instruction knows the |
| 5069 | // destination vreg to set, the condition code register to branch on, the |
| 5070 | // true/false values to select between, and a branch opcode to use. |
| 5071 | const BasicBlock *LLVM_BB = BB->getBasicBlock(); |
| 5072 | ilist<MachineBasicBlock>::iterator It = BB; |
| 5073 | ++It; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5074 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5075 | // thisMBB: |
| 5076 | // ... |
| 5077 | // TrueVal = ... |
| 5078 | // cmpTY ccX, r1, r2 |
| 5079 | // bCC copy1MBB |
| 5080 | // fallthrough --> copy0MBB |
| 5081 | MachineBasicBlock *thisMBB = BB; |
| 5082 | MachineBasicBlock *copy0MBB = new MachineBasicBlock(LLVM_BB); |
| 5083 | MachineBasicBlock *sinkMBB = new MachineBasicBlock(LLVM_BB); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5084 | unsigned Opc = |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 5085 | X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm()); |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 5086 | BuildMI(BB, TII->get(Opc)).addMBB(sinkMBB); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5087 | MachineFunction *F = BB->getParent(); |
| 5088 | F->getBasicBlockList().insert(It, copy0MBB); |
| 5089 | F->getBasicBlockList().insert(It, sinkMBB); |
| 5090 | // Update machine-CFG edges by first adding all successors of the current |
| 5091 | // block to the new block which will contain the Phi node for the select. |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5092 | for(MachineBasicBlock::succ_iterator i = BB->succ_begin(), |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5093 | e = BB->succ_end(); i != e; ++i) |
| 5094 | sinkMBB->addSuccessor(*i); |
| 5095 | // Next, remove all successors of the current block, and add the true |
| 5096 | // and fallthrough blocks as its successors. |
| 5097 | while(!BB->succ_empty()) |
| 5098 | BB->removeSuccessor(BB->succ_begin()); |
| 5099 | BB->addSuccessor(copy0MBB); |
| 5100 | BB->addSuccessor(sinkMBB); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5101 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5102 | // copy0MBB: |
| 5103 | // %FalseValue = ... |
| 5104 | // # fallthrough to sinkMBB |
| 5105 | BB = copy0MBB; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5106 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5107 | // Update machine-CFG edges |
| 5108 | BB->addSuccessor(sinkMBB); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5109 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5110 | // sinkMBB: |
| 5111 | // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] |
| 5112 | // ... |
| 5113 | BB = sinkMBB; |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 5114 | BuildMI(BB, TII->get(X86::PHI), MI->getOperand(0).getReg()) |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5115 | .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) |
| 5116 | .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); |
| 5117 | |
| 5118 | delete MI; // The pseudo instruction is gone now. |
| 5119 | return BB; |
| 5120 | } |
| 5121 | |
| 5122 | case X86::FP_TO_INT16_IN_MEM: |
| 5123 | case X86::FP_TO_INT32_IN_MEM: |
| 5124 | case X86::FP_TO_INT64_IN_MEM: { |
| 5125 | // Change the floating point control register to use "round towards zero" |
| 5126 | // mode when truncating to an integer value. |
| 5127 | MachineFunction *F = BB->getParent(); |
| 5128 | int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2); |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 5129 | addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5130 | |
| 5131 | // Load the old value of the high byte of the control word... |
| 5132 | unsigned OldCW = |
| 5133 | F->getSSARegMap()->createVirtualRegister(X86::GR16RegisterClass); |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 5134 | addFrameReference(BuildMI(BB, TII->get(X86::MOV16rm), OldCW), CWFrameIdx); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5135 | |
| 5136 | // Set the high part to be round to zero... |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 5137 | addFrameReference(BuildMI(BB, TII->get(X86::MOV16mi)), CWFrameIdx) |
| 5138 | .addImm(0xC7F); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5139 | |
| 5140 | // Reload the modified control word now... |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 5141 | addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5142 | |
| 5143 | // Restore the memory image of control word to original value |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 5144 | addFrameReference(BuildMI(BB, TII->get(X86::MOV16mr)), CWFrameIdx) |
| 5145 | .addReg(OldCW); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5146 | |
| 5147 | // Get the X86 opcode to use. |
| 5148 | unsigned Opc; |
| 5149 | switch (MI->getOpcode()) { |
| 5150 | default: assert(0 && "illegal opcode!"); |
| 5151 | case X86::FP_TO_INT16_IN_MEM: Opc = X86::FpIST16m; break; |
| 5152 | case X86::FP_TO_INT32_IN_MEM: Opc = X86::FpIST32m; break; |
| 5153 | case X86::FP_TO_INT64_IN_MEM: Opc = X86::FpIST64m; break; |
| 5154 | } |
| 5155 | |
| 5156 | X86AddressMode AM; |
| 5157 | MachineOperand &Op = MI->getOperand(0); |
| 5158 | if (Op.isRegister()) { |
| 5159 | AM.BaseType = X86AddressMode::RegBase; |
| 5160 | AM.Base.Reg = Op.getReg(); |
| 5161 | } else { |
| 5162 | AM.BaseType = X86AddressMode::FrameIndexBase; |
| 5163 | AM.Base.FrameIndex = Op.getFrameIndex(); |
| 5164 | } |
| 5165 | Op = MI->getOperand(1); |
| 5166 | if (Op.isImmediate()) |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 5167 | AM.Scale = Op.getImm(); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5168 | Op = MI->getOperand(2); |
| 5169 | if (Op.isImmediate()) |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 5170 | AM.IndexReg = Op.getImm(); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5171 | Op = MI->getOperand(3); |
| 5172 | if (Op.isGlobalAddress()) { |
| 5173 | AM.GV = Op.getGlobal(); |
| 5174 | } else { |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 5175 | AM.Disp = Op.getImm(); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5176 | } |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 5177 | addFullAddress(BuildMI(BB, TII->get(Opc)), AM) |
| 5178 | .addReg(MI->getOperand(4).getReg()); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5179 | |
| 5180 | // Reload the original control word now. |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 5181 | addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5182 | |
| 5183 | delete MI; // The pseudo instruction is gone now. |
| 5184 | return BB; |
| 5185 | } |
| 5186 | } |
| 5187 | } |
| 5188 | |
| 5189 | //===----------------------------------------------------------------------===// |
| 5190 | // X86 Optimization Hooks |
| 5191 | //===----------------------------------------------------------------------===// |
| 5192 | |
Nate Begeman | 368e18d | 2006-02-16 21:11:51 +0000 | [diff] [blame] | 5193 | void X86TargetLowering::computeMaskedBitsForTargetNode(const SDOperand Op, |
| 5194 | uint64_t Mask, |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5195 | uint64_t &KnownZero, |
Nate Begeman | 368e18d | 2006-02-16 21:11:51 +0000 | [diff] [blame] | 5196 | uint64_t &KnownOne, |
| 5197 | unsigned Depth) const { |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 5198 | unsigned Opc = Op.getOpcode(); |
Evan Cheng | 865f060 | 2006-04-05 06:11:20 +0000 | [diff] [blame] | 5199 | assert((Opc >= ISD::BUILTIN_OP_END || |
| 5200 | Opc == ISD::INTRINSIC_WO_CHAIN || |
| 5201 | Opc == ISD::INTRINSIC_W_CHAIN || |
| 5202 | Opc == ISD::INTRINSIC_VOID) && |
| 5203 | "Should use MaskedValueIsZero if you don't know whether Op" |
| 5204 | " is a target node!"); |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 5205 | |
Evan Cheng | 865f060 | 2006-04-05 06:11:20 +0000 | [diff] [blame] | 5206 | KnownZero = KnownOne = 0; // Don't know anything. |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 5207 | switch (Opc) { |
Evan Cheng | 865f060 | 2006-04-05 06:11:20 +0000 | [diff] [blame] | 5208 | default: break; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5209 | case X86ISD::SETCC: |
Nate Begeman | 368e18d | 2006-02-16 21:11:51 +0000 | [diff] [blame] | 5210 | KnownZero |= (MVT::getIntVTBitMask(Op.getValueType()) ^ 1ULL); |
| 5211 | break; |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 5212 | } |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 5213 | } |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 5214 | |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5215 | /// getShuffleScalarElt - Returns the scalar element that will make up the ith |
| 5216 | /// element of the result of the vector shuffle. |
| 5217 | static SDOperand getShuffleScalarElt(SDNode *N, unsigned i, SelectionDAG &DAG) { |
| 5218 | MVT::ValueType VT = N->getValueType(0); |
| 5219 | SDOperand PermMask = N->getOperand(2); |
| 5220 | unsigned NumElems = PermMask.getNumOperands(); |
| 5221 | SDOperand V = (i < NumElems) ? N->getOperand(0) : N->getOperand(1); |
| 5222 | i %= NumElems; |
| 5223 | if (V.getOpcode() == ISD::SCALAR_TO_VECTOR) { |
| 5224 | return (i == 0) |
| 5225 | ? V.getOperand(0) : DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(VT)); |
| 5226 | } else if (V.getOpcode() == ISD::VECTOR_SHUFFLE) { |
| 5227 | SDOperand Idx = PermMask.getOperand(i); |
| 5228 | if (Idx.getOpcode() == ISD::UNDEF) |
| 5229 | return DAG.getNode(ISD::UNDEF, MVT::getVectorBaseType(VT)); |
| 5230 | return getShuffleScalarElt(V.Val,cast<ConstantSDNode>(Idx)->getValue(),DAG); |
| 5231 | } |
| 5232 | return SDOperand(); |
| 5233 | } |
| 5234 | |
| 5235 | /// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the |
| 5236 | /// node is a GlobalAddress + an offset. |
| 5237 | static bool isGAPlusOffset(SDNode *N, GlobalValue* &GA, int64_t &Offset) { |
Evan Cheng | 0085a28 | 2006-11-30 21:55:46 +0000 | [diff] [blame] | 5238 | unsigned Opc = N->getOpcode(); |
Evan Cheng | 19f2ffc | 2006-12-05 04:01:03 +0000 | [diff] [blame] | 5239 | if (Opc == X86ISD::Wrapper) { |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5240 | if (dyn_cast<GlobalAddressSDNode>(N->getOperand(0))) { |
| 5241 | GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal(); |
| 5242 | return true; |
| 5243 | } |
Evan Cheng | 0085a28 | 2006-11-30 21:55:46 +0000 | [diff] [blame] | 5244 | } else if (Opc == ISD::ADD) { |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5245 | SDOperand N1 = N->getOperand(0); |
| 5246 | SDOperand N2 = N->getOperand(1); |
| 5247 | if (isGAPlusOffset(N1.Val, GA, Offset)) { |
| 5248 | ConstantSDNode *V = dyn_cast<ConstantSDNode>(N2); |
| 5249 | if (V) { |
| 5250 | Offset += V->getSignExtended(); |
| 5251 | return true; |
| 5252 | } |
| 5253 | } else if (isGAPlusOffset(N2.Val, GA, Offset)) { |
| 5254 | ConstantSDNode *V = dyn_cast<ConstantSDNode>(N1); |
| 5255 | if (V) { |
| 5256 | Offset += V->getSignExtended(); |
| 5257 | return true; |
| 5258 | } |
| 5259 | } |
| 5260 | } |
| 5261 | return false; |
| 5262 | } |
| 5263 | |
| 5264 | /// isConsecutiveLoad - Returns true if N is loading from an address of Base |
| 5265 | /// + Dist * Size. |
| 5266 | static bool isConsecutiveLoad(SDNode *N, SDNode *Base, int Dist, int Size, |
| 5267 | MachineFrameInfo *MFI) { |
| 5268 | if (N->getOperand(0).Val != Base->getOperand(0).Val) |
| 5269 | return false; |
| 5270 | |
| 5271 | SDOperand Loc = N->getOperand(1); |
| 5272 | SDOperand BaseLoc = Base->getOperand(1); |
| 5273 | if (Loc.getOpcode() == ISD::FrameIndex) { |
| 5274 | if (BaseLoc.getOpcode() != ISD::FrameIndex) |
| 5275 | return false; |
| 5276 | int FI = dyn_cast<FrameIndexSDNode>(Loc)->getIndex(); |
| 5277 | int BFI = dyn_cast<FrameIndexSDNode>(BaseLoc)->getIndex(); |
| 5278 | int FS = MFI->getObjectSize(FI); |
| 5279 | int BFS = MFI->getObjectSize(BFI); |
| 5280 | if (FS != BFS || FS != Size) return false; |
| 5281 | return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Size); |
| 5282 | } else { |
| 5283 | GlobalValue *GV1 = NULL; |
| 5284 | GlobalValue *GV2 = NULL; |
| 5285 | int64_t Offset1 = 0; |
| 5286 | int64_t Offset2 = 0; |
| 5287 | bool isGA1 = isGAPlusOffset(Loc.Val, GV1, Offset1); |
| 5288 | bool isGA2 = isGAPlusOffset(BaseLoc.Val, GV2, Offset2); |
| 5289 | if (isGA1 && isGA2 && GV1 == GV2) |
| 5290 | return Offset1 == (Offset2 + Dist*Size); |
| 5291 | } |
| 5292 | |
| 5293 | return false; |
| 5294 | } |
| 5295 | |
Evan Cheng | 1e60c09 | 2006-07-10 21:37:44 +0000 | [diff] [blame] | 5296 | static bool isBaseAlignment16(SDNode *Base, MachineFrameInfo *MFI, |
| 5297 | const X86Subtarget *Subtarget) { |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5298 | GlobalValue *GV; |
| 5299 | int64_t Offset; |
| 5300 | if (isGAPlusOffset(Base, GV, Offset)) |
| 5301 | return (GV->getAlignment() >= 16 && (Offset % 16) == 0); |
| 5302 | else { |
| 5303 | assert(Base->getOpcode() == ISD::FrameIndex && "Unexpected base node!"); |
| 5304 | int BFI = dyn_cast<FrameIndexSDNode>(Base)->getIndex(); |
Evan Cheng | 1e60c09 | 2006-07-10 21:37:44 +0000 | [diff] [blame] | 5305 | if (BFI < 0) |
| 5306 | // Fixed objects do not specify alignment, however the offsets are known. |
| 5307 | return ((Subtarget->getStackAlignment() % 16) == 0 && |
| 5308 | (MFI->getObjectOffset(BFI) % 16) == 0); |
| 5309 | else |
| 5310 | return MFI->getObjectAlignment(BFI) >= 16; |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5311 | } |
| 5312 | return false; |
| 5313 | } |
| 5314 | |
| 5315 | |
| 5316 | /// PerformShuffleCombine - Combine a vector_shuffle that is equal to |
| 5317 | /// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load |
| 5318 | /// if the load addresses are consecutive, non-overlapping, and in the right |
| 5319 | /// order. |
Evan Cheng | 1e60c09 | 2006-07-10 21:37:44 +0000 | [diff] [blame] | 5320 | static SDOperand PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, |
| 5321 | const X86Subtarget *Subtarget) { |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5322 | MachineFunction &MF = DAG.getMachineFunction(); |
| 5323 | MachineFrameInfo *MFI = MF.getFrameInfo(); |
| 5324 | MVT::ValueType VT = N->getValueType(0); |
| 5325 | MVT::ValueType EVT = MVT::getVectorBaseType(VT); |
| 5326 | SDOperand PermMask = N->getOperand(2); |
| 5327 | int NumElems = (int)PermMask.getNumOperands(); |
| 5328 | SDNode *Base = NULL; |
| 5329 | for (int i = 0; i < NumElems; ++i) { |
| 5330 | SDOperand Idx = PermMask.getOperand(i); |
| 5331 | if (Idx.getOpcode() == ISD::UNDEF) { |
| 5332 | if (!Base) return SDOperand(); |
| 5333 | } else { |
| 5334 | SDOperand Arg = |
| 5335 | getShuffleScalarElt(N, cast<ConstantSDNode>(Idx)->getValue(), DAG); |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 5336 | if (!Arg.Val || !ISD::isNON_EXTLoad(Arg.Val)) |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5337 | return SDOperand(); |
| 5338 | if (!Base) |
| 5339 | Base = Arg.Val; |
| 5340 | else if (!isConsecutiveLoad(Arg.Val, Base, |
| 5341 | i, MVT::getSizeInBits(EVT)/8,MFI)) |
| 5342 | return SDOperand(); |
| 5343 | } |
| 5344 | } |
| 5345 | |
Evan Cheng | 1e60c09 | 2006-07-10 21:37:44 +0000 | [diff] [blame] | 5346 | bool isAlign16 = isBaseAlignment16(Base->getOperand(1).Val, MFI, Subtarget); |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 5347 | if (isAlign16) { |
| 5348 | LoadSDNode *LD = cast<LoadSDNode>(Base); |
| 5349 | return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), |
| 5350 | LD->getSrcValueOffset()); |
| 5351 | } else { |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5352 | // Just use movups, it's shorter. |
Evan Cheng | 64a752f | 2006-08-11 09:08:15 +0000 | [diff] [blame] | 5353 | std::vector<MVT::ValueType> Tys; |
| 5354 | Tys.push_back(MVT::v4f32); |
| 5355 | Tys.push_back(MVT::Other); |
| 5356 | SmallVector<SDOperand, 3> Ops; |
| 5357 | Ops.push_back(Base->getOperand(0)); |
| 5358 | Ops.push_back(Base->getOperand(1)); |
| 5359 | Ops.push_back(Base->getOperand(2)); |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5360 | return DAG.getNode(ISD::BIT_CONVERT, VT, |
Evan Cheng | 64a752f | 2006-08-11 09:08:15 +0000 | [diff] [blame] | 5361 | DAG.getNode(X86ISD::LOAD_UA, Tys, &Ops[0], Ops.size())); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 5362 | } |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5363 | } |
| 5364 | |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 5365 | /// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes. |
| 5366 | static SDOperand PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, |
| 5367 | const X86Subtarget *Subtarget) { |
| 5368 | SDOperand Cond = N->getOperand(0); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5369 | |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 5370 | // If we have SSE[12] support, try to form min/max nodes. |
| 5371 | if (Subtarget->hasSSE2() && |
| 5372 | (N->getValueType(0) == MVT::f32 || N->getValueType(0) == MVT::f64)) { |
| 5373 | if (Cond.getOpcode() == ISD::SETCC) { |
| 5374 | // Get the LHS/RHS of the select. |
| 5375 | SDOperand LHS = N->getOperand(1); |
| 5376 | SDOperand RHS = N->getOperand(2); |
| 5377 | ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5378 | |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 5379 | unsigned Opcode = 0; |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 5380 | if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) { |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 5381 | switch (CC) { |
| 5382 | default: break; |
| 5383 | case ISD::SETOLE: // (X <= Y) ? X : Y -> min |
| 5384 | case ISD::SETULE: |
| 5385 | case ISD::SETLE: |
| 5386 | if (!UnsafeFPMath) break; |
| 5387 | // FALL THROUGH. |
| 5388 | case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min |
| 5389 | case ISD::SETLT: |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 5390 | Opcode = X86ISD::FMIN; |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 5391 | break; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5392 | |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 5393 | case ISD::SETOGT: // (X > Y) ? X : Y -> max |
| 5394 | case ISD::SETUGT: |
| 5395 | case ISD::SETGT: |
| 5396 | if (!UnsafeFPMath) break; |
| 5397 | // FALL THROUGH. |
| 5398 | case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max |
| 5399 | case ISD::SETGE: |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 5400 | Opcode = X86ISD::FMAX; |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 5401 | break; |
| 5402 | } |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 5403 | } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) { |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 5404 | switch (CC) { |
| 5405 | default: break; |
| 5406 | case ISD::SETOGT: // (X > Y) ? Y : X -> min |
| 5407 | case ISD::SETUGT: |
| 5408 | case ISD::SETGT: |
| 5409 | if (!UnsafeFPMath) break; |
| 5410 | // FALL THROUGH. |
| 5411 | case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min |
| 5412 | case ISD::SETGE: |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 5413 | Opcode = X86ISD::FMIN; |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 5414 | break; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5415 | |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 5416 | case ISD::SETOLE: // (X <= Y) ? Y : X -> max |
| 5417 | case ISD::SETULE: |
| 5418 | case ISD::SETLE: |
| 5419 | if (!UnsafeFPMath) break; |
| 5420 | // FALL THROUGH. |
| 5421 | case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max |
| 5422 | case ISD::SETLT: |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 5423 | Opcode = X86ISD::FMAX; |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 5424 | break; |
| 5425 | } |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 5426 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5427 | |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 5428 | if (Opcode) |
| 5429 | return DAG.getNode(Opcode, N->getValueType(0), LHS, RHS); |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 5430 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5431 | |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 5432 | } |
| 5433 | |
| 5434 | return SDOperand(); |
| 5435 | } |
| 5436 | |
| 5437 | |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5438 | SDOperand X86TargetLowering::PerformDAGCombine(SDNode *N, |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5439 | DAGCombinerInfo &DCI) const { |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5440 | SelectionDAG &DAG = DCI.DAG; |
| 5441 | switch (N->getOpcode()) { |
| 5442 | default: break; |
| 5443 | case ISD::VECTOR_SHUFFLE: |
Evan Cheng | 1e60c09 | 2006-07-10 21:37:44 +0000 | [diff] [blame] | 5444 | return PerformShuffleCombine(N, DAG, Subtarget); |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 5445 | case ISD::SELECT: |
| 5446 | return PerformSELECTCombine(N, DAG, Subtarget); |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 5447 | } |
| 5448 | |
| 5449 | return SDOperand(); |
| 5450 | } |
| 5451 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 5452 | //===----------------------------------------------------------------------===// |
| 5453 | // X86 Inline Assembly Support |
| 5454 | //===----------------------------------------------------------------------===// |
| 5455 | |
Chris Lattner | f4dff84 | 2006-07-11 02:54:03 +0000 | [diff] [blame] | 5456 | /// getConstraintType - Given a constraint letter, return the type of |
| 5457 | /// constraint it is for this target. |
| 5458 | X86TargetLowering::ConstraintType |
| 5459 | X86TargetLowering::getConstraintType(char ConstraintLetter) const { |
| 5460 | switch (ConstraintLetter) { |
Chris Lattner | 6d34657 | 2006-07-12 16:59:49 +0000 | [diff] [blame] | 5461 | case 'A': |
| 5462 | case 'r': |
| 5463 | case 'R': |
| 5464 | case 'l': |
| 5465 | case 'q': |
| 5466 | case 'Q': |
| 5467 | case 'x': |
| 5468 | case 'Y': |
| 5469 | return C_RegisterClass; |
Chris Lattner | f4dff84 | 2006-07-11 02:54:03 +0000 | [diff] [blame] | 5470 | default: return TargetLowering::getConstraintType(ConstraintLetter); |
| 5471 | } |
| 5472 | } |
| 5473 | |
Chris Lattner | 22aaf1d | 2006-10-31 20:13:11 +0000 | [diff] [blame] | 5474 | /// isOperandValidForConstraint - Return the specified operand (possibly |
| 5475 | /// modified) if the specified SDOperand is valid for the specified target |
| 5476 | /// constraint letter, otherwise return null. |
| 5477 | SDOperand X86TargetLowering:: |
| 5478 | isOperandValidForConstraint(SDOperand Op, char Constraint, SelectionDAG &DAG) { |
| 5479 | switch (Constraint) { |
| 5480 | default: break; |
| 5481 | case 'i': |
| 5482 | // Literal immediates are always ok. |
| 5483 | if (isa<ConstantSDNode>(Op)) return Op; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5484 | |
Chris Lattner | 22aaf1d | 2006-10-31 20:13:11 +0000 | [diff] [blame] | 5485 | // If we are in non-pic codegen mode, we allow the address of a global to |
| 5486 | // be used with 'i'. |
| 5487 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) { |
| 5488 | if (getTargetMachine().getRelocationModel() == Reloc::PIC_) |
| 5489 | return SDOperand(0, 0); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5490 | |
Chris Lattner | 22aaf1d | 2006-10-31 20:13:11 +0000 | [diff] [blame] | 5491 | if (GA->getOpcode() != ISD::TargetGlobalAddress) |
| 5492 | Op = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0), |
| 5493 | GA->getOffset()); |
| 5494 | return Op; |
| 5495 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5496 | |
Chris Lattner | 22aaf1d | 2006-10-31 20:13:11 +0000 | [diff] [blame] | 5497 | // Otherwise, not valid for this mode. |
| 5498 | return SDOperand(0, 0); |
| 5499 | } |
| 5500 | return TargetLowering::isOperandValidForConstraint(Op, Constraint, DAG); |
| 5501 | } |
| 5502 | |
| 5503 | |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 5504 | std::vector<unsigned> X86TargetLowering:: |
Chris Lattner | 1efa40f | 2006-02-22 00:56:39 +0000 | [diff] [blame] | 5505 | getRegClassForInlineAsmConstraint(const std::string &Constraint, |
| 5506 | MVT::ValueType VT) const { |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 5507 | if (Constraint.size() == 1) { |
| 5508 | // FIXME: not handling fp-stack yet! |
| 5509 | // FIXME: not handling MMX registers yet ('y' constraint). |
| 5510 | switch (Constraint[0]) { // GCC X86 Constraint Letters |
Chris Lattner | f4dff84 | 2006-07-11 02:54:03 +0000 | [diff] [blame] | 5511 | default: break; // Unknown constraint letter |
| 5512 | case 'A': // EAX/EDX |
| 5513 | if (VT == MVT::i32 || VT == MVT::i64) |
| 5514 | return make_vector<unsigned>(X86::EAX, X86::EDX, 0); |
| 5515 | break; |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 5516 | case 'r': // GENERAL_REGS |
| 5517 | case 'R': // LEGACY_REGS |
Chris Lattner | 98ae09c | 2006-12-04 22:38:21 +0000 | [diff] [blame] | 5518 | if (VT == MVT::i64 && Subtarget->is64Bit()) |
| 5519 | return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, |
| 5520 | X86::RSI, X86::RDI, X86::RBP, X86::RSP, |
| 5521 | X86::R8, X86::R9, X86::R10, X86::R11, |
| 5522 | X86::R12, X86::R13, X86::R14, X86::R15, 0); |
Chris Lattner | 80a7ecc | 2006-05-06 00:29:37 +0000 | [diff] [blame] | 5523 | if (VT == MVT::i32) |
| 5524 | return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, |
| 5525 | X86::ESI, X86::EDI, X86::EBP, X86::ESP, 0); |
| 5526 | else if (VT == MVT::i16) |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5527 | return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, |
Chris Lattner | 80a7ecc | 2006-05-06 00:29:37 +0000 | [diff] [blame] | 5528 | X86::SI, X86::DI, X86::BP, X86::SP, 0); |
| 5529 | else if (VT == MVT::i8) |
Chris Lattner | af21f4f | 2006-12-05 17:29:40 +0000 | [diff] [blame] | 5530 | return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0); |
Chris Lattner | 80a7ecc | 2006-05-06 00:29:37 +0000 | [diff] [blame] | 5531 | break; |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 5532 | case 'l': // INDEX_REGS |
Chris Lattner | 80a7ecc | 2006-05-06 00:29:37 +0000 | [diff] [blame] | 5533 | if (VT == MVT::i32) |
| 5534 | return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, |
| 5535 | X86::ESI, X86::EDI, X86::EBP, 0); |
| 5536 | else if (VT == MVT::i16) |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5537 | return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, |
Chris Lattner | 80a7ecc | 2006-05-06 00:29:37 +0000 | [diff] [blame] | 5538 | X86::SI, X86::DI, X86::BP, 0); |
| 5539 | else if (VT == MVT::i8) |
| 5540 | return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::DL, 0); |
| 5541 | break; |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 5542 | case 'q': // Q_REGS (GENERAL_REGS in 64-bit mode) |
| 5543 | case 'Q': // Q_REGS |
Chris Lattner | 80a7ecc | 2006-05-06 00:29:37 +0000 | [diff] [blame] | 5544 | if (VT == MVT::i32) |
| 5545 | return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0); |
| 5546 | else if (VT == MVT::i16) |
| 5547 | return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0); |
| 5548 | else if (VT == MVT::i8) |
| 5549 | return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::DL, 0); |
| 5550 | break; |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 5551 | case 'x': // SSE_REGS if SSE1 allowed |
| 5552 | if (Subtarget->hasSSE1()) |
| 5553 | return make_vector<unsigned>(X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, |
| 5554 | X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7, |
| 5555 | 0); |
| 5556 | return std::vector<unsigned>(); |
| 5557 | case 'Y': // SSE_REGS if SSE2 allowed |
| 5558 | if (Subtarget->hasSSE2()) |
| 5559 | return make_vector<unsigned>(X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, |
| 5560 | X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7, |
| 5561 | 0); |
| 5562 | return std::vector<unsigned>(); |
| 5563 | } |
| 5564 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5565 | |
Chris Lattner | 1efa40f | 2006-02-22 00:56:39 +0000 | [diff] [blame] | 5566 | return std::vector<unsigned>(); |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 5567 | } |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 5568 | |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5569 | std::pair<unsigned, const TargetRegisterClass*> |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 5570 | X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, |
| 5571 | MVT::ValueType VT) const { |
| 5572 | // Use the default implementation in TargetLowering to convert the register |
| 5573 | // constraint into a member of a register class. |
| 5574 | std::pair<unsigned, const TargetRegisterClass*> Res; |
| 5575 | Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); |
Chris Lattner | 1a60aa7 | 2006-10-31 19:42:44 +0000 | [diff] [blame] | 5576 | |
| 5577 | // Not found as a standard register? |
| 5578 | if (Res.second == 0) { |
| 5579 | // GCC calls "st(0)" just plain "st". |
| 5580 | if (StringsEqualNoCase("{st}", Constraint)) { |
| 5581 | Res.first = X86::ST0; |
| 5582 | Res.second = X86::RSTRegisterClass; |
| 5583 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5584 | |
Chris Lattner | 1a60aa7 | 2006-10-31 19:42:44 +0000 | [diff] [blame] | 5585 | return Res; |
| 5586 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5587 | |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 5588 | // Otherwise, check to see if this is a register class of the wrong value |
| 5589 | // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to |
| 5590 | // turn into {ax},{dx}. |
| 5591 | if (Res.second->hasType(VT)) |
| 5592 | return Res; // Correct type already, nothing to do. |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5593 | |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 5594 | // All of the single-register GCC register classes map their values onto |
| 5595 | // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we |
| 5596 | // really want an 8-bit or 32-bit register, map to the appropriate register |
| 5597 | // class and return the appropriate register. |
| 5598 | if (Res.second != X86::GR16RegisterClass) |
| 5599 | return Res; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5600 | |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 5601 | if (VT == MVT::i8) { |
| 5602 | unsigned DestReg = 0; |
| 5603 | switch (Res.first) { |
| 5604 | default: break; |
| 5605 | case X86::AX: DestReg = X86::AL; break; |
| 5606 | case X86::DX: DestReg = X86::DL; break; |
| 5607 | case X86::CX: DestReg = X86::CL; break; |
| 5608 | case X86::BX: DestReg = X86::BL; break; |
| 5609 | } |
| 5610 | if (DestReg) { |
| 5611 | Res.first = DestReg; |
| 5612 | Res.second = Res.second = X86::GR8RegisterClass; |
| 5613 | } |
| 5614 | } else if (VT == MVT::i32) { |
| 5615 | unsigned DestReg = 0; |
| 5616 | switch (Res.first) { |
| 5617 | default: break; |
| 5618 | case X86::AX: DestReg = X86::EAX; break; |
| 5619 | case X86::DX: DestReg = X86::EDX; break; |
| 5620 | case X86::CX: DestReg = X86::ECX; break; |
| 5621 | case X86::BX: DestReg = X86::EBX; break; |
| 5622 | case X86::SI: DestReg = X86::ESI; break; |
| 5623 | case X86::DI: DestReg = X86::EDI; break; |
| 5624 | case X86::BP: DestReg = X86::EBP; break; |
| 5625 | case X86::SP: DestReg = X86::ESP; break; |
| 5626 | } |
| 5627 | if (DestReg) { |
| 5628 | Res.first = DestReg; |
| 5629 | Res.second = Res.second = X86::GR32RegisterClass; |
| 5630 | } |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5631 | } else if (VT == MVT::i64) { |
| 5632 | unsigned DestReg = 0; |
| 5633 | switch (Res.first) { |
| 5634 | default: break; |
| 5635 | case X86::AX: DestReg = X86::RAX; break; |
| 5636 | case X86::DX: DestReg = X86::RDX; break; |
| 5637 | case X86::CX: DestReg = X86::RCX; break; |
| 5638 | case X86::BX: DestReg = X86::RBX; break; |
| 5639 | case X86::SI: DestReg = X86::RSI; break; |
| 5640 | case X86::DI: DestReg = X86::RDI; break; |
| 5641 | case X86::BP: DestReg = X86::RBP; break; |
| 5642 | case X86::SP: DestReg = X86::RSP; break; |
| 5643 | } |
| 5644 | if (DestReg) { |
| 5645 | Res.first = DestReg; |
| 5646 | Res.second = Res.second = X86::GR64RegisterClass; |
| 5647 | } |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 5648 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5649 | |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 5650 | return Res; |
| 5651 | } |