Arnold Schwaighofer | 92226dd | 2007-10-12 21:53:12 +0000 | [diff] [blame] | 1 | //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===// |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
Chris Lattner | 4ee451d | 2007-12-29 20:36:04 +0000 | [diff] [blame] | 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file defines the interfaces that X86 uses to lower LLVM code into a |
| 11 | // selection DAG. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "X86.h" |
Evan Cheng | 0cc3945 | 2006-01-16 21:21:29 +0000 | [diff] [blame] | 16 | #include "X86InstrBuilder.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 17 | #include "X86ISelLowering.h" |
Evan Cheng | e8bd0a3 | 2006-06-06 23:30:24 +0000 | [diff] [blame] | 18 | #include "X86MachineFunctionInfo.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 19 | #include "X86TargetMachine.h" |
| 20 | #include "llvm/CallingConv.h" |
Evan Cheng | 223547a | 2006-01-31 22:28:30 +0000 | [diff] [blame] | 21 | #include "llvm/Constants.h" |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 22 | #include "llvm/DerivedTypes.h" |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 23 | #include "llvm/GlobalVariable.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 24 | #include "llvm/Function.h" |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 25 | #include "llvm/Intrinsics.h" |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 26 | #include "llvm/ADT/BitVector.h" |
Evan Cheng | 30b37b5 | 2006-03-13 23:18:16 +0000 | [diff] [blame] | 27 | #include "llvm/ADT/VectorExtras.h" |
Chris Lattner | 362e98a | 2007-02-27 04:43:02 +0000 | [diff] [blame] | 28 | #include "llvm/CodeGen/CallingConvLower.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 29 | #include "llvm/CodeGen/MachineFrameInfo.h" |
Evan Cheng | 4a46080 | 2006-01-11 00:33:36 +0000 | [diff] [blame] | 30 | #include "llvm/CodeGen/MachineFunction.h" |
| 31 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
Evan Cheng | a844bde | 2008-02-02 04:07:54 +0000 | [diff] [blame] | 32 | #include "llvm/CodeGen/MachineModuleInfo.h" |
Chris Lattner | 84bc542 | 2007-12-31 04:13:23 +0000 | [diff] [blame] | 33 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 34 | #include "llvm/CodeGen/PseudoSourceValue.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 35 | #include "llvm/CodeGen/SelectionDAG.h" |
Evan Cheng | ef6ffb1 | 2006-01-31 03:14:29 +0000 | [diff] [blame] | 36 | #include "llvm/Support/MathExtras.h" |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 37 | #include "llvm/Support/Debug.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 38 | #include "llvm/Target/TargetOptions.h" |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 39 | #include "llvm/ADT/SmallSet.h" |
Chris Lattner | 1a60aa7 | 2006-10-31 19:42:44 +0000 | [diff] [blame] | 40 | #include "llvm/ADT/StringExtras.h" |
Mon P Wang | 3c81d35 | 2008-11-23 04:37:22 +0000 | [diff] [blame] | 41 | #include "llvm/Support/CommandLine.h" |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 42 | using namespace llvm; |
| 43 | |
Mon P Wang | 3c81d35 | 2008-11-23 04:37:22 +0000 | [diff] [blame] | 44 | static cl::opt<bool> |
Mon P Wang | 9f22a4a | 2008-11-24 02:10:43 +0000 | [diff] [blame] | 45 | DisableMMX("disable-mmx", cl::Hidden, cl::desc("Disable use of MMX")); |
Mon P Wang | 3c81d35 | 2008-11-23 04:37:22 +0000 | [diff] [blame] | 46 | |
Evan Cheng | 10e8642 | 2008-04-25 19:11:04 +0000 | [diff] [blame] | 47 | // Forward declarations. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 48 | static SDValue getMOVLMask(unsigned NumElems, SelectionDAG &DAG); |
Evan Cheng | 10e8642 | 2008-04-25 19:11:04 +0000 | [diff] [blame] | 49 | |
Dan Gohman | c9f5f3f | 2008-05-14 01:58:56 +0000 | [diff] [blame] | 50 | X86TargetLowering::X86TargetLowering(X86TargetMachine &TM) |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 51 | : TargetLowering(TM) { |
Evan Cheng | 559806f | 2006-01-27 08:10:46 +0000 | [diff] [blame] | 52 | Subtarget = &TM.getSubtarget<X86Subtarget>(); |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 53 | X86ScalarSSEf64 = Subtarget->hasSSE2(); |
| 54 | X86ScalarSSEf32 = Subtarget->hasSSE1(); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 55 | X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP; |
Anton Korobeynikov | bff66b0 | 2008-09-09 18:22:57 +0000 | [diff] [blame] | 56 | |
Chris Lattner | d43d00c | 2008-01-24 08:07:48 +0000 | [diff] [blame] | 57 | bool Fast = false; |
Evan Cheng | 559806f | 2006-01-27 08:10:46 +0000 | [diff] [blame] | 58 | |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 59 | RegInfo = TM.getRegisterInfo(); |
Anton Korobeynikov | bff66b0 | 2008-09-09 18:22:57 +0000 | [diff] [blame] | 60 | TD = getTargetData(); |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 61 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 62 | // Set up the TargetLowering object. |
| 63 | |
| 64 | // X86 is weird, it always uses i8 for shift amounts and setcc results. |
| 65 | setShiftAmountType(MVT::i8); |
Duncan Sands | 0322808 | 2008-11-23 15:47:28 +0000 | [diff] [blame] | 66 | setBooleanContents(ZeroOrOneBooleanContent); |
Evan Cheng | 0b2afbd | 2006-01-25 09:15:17 +0000 | [diff] [blame] | 67 | setSchedulingPreference(SchedulingForRegPressure); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 68 | setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0 |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 69 | setStackPointerRegisterToSaveRestore(X86StackPtr); |
Evan Cheng | 714554d | 2006-03-16 21:47:42 +0000 | [diff] [blame] | 70 | |
Anton Korobeynikov | d27a258 | 2006-12-10 23:12:42 +0000 | [diff] [blame] | 71 | if (Subtarget->isTargetDarwin()) { |
Evan Cheng | df57fa0 | 2006-03-17 20:31:41 +0000 | [diff] [blame] | 72 | // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp. |
Anton Korobeynikov | d27a258 | 2006-12-10 23:12:42 +0000 | [diff] [blame] | 73 | setUseUnderscoreSetJmp(false); |
| 74 | setUseUnderscoreLongJmp(false); |
Anton Korobeynikov | 317848f | 2007-01-03 11:43:14 +0000 | [diff] [blame] | 75 | } else if (Subtarget->isTargetMingw()) { |
Anton Korobeynikov | d27a258 | 2006-12-10 23:12:42 +0000 | [diff] [blame] | 76 | // MS runtime is weird: it exports _setjmp, but longjmp! |
| 77 | setUseUnderscoreSetJmp(true); |
| 78 | setUseUnderscoreLongJmp(false); |
| 79 | } else { |
| 80 | setUseUnderscoreSetJmp(true); |
| 81 | setUseUnderscoreLongJmp(true); |
| 82 | } |
| 83 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 84 | // Set up the register classes. |
Evan Cheng | 069287d | 2006-05-16 07:21:53 +0000 | [diff] [blame] | 85 | addRegisterClass(MVT::i8, X86::GR8RegisterClass); |
| 86 | addRegisterClass(MVT::i16, X86::GR16RegisterClass); |
| 87 | addRegisterClass(MVT::i32, X86::GR32RegisterClass); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 88 | if (Subtarget->is64Bit()) |
| 89 | addRegisterClass(MVT::i64, X86::GR64RegisterClass); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 90 | |
Evan Cheng | 0329466 | 2008-10-14 21:26:46 +0000 | [diff] [blame] | 91 | setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); |
Evan Cheng | c548428 | 2006-10-04 00:56:09 +0000 | [diff] [blame] | 92 | |
Chris Lattner | ddf8956 | 2008-01-17 19:59:44 +0000 | [diff] [blame] | 93 | // We don't accept any truncstore of integer registers. |
| 94 | setTruncStoreAction(MVT::i64, MVT::i32, Expand); |
| 95 | setTruncStoreAction(MVT::i64, MVT::i16, Expand); |
| 96 | setTruncStoreAction(MVT::i64, MVT::i8 , Expand); |
| 97 | setTruncStoreAction(MVT::i32, MVT::i16, Expand); |
| 98 | setTruncStoreAction(MVT::i32, MVT::i8 , Expand); |
Evan Cheng | 7f04268 | 2008-10-15 02:05:31 +0000 | [diff] [blame] | 99 | setTruncStoreAction(MVT::i16, MVT::i8, Expand); |
| 100 | |
| 101 | // SETOEQ and SETUNE require checking two conditions. |
| 102 | setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand); |
| 103 | setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand); |
| 104 | setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand); |
| 105 | setCondCodeAction(ISD::SETUNE, MVT::f32, Expand); |
| 106 | setCondCodeAction(ISD::SETUNE, MVT::f64, Expand); |
| 107 | setCondCodeAction(ISD::SETUNE, MVT::f80, Expand); |
Chris Lattner | ddf8956 | 2008-01-17 19:59:44 +0000 | [diff] [blame] | 108 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 109 | // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this |
| 110 | // operation. |
| 111 | setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); |
| 112 | setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); |
| 113 | setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); |
Evan Cheng | 6892f28 | 2006-01-17 02:32:49 +0000 | [diff] [blame] | 114 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 115 | if (Subtarget->is64Bit()) { |
| 116 | setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Expand); |
Evan Cheng | 6892f28 | 2006-01-17 02:32:49 +0000 | [diff] [blame] | 117 | setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 118 | } else { |
Dale Johannesen | 1c15bf5 | 2008-10-21 20:50:01 +0000 | [diff] [blame] | 119 | if (X86ScalarSSEf64) { |
| 120 | // We have an impenetrably clever algorithm for ui64->double only. |
| 121 | setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 122 | // If SSE i64 SINT_TO_FP is not available, expand i32 UINT_TO_FP. |
| 123 | setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand); |
Dale Johannesen | 1c15bf5 | 2008-10-21 20:50:01 +0000 | [diff] [blame] | 124 | } else |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 125 | setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); |
| 126 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 127 | |
| 128 | // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have |
| 129 | // this operation. |
| 130 | setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); |
| 131 | setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 132 | // SSE has no i16 to fp conversion, only i32 |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 133 | if (X86ScalarSSEf32) { |
Evan Cheng | 02568ff | 2006-01-30 22:13:22 +0000 | [diff] [blame] | 134 | setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); |
Dale Johannesen | 9e3d3ab | 2007-09-14 22:26:36 +0000 | [diff] [blame] | 135 | // f32 and f64 cases are Legal, f80 case is not |
| 136 | setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); |
| 137 | } else { |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 138 | setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); |
| 139 | setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); |
| 140 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 141 | |
Dale Johannesen | 73328d1 | 2007-09-19 23:55:34 +0000 | [diff] [blame] | 142 | // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 |
| 143 | // are Legal, f80 is custom lowered. |
| 144 | setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); |
| 145 | setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); |
Evan Cheng | 6dab053 | 2006-01-30 08:02:57 +0000 | [diff] [blame] | 146 | |
Evan Cheng | 02568ff | 2006-01-30 22:13:22 +0000 | [diff] [blame] | 147 | // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have |
| 148 | // this operation. |
| 149 | setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); |
| 150 | setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); |
| 151 | |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 152 | if (X86ScalarSSEf32) { |
Evan Cheng | 02568ff | 2006-01-30 22:13:22 +0000 | [diff] [blame] | 153 | setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); |
Dale Johannesen | 9e3d3ab | 2007-09-14 22:26:36 +0000 | [diff] [blame] | 154 | // f32 and f64 cases are Legal, f80 case is not |
| 155 | setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); |
Evan Cheng | 02568ff | 2006-01-30 22:13:22 +0000 | [diff] [blame] | 156 | } else { |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 157 | setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); |
Evan Cheng | 02568ff | 2006-01-30 22:13:22 +0000 | [diff] [blame] | 158 | setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 159 | } |
| 160 | |
| 161 | // Handle FP_TO_UINT by promoting the destination to a larger signed |
| 162 | // conversion. |
| 163 | setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); |
| 164 | setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); |
| 165 | setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); |
| 166 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 167 | if (Subtarget->is64Bit()) { |
| 168 | setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 169 | setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 170 | } else { |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 171 | if (X86ScalarSSEf32 && !Subtarget->hasSSE3()) |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 172 | // Expand FP_TO_UINT into a select. |
| 173 | // FIXME: We would like to use a Custom expander here eventually to do |
| 174 | // the optimal thing for SSE vs. the default expansion in the legalizer. |
| 175 | setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); |
| 176 | else |
| 177 | // With SSE3 we can use fisttpll to convert to a signed i64. |
| 178 | setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); |
| 179 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 180 | |
Chris Lattner | 399610a | 2006-12-05 18:22:22 +0000 | [diff] [blame] | 181 | // TODO: when we have SSE, these could be more efficient, by using movd/movq. |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 182 | if (!X86ScalarSSEf64) { |
Chris Lattner | f3597a1 | 2006-12-05 18:45:06 +0000 | [diff] [blame] | 183 | setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand); |
| 184 | setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand); |
| 185 | } |
Chris Lattner | 21f6685 | 2005-12-23 05:15:23 +0000 | [diff] [blame] | 186 | |
Dan Gohman | b00ee21 | 2008-02-18 19:34:53 +0000 | [diff] [blame] | 187 | // Scalar integer divide and remainder are lowered to use operations that |
| 188 | // produce two results, to match the available instructions. This exposes |
| 189 | // the two-result form to trivial CSE, which is able to combine x/y and x%y |
| 190 | // into a single instruction. |
| 191 | // |
| 192 | // Scalar integer multiply-high is also lowered to use two-result |
| 193 | // operations, to match the available instructions. However, plain multiply |
| 194 | // (low) operations are left as Legal, as there are single-result |
| 195 | // instructions for this in x86. Using the two-result multiply instructions |
| 196 | // when both high and low results are needed must be arranged by dagcombine. |
Dan Gohman | 525178c | 2007-10-08 18:33:35 +0000 | [diff] [blame] | 197 | setOperationAction(ISD::MULHS , MVT::i8 , Expand); |
| 198 | setOperationAction(ISD::MULHU , MVT::i8 , Expand); |
| 199 | setOperationAction(ISD::SDIV , MVT::i8 , Expand); |
| 200 | setOperationAction(ISD::UDIV , MVT::i8 , Expand); |
| 201 | setOperationAction(ISD::SREM , MVT::i8 , Expand); |
| 202 | setOperationAction(ISD::UREM , MVT::i8 , Expand); |
Dan Gohman | 525178c | 2007-10-08 18:33:35 +0000 | [diff] [blame] | 203 | setOperationAction(ISD::MULHS , MVT::i16 , Expand); |
| 204 | setOperationAction(ISD::MULHU , MVT::i16 , Expand); |
| 205 | setOperationAction(ISD::SDIV , MVT::i16 , Expand); |
| 206 | setOperationAction(ISD::UDIV , MVT::i16 , Expand); |
| 207 | setOperationAction(ISD::SREM , MVT::i16 , Expand); |
| 208 | setOperationAction(ISD::UREM , MVT::i16 , Expand); |
Dan Gohman | 525178c | 2007-10-08 18:33:35 +0000 | [diff] [blame] | 209 | setOperationAction(ISD::MULHS , MVT::i32 , Expand); |
| 210 | setOperationAction(ISD::MULHU , MVT::i32 , Expand); |
| 211 | setOperationAction(ISD::SDIV , MVT::i32 , Expand); |
| 212 | setOperationAction(ISD::UDIV , MVT::i32 , Expand); |
| 213 | setOperationAction(ISD::SREM , MVT::i32 , Expand); |
| 214 | setOperationAction(ISD::UREM , MVT::i32 , Expand); |
Dan Gohman | 525178c | 2007-10-08 18:33:35 +0000 | [diff] [blame] | 215 | setOperationAction(ISD::MULHS , MVT::i64 , Expand); |
| 216 | setOperationAction(ISD::MULHU , MVT::i64 , Expand); |
| 217 | setOperationAction(ISD::SDIV , MVT::i64 , Expand); |
| 218 | setOperationAction(ISD::UDIV , MVT::i64 , Expand); |
| 219 | setOperationAction(ISD::SREM , MVT::i64 , Expand); |
| 220 | setOperationAction(ISD::UREM , MVT::i64 , Expand); |
Dan Gohman | a37c9f7 | 2007-09-25 18:23:27 +0000 | [diff] [blame] | 221 | |
Evan Cheng | c35497f | 2006-10-30 08:02:39 +0000 | [diff] [blame] | 222 | setOperationAction(ISD::BR_JT , MVT::Other, Expand); |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 223 | setOperationAction(ISD::BRCOND , MVT::Other, Custom); |
Nate Begeman | 750ac1b | 2006-02-01 07:19:44 +0000 | [diff] [blame] | 224 | setOperationAction(ISD::BR_CC , MVT::Other, Expand); |
| 225 | setOperationAction(ISD::SELECT_CC , MVT::Other, Expand); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 226 | if (Subtarget->is64Bit()) |
Christopher Lamb | c59e521 | 2007-08-10 21:48:46 +0000 | [diff] [blame] | 227 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); |
| 228 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); |
| 229 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 230 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); |
| 231 | setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); |
Chris Lattner | d110822 | 2008-03-07 06:36:32 +0000 | [diff] [blame] | 232 | setOperationAction(ISD::FREM , MVT::f32 , Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 233 | setOperationAction(ISD::FREM , MVT::f64 , Expand); |
Chris Lattner | d110822 | 2008-03-07 06:36:32 +0000 | [diff] [blame] | 234 | setOperationAction(ISD::FREM , MVT::f80 , Expand); |
Dan Gohman | 1a02486 | 2008-01-31 00:41:03 +0000 | [diff] [blame] | 235 | setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); |
Anton Korobeynikov | 45b22fa | 2007-11-16 01:31:51 +0000 | [diff] [blame] | 236 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 237 | setOperationAction(ISD::CTPOP , MVT::i8 , Expand); |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 238 | setOperationAction(ISD::CTTZ , MVT::i8 , Custom); |
| 239 | setOperationAction(ISD::CTLZ , MVT::i8 , Custom); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 240 | setOperationAction(ISD::CTPOP , MVT::i16 , Expand); |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 241 | setOperationAction(ISD::CTTZ , MVT::i16 , Custom); |
| 242 | setOperationAction(ISD::CTLZ , MVT::i16 , Custom); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 243 | setOperationAction(ISD::CTPOP , MVT::i32 , Expand); |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 244 | setOperationAction(ISD::CTTZ , MVT::i32 , Custom); |
| 245 | setOperationAction(ISD::CTLZ , MVT::i32 , Custom); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 246 | if (Subtarget->is64Bit()) { |
| 247 | setOperationAction(ISD::CTPOP , MVT::i64 , Expand); |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 248 | setOperationAction(ISD::CTTZ , MVT::i64 , Custom); |
| 249 | setOperationAction(ISD::CTLZ , MVT::i64 , Custom); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 250 | } |
| 251 | |
Andrew Lenharth | b873ff3 | 2005-11-20 21:41:10 +0000 | [diff] [blame] | 252 | setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); |
Nate Begeman | d88fc03 | 2006-01-14 03:14:10 +0000 | [diff] [blame] | 253 | setOperationAction(ISD::BSWAP , MVT::i16 , Expand); |
Nate Begeman | 35ef913 | 2006-01-11 21:21:00 +0000 | [diff] [blame] | 254 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 255 | // These should be promoted to a larger select which is supported. |
| 256 | setOperationAction(ISD::SELECT , MVT::i1 , Promote); |
| 257 | setOperationAction(ISD::SELECT , MVT::i8 , Promote); |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 258 | // X86 wants to expand cmov itself. |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 259 | setOperationAction(ISD::SELECT , MVT::i16 , Custom); |
| 260 | setOperationAction(ISD::SELECT , MVT::i32 , Custom); |
| 261 | setOperationAction(ISD::SELECT , MVT::f32 , Custom); |
| 262 | setOperationAction(ISD::SELECT , MVT::f64 , Custom); |
Dale Johannesen | 9e3d3ab | 2007-09-14 22:26:36 +0000 | [diff] [blame] | 263 | setOperationAction(ISD::SELECT , MVT::f80 , Custom); |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 264 | setOperationAction(ISD::SETCC , MVT::i8 , Custom); |
| 265 | setOperationAction(ISD::SETCC , MVT::i16 , Custom); |
| 266 | setOperationAction(ISD::SETCC , MVT::i32 , Custom); |
| 267 | setOperationAction(ISD::SETCC , MVT::f32 , Custom); |
| 268 | setOperationAction(ISD::SETCC , MVT::f64 , Custom); |
Dale Johannesen | 9e3d3ab | 2007-09-14 22:26:36 +0000 | [diff] [blame] | 269 | setOperationAction(ISD::SETCC , MVT::f80 , Custom); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 270 | if (Subtarget->is64Bit()) { |
| 271 | setOperationAction(ISD::SELECT , MVT::i64 , Custom); |
| 272 | setOperationAction(ISD::SETCC , MVT::i64 , Custom); |
| 273 | } |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 274 | // X86 ret instruction may pop stack. |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 275 | setOperationAction(ISD::RET , MVT::Other, Custom); |
Anton Korobeynikov | 260a6b8 | 2008-09-08 21:12:11 +0000 | [diff] [blame] | 276 | setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 277 | |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 278 | // Darwin ABI issue. |
Evan Cheng | 7ccced6 | 2006-02-18 00:15:05 +0000 | [diff] [blame] | 279 | setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); |
Nate Begeman | 37efe67 | 2006-04-22 18:53:45 +0000 | [diff] [blame] | 280 | setOperationAction(ISD::JumpTable , MVT::i32 , Custom); |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 281 | setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 282 | setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 283 | if (Subtarget->is64Bit()) |
| 284 | setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); |
Bill Wendling | 056292f | 2008-09-16 21:48:12 +0000 | [diff] [blame] | 285 | setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 286 | if (Subtarget->is64Bit()) { |
| 287 | setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); |
| 288 | setOperationAction(ISD::JumpTable , MVT::i64 , Custom); |
| 289 | setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); |
Bill Wendling | 056292f | 2008-09-16 21:48:12 +0000 | [diff] [blame] | 290 | setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 291 | } |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 292 | // 64-bit addm sub, shl, sra, srl (iff 32-bit x86) |
Evan Cheng | 5298bcc | 2006-02-17 07:01:52 +0000 | [diff] [blame] | 293 | setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); |
| 294 | setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); |
| 295 | setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); |
Dan Gohman | 4c1fa61 | 2008-03-03 22:22:09 +0000 | [diff] [blame] | 296 | if (Subtarget->is64Bit()) { |
| 297 | setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); |
| 298 | setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); |
| 299 | setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); |
| 300 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 301 | |
Evan Cheng | d2cde68 | 2008-03-10 19:38:10 +0000 | [diff] [blame] | 302 | if (Subtarget->hasSSE1()) |
| 303 | setOperationAction(ISD::PREFETCH , MVT::Other, Legal); |
Evan Cheng | 27b7db5 | 2008-03-08 00:58:38 +0000 | [diff] [blame] | 304 | |
Andrew Lenharth | d497d9f | 2008-02-16 14:46:26 +0000 | [diff] [blame] | 305 | if (!Subtarget->hasSSE2()) |
| 306 | setOperationAction(ISD::MEMBARRIER , MVT::Other, Expand); |
| 307 | |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 308 | // Expand certain atomics |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 309 | setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, Custom); |
| 310 | setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, Custom); |
| 311 | setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); |
| 312 | setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom); |
Bill Wendling | 5bf1b4e | 2008-08-20 00:28:16 +0000 | [diff] [blame] | 313 | |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 314 | setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i8, Custom); |
| 315 | setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i16, Custom); |
| 316 | setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); |
| 317 | setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom); |
Andrew Lenharth | d497d9f | 2008-02-16 14:46:26 +0000 | [diff] [blame] | 318 | |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 319 | if (!Subtarget->is64Bit()) { |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 320 | setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, Custom); |
| 321 | setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom); |
| 322 | setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom); |
| 323 | setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, Custom); |
| 324 | setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, Custom); |
| 325 | setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i64, Custom); |
| 326 | setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, Custom); |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 327 | } |
| 328 | |
Dan Gohman | 7f46020 | 2008-06-30 20:59:49 +0000 | [diff] [blame] | 329 | // Use the default ISD::DBG_STOPPOINT, ISD::DECLARE expansion. |
| 330 | setOperationAction(ISD::DBG_STOPPOINT, MVT::Other, Expand); |
Evan Cheng | 3c992d2 | 2006-03-07 02:02:57 +0000 | [diff] [blame] | 331 | // FIXME - use subtarget debug flags |
Anton Korobeynikov | ab4022f | 2006-10-31 08:31:24 +0000 | [diff] [blame] | 332 | if (!Subtarget->isTargetDarwin() && |
| 333 | !Subtarget->isTargetELF() && |
Dan Gohman | 4406604 | 2008-07-01 00:05:16 +0000 | [diff] [blame] | 334 | !Subtarget->isTargetCygMing()) { |
| 335 | setOperationAction(ISD::DBG_LABEL, MVT::Other, Expand); |
| 336 | setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); |
| 337 | } |
Chris Lattner | f73bae1 | 2005-11-29 06:16:21 +0000 | [diff] [blame] | 338 | |
Anton Korobeynikov | ce3b465 | 2007-05-02 19:53:33 +0000 | [diff] [blame] | 339 | setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand); |
| 340 | setOperationAction(ISD::EHSELECTION, MVT::i64, Expand); |
| 341 | setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand); |
| 342 | setOperationAction(ISD::EHSELECTION, MVT::i32, Expand); |
| 343 | if (Subtarget->is64Bit()) { |
Anton Korobeynikov | ce3b465 | 2007-05-02 19:53:33 +0000 | [diff] [blame] | 344 | setExceptionPointerRegister(X86::RAX); |
| 345 | setExceptionSelectorRegister(X86::RDX); |
| 346 | } else { |
| 347 | setExceptionPointerRegister(X86::EAX); |
| 348 | setExceptionSelectorRegister(X86::EDX); |
| 349 | } |
Anton Korobeynikov | 3825262 | 2007-09-03 00:36:06 +0000 | [diff] [blame] | 350 | setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); |
Anton Korobeynikov | 260a6b8 | 2008-09-08 21:12:11 +0000 | [diff] [blame] | 351 | setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); |
| 352 | |
Duncan Sands | f7331b3 | 2007-09-11 14:10:23 +0000 | [diff] [blame] | 353 | setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom); |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 354 | |
Chris Lattner | da68d30 | 2008-01-15 21:58:22 +0000 | [diff] [blame] | 355 | setOperationAction(ISD::TRAP, MVT::Other, Legal); |
Anton Korobeynikov | 66fac79 | 2008-01-15 07:02:33 +0000 | [diff] [blame] | 356 | |
Nate Begeman | acc398c | 2006-01-25 18:21:52 +0000 | [diff] [blame] | 357 | // VASTART needs to be custom lowered to use the VarArgsFrameIndex |
| 358 | setOperationAction(ISD::VASTART , MVT::Other, Custom); |
Nate Begeman | acc398c | 2006-01-25 18:21:52 +0000 | [diff] [blame] | 359 | setOperationAction(ISD::VAEND , MVT::Other, Expand); |
Dan Gohman | 9018e83 | 2008-05-10 01:26:14 +0000 | [diff] [blame] | 360 | if (Subtarget->is64Bit()) { |
| 361 | setOperationAction(ISD::VAARG , MVT::Other, Custom); |
Evan Cheng | ae64219 | 2007-03-02 23:16:35 +0000 | [diff] [blame] | 362 | setOperationAction(ISD::VACOPY , MVT::Other, Custom); |
Dan Gohman | 9018e83 | 2008-05-10 01:26:14 +0000 | [diff] [blame] | 363 | } else { |
| 364 | setOperationAction(ISD::VAARG , MVT::Other, Expand); |
Evan Cheng | ae64219 | 2007-03-02 23:16:35 +0000 | [diff] [blame] | 365 | setOperationAction(ISD::VACOPY , MVT::Other, Expand); |
Dan Gohman | 9018e83 | 2008-05-10 01:26:14 +0000 | [diff] [blame] | 366 | } |
Evan Cheng | ae64219 | 2007-03-02 23:16:35 +0000 | [diff] [blame] | 367 | |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 368 | setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); |
Chris Lattner | e112552 | 2006-01-15 09:00:21 +0000 | [diff] [blame] | 369 | setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 370 | if (Subtarget->is64Bit()) |
| 371 | setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); |
Anton Korobeynikov | 57fc00d | 2007-04-17 09:20:00 +0000 | [diff] [blame] | 372 | if (Subtarget->isTargetCygMing()) |
| 373 | setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); |
| 374 | else |
| 375 | setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); |
Chris Lattner | b99329e | 2006-01-13 02:42:53 +0000 | [diff] [blame] | 376 | |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 377 | if (X86ScalarSSEf64) { |
| 378 | // f32 and f64 use SSE. |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 379 | // Set up the FP register classes. |
Evan Cheng | 5ee4ccc | 2006-01-12 08:27:59 +0000 | [diff] [blame] | 380 | addRegisterClass(MVT::f32, X86::FR32RegisterClass); |
| 381 | addRegisterClass(MVT::f64, X86::FR64RegisterClass); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 382 | |
Evan Cheng | 223547a | 2006-01-31 22:28:30 +0000 | [diff] [blame] | 383 | // Use ANDPD to simulate FABS. |
| 384 | setOperationAction(ISD::FABS , MVT::f64, Custom); |
| 385 | setOperationAction(ISD::FABS , MVT::f32, Custom); |
| 386 | |
| 387 | // Use XORP to simulate FNEG. |
| 388 | setOperationAction(ISD::FNEG , MVT::f64, Custom); |
| 389 | setOperationAction(ISD::FNEG , MVT::f32, Custom); |
| 390 | |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 391 | // Use ANDPD and ORPD to simulate FCOPYSIGN. |
| 392 | setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); |
| 393 | setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); |
| 394 | |
Evan Cheng | d25e9e8 | 2006-02-02 00:28:23 +0000 | [diff] [blame] | 395 | // We don't support sin/cos/fmod |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 396 | setOperationAction(ISD::FSIN , MVT::f64, Expand); |
| 397 | setOperationAction(ISD::FCOS , MVT::f64, Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 398 | setOperationAction(ISD::FSIN , MVT::f32, Expand); |
| 399 | setOperationAction(ISD::FCOS , MVT::f32, Expand); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 400 | |
Chris Lattner | a54aa94 | 2006-01-29 06:26:08 +0000 | [diff] [blame] | 401 | // Expand FP immediates into loads from the stack, except for the special |
| 402 | // cases we handle. |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 403 | addLegalFPImmediate(APFloat(+0.0)); // xorpd |
| 404 | addLegalFPImmediate(APFloat(+0.0f)); // xorps |
Dale Johannesen | 5411a39 | 2007-08-09 01:04:01 +0000 | [diff] [blame] | 405 | |
Chris Lattner | d43d00c | 2008-01-24 08:07:48 +0000 | [diff] [blame] | 406 | // Floating truncations from f80 and extensions to f80 go through memory. |
| 407 | // If optimizing, we lie about this though and handle it in |
| 408 | // InstructionSelectPreprocess so that dagcombine2 can hack on these. |
| 409 | if (Fast) { |
| 410 | setConvertAction(MVT::f32, MVT::f80, Expand); |
| 411 | setConvertAction(MVT::f64, MVT::f80, Expand); |
| 412 | setConvertAction(MVT::f80, MVT::f32, Expand); |
| 413 | setConvertAction(MVT::f80, MVT::f64, Expand); |
| 414 | } |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 415 | } else if (X86ScalarSSEf32) { |
| 416 | // Use SSE for f32, x87 for f64. |
| 417 | // Set up the FP register classes. |
| 418 | addRegisterClass(MVT::f32, X86::FR32RegisterClass); |
| 419 | addRegisterClass(MVT::f64, X86::RFP64RegisterClass); |
| 420 | |
| 421 | // Use ANDPS to simulate FABS. |
| 422 | setOperationAction(ISD::FABS , MVT::f32, Custom); |
| 423 | |
| 424 | // Use XORP to simulate FNEG. |
| 425 | setOperationAction(ISD::FNEG , MVT::f32, Custom); |
| 426 | |
| 427 | setOperationAction(ISD::UNDEF, MVT::f64, Expand); |
| 428 | |
| 429 | // Use ANDPS and ORPS to simulate FCOPYSIGN. |
| 430 | setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); |
| 431 | setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); |
| 432 | |
| 433 | // We don't support sin/cos/fmod |
| 434 | setOperationAction(ISD::FSIN , MVT::f32, Expand); |
| 435 | setOperationAction(ISD::FCOS , MVT::f32, Expand); |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 436 | |
Nate Begeman | e179584 | 2008-02-14 08:57:00 +0000 | [diff] [blame] | 437 | // Special cases we handle for FP constants. |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 438 | addLegalFPImmediate(APFloat(+0.0f)); // xorps |
| 439 | addLegalFPImmediate(APFloat(+0.0)); // FLD0 |
| 440 | addLegalFPImmediate(APFloat(+1.0)); // FLD1 |
| 441 | addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS |
| 442 | addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS |
| 443 | |
Chris Lattner | d43d00c | 2008-01-24 08:07:48 +0000 | [diff] [blame] | 444 | // SSE <-> X87 conversions go through memory. If optimizing, we lie about |
| 445 | // this though and handle it in InstructionSelectPreprocess so that |
| 446 | // dagcombine2 can hack on these. |
| 447 | if (Fast) { |
| 448 | setConvertAction(MVT::f32, MVT::f64, Expand); |
| 449 | setConvertAction(MVT::f32, MVT::f80, Expand); |
| 450 | setConvertAction(MVT::f80, MVT::f32, Expand); |
| 451 | setConvertAction(MVT::f64, MVT::f32, Expand); |
| 452 | // And x87->x87 truncations also. |
| 453 | setConvertAction(MVT::f80, MVT::f64, Expand); |
| 454 | } |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 455 | |
| 456 | if (!UnsafeFPMath) { |
| 457 | setOperationAction(ISD::FSIN , MVT::f64 , Expand); |
| 458 | setOperationAction(ISD::FCOS , MVT::f64 , Expand); |
| 459 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 460 | } else { |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 461 | // f32 and f64 in x87. |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 462 | // Set up the FP register classes. |
Dale Johannesen | 849f214 | 2007-07-03 00:53:03 +0000 | [diff] [blame] | 463 | addRegisterClass(MVT::f64, X86::RFP64RegisterClass); |
| 464 | addRegisterClass(MVT::f32, X86::RFP32RegisterClass); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 465 | |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 466 | setOperationAction(ISD::UNDEF, MVT::f64, Expand); |
Dale Johannesen | 849f214 | 2007-07-03 00:53:03 +0000 | [diff] [blame] | 467 | setOperationAction(ISD::UNDEF, MVT::f32, Expand); |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 468 | setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); |
| 469 | setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); |
Dale Johannesen | 5411a39 | 2007-08-09 01:04:01 +0000 | [diff] [blame] | 470 | |
Chris Lattner | d43d00c | 2008-01-24 08:07:48 +0000 | [diff] [blame] | 471 | // Floating truncations go through memory. If optimizing, we lie about |
| 472 | // this though and handle it in InstructionSelectPreprocess so that |
| 473 | // dagcombine2 can hack on these. |
| 474 | if (Fast) { |
| 475 | setConvertAction(MVT::f80, MVT::f32, Expand); |
| 476 | setConvertAction(MVT::f64, MVT::f32, Expand); |
| 477 | setConvertAction(MVT::f80, MVT::f64, Expand); |
| 478 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 479 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 480 | if (!UnsafeFPMath) { |
| 481 | setOperationAction(ISD::FSIN , MVT::f64 , Expand); |
| 482 | setOperationAction(ISD::FCOS , MVT::f64 , Expand); |
| 483 | } |
Dale Johannesen | f04afdb | 2007-08-30 00:23:21 +0000 | [diff] [blame] | 484 | addLegalFPImmediate(APFloat(+0.0)); // FLD0 |
| 485 | addLegalFPImmediate(APFloat(+1.0)); // FLD1 |
| 486 | addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS |
| 487 | addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 488 | addLegalFPImmediate(APFloat(+0.0f)); // FLD0 |
| 489 | addLegalFPImmediate(APFloat(+1.0f)); // FLD1 |
| 490 | addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS |
| 491 | addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 492 | } |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 493 | |
Dale Johannesen | 59a5873 | 2007-08-05 18:49:15 +0000 | [diff] [blame] | 494 | // Long double always uses X87. |
| 495 | addRegisterClass(MVT::f80, X86::RFP80RegisterClass); |
Dale Johannesen | 9e3d3ab | 2007-09-14 22:26:36 +0000 | [diff] [blame] | 496 | setOperationAction(ISD::UNDEF, MVT::f80, Expand); |
| 497 | setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); |
Chris Lattner | 71d07a0 | 2008-01-27 06:19:31 +0000 | [diff] [blame] | 498 | { |
Dale Johannesen | 23a9855 | 2008-10-09 23:00:39 +0000 | [diff] [blame] | 499 | bool ignored; |
Chris Lattner | 71d07a0 | 2008-01-27 06:19:31 +0000 | [diff] [blame] | 500 | APFloat TmpFlt(+0.0); |
Dale Johannesen | 23a9855 | 2008-10-09 23:00:39 +0000 | [diff] [blame] | 501 | TmpFlt.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven, |
| 502 | &ignored); |
Chris Lattner | 71d07a0 | 2008-01-27 06:19:31 +0000 | [diff] [blame] | 503 | addLegalFPImmediate(TmpFlt); // FLD0 |
| 504 | TmpFlt.changeSign(); |
| 505 | addLegalFPImmediate(TmpFlt); // FLD0/FCHS |
| 506 | APFloat TmpFlt2(+1.0); |
Dale Johannesen | 23a9855 | 2008-10-09 23:00:39 +0000 | [diff] [blame] | 507 | TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven, |
| 508 | &ignored); |
Chris Lattner | 71d07a0 | 2008-01-27 06:19:31 +0000 | [diff] [blame] | 509 | addLegalFPImmediate(TmpFlt2); // FLD1 |
| 510 | TmpFlt2.changeSign(); |
| 511 | addLegalFPImmediate(TmpFlt2); // FLD1/FCHS |
| 512 | } |
| 513 | |
Dale Johannesen | 2f42901 | 2007-09-26 21:10:55 +0000 | [diff] [blame] | 514 | if (!UnsafeFPMath) { |
| 515 | setOperationAction(ISD::FSIN , MVT::f80 , Expand); |
| 516 | setOperationAction(ISD::FCOS , MVT::f80 , Expand); |
| 517 | } |
Dale Johannesen | 59a5873 | 2007-08-05 18:49:15 +0000 | [diff] [blame] | 518 | |
Dan Gohman | f96e4de | 2007-10-11 23:21:31 +0000 | [diff] [blame] | 519 | // Always use a library call for pow. |
| 520 | setOperationAction(ISD::FPOW , MVT::f32 , Expand); |
| 521 | setOperationAction(ISD::FPOW , MVT::f64 , Expand); |
| 522 | setOperationAction(ISD::FPOW , MVT::f80 , Expand); |
| 523 | |
Dale Johannesen | 7794f2a | 2008-09-04 00:47:13 +0000 | [diff] [blame] | 524 | setOperationAction(ISD::FLOG, MVT::f80, Expand); |
Dale Johannesen | 7794f2a | 2008-09-04 00:47:13 +0000 | [diff] [blame] | 525 | setOperationAction(ISD::FLOG2, MVT::f80, Expand); |
Dale Johannesen | 7794f2a | 2008-09-04 00:47:13 +0000 | [diff] [blame] | 526 | setOperationAction(ISD::FLOG10, MVT::f80, Expand); |
Dale Johannesen | 7794f2a | 2008-09-04 00:47:13 +0000 | [diff] [blame] | 527 | setOperationAction(ISD::FEXP, MVT::f80, Expand); |
Dale Johannesen | 7794f2a | 2008-09-04 00:47:13 +0000 | [diff] [blame] | 528 | setOperationAction(ISD::FEXP2, MVT::f80, Expand); |
| 529 | |
Mon P Wang | f007a8b | 2008-11-06 05:31:54 +0000 | [diff] [blame] | 530 | // First set operation action for all vector types to either promote |
Mon P Wang | 0c39719 | 2008-10-30 08:01:45 +0000 | [diff] [blame] | 531 | // (for widening) or expand (for scalarization). Then we will selectively |
| 532 | // turn on ones that can be effectively codegen'd. |
Dan Gohman | fa0f77d | 2007-05-18 18:44:07 +0000 | [diff] [blame] | 533 | for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; |
| 534 | VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 535 | setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand); |
| 536 | setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand); |
| 537 | setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand); |
| 538 | setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand); |
| 539 | setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand); |
| 540 | setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand); |
| 541 | setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand); |
| 542 | setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand); |
| 543 | setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand); |
| 544 | setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand); |
| 545 | setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand); |
| 546 | setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand); |
| 547 | setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand); |
Gabor Greif | 327ef03 | 2008-08-28 23:19:51 +0000 | [diff] [blame] | 548 | setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand); |
| 549 | setOperationAction(ISD::EXTRACT_VECTOR_ELT,(MVT::SimpleValueType)VT,Expand); |
| 550 | setOperationAction(ISD::INSERT_VECTOR_ELT,(MVT::SimpleValueType)VT, Expand); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 551 | setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand); |
| 552 | setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand); |
| 553 | setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand); |
| 554 | setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand); |
| 555 | setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand); |
| 556 | setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand); |
| 557 | setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand); |
| 558 | setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand); |
| 559 | setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand); |
| 560 | setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand); |
| 561 | setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand); |
| 562 | setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand); |
| 563 | setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand); |
| 564 | setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand); |
| 565 | setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand); |
| 566 | setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand); |
| 567 | setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand); |
| 568 | setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand); |
| 569 | setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand); |
| 570 | setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand); |
| 571 | setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand); |
| 572 | setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand); |
Dale Johannesen | fb0e132 | 2008-09-10 17:31:40 +0000 | [diff] [blame] | 573 | setOperationAction(ISD::FLOG, (MVT::SimpleValueType)VT, Expand); |
| 574 | setOperationAction(ISD::FLOG2, (MVT::SimpleValueType)VT, Expand); |
| 575 | setOperationAction(ISD::FLOG10, (MVT::SimpleValueType)VT, Expand); |
| 576 | setOperationAction(ISD::FEXP, (MVT::SimpleValueType)VT, Expand); |
| 577 | setOperationAction(ISD::FEXP2, (MVT::SimpleValueType)VT, Expand); |
Evan Cheng | d30bf01 | 2006-03-01 01:11:20 +0000 | [diff] [blame] | 578 | } |
| 579 | |
Mon P Wang | 3c81d35 | 2008-11-23 04:37:22 +0000 | [diff] [blame] | 580 | if (!DisableMMX && Subtarget->hasMMX()) { |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 581 | addRegisterClass(MVT::v8i8, X86::VR64RegisterClass); |
| 582 | addRegisterClass(MVT::v4i16, X86::VR64RegisterClass); |
| 583 | addRegisterClass(MVT::v2i32, X86::VR64RegisterClass); |
Dale Johannesen | a68f901 | 2008-06-24 22:01:44 +0000 | [diff] [blame] | 584 | addRegisterClass(MVT::v2f32, X86::VR64RegisterClass); |
Bill Wendling | eebc8a1 | 2007-03-26 07:53:08 +0000 | [diff] [blame] | 585 | addRegisterClass(MVT::v1i64, X86::VR64RegisterClass); |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 586 | |
Evan Cheng | d30bf01 | 2006-03-01 01:11:20 +0000 | [diff] [blame] | 587 | // FIXME: add MMX packed arithmetics |
Bill Wendling | bc9bffa | 2007-03-07 05:43:18 +0000 | [diff] [blame] | 588 | |
Bill Wendling | 2f88dcd | 2007-03-08 22:09:11 +0000 | [diff] [blame] | 589 | setOperationAction(ISD::ADD, MVT::v8i8, Legal); |
| 590 | setOperationAction(ISD::ADD, MVT::v4i16, Legal); |
| 591 | setOperationAction(ISD::ADD, MVT::v2i32, Legal); |
Chris Lattner | 6c284d7 | 2007-04-12 04:14:49 +0000 | [diff] [blame] | 592 | setOperationAction(ISD::ADD, MVT::v1i64, Legal); |
Bill Wendling | 2f88dcd | 2007-03-08 22:09:11 +0000 | [diff] [blame] | 593 | |
Bill Wendling | c1fb047 | 2007-03-10 09:57:05 +0000 | [diff] [blame] | 594 | setOperationAction(ISD::SUB, MVT::v8i8, Legal); |
| 595 | setOperationAction(ISD::SUB, MVT::v4i16, Legal); |
| 596 | setOperationAction(ISD::SUB, MVT::v2i32, Legal); |
Dale Johannesen | 8d26e59 | 2007-10-30 01:18:38 +0000 | [diff] [blame] | 597 | setOperationAction(ISD::SUB, MVT::v1i64, Legal); |
Bill Wendling | c1fb047 | 2007-03-10 09:57:05 +0000 | [diff] [blame] | 598 | |
Bill Wendling | 74027e9 | 2007-03-15 21:24:36 +0000 | [diff] [blame] | 599 | setOperationAction(ISD::MULHS, MVT::v4i16, Legal); |
| 600 | setOperationAction(ISD::MUL, MVT::v4i16, Legal); |
| 601 | |
Bill Wendling | 1b7a81d | 2007-03-16 09:44:46 +0000 | [diff] [blame] | 602 | setOperationAction(ISD::AND, MVT::v8i8, Promote); |
Bill Wendling | ab5b49d | 2007-03-26 08:03:33 +0000 | [diff] [blame] | 603 | AddPromotedToType (ISD::AND, MVT::v8i8, MVT::v1i64); |
Bill Wendling | 1b7a81d | 2007-03-16 09:44:46 +0000 | [diff] [blame] | 604 | setOperationAction(ISD::AND, MVT::v4i16, Promote); |
Bill Wendling | ab5b49d | 2007-03-26 08:03:33 +0000 | [diff] [blame] | 605 | AddPromotedToType (ISD::AND, MVT::v4i16, MVT::v1i64); |
| 606 | setOperationAction(ISD::AND, MVT::v2i32, Promote); |
| 607 | AddPromotedToType (ISD::AND, MVT::v2i32, MVT::v1i64); |
| 608 | setOperationAction(ISD::AND, MVT::v1i64, Legal); |
Bill Wendling | 1b7a81d | 2007-03-16 09:44:46 +0000 | [diff] [blame] | 609 | |
| 610 | setOperationAction(ISD::OR, MVT::v8i8, Promote); |
Bill Wendling | ab5b49d | 2007-03-26 08:03:33 +0000 | [diff] [blame] | 611 | AddPromotedToType (ISD::OR, MVT::v8i8, MVT::v1i64); |
Bill Wendling | 1b7a81d | 2007-03-16 09:44:46 +0000 | [diff] [blame] | 612 | setOperationAction(ISD::OR, MVT::v4i16, Promote); |
Bill Wendling | ab5b49d | 2007-03-26 08:03:33 +0000 | [diff] [blame] | 613 | AddPromotedToType (ISD::OR, MVT::v4i16, MVT::v1i64); |
| 614 | setOperationAction(ISD::OR, MVT::v2i32, Promote); |
| 615 | AddPromotedToType (ISD::OR, MVT::v2i32, MVT::v1i64); |
| 616 | setOperationAction(ISD::OR, MVT::v1i64, Legal); |
Bill Wendling | 1b7a81d | 2007-03-16 09:44:46 +0000 | [diff] [blame] | 617 | |
| 618 | setOperationAction(ISD::XOR, MVT::v8i8, Promote); |
Bill Wendling | ab5b49d | 2007-03-26 08:03:33 +0000 | [diff] [blame] | 619 | AddPromotedToType (ISD::XOR, MVT::v8i8, MVT::v1i64); |
Bill Wendling | 1b7a81d | 2007-03-16 09:44:46 +0000 | [diff] [blame] | 620 | setOperationAction(ISD::XOR, MVT::v4i16, Promote); |
Bill Wendling | ab5b49d | 2007-03-26 08:03:33 +0000 | [diff] [blame] | 621 | AddPromotedToType (ISD::XOR, MVT::v4i16, MVT::v1i64); |
| 622 | setOperationAction(ISD::XOR, MVT::v2i32, Promote); |
| 623 | AddPromotedToType (ISD::XOR, MVT::v2i32, MVT::v1i64); |
| 624 | setOperationAction(ISD::XOR, MVT::v1i64, Legal); |
Bill Wendling | 1b7a81d | 2007-03-16 09:44:46 +0000 | [diff] [blame] | 625 | |
Bill Wendling | 2f88dcd | 2007-03-08 22:09:11 +0000 | [diff] [blame] | 626 | setOperationAction(ISD::LOAD, MVT::v8i8, Promote); |
Bill Wendling | eebc8a1 | 2007-03-26 07:53:08 +0000 | [diff] [blame] | 627 | AddPromotedToType (ISD::LOAD, MVT::v8i8, MVT::v1i64); |
Bill Wendling | 2f88dcd | 2007-03-08 22:09:11 +0000 | [diff] [blame] | 628 | setOperationAction(ISD::LOAD, MVT::v4i16, Promote); |
Bill Wendling | eebc8a1 | 2007-03-26 07:53:08 +0000 | [diff] [blame] | 629 | AddPromotedToType (ISD::LOAD, MVT::v4i16, MVT::v1i64); |
| 630 | setOperationAction(ISD::LOAD, MVT::v2i32, Promote); |
| 631 | AddPromotedToType (ISD::LOAD, MVT::v2i32, MVT::v1i64); |
Dale Johannesen | a68f901 | 2008-06-24 22:01:44 +0000 | [diff] [blame] | 632 | setOperationAction(ISD::LOAD, MVT::v2f32, Promote); |
| 633 | AddPromotedToType (ISD::LOAD, MVT::v2f32, MVT::v1i64); |
Bill Wendling | eebc8a1 | 2007-03-26 07:53:08 +0000 | [diff] [blame] | 634 | setOperationAction(ISD::LOAD, MVT::v1i64, Legal); |
Bill Wendling | 2f88dcd | 2007-03-08 22:09:11 +0000 | [diff] [blame] | 635 | |
Bill Wendling | ccc44ad | 2007-03-27 20:22:40 +0000 | [diff] [blame] | 636 | setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Custom); |
| 637 | setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); |
| 638 | setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Custom); |
Dale Johannesen | a68f901 | 2008-06-24 22:01:44 +0000 | [diff] [blame] | 639 | setOperationAction(ISD::BUILD_VECTOR, MVT::v2f32, Custom); |
Bill Wendling | ccc44ad | 2007-03-27 20:22:40 +0000 | [diff] [blame] | 640 | setOperationAction(ISD::BUILD_VECTOR, MVT::v1i64, Custom); |
Bill Wendling | a348c56 | 2007-03-22 18:42:45 +0000 | [diff] [blame] | 641 | |
| 642 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); |
| 643 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); |
| 644 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i32, Custom); |
Bill Wendling | ccc44ad | 2007-03-27 20:22:40 +0000 | [diff] [blame] | 645 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v1i64, Custom); |
Bill Wendling | 826f36f | 2007-03-28 00:57:11 +0000 | [diff] [blame] | 646 | |
Evan Cheng | 52672b8 | 2008-07-22 18:39:19 +0000 | [diff] [blame] | 647 | setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f32, Custom); |
Bill Wendling | 826f36f | 2007-03-28 00:57:11 +0000 | [diff] [blame] | 648 | setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Custom); |
| 649 | setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Custom); |
Bill Wendling | 2f9bb1a | 2007-04-24 21:16:55 +0000 | [diff] [blame] | 650 | setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Custom); |
Bill Wendling | 3180e20 | 2008-07-20 02:32:23 +0000 | [diff] [blame] | 651 | |
| 652 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); |
Mon P Wang | 9e5ecb8 | 2008-12-12 01:25:51 +0000 | [diff] [blame] | 653 | |
| 654 | setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand); |
| 655 | setOperationAction(ISD::TRUNCATE, MVT::v8i8, Expand); |
| 656 | setOperationAction(ISD::SELECT, MVT::v8i8, Promote); |
| 657 | setOperationAction(ISD::SELECT, MVT::v4i16, Promote); |
| 658 | setOperationAction(ISD::SELECT, MVT::v2i32, Promote); |
| 659 | setOperationAction(ISD::SELECT, MVT::v1i64, Custom); |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 660 | } |
| 661 | |
Evan Cheng | a88973f | 2006-03-22 19:22:18 +0000 | [diff] [blame] | 662 | if (Subtarget->hasSSE1()) { |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 663 | addRegisterClass(MVT::v4f32, X86::VR128RegisterClass); |
| 664 | |
Evan Cheng | 6bdb3f6 | 2006-10-27 18:49:08 +0000 | [diff] [blame] | 665 | setOperationAction(ISD::FADD, MVT::v4f32, Legal); |
| 666 | setOperationAction(ISD::FSUB, MVT::v4f32, Legal); |
| 667 | setOperationAction(ISD::FMUL, MVT::v4f32, Legal); |
| 668 | setOperationAction(ISD::FDIV, MVT::v4f32, Legal); |
Dan Gohman | 2038252 | 2007-07-10 00:05:58 +0000 | [diff] [blame] | 669 | setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); |
| 670 | setOperationAction(ISD::FNEG, MVT::v4f32, Custom); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 671 | setOperationAction(ISD::LOAD, MVT::v4f32, Legal); |
| 672 | setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); |
| 673 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); |
Evan Cheng | 11e15b3 | 2006-04-03 20:53:28 +0000 | [diff] [blame] | 674 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 675 | setOperationAction(ISD::SELECT, MVT::v4f32, Custom); |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 676 | setOperationAction(ISD::VSETCC, MVT::v4f32, Custom); |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 677 | } |
| 678 | |
Evan Cheng | a88973f | 2006-03-22 19:22:18 +0000 | [diff] [blame] | 679 | if (Subtarget->hasSSE2()) { |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 680 | addRegisterClass(MVT::v2f64, X86::VR128RegisterClass); |
| 681 | addRegisterClass(MVT::v16i8, X86::VR128RegisterClass); |
| 682 | addRegisterClass(MVT::v8i16, X86::VR128RegisterClass); |
| 683 | addRegisterClass(MVT::v4i32, X86::VR128RegisterClass); |
| 684 | addRegisterClass(MVT::v2i64, X86::VR128RegisterClass); |
| 685 | |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 686 | setOperationAction(ISD::ADD, MVT::v16i8, Legal); |
| 687 | setOperationAction(ISD::ADD, MVT::v8i16, Legal); |
| 688 | setOperationAction(ISD::ADD, MVT::v4i32, Legal); |
Evan Cheng | 37e8856 | 2007-03-12 22:58:52 +0000 | [diff] [blame] | 689 | setOperationAction(ISD::ADD, MVT::v2i64, Legal); |
Mon P Wang | af9b952 | 2008-12-18 21:42:19 +0000 | [diff] [blame] | 690 | setOperationAction(ISD::MUL, MVT::v2i64, Custom); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 691 | setOperationAction(ISD::SUB, MVT::v16i8, Legal); |
| 692 | setOperationAction(ISD::SUB, MVT::v8i16, Legal); |
| 693 | setOperationAction(ISD::SUB, MVT::v4i32, Legal); |
Evan Cheng | 37e8856 | 2007-03-12 22:58:52 +0000 | [diff] [blame] | 694 | setOperationAction(ISD::SUB, MVT::v2i64, Legal); |
Evan Cheng | f998984 | 2006-04-13 05:10:25 +0000 | [diff] [blame] | 695 | setOperationAction(ISD::MUL, MVT::v8i16, Legal); |
Evan Cheng | 6bdb3f6 | 2006-10-27 18:49:08 +0000 | [diff] [blame] | 696 | setOperationAction(ISD::FADD, MVT::v2f64, Legal); |
| 697 | setOperationAction(ISD::FSUB, MVT::v2f64, Legal); |
| 698 | setOperationAction(ISD::FMUL, MVT::v2f64, Legal); |
| 699 | setOperationAction(ISD::FDIV, MVT::v2f64, Legal); |
Dan Gohman | 2038252 | 2007-07-10 00:05:58 +0000 | [diff] [blame] | 700 | setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); |
| 701 | setOperationAction(ISD::FNEG, MVT::v2f64, Custom); |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 702 | |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 703 | setOperationAction(ISD::VSETCC, MVT::v2f64, Custom); |
| 704 | setOperationAction(ISD::VSETCC, MVT::v16i8, Custom); |
| 705 | setOperationAction(ISD::VSETCC, MVT::v8i16, Custom); |
| 706 | setOperationAction(ISD::VSETCC, MVT::v4i32, Custom); |
Nate Begeman | c2616e4 | 2008-05-12 20:34:32 +0000 | [diff] [blame] | 707 | |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 708 | setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); |
| 709 | setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); |
Evan Cheng | b067a1e | 2006-03-31 19:22:53 +0000 | [diff] [blame] | 710 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); |
Evan Cheng | 5edb8d2 | 2006-04-17 22:04:06 +0000 | [diff] [blame] | 711 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); |
Evan Cheng | 5edb8d2 | 2006-04-17 22:04:06 +0000 | [diff] [blame] | 712 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 713 | |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 714 | // Custom lower build_vector, vector_shuffle, and extract_vector_elt. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 715 | for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; ++i) { |
| 716 | MVT VT = (MVT::SimpleValueType)i; |
Nate Begeman | 844e0f9 | 2007-12-11 01:41:33 +0000 | [diff] [blame] | 717 | // Do not attempt to custom lower non-power-of-2 vectors |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 718 | if (!isPowerOf2_32(VT.getVectorNumElements())) |
Nate Begeman | 844e0f9 | 2007-12-11 01:41:33 +0000 | [diff] [blame] | 719 | continue; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 720 | setOperationAction(ISD::BUILD_VECTOR, VT, Custom); |
| 721 | setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); |
| 722 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 723 | } |
| 724 | setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); |
| 725 | setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); |
| 726 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); |
| 727 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); |
Nate Begeman | cdd1eec | 2008-02-12 22:51:28 +0000 | [diff] [blame] | 728 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 729 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); |
Nate Begeman | cdd1eec | 2008-02-12 22:51:28 +0000 | [diff] [blame] | 730 | if (Subtarget->is64Bit()) { |
| 731 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); |
Dale Johannesen | 25f1d08 | 2007-10-31 00:32:36 +0000 | [diff] [blame] | 732 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); |
Nate Begeman | cdd1eec | 2008-02-12 22:51:28 +0000 | [diff] [blame] | 733 | } |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 734 | |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 735 | // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64. |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 736 | for (unsigned VT = (unsigned)MVT::v16i8; VT != (unsigned)MVT::v2i64; VT++) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 737 | setOperationAction(ISD::AND, (MVT::SimpleValueType)VT, Promote); |
| 738 | AddPromotedToType (ISD::AND, (MVT::SimpleValueType)VT, MVT::v2i64); |
| 739 | setOperationAction(ISD::OR, (MVT::SimpleValueType)VT, Promote); |
| 740 | AddPromotedToType (ISD::OR, (MVT::SimpleValueType)VT, MVT::v2i64); |
| 741 | setOperationAction(ISD::XOR, (MVT::SimpleValueType)VT, Promote); |
| 742 | AddPromotedToType (ISD::XOR, (MVT::SimpleValueType)VT, MVT::v2i64); |
| 743 | setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Promote); |
| 744 | AddPromotedToType (ISD::LOAD, (MVT::SimpleValueType)VT, MVT::v2i64); |
| 745 | setOperationAction(ISD::SELECT, (MVT::SimpleValueType)VT, Promote); |
| 746 | AddPromotedToType (ISD::SELECT, (MVT::SimpleValueType)VT, MVT::v2i64); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 747 | } |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 748 | |
Chris Lattner | ddf8956 | 2008-01-17 19:59:44 +0000 | [diff] [blame] | 749 | setTruncStoreAction(MVT::f64, MVT::f32, Expand); |
Chris Lattner | d43d00c | 2008-01-24 08:07:48 +0000 | [diff] [blame] | 750 | |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 751 | // Custom lower v2i64 and v2f64 selects. |
| 752 | setOperationAction(ISD::LOAD, MVT::v2f64, Legal); |
Evan Cheng | 91b740d | 2006-04-12 17:12:36 +0000 | [diff] [blame] | 753 | setOperationAction(ISD::LOAD, MVT::v2i64, Legal); |
Evan Cheng | f7c378e | 2006-04-10 07:23:14 +0000 | [diff] [blame] | 754 | setOperationAction(ISD::SELECT, MVT::v2f64, Custom); |
Evan Cheng | 2c3ae37 | 2006-04-12 21:21:57 +0000 | [diff] [blame] | 755 | setOperationAction(ISD::SELECT, MVT::v2i64, Custom); |
Nate Begeman | c2616e4 | 2008-05-12 20:34:32 +0000 | [diff] [blame] | 756 | |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 757 | } |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 758 | |
| 759 | if (Subtarget->hasSSE41()) { |
| 760 | // FIXME: Do we need to handle scalar-to-vector here? |
| 761 | setOperationAction(ISD::MUL, MVT::v4i32, Legal); |
| 762 | |
| 763 | // i8 and i16 vectors are custom , because the source register and source |
| 764 | // source memory operand types are not the same width. f32 vectors are |
| 765 | // custom since the immediate controlling the insert encodes additional |
| 766 | // information. |
| 767 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); |
| 768 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); |
| 769 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Legal); |
| 770 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); |
| 771 | |
| 772 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); |
| 773 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); |
| 774 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); |
Evan Cheng | 62a3f15 | 2008-03-24 21:52:23 +0000 | [diff] [blame] | 775 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 776 | |
| 777 | if (Subtarget->is64Bit()) { |
Nate Begeman | cdd1eec | 2008-02-12 22:51:28 +0000 | [diff] [blame] | 778 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal); |
| 779 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 780 | } |
| 781 | } |
Evan Cheng | 470a6ad | 2006-02-22 02:26:30 +0000 | [diff] [blame] | 782 | |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 783 | if (Subtarget->hasSSE42()) { |
| 784 | setOperationAction(ISD::VSETCC, MVT::v2i64, Custom); |
| 785 | } |
| 786 | |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 787 | // We want to custom lower some of our intrinsics. |
| 788 | setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); |
| 789 | |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 790 | // Add/Sub/Mul with overflow operations are custom lowered. |
Bill Wendling | 41ea7e7 | 2008-11-24 19:21:46 +0000 | [diff] [blame] | 791 | setOperationAction(ISD::SADDO, MVT::i32, Custom); |
| 792 | setOperationAction(ISD::SADDO, MVT::i64, Custom); |
| 793 | setOperationAction(ISD::UADDO, MVT::i32, Custom); |
| 794 | setOperationAction(ISD::UADDO, MVT::i64, Custom); |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 795 | setOperationAction(ISD::SSUBO, MVT::i32, Custom); |
| 796 | setOperationAction(ISD::SSUBO, MVT::i64, Custom); |
| 797 | setOperationAction(ISD::USUBO, MVT::i32, Custom); |
| 798 | setOperationAction(ISD::USUBO, MVT::i64, Custom); |
| 799 | setOperationAction(ISD::SMULO, MVT::i32, Custom); |
| 800 | setOperationAction(ISD::SMULO, MVT::i64, Custom); |
| 801 | setOperationAction(ISD::UMULO, MVT::i32, Custom); |
| 802 | setOperationAction(ISD::UMULO, MVT::i64, Custom); |
Bill Wendling | 41ea7e7 | 2008-11-24 19:21:46 +0000 | [diff] [blame] | 803 | |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 804 | // We have target-specific dag combine patterns for the following nodes: |
| 805 | setTargetDAGCombine(ISD::VECTOR_SHUFFLE); |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 806 | setTargetDAGCombine(ISD::BUILD_VECTOR); |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 807 | setTargetDAGCombine(ISD::SELECT); |
Chris Lattner | 149a4e5 | 2008-02-22 02:09:43 +0000 | [diff] [blame] | 808 | setTargetDAGCombine(ISD::STORE); |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 809 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 810 | computeRegisterProperties(); |
| 811 | |
Evan Cheng | 87ed716 | 2006-02-14 08:25:08 +0000 | [diff] [blame] | 812 | // FIXME: These should be based on subtarget info. Plus, the values should |
| 813 | // be smaller when we are in optimizing for size mode. |
Dan Gohman | 87060f5 | 2008-06-30 21:00:56 +0000 | [diff] [blame] | 814 | maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores |
| 815 | maxStoresPerMemcpy = 16; // For @llvm.memcpy -> sequence of stores |
| 816 | maxStoresPerMemmove = 3; // For @llvm.memmove -> sequence of stores |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 817 | allowUnalignedMemoryAccesses = true; // x86 supports it! |
Evan Cheng | fb8075d | 2008-02-28 00:43:03 +0000 | [diff] [blame] | 818 | setPrefLoopAlignment(16); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 819 | } |
| 820 | |
Scott Michel | 5b8f82e | 2008-03-10 15:42:14 +0000 | [diff] [blame] | 821 | |
Duncan Sands | 5480c04 | 2009-01-01 15:52:00 +0000 | [diff] [blame^] | 822 | MVT X86TargetLowering::getSetCCResultType(MVT VT) const { |
Scott Michel | 5b8f82e | 2008-03-10 15:42:14 +0000 | [diff] [blame] | 823 | return MVT::i8; |
| 824 | } |
| 825 | |
| 826 | |
Evan Cheng | 2928650 | 2008-01-23 23:17:41 +0000 | [diff] [blame] | 827 | /// getMaxByValAlign - Helper for getByValTypeAlignment to determine |
| 828 | /// the desired ByVal argument alignment. |
| 829 | static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) { |
| 830 | if (MaxAlign == 16) |
| 831 | return; |
| 832 | if (const VectorType *VTy = dyn_cast<VectorType>(Ty)) { |
| 833 | if (VTy->getBitWidth() == 128) |
| 834 | MaxAlign = 16; |
Evan Cheng | 2928650 | 2008-01-23 23:17:41 +0000 | [diff] [blame] | 835 | } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { |
| 836 | unsigned EltAlign = 0; |
| 837 | getMaxByValAlign(ATy->getElementType(), EltAlign); |
| 838 | if (EltAlign > MaxAlign) |
| 839 | MaxAlign = EltAlign; |
| 840 | } else if (const StructType *STy = dyn_cast<StructType>(Ty)) { |
| 841 | for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { |
| 842 | unsigned EltAlign = 0; |
| 843 | getMaxByValAlign(STy->getElementType(i), EltAlign); |
| 844 | if (EltAlign > MaxAlign) |
| 845 | MaxAlign = EltAlign; |
| 846 | if (MaxAlign == 16) |
| 847 | break; |
| 848 | } |
| 849 | } |
| 850 | return; |
| 851 | } |
| 852 | |
| 853 | /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate |
| 854 | /// function arguments in the caller parameter area. For X86, aggregates |
Dale Johannesen | 0c19187 | 2008-02-08 19:48:20 +0000 | [diff] [blame] | 855 | /// that contain SSE vectors are placed at 16-byte boundaries while the rest |
| 856 | /// are at 4-byte boundaries. |
Evan Cheng | 2928650 | 2008-01-23 23:17:41 +0000 | [diff] [blame] | 857 | unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const { |
Evan Cheng | 1887c1c | 2008-08-21 21:00:15 +0000 | [diff] [blame] | 858 | if (Subtarget->is64Bit()) { |
| 859 | // Max of 8 and alignment of type. |
Anton Korobeynikov | bff66b0 | 2008-09-09 18:22:57 +0000 | [diff] [blame] | 860 | unsigned TyAlign = TD->getABITypeAlignment(Ty); |
Evan Cheng | 1887c1c | 2008-08-21 21:00:15 +0000 | [diff] [blame] | 861 | if (TyAlign > 8) |
| 862 | return TyAlign; |
| 863 | return 8; |
| 864 | } |
| 865 | |
Evan Cheng | 2928650 | 2008-01-23 23:17:41 +0000 | [diff] [blame] | 866 | unsigned Align = 4; |
Dale Johannesen | 0c19187 | 2008-02-08 19:48:20 +0000 | [diff] [blame] | 867 | if (Subtarget->hasSSE1()) |
| 868 | getMaxByValAlign(Ty, Align); |
Evan Cheng | 2928650 | 2008-01-23 23:17:41 +0000 | [diff] [blame] | 869 | return Align; |
| 870 | } |
Chris Lattner | 2b02a44 | 2007-02-25 08:29:00 +0000 | [diff] [blame] | 871 | |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 872 | /// getOptimalMemOpType - Returns the target specific optimal type for load |
Evan Cheng | 0ef8de3 | 2008-05-15 22:13:02 +0000 | [diff] [blame] | 873 | /// and store operations as a result of memset, memcpy, and memmove |
| 874 | /// lowering. It returns MVT::iAny if SelectionDAG should be responsible for |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 875 | /// determining it. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 876 | MVT |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 877 | X86TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned Align, |
| 878 | bool isSrcConst, bool isSrcStr) const { |
Chris Lattner | 4002a1b | 2008-10-28 05:49:35 +0000 | [diff] [blame] | 879 | // FIXME: This turns off use of xmm stores for memset/memcpy on targets like |
| 880 | // linux. This is because the stack realignment code can't handle certain |
| 881 | // cases like PR2962. This should be removed when PR2962 is fixed. |
| 882 | if (Subtarget->getStackAlignment() >= 16) { |
| 883 | if ((isSrcConst || isSrcStr) && Subtarget->hasSSE2() && Size >= 16) |
| 884 | return MVT::v4i32; |
| 885 | if ((isSrcConst || isSrcStr) && Subtarget->hasSSE1() && Size >= 16) |
| 886 | return MVT::v4f32; |
| 887 | } |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 888 | if (Subtarget->is64Bit() && Size >= 8) |
| 889 | return MVT::i64; |
| 890 | return MVT::i32; |
| 891 | } |
| 892 | |
| 893 | |
Evan Cheng | cc41586 | 2007-11-09 01:32:10 +0000 | [diff] [blame] | 894 | /// getPICJumpTableRelocaBase - Returns relocation base for the given PIC |
| 895 | /// jumptable. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 896 | SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table, |
Evan Cheng | cc41586 | 2007-11-09 01:32:10 +0000 | [diff] [blame] | 897 | SelectionDAG &DAG) const { |
| 898 | if (usesGlobalOffsetTable()) |
| 899 | return DAG.getNode(ISD::GLOBAL_OFFSET_TABLE, getPointerTy()); |
| 900 | if (!Subtarget->isPICStyleRIPRel()) |
| 901 | return DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()); |
| 902 | return Table; |
| 903 | } |
| 904 | |
Chris Lattner | 2b02a44 | 2007-02-25 08:29:00 +0000 | [diff] [blame] | 905 | //===----------------------------------------------------------------------===// |
| 906 | // Return Value Calling Convention Implementation |
| 907 | //===----------------------------------------------------------------------===// |
| 908 | |
Chris Lattner | 59ed56b | 2007-02-28 04:55:35 +0000 | [diff] [blame] | 909 | #include "X86GenCallingConv.inc" |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 910 | |
Chris Lattner | 2a9bdd7 | 2007-02-25 09:12:39 +0000 | [diff] [blame] | 911 | /// LowerRET - Lower an ISD::RET node. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 912 | SDValue X86TargetLowering::LowerRET(SDValue Op, SelectionDAG &DAG) { |
Chris Lattner | 2a9bdd7 | 2007-02-25 09:12:39 +0000 | [diff] [blame] | 913 | assert((Op.getNumOperands() & 1) == 1 && "ISD::RET should have odd # args"); |
| 914 | |
Chris Lattner | 9774c91 | 2007-02-27 05:28:59 +0000 | [diff] [blame] | 915 | SmallVector<CCValAssign, 16> RVLocs; |
| 916 | unsigned CC = DAG.getMachineFunction().getFunction()->getCallingConv(); |
Chris Lattner | 52387be | 2007-06-19 00:13:10 +0000 | [diff] [blame] | 917 | bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg(); |
| 918 | CCState CCInfo(CC, isVarArg, getTargetMachine(), RVLocs); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 919 | CCInfo.AnalyzeReturn(Op.getNode(), RetCC_X86); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 920 | |
Chris Lattner | 2a9bdd7 | 2007-02-25 09:12:39 +0000 | [diff] [blame] | 921 | // If this is the first return lowered for this function, add the regs to the |
| 922 | // liveout set for the function. |
Chris Lattner | 84bc542 | 2007-12-31 04:13:23 +0000 | [diff] [blame] | 923 | if (DAG.getMachineFunction().getRegInfo().liveout_empty()) { |
Chris Lattner | 9774c91 | 2007-02-27 05:28:59 +0000 | [diff] [blame] | 924 | for (unsigned i = 0; i != RVLocs.size(); ++i) |
| 925 | if (RVLocs[i].isRegLoc()) |
Chris Lattner | 84bc542 | 2007-12-31 04:13:23 +0000 | [diff] [blame] | 926 | DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg()); |
Chris Lattner | 2a9bdd7 | 2007-02-25 09:12:39 +0000 | [diff] [blame] | 927 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 928 | SDValue Chain = Op.getOperand(0); |
Chris Lattner | 2a9bdd7 | 2007-02-25 09:12:39 +0000 | [diff] [blame] | 929 | |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 930 | // Handle tail call return. |
Arnold Schwaighofer | 30e62c0 | 2008-04-30 09:16:33 +0000 | [diff] [blame] | 931 | Chain = GetPossiblePreceedingTailCall(Chain, X86ISD::TAILCALL); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 932 | if (Chain.getOpcode() == X86ISD::TAILCALL) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 933 | SDValue TailCall = Chain; |
| 934 | SDValue TargetAddress = TailCall.getOperand(1); |
| 935 | SDValue StackAdjustment = TailCall.getOperand(2); |
Chris Lattner | b4a6eaa | 2008-01-16 05:52:18 +0000 | [diff] [blame] | 936 | assert(((TargetAddress.getOpcode() == ISD::Register && |
Arnold Schwaighofer | 290ae03 | 2008-09-22 14:50:07 +0000 | [diff] [blame] | 937 | (cast<RegisterSDNode>(TargetAddress)->getReg() == X86::EAX || |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 938 | cast<RegisterSDNode>(TargetAddress)->getReg() == X86::R9)) || |
Bill Wendling | 056292f | 2008-09-16 21:48:12 +0000 | [diff] [blame] | 939 | TargetAddress.getOpcode() == ISD::TargetExternalSymbol || |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 940 | TargetAddress.getOpcode() == ISD::TargetGlobalAddress) && |
| 941 | "Expecting an global address, external symbol, or register"); |
Chris Lattner | b4a6eaa | 2008-01-16 05:52:18 +0000 | [diff] [blame] | 942 | assert(StackAdjustment.getOpcode() == ISD::Constant && |
| 943 | "Expecting a const value"); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 944 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 945 | SmallVector<SDValue,8> Operands; |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 946 | Operands.push_back(Chain.getOperand(0)); |
| 947 | Operands.push_back(TargetAddress); |
| 948 | Operands.push_back(StackAdjustment); |
| 949 | // Copy registers used by the call. Last operand is a flag so it is not |
| 950 | // copied. |
Arnold Schwaighofer | 448175f | 2007-10-16 09:05:00 +0000 | [diff] [blame] | 951 | for (unsigned i=3; i < TailCall.getNumOperands()-1; i++) { |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 952 | Operands.push_back(Chain.getOperand(i)); |
| 953 | } |
Arnold Schwaighofer | 448175f | 2007-10-16 09:05:00 +0000 | [diff] [blame] | 954 | return DAG.getNode(X86ISD::TC_RETURN, MVT::Other, &Operands[0], |
| 955 | Operands.size()); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 956 | } |
| 957 | |
| 958 | // Regular return. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 959 | SDValue Flag; |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 960 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 961 | SmallVector<SDValue, 6> RetOps; |
Chris Lattner | 447ff68 | 2008-03-11 03:23:40 +0000 | [diff] [blame] | 962 | RetOps.push_back(Chain); // Operand #0 = Chain (updated below) |
| 963 | // Operand #1 = Bytes To Pop |
| 964 | RetOps.push_back(DAG.getConstant(getBytesToPopOnReturn(), MVT::i16)); |
| 965 | |
Chris Lattner | 2a9bdd7 | 2007-02-25 09:12:39 +0000 | [diff] [blame] | 966 | // Copy the result values into the output registers. |
Chris Lattner | 8e6da15 | 2008-03-10 21:08:41 +0000 | [diff] [blame] | 967 | for (unsigned i = 0; i != RVLocs.size(); ++i) { |
| 968 | CCValAssign &VA = RVLocs[i]; |
| 969 | assert(VA.isRegLoc() && "Can only return in registers!"); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 970 | SDValue ValToCopy = Op.getOperand(i*2+1); |
Chris Lattner | 2a9bdd7 | 2007-02-25 09:12:39 +0000 | [diff] [blame] | 971 | |
Chris Lattner | 447ff68 | 2008-03-11 03:23:40 +0000 | [diff] [blame] | 972 | // Returns in ST0/ST1 are handled specially: these are pushed as operands to |
| 973 | // the RET instruction and handled by the FP Stackifier. |
| 974 | if (RVLocs[i].getLocReg() == X86::ST0 || |
| 975 | RVLocs[i].getLocReg() == X86::ST1) { |
| 976 | // If this is a copy from an xmm register to ST(0), use an FPExtend to |
| 977 | // change the value to the FP stack register class. |
| 978 | if (isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) |
| 979 | ValToCopy = DAG.getNode(ISD::FP_EXTEND, MVT::f80, ValToCopy); |
| 980 | RetOps.push_back(ValToCopy); |
| 981 | // Don't emit a copytoreg. |
| 982 | continue; |
| 983 | } |
Dale Johannesen | a68f901 | 2008-06-24 22:01:44 +0000 | [diff] [blame] | 984 | |
Chris Lattner | 8e6da15 | 2008-03-10 21:08:41 +0000 | [diff] [blame] | 985 | Chain = DAG.getCopyToReg(Chain, VA.getLocReg(), ValToCopy, Flag); |
Chris Lattner | 2a9bdd7 | 2007-02-25 09:12:39 +0000 | [diff] [blame] | 986 | Flag = Chain.getValue(1); |
| 987 | } |
Dan Gohman | 61a9213 | 2008-04-21 23:59:07 +0000 | [diff] [blame] | 988 | |
| 989 | // The x86-64 ABI for returning structs by value requires that we copy |
| 990 | // the sret argument into %rax for the return. We saved the argument into |
| 991 | // a virtual register in the entry block, so now we copy the value out |
| 992 | // and into %rax. |
| 993 | if (Subtarget->is64Bit() && |
| 994 | DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { |
| 995 | MachineFunction &MF = DAG.getMachineFunction(); |
| 996 | X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); |
| 997 | unsigned Reg = FuncInfo->getSRetReturnReg(); |
| 998 | if (!Reg) { |
| 999 | Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); |
| 1000 | FuncInfo->setSRetReturnReg(Reg); |
| 1001 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1002 | SDValue Val = DAG.getCopyFromReg(Chain, Reg, getPointerTy()); |
Dan Gohman | 61a9213 | 2008-04-21 23:59:07 +0000 | [diff] [blame] | 1003 | |
| 1004 | Chain = DAG.getCopyToReg(Chain, X86::RAX, Val, Flag); |
| 1005 | Flag = Chain.getValue(1); |
| 1006 | } |
Chris Lattner | 2a9bdd7 | 2007-02-25 09:12:39 +0000 | [diff] [blame] | 1007 | |
Chris Lattner | 447ff68 | 2008-03-11 03:23:40 +0000 | [diff] [blame] | 1008 | RetOps[0] = Chain; // Update chain. |
| 1009 | |
| 1010 | // Add the flag if we have it. |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 1011 | if (Flag.getNode()) |
Chris Lattner | 447ff68 | 2008-03-11 03:23:40 +0000 | [diff] [blame] | 1012 | RetOps.push_back(Flag); |
| 1013 | |
| 1014 | return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, &RetOps[0], RetOps.size()); |
Chris Lattner | 2a9bdd7 | 2007-02-25 09:12:39 +0000 | [diff] [blame] | 1015 | } |
| 1016 | |
| 1017 | |
Chris Lattner | 3085e15 | 2007-02-25 08:59:22 +0000 | [diff] [blame] | 1018 | /// LowerCallResult - Lower the result values of an ISD::CALL into the |
| 1019 | /// appropriate copies out of appropriate physical registers. This assumes that |
| 1020 | /// Chain/InFlag are the input chain/flag to use, and that TheCall is the call |
| 1021 | /// being lowered. The returns a SDNode with the same number of values as the |
| 1022 | /// ISD::CALL. |
| 1023 | SDNode *X86TargetLowering:: |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1024 | LowerCallResult(SDValue Chain, SDValue InFlag, CallSDNode *TheCall, |
Chris Lattner | 3085e15 | 2007-02-25 08:59:22 +0000 | [diff] [blame] | 1025 | unsigned CallingConv, SelectionDAG &DAG) { |
Chris Lattner | e32bbf6 | 2007-02-28 07:09:55 +0000 | [diff] [blame] | 1026 | |
| 1027 | // Assign locations to each value returned by this call. |
Chris Lattner | 9774c91 | 2007-02-27 05:28:59 +0000 | [diff] [blame] | 1028 | SmallVector<CCValAssign, 16> RVLocs; |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1029 | bool isVarArg = TheCall->isVarArg(); |
Chris Lattner | 52387be | 2007-06-19 00:13:10 +0000 | [diff] [blame] | 1030 | CCState CCInfo(CallingConv, isVarArg, getTargetMachine(), RVLocs); |
Chris Lattner | e32bbf6 | 2007-02-28 07:09:55 +0000 | [diff] [blame] | 1031 | CCInfo.AnalyzeCallResult(TheCall, RetCC_X86); |
| 1032 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1033 | SmallVector<SDValue, 8> ResultVals; |
Chris Lattner | 3085e15 | 2007-02-25 08:59:22 +0000 | [diff] [blame] | 1034 | |
| 1035 | // Copy all of the result registers out of their specified physreg. |
Chris Lattner | 8e6da15 | 2008-03-10 21:08:41 +0000 | [diff] [blame] | 1036 | for (unsigned i = 0; i != RVLocs.size(); ++i) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 1037 | MVT CopyVT = RVLocs[i].getValVT(); |
Chris Lattner | 8e6da15 | 2008-03-10 21:08:41 +0000 | [diff] [blame] | 1038 | |
| 1039 | // If this is a call to a function that returns an fp value on the floating |
| 1040 | // point stack, but where we prefer to use the value in xmm registers, copy |
| 1041 | // it out as F80 and use a truncate to move it from fp stack reg to xmm reg. |
Mon P Wang | 7e66510 | 2008-08-21 19:54:16 +0000 | [diff] [blame] | 1042 | if ((RVLocs[i].getLocReg() == X86::ST0 || |
| 1043 | RVLocs[i].getLocReg() == X86::ST1) && |
Chris Lattner | 8e6da15 | 2008-03-10 21:08:41 +0000 | [diff] [blame] | 1044 | isScalarFPTypeInSSEReg(RVLocs[i].getValVT())) { |
| 1045 | CopyVT = MVT::f80; |
Chris Lattner | 3085e15 | 2007-02-25 08:59:22 +0000 | [diff] [blame] | 1046 | } |
Chris Lattner | 3085e15 | 2007-02-25 08:59:22 +0000 | [diff] [blame] | 1047 | |
Chris Lattner | 8e6da15 | 2008-03-10 21:08:41 +0000 | [diff] [blame] | 1048 | Chain = DAG.getCopyFromReg(Chain, RVLocs[i].getLocReg(), |
| 1049 | CopyVT, InFlag).getValue(1); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1050 | SDValue Val = Chain.getValue(0); |
Chris Lattner | 8e6da15 | 2008-03-10 21:08:41 +0000 | [diff] [blame] | 1051 | InFlag = Chain.getValue(2); |
Chris Lattner | 112dedc | 2007-12-29 06:41:28 +0000 | [diff] [blame] | 1052 | |
Chris Lattner | 8e6da15 | 2008-03-10 21:08:41 +0000 | [diff] [blame] | 1053 | if (CopyVT != RVLocs[i].getValVT()) { |
| 1054 | // Round the F80 the right size, which also moves to the appropriate xmm |
| 1055 | // register. |
| 1056 | Val = DAG.getNode(ISD::FP_ROUND, RVLocs[i].getValVT(), Val, |
| 1057 | // This truncation won't change the value. |
| 1058 | DAG.getIntPtrConstant(1)); |
| 1059 | } |
Chris Lattner | d43d00c | 2008-01-24 08:07:48 +0000 | [diff] [blame] | 1060 | |
Chris Lattner | 8e6da15 | 2008-03-10 21:08:41 +0000 | [diff] [blame] | 1061 | ResultVals.push_back(Val); |
Chris Lattner | 3085e15 | 2007-02-25 08:59:22 +0000 | [diff] [blame] | 1062 | } |
Duncan Sands | 4bdcb61 | 2008-07-02 17:40:58 +0000 | [diff] [blame] | 1063 | |
Chris Lattner | 3085e15 | 2007-02-25 08:59:22 +0000 | [diff] [blame] | 1064 | // Merge everything together with a MERGE_VALUES node. |
| 1065 | ResultVals.push_back(Chain); |
Duncan Sands | aaffa05 | 2008-12-01 11:41:29 +0000 | [diff] [blame] | 1066 | return DAG.getNode(ISD::MERGE_VALUES, TheCall->getVTList(), &ResultVals[0], |
| 1067 | ResultVals.size()).getNode(); |
Chris Lattner | 2b02a44 | 2007-02-25 08:29:00 +0000 | [diff] [blame] | 1068 | } |
| 1069 | |
| 1070 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1071 | //===----------------------------------------------------------------------===// |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1072 | // C & StdCall & Fast Calling Convention implementation |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1073 | //===----------------------------------------------------------------------===// |
Anton Korobeynikov | b10308e | 2007-01-28 13:31:35 +0000 | [diff] [blame] | 1074 | // StdCall calling convention seems to be standard for many Windows' API |
| 1075 | // routines and around. It differs from C calling convention just a little: |
| 1076 | // callee should clean up the stack, not caller. Symbols should be also |
| 1077 | // decorated in some fancy way :) It doesn't support any vector arguments. |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1078 | // For info on fast calling convention see Fast Calling Convention (tail call) |
| 1079 | // implementation LowerX86_32FastCCCallTo. |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1080 | |
Evan Cheng | 85e3800 | 2006-04-27 05:35:28 +0000 | [diff] [blame] | 1081 | /// AddLiveIn - This helper function adds the specified physical register to the |
| 1082 | /// MachineFunction as a live in value. It also creates a corresponding virtual |
| 1083 | /// register for it. |
| 1084 | static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg, |
Anton Korobeynikov | b10308e | 2007-01-28 13:31:35 +0000 | [diff] [blame] | 1085 | const TargetRegisterClass *RC) { |
Evan Cheng | 85e3800 | 2006-04-27 05:35:28 +0000 | [diff] [blame] | 1086 | assert(RC->contains(PReg) && "Not the correct regclass!"); |
Chris Lattner | 84bc542 | 2007-12-31 04:13:23 +0000 | [diff] [blame] | 1087 | unsigned VReg = MF.getRegInfo().createVirtualRegister(RC); |
| 1088 | MF.getRegInfo().addLiveIn(PReg, VReg); |
Evan Cheng | 85e3800 | 2006-04-27 05:35:28 +0000 | [diff] [blame] | 1089 | return VReg; |
| 1090 | } |
| 1091 | |
Arnold Schwaighofer | 16a3e52 | 2008-02-26 17:50:59 +0000 | [diff] [blame] | 1092 | /// CallIsStructReturn - Determines whether a CALL node uses struct return |
| 1093 | /// semantics. |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1094 | static bool CallIsStructReturn(CallSDNode *TheCall) { |
| 1095 | unsigned NumOps = TheCall->getNumArgs(); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1096 | if (!NumOps) |
| 1097 | return false; |
Duncan Sands | 276dcbd | 2008-03-21 09:14:45 +0000 | [diff] [blame] | 1098 | |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1099 | return TheCall->getArgFlags(0).isSRet(); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1100 | } |
| 1101 | |
Arnold Schwaighofer | 16a3e52 | 2008-02-26 17:50:59 +0000 | [diff] [blame] | 1102 | /// ArgsAreStructReturn - Determines whether a FORMAL_ARGUMENTS node uses struct |
| 1103 | /// return semantics. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1104 | static bool ArgsAreStructReturn(SDValue Op) { |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 1105 | unsigned NumArgs = Op.getNode()->getNumValues() - 1; |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1106 | if (!NumArgs) |
| 1107 | return false; |
Duncan Sands | 276dcbd | 2008-03-21 09:14:45 +0000 | [diff] [blame] | 1108 | |
| 1109 | return cast<ARG_FLAGSSDNode>(Op.getOperand(3))->getArgFlags().isSRet(); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1110 | } |
| 1111 | |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1112 | /// IsCalleePop - Determines whether a CALL or FORMAL_ARGUMENTS node requires |
| 1113 | /// the callee to pop its own arguments. Callee pop is necessary to support tail |
Arnold Schwaighofer | 16a3e52 | 2008-02-26 17:50:59 +0000 | [diff] [blame] | 1114 | /// calls. |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1115 | bool X86TargetLowering::IsCalleePop(bool IsVarArg, unsigned CallingConv) { |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1116 | if (IsVarArg) |
| 1117 | return false; |
| 1118 | |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1119 | switch (CallingConv) { |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1120 | default: |
| 1121 | return false; |
| 1122 | case CallingConv::X86_StdCall: |
| 1123 | return !Subtarget->is64Bit(); |
| 1124 | case CallingConv::X86_FastCall: |
| 1125 | return !Subtarget->is64Bit(); |
| 1126 | case CallingConv::Fast: |
| 1127 | return PerformTailCallOpt; |
| 1128 | } |
| 1129 | } |
| 1130 | |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1131 | /// CCAssignFnForNode - Selects the correct CCAssignFn for a the |
| 1132 | /// given CallingConvention value. |
| 1133 | CCAssignFn *X86TargetLowering::CCAssignFnForNode(unsigned CC) const { |
Anton Korobeynikov | 7c1c261 | 2008-02-20 11:22:39 +0000 | [diff] [blame] | 1134 | if (Subtarget->is64Bit()) { |
Anton Korobeynikov | 1a979d9 | 2008-03-22 20:57:27 +0000 | [diff] [blame] | 1135 | if (Subtarget->isTargetWin64()) |
Anton Korobeynikov | 8f88cb0 | 2008-03-22 20:37:30 +0000 | [diff] [blame] | 1136 | return CC_X86_Win64_C; |
Evan Cheng | e9ac9e6 | 2008-09-07 09:07:23 +0000 | [diff] [blame] | 1137 | else if (CC == CallingConv::Fast && PerformTailCallOpt) |
| 1138 | return CC_X86_64_TailCall; |
| 1139 | else |
| 1140 | return CC_X86_64_C; |
Anton Korobeynikov | 7c1c261 | 2008-02-20 11:22:39 +0000 | [diff] [blame] | 1141 | } |
| 1142 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1143 | if (CC == CallingConv::X86_FastCall) |
| 1144 | return CC_X86_32_FastCall; |
Evan Cheng | b188dd9 | 2008-09-10 18:25:29 +0000 | [diff] [blame] | 1145 | else if (CC == CallingConv::Fast) |
| 1146 | return CC_X86_32_FastCC; |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1147 | else |
| 1148 | return CC_X86_32_C; |
| 1149 | } |
| 1150 | |
Arnold Schwaighofer | 16a3e52 | 2008-02-26 17:50:59 +0000 | [diff] [blame] | 1151 | /// NameDecorationForFORMAL_ARGUMENTS - Selects the appropriate decoration to |
| 1152 | /// apply to a MachineFunction containing a given FORMAL_ARGUMENTS node. |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1153 | NameDecorationStyle |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1154 | X86TargetLowering::NameDecorationForFORMAL_ARGUMENTS(SDValue Op) { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 1155 | unsigned CC = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1156 | if (CC == CallingConv::X86_FastCall) |
| 1157 | return FastCall; |
| 1158 | else if (CC == CallingConv::X86_StdCall) |
| 1159 | return StdCall; |
| 1160 | return None; |
| 1161 | } |
| 1162 | |
Arnold Schwaighofer | c8ab8cd | 2008-01-11 16:49:42 +0000 | [diff] [blame] | 1163 | |
Arnold Schwaighofer | 258bb1b | 2008-02-26 22:21:54 +0000 | [diff] [blame] | 1164 | /// CallRequiresGOTInRegister - Check whether the call requires the GOT pointer |
| 1165 | /// in a register before calling. |
| 1166 | bool X86TargetLowering::CallRequiresGOTPtrInReg(bool Is64Bit, bool IsTailCall) { |
| 1167 | return !IsTailCall && !Is64Bit && |
| 1168 | getTargetMachine().getRelocationModel() == Reloc::PIC_ && |
| 1169 | Subtarget->isPICStyleGOT(); |
| 1170 | } |
| 1171 | |
Arnold Schwaighofer | 258bb1b | 2008-02-26 22:21:54 +0000 | [diff] [blame] | 1172 | /// CallRequiresFnAddressInReg - Check whether the call requires the function |
| 1173 | /// address to be loaded in a register. |
| 1174 | bool |
| 1175 | X86TargetLowering::CallRequiresFnAddressInReg(bool Is64Bit, bool IsTailCall) { |
| 1176 | return !Is64Bit && IsTailCall && |
| 1177 | getTargetMachine().getRelocationModel() == Reloc::PIC_ && |
| 1178 | Subtarget->isPICStyleGOT(); |
| 1179 | } |
| 1180 | |
Arnold Schwaighofer | 16a3e52 | 2008-02-26 17:50:59 +0000 | [diff] [blame] | 1181 | /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified |
| 1182 | /// by "Src" to address "Dst" with size and alignment information specified by |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1183 | /// the specific parameter attribute. The copy will be passed as a byval |
| 1184 | /// function parameter. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1185 | static SDValue |
| 1186 | CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, |
Duncan Sands | 276dcbd | 2008-03-21 09:14:45 +0000 | [diff] [blame] | 1187 | ISD::ArgFlagsTy Flags, SelectionDAG &DAG) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1188 | SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32); |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 1189 | return DAG.getMemcpy(Chain, Dst, Src, SizeNode, Flags.getByValAlign(), |
Arnold Schwaighofer | 30e62c0 | 2008-04-30 09:16:33 +0000 | [diff] [blame] | 1190 | /*AlwaysInline=*/true, NULL, 0, NULL, 0); |
Arnold Schwaighofer | c8ab8cd | 2008-01-11 16:49:42 +0000 | [diff] [blame] | 1191 | } |
| 1192 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1193 | SDValue X86TargetLowering::LowerMemArgument(SDValue Op, SelectionDAG &DAG, |
Rafael Espindola | 7effac5 | 2007-09-14 15:48:13 +0000 | [diff] [blame] | 1194 | const CCValAssign &VA, |
| 1195 | MachineFrameInfo *MFI, |
Arnold Schwaighofer | 865c681 | 2008-02-26 09:19:59 +0000 | [diff] [blame] | 1196 | unsigned CC, |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1197 | SDValue Root, unsigned i) { |
Rafael Espindola | 7effac5 | 2007-09-14 15:48:13 +0000 | [diff] [blame] | 1198 | // Create the nodes corresponding to a load from this parameter slot. |
Duncan Sands | 276dcbd | 2008-03-21 09:14:45 +0000 | [diff] [blame] | 1199 | ISD::ArgFlagsTy Flags = |
| 1200 | cast<ARG_FLAGSSDNode>(Op.getOperand(3 + i))->getArgFlags(); |
Arnold Schwaighofer | 865c681 | 2008-02-26 09:19:59 +0000 | [diff] [blame] | 1201 | bool AlwaysUseMutable = (CC==CallingConv::Fast) && PerformTailCallOpt; |
Duncan Sands | 276dcbd | 2008-03-21 09:14:45 +0000 | [diff] [blame] | 1202 | bool isImmutable = !AlwaysUseMutable && !Flags.isByVal(); |
Evan Cheng | e70bb59 | 2008-01-10 02:24:25 +0000 | [diff] [blame] | 1203 | |
Arnold Schwaighofer | 865c681 | 2008-02-26 09:19:59 +0000 | [diff] [blame] | 1204 | // FIXME: For now, all byval parameter objects are marked mutable. This can be |
| 1205 | // changed with more analysis. |
| 1206 | // In case of tail call optimization mark all arguments mutable. Since they |
| 1207 | // could be overwritten by lowering of arguments in case of a tail call. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 1208 | int FI = MFI->CreateFixedObject(VA.getValVT().getSizeInBits()/8, |
Arnold Schwaighofer | 865c681 | 2008-02-26 09:19:59 +0000 | [diff] [blame] | 1209 | VA.getLocMemOffset(), isImmutable); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1210 | SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); |
Duncan Sands | 276dcbd | 2008-03-21 09:14:45 +0000 | [diff] [blame] | 1211 | if (Flags.isByVal()) |
Rafael Espindola | 7effac5 | 2007-09-14 15:48:13 +0000 | [diff] [blame] | 1212 | return FIN; |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 1213 | return DAG.getLoad(VA.getValVT(), Root, FIN, |
Dan Gohman | a54cf17 | 2008-07-11 22:44:52 +0000 | [diff] [blame] | 1214 | PseudoSourceValue::getFixedStack(FI), 0); |
Rafael Espindola | 7effac5 | 2007-09-14 15:48:13 +0000 | [diff] [blame] | 1215 | } |
| 1216 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1217 | SDValue |
| 1218 | X86TargetLowering::LowerFORMAL_ARGUMENTS(SDValue Op, SelectionDAG &DAG) { |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 1219 | MachineFunction &MF = DAG.getMachineFunction(); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1220 | X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); |
| 1221 | |
| 1222 | const Function* Fn = MF.getFunction(); |
| 1223 | if (Fn->hasExternalLinkage() && |
| 1224 | Subtarget->isTargetCygMing() && |
| 1225 | Fn->getName() == "main") |
| 1226 | FuncInfo->setForceFramePointer(true); |
| 1227 | |
| 1228 | // Decorate the function name. |
| 1229 | FuncInfo->setDecorationStyle(NameDecorationForFORMAL_ARGUMENTS(Op)); |
| 1230 | |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 1231 | MachineFrameInfo *MFI = MF.getFrameInfo(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1232 | SDValue Root = Op.getOperand(0); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 1233 | bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue() != 0; |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1234 | unsigned CC = MF.getFunction()->getCallingConv(); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1235 | bool Is64Bit = Subtarget->is64Bit(); |
Anton Korobeynikov | 998a5bc | 2008-04-27 23:15:03 +0000 | [diff] [blame] | 1236 | bool IsWin64 = Subtarget->isTargetWin64(); |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1237 | |
| 1238 | assert(!(isVarArg && CC == CallingConv::Fast) && |
| 1239 | "Var args not supported with calling convention fastcc"); |
| 1240 | |
Chris Lattner | 638402b | 2007-02-28 07:00:42 +0000 | [diff] [blame] | 1241 | // Assign locations to all of the incoming arguments. |
Chris Lattner | f39f771 | 2007-02-28 05:46:49 +0000 | [diff] [blame] | 1242 | SmallVector<CCValAssign, 16> ArgLocs; |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1243 | CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1244 | CCInfo.AnalyzeFormalArguments(Op.getNode(), CCAssignFnForNode(CC)); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1245 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1246 | SmallVector<SDValue, 8> ArgValues; |
Chris Lattner | f39f771 | 2007-02-28 05:46:49 +0000 | [diff] [blame] | 1247 | unsigned LastVal = ~0U; |
| 1248 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { |
| 1249 | CCValAssign &VA = ArgLocs[i]; |
| 1250 | // TODO: If an arg is passed in two places (e.g. reg and stack), skip later |
| 1251 | // places. |
| 1252 | assert(VA.getValNo() != LastVal && |
| 1253 | "Don't support value assigned to multiple locs yet"); |
| 1254 | LastVal = VA.getValNo(); |
| 1255 | |
| 1256 | if (VA.isRegLoc()) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 1257 | MVT RegVT = VA.getLocVT(); |
Chris Lattner | f39f771 | 2007-02-28 05:46:49 +0000 | [diff] [blame] | 1258 | TargetRegisterClass *RC; |
| 1259 | if (RegVT == MVT::i32) |
| 1260 | RC = X86::GR32RegisterClass; |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1261 | else if (Is64Bit && RegVT == MVT::i64) |
| 1262 | RC = X86::GR64RegisterClass; |
Dale Johannesen | e672af1 | 2008-02-05 20:46:33 +0000 | [diff] [blame] | 1263 | else if (RegVT == MVT::f32) |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1264 | RC = X86::FR32RegisterClass; |
Dale Johannesen | e672af1 | 2008-02-05 20:46:33 +0000 | [diff] [blame] | 1265 | else if (RegVT == MVT::f64) |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1266 | RC = X86::FR64RegisterClass; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 1267 | else if (RegVT.isVector() && RegVT.getSizeInBits() == 128) |
Evan Cheng | ee472b1 | 2008-04-25 07:56:45 +0000 | [diff] [blame] | 1268 | RC = X86::VR128RegisterClass; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 1269 | else if (RegVT.isVector()) { |
| 1270 | assert(RegVT.getSizeInBits() == 64); |
Evan Cheng | ee472b1 | 2008-04-25 07:56:45 +0000 | [diff] [blame] | 1271 | if (!Is64Bit) |
| 1272 | RC = X86::VR64RegisterClass; // MMX values are passed in MMXs. |
| 1273 | else { |
| 1274 | // Darwin calling convention passes MMX values in either GPRs or |
| 1275 | // XMMs in x86-64. Other targets pass them in memory. |
| 1276 | if (RegVT != MVT::v1i64 && Subtarget->hasSSE2()) { |
| 1277 | RC = X86::VR128RegisterClass; // MMX values are passed in XMMs. |
| 1278 | RegVT = MVT::v2i64; |
| 1279 | } else { |
| 1280 | RC = X86::GR64RegisterClass; // v1i64 values are passed in GPRs. |
| 1281 | RegVT = MVT::i64; |
| 1282 | } |
| 1283 | } |
| 1284 | } else { |
| 1285 | assert(0 && "Unknown argument type!"); |
Anton Korobeynikov | b10308e | 2007-01-28 13:31:35 +0000 | [diff] [blame] | 1286 | } |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1287 | |
Chris Lattner | 82932a5 | 2007-03-02 05:12:29 +0000 | [diff] [blame] | 1288 | unsigned Reg = AddLiveIn(DAG.getMachineFunction(), VA.getLocReg(), RC); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1289 | SDValue ArgValue = DAG.getCopyFromReg(Root, Reg, RegVT); |
Chris Lattner | f39f771 | 2007-02-28 05:46:49 +0000 | [diff] [blame] | 1290 | |
| 1291 | // If this is an 8 or 16-bit value, it is really passed promoted to 32 |
| 1292 | // bits. Insert an assert[sz]ext to capture this, then truncate to the |
| 1293 | // right size. |
| 1294 | if (VA.getLocInfo() == CCValAssign::SExt) |
| 1295 | ArgValue = DAG.getNode(ISD::AssertSext, RegVT, ArgValue, |
| 1296 | DAG.getValueType(VA.getValVT())); |
| 1297 | else if (VA.getLocInfo() == CCValAssign::ZExt) |
| 1298 | ArgValue = DAG.getNode(ISD::AssertZext, RegVT, ArgValue, |
| 1299 | DAG.getValueType(VA.getValVT())); |
| 1300 | |
| 1301 | if (VA.getLocInfo() != CCValAssign::Full) |
| 1302 | ArgValue = DAG.getNode(ISD::TRUNCATE, VA.getValVT(), ArgValue); |
| 1303 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1304 | // Handle MMX values passed in GPRs. |
Evan Cheng | 44c0fd1 | 2008-04-25 20:13:28 +0000 | [diff] [blame] | 1305 | if (Is64Bit && RegVT != VA.getLocVT()) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 1306 | if (RegVT.getSizeInBits() == 64 && RC == X86::GR64RegisterClass) |
Evan Cheng | 44c0fd1 | 2008-04-25 20:13:28 +0000 | [diff] [blame] | 1307 | ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); |
| 1308 | else if (RC == X86::VR128RegisterClass) { |
| 1309 | ArgValue = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i64, ArgValue, |
| 1310 | DAG.getConstant(0, MVT::i64)); |
| 1311 | ArgValue = DAG.getNode(ISD::BIT_CONVERT, VA.getLocVT(), ArgValue); |
| 1312 | } |
| 1313 | } |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1314 | |
Chris Lattner | f39f771 | 2007-02-28 05:46:49 +0000 | [diff] [blame] | 1315 | ArgValues.push_back(ArgValue); |
| 1316 | } else { |
| 1317 | assert(VA.isMemLoc()); |
Arnold Schwaighofer | 865c681 | 2008-02-26 09:19:59 +0000 | [diff] [blame] | 1318 | ArgValues.push_back(LowerMemArgument(Op, DAG, VA, MFI, CC, Root, i)); |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 1319 | } |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 1320 | } |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1321 | |
Dan Gohman | 61a9213 | 2008-04-21 23:59:07 +0000 | [diff] [blame] | 1322 | // The x86-64 ABI for returning structs by value requires that we copy |
| 1323 | // the sret argument into %rax for the return. Save the argument into |
| 1324 | // a virtual register so that we can access it from the return points. |
| 1325 | if (Is64Bit && DAG.getMachineFunction().getFunction()->hasStructRetAttr()) { |
| 1326 | MachineFunction &MF = DAG.getMachineFunction(); |
| 1327 | X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); |
| 1328 | unsigned Reg = FuncInfo->getSRetReturnReg(); |
| 1329 | if (!Reg) { |
| 1330 | Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64)); |
| 1331 | FuncInfo->setSRetReturnReg(Reg); |
| 1332 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1333 | SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), Reg, ArgValues[0]); |
Dan Gohman | 61a9213 | 2008-04-21 23:59:07 +0000 | [diff] [blame] | 1334 | Root = DAG.getNode(ISD::TokenFactor, MVT::Other, Copy, Root); |
| 1335 | } |
| 1336 | |
Chris Lattner | f39f771 | 2007-02-28 05:46:49 +0000 | [diff] [blame] | 1337 | unsigned StackSize = CCInfo.getNextStackOffset(); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1338 | // align stack specially for tail calls |
Evan Cheng | e9ac9e6 | 2008-09-07 09:07:23 +0000 | [diff] [blame] | 1339 | if (PerformTailCallOpt && CC == CallingConv::Fast) |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1340 | StackSize = GetAlignedArgumentStackSize(StackSize, DAG); |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1341 | |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 1342 | // If the function takes variable number of arguments, make a frame index for |
| 1343 | // the start of the first vararg value... for expansion of llvm.va_start. |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1344 | if (isVarArg) { |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1345 | if (Is64Bit || CC != CallingConv::X86_FastCall) { |
| 1346 | VarArgsFrameIndex = MFI->CreateFixedObject(1, StackSize); |
| 1347 | } |
| 1348 | if (Is64Bit) { |
Anton Korobeynikov | 998a5bc | 2008-04-27 23:15:03 +0000 | [diff] [blame] | 1349 | unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0; |
| 1350 | |
| 1351 | // FIXME: We should really autogenerate these arrays |
| 1352 | static const unsigned GPR64ArgRegsWin64[] = { |
| 1353 | X86::RCX, X86::RDX, X86::R8, X86::R9 |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1354 | }; |
Anton Korobeynikov | 998a5bc | 2008-04-27 23:15:03 +0000 | [diff] [blame] | 1355 | static const unsigned XMMArgRegsWin64[] = { |
| 1356 | X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3 |
| 1357 | }; |
| 1358 | static const unsigned GPR64ArgRegs64Bit[] = { |
| 1359 | X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9 |
| 1360 | }; |
| 1361 | static const unsigned XMMArgRegs64Bit[] = { |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1362 | X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, |
| 1363 | X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 |
| 1364 | }; |
Anton Korobeynikov | 998a5bc | 2008-04-27 23:15:03 +0000 | [diff] [blame] | 1365 | const unsigned *GPR64ArgRegs, *XMMArgRegs; |
| 1366 | |
| 1367 | if (IsWin64) { |
| 1368 | TotalNumIntRegs = 4; TotalNumXMMRegs = 4; |
| 1369 | GPR64ArgRegs = GPR64ArgRegsWin64; |
| 1370 | XMMArgRegs = XMMArgRegsWin64; |
| 1371 | } else { |
| 1372 | TotalNumIntRegs = 6; TotalNumXMMRegs = 8; |
| 1373 | GPR64ArgRegs = GPR64ArgRegs64Bit; |
| 1374 | XMMArgRegs = XMMArgRegs64Bit; |
| 1375 | } |
| 1376 | unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs, |
| 1377 | TotalNumIntRegs); |
| 1378 | unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, |
| 1379 | TotalNumXMMRegs); |
| 1380 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1381 | // For X86-64, if there are vararg parameters that are passed via |
| 1382 | // registers, then we must store them to their spots on the stack so they |
| 1383 | // may be loaded by deferencing the result of va_next. |
| 1384 | VarArgsGPOffset = NumIntRegs * 8; |
Anton Korobeynikov | 998a5bc | 2008-04-27 23:15:03 +0000 | [diff] [blame] | 1385 | VarArgsFPOffset = TotalNumIntRegs * 8 + NumXMMRegs * 16; |
| 1386 | RegSaveFrameIndex = MFI->CreateStackObject(TotalNumIntRegs * 8 + |
| 1387 | TotalNumXMMRegs * 16, 16); |
| 1388 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1389 | // Store the integer parameter registers. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1390 | SmallVector<SDValue, 8> MemOps; |
| 1391 | SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); |
| 1392 | SDValue FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 1393 | DAG.getIntPtrConstant(VarArgsGPOffset)); |
Anton Korobeynikov | 998a5bc | 2008-04-27 23:15:03 +0000 | [diff] [blame] | 1394 | for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) { |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1395 | unsigned VReg = AddLiveIn(MF, GPR64ArgRegs[NumIntRegs], |
| 1396 | X86::GR64RegisterClass); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1397 | SDValue Val = DAG.getCopyFromReg(Root, VReg, MVT::i64); |
| 1398 | SDValue Store = |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 1399 | DAG.getStore(Val.getValue(1), Val, FIN, |
Dan Gohman | a54cf17 | 2008-07-11 22:44:52 +0000 | [diff] [blame] | 1400 | PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1401 | MemOps.push_back(Store); |
| 1402 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 1403 | DAG.getIntPtrConstant(8)); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1404 | } |
Anton Korobeynikov | 998a5bc | 2008-04-27 23:15:03 +0000 | [diff] [blame] | 1405 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1406 | // Now store the XMM (fp + vector) parameter registers. |
| 1407 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), RSFIN, |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 1408 | DAG.getIntPtrConstant(VarArgsFPOffset)); |
Anton Korobeynikov | 998a5bc | 2008-04-27 23:15:03 +0000 | [diff] [blame] | 1409 | for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) { |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1410 | unsigned VReg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], |
| 1411 | X86::VR128RegisterClass); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1412 | SDValue Val = DAG.getCopyFromReg(Root, VReg, MVT::v4f32); |
| 1413 | SDValue Store = |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 1414 | DAG.getStore(Val.getValue(1), Val, FIN, |
Dan Gohman | a54cf17 | 2008-07-11 22:44:52 +0000 | [diff] [blame] | 1415 | PseudoSourceValue::getFixedStack(RegSaveFrameIndex), 0); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1416 | MemOps.push_back(Store); |
| 1417 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 1418 | DAG.getIntPtrConstant(16)); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1419 | } |
| 1420 | if (!MemOps.empty()) |
| 1421 | Root = DAG.getNode(ISD::TokenFactor, MVT::Other, |
| 1422 | &MemOps[0], MemOps.size()); |
| 1423 | } |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1424 | } |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1425 | |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1426 | ArgValues.push_back(Root); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1427 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1428 | // Some CCs need callee pop. |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1429 | if (IsCalleePop(isVarArg, CC)) { |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1430 | BytesToPopOnReturn = StackSize; // Callee pops everything. |
Anton Korobeynikov | b10308e | 2007-01-28 13:31:35 +0000 | [diff] [blame] | 1431 | BytesCallerReserves = 0; |
| 1432 | } else { |
Anton Korobeynikov | 1d9bacc | 2007-03-06 08:12:33 +0000 | [diff] [blame] | 1433 | BytesToPopOnReturn = 0; // Callee pops nothing. |
Chris Lattner | f39f771 | 2007-02-28 05:46:49 +0000 | [diff] [blame] | 1434 | // If this is an sret function, the return should pop the hidden pointer. |
Evan Cheng | b188dd9 | 2008-09-10 18:25:29 +0000 | [diff] [blame] | 1435 | if (!Is64Bit && CC != CallingConv::Fast && ArgsAreStructReturn(Op)) |
Chris Lattner | f39f771 | 2007-02-28 05:46:49 +0000 | [diff] [blame] | 1436 | BytesToPopOnReturn = 4; |
Chris Lattner | f39f771 | 2007-02-28 05:46:49 +0000 | [diff] [blame] | 1437 | BytesCallerReserves = StackSize; |
Anton Korobeynikov | b10308e | 2007-01-28 13:31:35 +0000 | [diff] [blame] | 1438 | } |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1439 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1440 | if (!Is64Bit) { |
| 1441 | RegSaveFrameIndex = 0xAAAAAAA; // RegSaveFrameIndex is X86-64 only. |
| 1442 | if (CC == CallingConv::X86_FastCall) |
| 1443 | VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs. |
| 1444 | } |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1445 | |
Anton Korobeynikov | a2780e1 | 2007-08-15 17:12:32 +0000 | [diff] [blame] | 1446 | FuncInfo->setBytesToPopOnReturn(BytesToPopOnReturn); |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 1447 | |
Evan Cheng | 25caf63 | 2006-05-23 21:06:34 +0000 | [diff] [blame] | 1448 | // Return the new list of results. |
Duncan Sands | aaffa05 | 2008-12-01 11:41:29 +0000 | [diff] [blame] | 1449 | return DAG.getNode(ISD::MERGE_VALUES, Op.getNode()->getVTList(), |
| 1450 | &ArgValues[0], ArgValues.size()).getValue(Op.getResNo()); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1451 | } |
| 1452 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1453 | SDValue |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1454 | X86TargetLowering::LowerMemOpCallTo(CallSDNode *TheCall, SelectionDAG &DAG, |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1455 | const SDValue &StackPtr, |
Evan Cheng | dffbd83 | 2008-01-10 00:09:10 +0000 | [diff] [blame] | 1456 | const CCValAssign &VA, |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1457 | SDValue Chain, |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1458 | SDValue Arg, ISD::ArgFlagsTy Flags) { |
Dan Gohman | 4fdad17 | 2008-02-07 16:28:05 +0000 | [diff] [blame] | 1459 | unsigned LocMemOffset = VA.getLocMemOffset(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1460 | SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset); |
Evan Cheng | dffbd83 | 2008-01-10 00:09:10 +0000 | [diff] [blame] | 1461 | PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff); |
Duncan Sands | 276dcbd | 2008-03-21 09:14:45 +0000 | [diff] [blame] | 1462 | if (Flags.isByVal()) { |
Evan Cheng | 8e5712b | 2008-01-12 01:08:07 +0000 | [diff] [blame] | 1463 | return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG); |
Evan Cheng | dffbd83 | 2008-01-10 00:09:10 +0000 | [diff] [blame] | 1464 | } |
Dan Gohman | 4fdad17 | 2008-02-07 16:28:05 +0000 | [diff] [blame] | 1465 | return DAG.getStore(Chain, Arg, PtrOff, |
Dan Gohman | 3069b87 | 2008-02-07 18:41:25 +0000 | [diff] [blame] | 1466 | PseudoSourceValue::getStack(), LocMemOffset); |
Evan Cheng | dffbd83 | 2008-01-10 00:09:10 +0000 | [diff] [blame] | 1467 | } |
| 1468 | |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1469 | /// EmitTailCallLoadRetAddr - Emit a load of return adress if tail call |
| 1470 | /// optimization is performed and it is required. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1471 | SDValue |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1472 | X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG, |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1473 | SDValue &OutRetAddr, |
| 1474 | SDValue Chain, |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1475 | bool IsTailCall, |
| 1476 | bool Is64Bit, |
| 1477 | int FPDiff) { |
| 1478 | if (!IsTailCall || FPDiff==0) return Chain; |
| 1479 | |
| 1480 | // Adjust the Return address stack slot. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 1481 | MVT VT = getPointerTy(); |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1482 | OutRetAddr = getReturnAddressFrameIndex(DAG); |
| 1483 | // Load the "old" Return address. |
| 1484 | OutRetAddr = DAG.getLoad(VT, Chain,OutRetAddr, NULL, 0); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 1485 | return SDValue(OutRetAddr.getNode(), 1); |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1486 | } |
| 1487 | |
| 1488 | /// EmitTailCallStoreRetAddr - Emit a store of the return adress if tail call |
| 1489 | /// optimization is performed and it is required (FPDiff!=0). |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1490 | static SDValue |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1491 | EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF, |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1492 | SDValue Chain, SDValue RetAddrFrIdx, |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1493 | bool Is64Bit, int FPDiff) { |
| 1494 | // Store the return address to the appropriate stack slot. |
| 1495 | if (!FPDiff) return Chain; |
| 1496 | // Calculate the new stack slot for the return address. |
| 1497 | int SlotSize = Is64Bit ? 8 : 4; |
| 1498 | int NewReturnAddrFI = |
| 1499 | MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 1500 | MVT VT = Is64Bit ? MVT::i64 : MVT::i32; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1501 | SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT); |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1502 | Chain = DAG.getStore(Chain, RetAddrFrIdx, NewRetAddrFrIdx, |
Dan Gohman | a54cf17 | 2008-07-11 22:44:52 +0000 | [diff] [blame] | 1503 | PseudoSourceValue::getFixedStack(NewReturnAddrFI), 0); |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1504 | return Chain; |
| 1505 | } |
| 1506 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1507 | SDValue X86TargetLowering::LowerCALL(SDValue Op, SelectionDAG &DAG) { |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1508 | MachineFunction &MF = DAG.getMachineFunction(); |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1509 | CallSDNode *TheCall = cast<CallSDNode>(Op.getNode()); |
| 1510 | SDValue Chain = TheCall->getChain(); |
| 1511 | unsigned CC = TheCall->getCallingConv(); |
| 1512 | bool isVarArg = TheCall->isVarArg(); |
| 1513 | bool IsTailCall = TheCall->isTailCall() && |
| 1514 | CC == CallingConv::Fast && PerformTailCallOpt; |
| 1515 | SDValue Callee = TheCall->getCallee(); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1516 | bool Is64Bit = Subtarget->is64Bit(); |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1517 | bool IsStructRet = CallIsStructReturn(TheCall); |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1518 | |
| 1519 | assert(!(isVarArg && CC == CallingConv::Fast) && |
| 1520 | "Var args not supported with calling convention fastcc"); |
| 1521 | |
Chris Lattner | 638402b | 2007-02-28 07:00:42 +0000 | [diff] [blame] | 1522 | // Analyze operands of the call, assigning locations to each operand. |
Chris Lattner | 423c5f4 | 2007-02-28 05:31:48 +0000 | [diff] [blame] | 1523 | SmallVector<CCValAssign, 16> ArgLocs; |
Chris Lattner | 52387be | 2007-06-19 00:13:10 +0000 | [diff] [blame] | 1524 | CCState CCInfo(CC, isVarArg, getTargetMachine(), ArgLocs); |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1525 | CCInfo.AnalyzeCallOperands(TheCall, CCAssignFnForNode(CC)); |
Anton Korobeynikov | b10308e | 2007-01-28 13:31:35 +0000 | [diff] [blame] | 1526 | |
Chris Lattner | 423c5f4 | 2007-02-28 05:31:48 +0000 | [diff] [blame] | 1527 | // Get a count of how many bytes are to be pushed on the stack. |
| 1528 | unsigned NumBytes = CCInfo.getNextStackOffset(); |
Arnold Schwaighofer | 1fdc40f | 2008-09-11 20:28:43 +0000 | [diff] [blame] | 1529 | if (PerformTailCallOpt && CC == CallingConv::Fast) |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1530 | NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1531 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1532 | int FPDiff = 0; |
| 1533 | if (IsTailCall) { |
| 1534 | // Lower arguments at fp - stackoffset + fpdiff. |
| 1535 | unsigned NumBytesCallerPushed = |
| 1536 | MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn(); |
| 1537 | FPDiff = NumBytesCallerPushed - NumBytes; |
| 1538 | |
| 1539 | // Set the delta of movement of the returnaddr stackslot. |
| 1540 | // But only set if delta is greater than previous delta. |
| 1541 | if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta())) |
| 1542 | MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff); |
| 1543 | } |
| 1544 | |
Chris Lattner | e563bbc | 2008-10-11 22:08:30 +0000 | [diff] [blame] | 1545 | Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true)); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1546 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1547 | SDValue RetAddrFrIdx; |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1548 | // Load return adress for tail calls. |
| 1549 | Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, IsTailCall, Is64Bit, |
| 1550 | FPDiff); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1551 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1552 | SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; |
| 1553 | SmallVector<SDValue, 8> MemOpChains; |
| 1554 | SDValue StackPtr; |
Chris Lattner | 423c5f4 | 2007-02-28 05:31:48 +0000 | [diff] [blame] | 1555 | |
Arnold Schwaighofer | 30e62c0 | 2008-04-30 09:16:33 +0000 | [diff] [blame] | 1556 | // Walk the register/memloc assignments, inserting copies/loads. In the case |
| 1557 | // of tail call optimization arguments are handle later. |
Chris Lattner | 423c5f4 | 2007-02-28 05:31:48 +0000 | [diff] [blame] | 1558 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { |
| 1559 | CCValAssign &VA = ArgLocs[i]; |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1560 | SDValue Arg = TheCall->getArg(i); |
| 1561 | ISD::ArgFlagsTy Flags = TheCall->getArgFlags(i); |
| 1562 | bool isByVal = Flags.isByVal(); |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1563 | |
Chris Lattner | 423c5f4 | 2007-02-28 05:31:48 +0000 | [diff] [blame] | 1564 | // Promote the value if needed. |
| 1565 | switch (VA.getLocInfo()) { |
| 1566 | default: assert(0 && "Unknown loc info!"); |
| 1567 | case CCValAssign::Full: break; |
| 1568 | case CCValAssign::SExt: |
| 1569 | Arg = DAG.getNode(ISD::SIGN_EXTEND, VA.getLocVT(), Arg); |
| 1570 | break; |
| 1571 | case CCValAssign::ZExt: |
| 1572 | Arg = DAG.getNode(ISD::ZERO_EXTEND, VA.getLocVT(), Arg); |
| 1573 | break; |
| 1574 | case CCValAssign::AExt: |
| 1575 | Arg = DAG.getNode(ISD::ANY_EXTEND, VA.getLocVT(), Arg); |
| 1576 | break; |
Evan Cheng | 6b5783d | 2006-05-25 18:56:34 +0000 | [diff] [blame] | 1577 | } |
Chris Lattner | 423c5f4 | 2007-02-28 05:31:48 +0000 | [diff] [blame] | 1578 | |
| 1579 | if (VA.isRegLoc()) { |
Evan Cheng | 10e8642 | 2008-04-25 19:11:04 +0000 | [diff] [blame] | 1580 | if (Is64Bit) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 1581 | MVT RegVT = VA.getLocVT(); |
| 1582 | if (RegVT.isVector() && RegVT.getSizeInBits() == 64) |
Evan Cheng | 10e8642 | 2008-04-25 19:11:04 +0000 | [diff] [blame] | 1583 | switch (VA.getLocReg()) { |
| 1584 | default: |
| 1585 | break; |
| 1586 | case X86::RDI: case X86::RSI: case X86::RDX: case X86::RCX: |
| 1587 | case X86::R8: { |
| 1588 | // Special case: passing MMX values in GPR registers. |
| 1589 | Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); |
| 1590 | break; |
| 1591 | } |
| 1592 | case X86::XMM0: case X86::XMM1: case X86::XMM2: case X86::XMM3: |
| 1593 | case X86::XMM4: case X86::XMM5: case X86::XMM6: case X86::XMM7: { |
| 1594 | // Special case: passing MMX values in XMM registers. |
| 1595 | Arg = DAG.getNode(ISD::BIT_CONVERT, MVT::i64, Arg); |
| 1596 | Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Arg); |
| 1597 | Arg = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, |
| 1598 | DAG.getNode(ISD::UNDEF, MVT::v2i64), Arg, |
| 1599 | getMOVLMask(2, DAG)); |
| 1600 | break; |
| 1601 | } |
| 1602 | } |
| 1603 | } |
Chris Lattner | 423c5f4 | 2007-02-28 05:31:48 +0000 | [diff] [blame] | 1604 | RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); |
| 1605 | } else { |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1606 | if (!IsTailCall || (IsTailCall && isByVal)) { |
Arnold Schwaighofer | c8ab8cd | 2008-01-11 16:49:42 +0000 | [diff] [blame] | 1607 | assert(VA.isMemLoc()); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 1608 | if (StackPtr.getNode() == 0) |
Arnold Schwaighofer | c8ab8cd | 2008-01-11 16:49:42 +0000 | [diff] [blame] | 1609 | StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); |
| 1610 | |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1611 | MemOpChains.push_back(LowerMemOpCallTo(TheCall, DAG, StackPtr, VA, |
| 1612 | Chain, Arg, Flags)); |
Arnold Schwaighofer | c8ab8cd | 2008-01-11 16:49:42 +0000 | [diff] [blame] | 1613 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1614 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1615 | } |
Chris Lattner | c0bdf34 | 2007-02-28 05:39:26 +0000 | [diff] [blame] | 1616 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1617 | if (!MemOpChains.empty()) |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 1618 | Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, |
| 1619 | &MemOpChains[0], MemOpChains.size()); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1620 | |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 1621 | // Build a sequence of copy-to-reg nodes chained together with token chain |
| 1622 | // and flag operands which copy the outgoing args into registers. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1623 | SDValue InFlag; |
Arnold Schwaighofer | 30e62c0 | 2008-04-30 09:16:33 +0000 | [diff] [blame] | 1624 | // Tail call byval lowering might overwrite argument registers so in case of |
| 1625 | // tail call optimization the copies to registers are lowered later. |
| 1626 | if (!IsTailCall) |
| 1627 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { |
| 1628 | Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, |
| 1629 | InFlag); |
| 1630 | InFlag = Chain.getValue(1); |
| 1631 | } |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1632 | |
Evan Cheng | f468471 | 2007-02-21 21:18:14 +0000 | [diff] [blame] | 1633 | // ELF / PIC requires GOT in the EBX register before function calls via PLT |
Arnold Schwaighofer | a2a4b47 | 2008-02-26 10:21:54 +0000 | [diff] [blame] | 1634 | // GOT pointer. |
Arnold Schwaighofer | 258bb1b | 2008-02-26 22:21:54 +0000 | [diff] [blame] | 1635 | if (CallRequiresGOTPtrInReg(Is64Bit, IsTailCall)) { |
| 1636 | Chain = DAG.getCopyToReg(Chain, X86::EBX, |
| 1637 | DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), |
| 1638 | InFlag); |
| 1639 | InFlag = Chain.getValue(1); |
| 1640 | } |
Arnold Schwaighofer | a2a4b47 | 2008-02-26 10:21:54 +0000 | [diff] [blame] | 1641 | // If we are tail calling and generating PIC/GOT style code load the address |
| 1642 | // of the callee into ecx. The value in ecx is used as target of the tail |
| 1643 | // jump. This is done to circumvent the ebx/callee-saved problem for tail |
| 1644 | // calls on PIC/GOT architectures. Normally we would just put the address of |
| 1645 | // GOT into ebx and then call target@PLT. But for tail callss ebx would be |
| 1646 | // restored (since ebx is callee saved) before jumping to the target@PLT. |
Arnold Schwaighofer | 258bb1b | 2008-02-26 22:21:54 +0000 | [diff] [blame] | 1647 | if (CallRequiresFnAddressInReg(Is64Bit, IsTailCall)) { |
Arnold Schwaighofer | a2a4b47 | 2008-02-26 10:21:54 +0000 | [diff] [blame] | 1648 | // Note: The actual moving to ecx is done further down. |
| 1649 | GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); |
Evan Cheng | da43bcf | 2008-09-24 00:05:32 +0000 | [diff] [blame] | 1650 | if (G && !G->getGlobal()->hasHiddenVisibility() && |
Arnold Schwaighofer | a2a4b47 | 2008-02-26 10:21:54 +0000 | [diff] [blame] | 1651 | !G->getGlobal()->hasProtectedVisibility()) |
| 1652 | Callee = LowerGlobalAddress(Callee, DAG); |
Bill Wendling | 056292f | 2008-09-16 21:48:12 +0000 | [diff] [blame] | 1653 | else if (isa<ExternalSymbolSDNode>(Callee)) |
| 1654 | Callee = LowerExternalSymbol(Callee,DAG); |
Anton Korobeynikov | 7f70559 | 2007-01-12 19:20:47 +0000 | [diff] [blame] | 1655 | } |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1656 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1657 | if (Is64Bit && isVarArg) { |
| 1658 | // From AMD64 ABI document: |
| 1659 | // For calls that may call functions that use varargs or stdargs |
| 1660 | // (prototype-less calls or calls to functions containing ellipsis (...) in |
| 1661 | // the declaration) %al is used as hidden argument to specify the number |
| 1662 | // of SSE registers used. The contents of %al do not need to match exactly |
| 1663 | // the number of registers, but must be an ubound on the number of SSE |
| 1664 | // registers used and is in the range 0 - 8 inclusive. |
Anton Korobeynikov | 998a5bc | 2008-04-27 23:15:03 +0000 | [diff] [blame] | 1665 | |
| 1666 | // FIXME: Verify this on Win64 |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1667 | // Count the number of XMM registers allocated. |
| 1668 | static const unsigned XMMArgRegs[] = { |
| 1669 | X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3, |
| 1670 | X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7 |
| 1671 | }; |
| 1672 | unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8); |
| 1673 | |
| 1674 | Chain = DAG.getCopyToReg(Chain, X86::AL, |
| 1675 | DAG.getConstant(NumXMMRegs, MVT::i8), InFlag); |
| 1676 | InFlag = Chain.getValue(1); |
| 1677 | } |
| 1678 | |
Arnold Schwaighofer | 865c681 | 2008-02-26 09:19:59 +0000 | [diff] [blame] | 1679 | |
Arnold Schwaighofer | c8ab8cd | 2008-01-11 16:49:42 +0000 | [diff] [blame] | 1680 | // For tail calls lower the arguments to the 'real' stack slot. |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1681 | if (IsTailCall) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1682 | SmallVector<SDValue, 8> MemOpChains2; |
| 1683 | SDValue FIN; |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1684 | int FI = 0; |
Arnold Schwaighofer | 865c681 | 2008-02-26 09:19:59 +0000 | [diff] [blame] | 1685 | // Do not flag preceeding copytoreg stuff together with the following stuff. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1686 | InFlag = SDValue(); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1687 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { |
| 1688 | CCValAssign &VA = ArgLocs[i]; |
| 1689 | if (!VA.isRegLoc()) { |
Arnold Schwaighofer | c8ab8cd | 2008-01-11 16:49:42 +0000 | [diff] [blame] | 1690 | assert(VA.isMemLoc()); |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1691 | SDValue Arg = TheCall->getArg(i); |
| 1692 | ISD::ArgFlagsTy Flags = TheCall->getArgFlags(i); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1693 | // Create frame index. |
| 1694 | int32_t Offset = VA.getLocMemOffset()+FPDiff; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 1695 | uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8; |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1696 | FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset); |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1697 | FIN = DAG.getFrameIndex(FI, getPointerTy()); |
Arnold Schwaighofer | c8ab8cd | 2008-01-11 16:49:42 +0000 | [diff] [blame] | 1698 | |
Duncan Sands | 276dcbd | 2008-03-21 09:14:45 +0000 | [diff] [blame] | 1699 | if (Flags.isByVal()) { |
Evan Cheng | 8e5712b | 2008-01-12 01:08:07 +0000 | [diff] [blame] | 1700 | // Copy relative to framepointer. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1701 | SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset()); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 1702 | if (StackPtr.getNode() == 0) |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1703 | StackPtr = DAG.getCopyFromReg(Chain, X86StackPtr, getPointerTy()); |
| 1704 | Source = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, Source); |
| 1705 | |
| 1706 | MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN, Chain, |
Evan Cheng | 8e5712b | 2008-01-12 01:08:07 +0000 | [diff] [blame] | 1707 | Flags, DAG)); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1708 | } else { |
Evan Cheng | 8e5712b | 2008-01-12 01:08:07 +0000 | [diff] [blame] | 1709 | // Store relative to framepointer. |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 1710 | MemOpChains2.push_back( |
Arnold Schwaighofer | 865c681 | 2008-02-26 09:19:59 +0000 | [diff] [blame] | 1711 | DAG.getStore(Chain, Arg, FIN, |
Dan Gohman | a54cf17 | 2008-07-11 22:44:52 +0000 | [diff] [blame] | 1712 | PseudoSourceValue::getFixedStack(FI), 0)); |
Arnold Schwaighofer | c8ab8cd | 2008-01-11 16:49:42 +0000 | [diff] [blame] | 1713 | } |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1714 | } |
| 1715 | } |
| 1716 | |
| 1717 | if (!MemOpChains2.empty()) |
| 1718 | Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, |
Arnold Schwaighofer | 719eb02 | 2008-01-11 14:34:56 +0000 | [diff] [blame] | 1719 | &MemOpChains2[0], MemOpChains2.size()); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1720 | |
Arnold Schwaighofer | 30e62c0 | 2008-04-30 09:16:33 +0000 | [diff] [blame] | 1721 | // Copy arguments to their registers. |
| 1722 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { |
| 1723 | Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second, |
| 1724 | InFlag); |
| 1725 | InFlag = Chain.getValue(1); |
| 1726 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1727 | InFlag =SDValue(); |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1728 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1729 | // Store the return address to the appropriate stack slot. |
Arnold Schwaighofer | 4b5324a | 2008-04-12 18:11:06 +0000 | [diff] [blame] | 1730 | Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit, |
| 1731 | FPDiff); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1732 | } |
| 1733 | |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 1734 | // If the callee is a GlobalAddress node (quite common, every direct call is) |
| 1735 | // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. |
Anton Korobeynikov | a598685 | 2006-11-20 10:46:14 +0000 | [diff] [blame] | 1736 | if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { |
Anton Korobeynikov | 2b2bc68 | 2006-12-22 22:29:05 +0000 | [diff] [blame] | 1737 | // We should use extra load for direct calls to dllimported functions in |
| 1738 | // non-JIT mode. |
Evan Cheng | 817a6a9 | 2008-07-16 01:34:02 +0000 | [diff] [blame] | 1739 | if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(), |
| 1740 | getTargetMachine(), true)) |
Dan Gohman | 6520e20 | 2008-10-18 02:06:02 +0000 | [diff] [blame] | 1741 | Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy(), |
| 1742 | G->getOffset()); |
Bill Wendling | 056292f | 2008-09-16 21:48:12 +0000 | [diff] [blame] | 1743 | } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { |
| 1744 | Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy()); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1745 | } else if (IsTailCall) { |
Arnold Schwaighofer | 290ae03 | 2008-09-22 14:50:07 +0000 | [diff] [blame] | 1746 | unsigned Opc = Is64Bit ? X86::R9 : X86::EAX; |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1747 | |
| 1748 | Chain = DAG.getCopyToReg(Chain, |
Arnold Schwaighofer | a2a4b47 | 2008-02-26 10:21:54 +0000 | [diff] [blame] | 1749 | DAG.getRegister(Opc, getPointerTy()), |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1750 | Callee,InFlag); |
| 1751 | Callee = DAG.getRegister(Opc, getPointerTy()); |
| 1752 | // Add register as live out. |
| 1753 | DAG.getMachineFunction().getRegInfo().addLiveOut(Opc); |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1754 | } |
| 1755 | |
Chris Lattner | d96d072 | 2007-02-25 06:40:16 +0000 | [diff] [blame] | 1756 | // Returns a chain & a flag for retval copy to use. |
| 1757 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1758 | SmallVector<SDValue, 8> Ops; |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1759 | |
| 1760 | if (IsTailCall) { |
| 1761 | Ops.push_back(Chain); |
Chris Lattner | e563bbc | 2008-10-11 22:08:30 +0000 | [diff] [blame] | 1762 | Ops.push_back(DAG.getIntPtrConstant(NumBytes, true)); |
| 1763 | Ops.push_back(DAG.getIntPtrConstant(0, true)); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 1764 | if (InFlag.getNode()) |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1765 | Ops.push_back(InFlag); |
| 1766 | Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size()); |
| 1767 | InFlag = Chain.getValue(1); |
| 1768 | |
| 1769 | // Returns a chain & a flag for retval copy to use. |
| 1770 | NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); |
| 1771 | Ops.clear(); |
| 1772 | } |
| 1773 | |
Nate Begeman | 4c5dcf5 | 2006-02-17 00:03:04 +0000 | [diff] [blame] | 1774 | Ops.push_back(Chain); |
| 1775 | Ops.push_back(Callee); |
Evan Cheng | b69d113 | 2006-06-14 18:17:40 +0000 | [diff] [blame] | 1776 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1777 | if (IsTailCall) |
| 1778 | Ops.push_back(DAG.getConstant(FPDiff, MVT::i32)); |
Evan Cheng | f468471 | 2007-02-21 21:18:14 +0000 | [diff] [blame] | 1779 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1780 | // Add argument registers to the end of the list so that they are known live |
| 1781 | // into the call. |
Evan Cheng | 9b44944 | 2008-01-07 23:08:23 +0000 | [diff] [blame] | 1782 | for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) |
| 1783 | Ops.push_back(DAG.getRegister(RegsToPass[i].first, |
| 1784 | RegsToPass[i].second.getValueType())); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1785 | |
Evan Cheng | 586ccac | 2008-03-18 23:36:35 +0000 | [diff] [blame] | 1786 | // Add an implicit use GOT pointer in EBX. |
| 1787 | if (!IsTailCall && !Is64Bit && |
| 1788 | getTargetMachine().getRelocationModel() == Reloc::PIC_ && |
| 1789 | Subtarget->isPICStyleGOT()) |
| 1790 | Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy())); |
| 1791 | |
| 1792 | // Add an implicit use of AL for x86 vararg functions. |
| 1793 | if (Is64Bit && isVarArg) |
| 1794 | Ops.push_back(DAG.getRegister(X86::AL, MVT::i8)); |
| 1795 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 1796 | if (InFlag.getNode()) |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 1797 | Ops.push_back(InFlag); |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1798 | |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1799 | if (IsTailCall) { |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 1800 | assert(InFlag.getNode() && |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1801 | "Flag must be set. Depend on flag being set in LowerRET"); |
| 1802 | Chain = DAG.getNode(X86ISD::TAILCALL, |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1803 | TheCall->getVTList(), &Ops[0], Ops.size()); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1804 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 1805 | return SDValue(Chain.getNode(), Op.getResNo()); |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1806 | } |
| 1807 | |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1808 | Chain = DAG.getNode(X86ISD::CALL, NodeTys, &Ops[0], Ops.size()); |
Evan Cheng | 347d5f7 | 2006-04-28 21:29:37 +0000 | [diff] [blame] | 1809 | InFlag = Chain.getValue(1); |
Evan Cheng | d90eb7f | 2006-01-05 00:27:02 +0000 | [diff] [blame] | 1810 | |
Chris Lattner | 2d29709 | 2006-05-23 18:50:38 +0000 | [diff] [blame] | 1811 | // Create the CALLSEQ_END node. |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1812 | unsigned NumBytesForCalleeToPush; |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1813 | if (IsCalleePop(isVarArg, CC)) |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1814 | NumBytesForCalleeToPush = NumBytes; // Callee pops everything |
Evan Cheng | b188dd9 | 2008-09-10 18:25:29 +0000 | [diff] [blame] | 1815 | else if (!Is64Bit && CC != CallingConv::Fast && IsStructRet) |
Anton Korobeynikov | b10308e | 2007-01-28 13:31:35 +0000 | [diff] [blame] | 1816 | // If this is is a call to a struct-return function, the callee |
| 1817 | // pops the hidden struct pointer, so we have to push it back. |
| 1818 | // This is common for Darwin/X86, Linux & Mingw32 targets. |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1819 | NumBytesForCalleeToPush = 4; |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1820 | else |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1821 | NumBytesForCalleeToPush = 0; // Callee pops nothing. |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1822 | |
Gordon Henriksen | ae636f8 | 2008-01-03 16:47:34 +0000 | [diff] [blame] | 1823 | // Returns a flag for retval copy to use. |
Bill Wendling | 0f8d9c0 | 2007-11-13 00:44:25 +0000 | [diff] [blame] | 1824 | Chain = DAG.getCALLSEQ_END(Chain, |
Chris Lattner | e563bbc | 2008-10-11 22:08:30 +0000 | [diff] [blame] | 1825 | DAG.getIntPtrConstant(NumBytes, true), |
| 1826 | DAG.getIntPtrConstant(NumBytesForCalleeToPush, |
| 1827 | true), |
Bill Wendling | 0f8d9c0 | 2007-11-13 00:44:25 +0000 | [diff] [blame] | 1828 | InFlag); |
Chris Lattner | 3085e15 | 2007-02-25 08:59:22 +0000 | [diff] [blame] | 1829 | InFlag = Chain.getValue(1); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1830 | |
Chris Lattner | 3085e15 | 2007-02-25 08:59:22 +0000 | [diff] [blame] | 1831 | // Handle result values, copying them out of physregs into vregs that we |
| 1832 | // return. |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1833 | return SDValue(LowerCallResult(Chain, InFlag, TheCall, CC, DAG), |
Gabor Greif | 327ef03 | 2008-08-28 23:19:51 +0000 | [diff] [blame] | 1834 | Op.getResNo()); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1835 | } |
| 1836 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1837 | |
| 1838 | //===----------------------------------------------------------------------===// |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1839 | // Fast Calling Convention (tail call) implementation |
| 1840 | //===----------------------------------------------------------------------===// |
| 1841 | |
| 1842 | // Like std call, callee cleans arguments, convention except that ECX is |
| 1843 | // reserved for storing the tail called function address. Only 2 registers are |
| 1844 | // free for argument passing (inreg). Tail call optimization is performed |
| 1845 | // provided: |
| 1846 | // * tailcallopt is enabled |
| 1847 | // * caller/callee are fastcc |
Arnold Schwaighofer | a2a4b47 | 2008-02-26 10:21:54 +0000 | [diff] [blame] | 1848 | // On X86_64 architecture with GOT-style position independent code only local |
| 1849 | // (within module) calls are supported at the moment. |
Arnold Schwaighofer | 48abc5c | 2007-10-12 21:30:57 +0000 | [diff] [blame] | 1850 | // To keep the stack aligned according to platform abi the function |
| 1851 | // GetAlignedArgumentStackSize ensures that argument delta is always multiples |
| 1852 | // of stack alignment. (Dynamic linkers need this - darwin's dyld for example) |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1853 | // If a tail called function callee has more arguments than the caller the |
| 1854 | // caller needs to make sure that there is room to move the RETADDR to. This is |
Arnold Schwaighofer | 48abc5c | 2007-10-12 21:30:57 +0000 | [diff] [blame] | 1855 | // achieved by reserving an area the size of the argument delta right after the |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1856 | // original REtADDR, but before the saved framepointer or the spilled registers |
| 1857 | // e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4) |
| 1858 | // stack layout: |
| 1859 | // arg1 |
| 1860 | // arg2 |
| 1861 | // RETADDR |
| 1862 | // [ new RETADDR |
| 1863 | // move area ] |
| 1864 | // (possible EBP) |
| 1865 | // ESI |
| 1866 | // EDI |
| 1867 | // local1 .. |
| 1868 | |
| 1869 | /// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned |
| 1870 | /// for a 16 byte align requirement. |
| 1871 | unsigned X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize, |
| 1872 | SelectionDAG& DAG) { |
Evan Cheng | e9ac9e6 | 2008-09-07 09:07:23 +0000 | [diff] [blame] | 1873 | MachineFunction &MF = DAG.getMachineFunction(); |
| 1874 | const TargetMachine &TM = MF.getTarget(); |
| 1875 | const TargetFrameInfo &TFI = *TM.getFrameInfo(); |
| 1876 | unsigned StackAlignment = TFI.getStackAlignment(); |
| 1877 | uint64_t AlignMask = StackAlignment - 1; |
| 1878 | int64_t Offset = StackSize; |
Anton Korobeynikov | bff66b0 | 2008-09-09 18:22:57 +0000 | [diff] [blame] | 1879 | uint64_t SlotSize = TD->getPointerSize(); |
Evan Cheng | e9ac9e6 | 2008-09-07 09:07:23 +0000 | [diff] [blame] | 1880 | if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) { |
| 1881 | // Number smaller than 12 so just add the difference. |
| 1882 | Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask)); |
| 1883 | } else { |
| 1884 | // Mask out lower bits, add stackalignment once plus the 12 bytes. |
| 1885 | Offset = ((~AlignMask) & Offset) + StackAlignment + |
| 1886 | (StackAlignment-SlotSize); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1887 | } |
Evan Cheng | e9ac9e6 | 2008-09-07 09:07:23 +0000 | [diff] [blame] | 1888 | return Offset; |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1889 | } |
| 1890 | |
| 1891 | /// IsEligibleForTailCallElimination - Check to see whether the next instruction |
Evan Cheng | 9df7dc5 | 2007-11-02 01:26:22 +0000 | [diff] [blame] | 1892 | /// following the call is a return. A function is eligible if caller/callee |
| 1893 | /// calling conventions match, currently only fastcc supports tail calls, and |
| 1894 | /// the function CALL is immediatly followed by a RET. |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1895 | bool X86TargetLowering::IsEligibleForTailCallOptimization(CallSDNode *TheCall, |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1896 | SDValue Ret, |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1897 | SelectionDAG& DAG) const { |
Evan Cheng | 9df7dc5 | 2007-11-02 01:26:22 +0000 | [diff] [blame] | 1898 | if (!PerformTailCallOpt) |
| 1899 | return false; |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1900 | |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1901 | if (CheckTailCallReturnConstraints(TheCall, Ret)) { |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1902 | MachineFunction &MF = DAG.getMachineFunction(); |
| 1903 | unsigned CallerCC = MF.getFunction()->getCallingConv(); |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1904 | unsigned CalleeCC= TheCall->getCallingConv(); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1905 | if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { |
Dan Gohman | 095cc29 | 2008-09-13 01:54:27 +0000 | [diff] [blame] | 1906 | SDValue Callee = TheCall->getCallee(); |
Arnold Schwaighofer | a2a4b47 | 2008-02-26 10:21:54 +0000 | [diff] [blame] | 1907 | // On x86/32Bit PIC/GOT tail calls are supported. |
Evan Cheng | 9df7dc5 | 2007-11-02 01:26:22 +0000 | [diff] [blame] | 1908 | if (getTargetMachine().getRelocationModel() != Reloc::PIC_ || |
Arnold Schwaighofer | a2a4b47 | 2008-02-26 10:21:54 +0000 | [diff] [blame] | 1909 | !Subtarget->isPICStyleGOT()|| !Subtarget->is64Bit()) |
Evan Cheng | 9df7dc5 | 2007-11-02 01:26:22 +0000 | [diff] [blame] | 1910 | return true; |
| 1911 | |
Arnold Schwaighofer | a2a4b47 | 2008-02-26 10:21:54 +0000 | [diff] [blame] | 1912 | // Can only do local tail calls (in same module, hidden or protected) on |
| 1913 | // x86_64 PIC/GOT at the moment. |
Gordon Henriksen | 8673766 | 2008-01-05 16:56:59 +0000 | [diff] [blame] | 1914 | if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) |
| 1915 | return G->getGlobal()->hasHiddenVisibility() |
| 1916 | || G->getGlobal()->hasProtectedVisibility(); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1917 | } |
| 1918 | } |
Evan Cheng | 9df7dc5 | 2007-11-02 01:26:22 +0000 | [diff] [blame] | 1919 | |
| 1920 | return false; |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 1921 | } |
| 1922 | |
Dan Gohman | 3df24e6 | 2008-09-03 23:12:08 +0000 | [diff] [blame] | 1923 | FastISel * |
| 1924 | X86TargetLowering::createFastISel(MachineFunction &mf, |
Dan Gohman | d57dd5f | 2008-09-23 21:53:34 +0000 | [diff] [blame] | 1925 | MachineModuleInfo *mmo, |
Dan Gohman | 3df24e6 | 2008-09-03 23:12:08 +0000 | [diff] [blame] | 1926 | DenseMap<const Value *, unsigned> &vm, |
| 1927 | DenseMap<const BasicBlock *, |
Dan Gohman | 0586d91 | 2008-09-10 20:11:02 +0000 | [diff] [blame] | 1928 | MachineBasicBlock *> &bm, |
Dan Gohman | dd5b58a | 2008-10-14 23:54:11 +0000 | [diff] [blame] | 1929 | DenseMap<const AllocaInst *, int> &am |
| 1930 | #ifndef NDEBUG |
| 1931 | , SmallSet<Instruction*, 8> &cil |
| 1932 | #endif |
| 1933 | ) { |
| 1934 | return X86::createFastISel(mf, mmo, vm, bm, am |
| 1935 | #ifndef NDEBUG |
| 1936 | , cil |
| 1937 | #endif |
| 1938 | ); |
Dan Gohman | d9f3c48 | 2008-08-19 21:32:53 +0000 | [diff] [blame] | 1939 | } |
| 1940 | |
| 1941 | |
Chris Lattner | fcf1a3d | 2007-02-28 06:10:12 +0000 | [diff] [blame] | 1942 | //===----------------------------------------------------------------------===// |
| 1943 | // Other Lowering Hooks |
| 1944 | //===----------------------------------------------------------------------===// |
| 1945 | |
| 1946 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 1947 | SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) { |
Anton Korobeynikov | a2780e1 | 2007-08-15 17:12:32 +0000 | [diff] [blame] | 1948 | MachineFunction &MF = DAG.getMachineFunction(); |
| 1949 | X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); |
| 1950 | int ReturnAddrIndex = FuncInfo->getRAIndex(); |
Anton Korobeynikov | bff66b0 | 2008-09-09 18:22:57 +0000 | [diff] [blame] | 1951 | uint64_t SlotSize = TD->getPointerSize(); |
Anton Korobeynikov | a2780e1 | 2007-08-15 17:12:32 +0000 | [diff] [blame] | 1952 | |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1953 | if (ReturnAddrIndex == 0) { |
| 1954 | // Set up a frame object for the return address. |
Anton Korobeynikov | bff66b0 | 2008-09-09 18:22:57 +0000 | [diff] [blame] | 1955 | ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(SlotSize, -SlotSize); |
Anton Korobeynikov | a2780e1 | 2007-08-15 17:12:32 +0000 | [diff] [blame] | 1956 | FuncInfo->setRAIndex(ReturnAddrIndex); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1957 | } |
| 1958 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 1959 | return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy()); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 1960 | } |
| 1961 | |
| 1962 | |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 1963 | /// TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86 |
| 1964 | /// specific condition code, returning the condition code and the LHS/RHS of the |
| 1965 | /// comparison to make. |
| 1966 | static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP, |
| 1967 | SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) { |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 1968 | if (!isFP) { |
Chris Lattner | bfd68a7 | 2006-09-13 17:04:54 +0000 | [diff] [blame] | 1969 | if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { |
| 1970 | if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { |
| 1971 | // X > -1 -> X == 0, jump !sign. |
| 1972 | RHS = DAG.getConstant(0, RHS.getValueType()); |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 1973 | return X86::COND_NS; |
Chris Lattner | bfd68a7 | 2006-09-13 17:04:54 +0000 | [diff] [blame] | 1974 | } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { |
| 1975 | // X < 0 -> X == 0, jump on sign. |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 1976 | return X86::COND_S; |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 1977 | } else if (SetCCOpcode == ISD::SETLT && RHSC->getZExtValue() == 1) { |
Dan Gohman | 5f6913c | 2007-09-17 14:49:27 +0000 | [diff] [blame] | 1978 | // X < 1 -> X <= 0 |
| 1979 | RHS = DAG.getConstant(0, RHS.getValueType()); |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 1980 | return X86::COND_LE; |
Chris Lattner | bfd68a7 | 2006-09-13 17:04:54 +0000 | [diff] [blame] | 1981 | } |
Chris Lattner | f957051 | 2006-09-13 03:22:10 +0000 | [diff] [blame] | 1982 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 1983 | |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 1984 | switch (SetCCOpcode) { |
Chris Lattner | 4c78e02 | 2008-12-23 23:42:27 +0000 | [diff] [blame] | 1985 | default: assert(0 && "Invalid integer condition!"); |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 1986 | case ISD::SETEQ: return X86::COND_E; |
| 1987 | case ISD::SETGT: return X86::COND_G; |
| 1988 | case ISD::SETGE: return X86::COND_GE; |
| 1989 | case ISD::SETLT: return X86::COND_L; |
| 1990 | case ISD::SETLE: return X86::COND_LE; |
| 1991 | case ISD::SETNE: return X86::COND_NE; |
| 1992 | case ISD::SETULT: return X86::COND_B; |
| 1993 | case ISD::SETUGT: return X86::COND_A; |
| 1994 | case ISD::SETULE: return X86::COND_BE; |
| 1995 | case ISD::SETUGE: return X86::COND_AE; |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 1996 | } |
Chris Lattner | 4c78e02 | 2008-12-23 23:42:27 +0000 | [diff] [blame] | 1997 | } |
| 1998 | |
| 1999 | // First determine if it is required or is profitable to flip the operands. |
Duncan Sands | 4047f4a | 2008-10-24 13:03:10 +0000 | [diff] [blame] | 2000 | |
Chris Lattner | 4c78e02 | 2008-12-23 23:42:27 +0000 | [diff] [blame] | 2001 | // If LHS is a foldable load, but RHS is not, flip the condition. |
| 2002 | if ((ISD::isNON_EXTLoad(LHS.getNode()) && LHS.hasOneUse()) && |
| 2003 | !(ISD::isNON_EXTLoad(RHS.getNode()) && RHS.hasOneUse())) { |
| 2004 | SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode); |
| 2005 | std::swap(LHS, RHS); |
Evan Cheng | 4d46d0a | 2008-08-28 23:48:31 +0000 | [diff] [blame] | 2006 | } |
| 2007 | |
Chris Lattner | 4c78e02 | 2008-12-23 23:42:27 +0000 | [diff] [blame] | 2008 | switch (SetCCOpcode) { |
| 2009 | default: break; |
| 2010 | case ISD::SETOLT: |
| 2011 | case ISD::SETOLE: |
| 2012 | case ISD::SETUGT: |
| 2013 | case ISD::SETUGE: |
| 2014 | std::swap(LHS, RHS); |
| 2015 | break; |
| 2016 | } |
| 2017 | |
| 2018 | // On a floating point condition, the flags are set as follows: |
| 2019 | // ZF PF CF op |
| 2020 | // 0 | 0 | 0 | X > Y |
| 2021 | // 0 | 0 | 1 | X < Y |
| 2022 | // 1 | 0 | 0 | X == Y |
| 2023 | // 1 | 1 | 1 | unordered |
| 2024 | switch (SetCCOpcode) { |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 2025 | default: assert(0 && "Condcode should be pre-legalized away"); |
Chris Lattner | 4c78e02 | 2008-12-23 23:42:27 +0000 | [diff] [blame] | 2026 | case ISD::SETUEQ: |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 2027 | case ISD::SETEQ: return X86::COND_E; |
Chris Lattner | 4c78e02 | 2008-12-23 23:42:27 +0000 | [diff] [blame] | 2028 | case ISD::SETOLT: // flipped |
| 2029 | case ISD::SETOGT: |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 2030 | case ISD::SETGT: return X86::COND_A; |
Chris Lattner | 4c78e02 | 2008-12-23 23:42:27 +0000 | [diff] [blame] | 2031 | case ISD::SETOLE: // flipped |
| 2032 | case ISD::SETOGE: |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 2033 | case ISD::SETGE: return X86::COND_AE; |
Chris Lattner | 4c78e02 | 2008-12-23 23:42:27 +0000 | [diff] [blame] | 2034 | case ISD::SETUGT: // flipped |
| 2035 | case ISD::SETULT: |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 2036 | case ISD::SETLT: return X86::COND_B; |
Chris Lattner | 4c78e02 | 2008-12-23 23:42:27 +0000 | [diff] [blame] | 2037 | case ISD::SETUGE: // flipped |
| 2038 | case ISD::SETULE: |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 2039 | case ISD::SETLE: return X86::COND_BE; |
Chris Lattner | 4c78e02 | 2008-12-23 23:42:27 +0000 | [diff] [blame] | 2040 | case ISD::SETONE: |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 2041 | case ISD::SETNE: return X86::COND_NE; |
| 2042 | case ISD::SETUO: return X86::COND_P; |
| 2043 | case ISD::SETO: return X86::COND_NP; |
Chris Lattner | 4c78e02 | 2008-12-23 23:42:27 +0000 | [diff] [blame] | 2044 | } |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 2045 | } |
| 2046 | |
Evan Cheng | 4a46080 | 2006-01-11 00:33:36 +0000 | [diff] [blame] | 2047 | /// hasFPCMov - is there a floating point cmov for the specific X86 condition |
| 2048 | /// code. Current x86 isa includes the following FP cmov instructions: |
Evan Cheng | aaca22c | 2006-01-10 20:26:56 +0000 | [diff] [blame] | 2049 | /// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. |
Evan Cheng | 4a46080 | 2006-01-11 00:33:36 +0000 | [diff] [blame] | 2050 | static bool hasFPCMov(unsigned X86CC) { |
Evan Cheng | aaca22c | 2006-01-10 20:26:56 +0000 | [diff] [blame] | 2051 | switch (X86CC) { |
| 2052 | default: |
| 2053 | return false; |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 2054 | case X86::COND_B: |
| 2055 | case X86::COND_BE: |
| 2056 | case X86::COND_E: |
| 2057 | case X86::COND_P: |
| 2058 | case X86::COND_A: |
| 2059 | case X86::COND_AE: |
| 2060 | case X86::COND_NE: |
| 2061 | case X86::COND_NP: |
Evan Cheng | aaca22c | 2006-01-10 20:26:56 +0000 | [diff] [blame] | 2062 | return true; |
| 2063 | } |
| 2064 | } |
| 2065 | |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2066 | /// isUndefOrInRange - Op is either an undef node or a ConstantSDNode. Return |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2067 | /// true if Op is undef or if its value falls within the specified range (L, H]. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2068 | static bool isUndefOrInRange(SDValue Op, unsigned Low, unsigned Hi) { |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2069 | if (Op.getOpcode() == ISD::UNDEF) |
| 2070 | return true; |
| 2071 | |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2072 | unsigned Val = cast<ConstantSDNode>(Op)->getZExtValue(); |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2073 | return (Val >= Low && Val < Hi); |
| 2074 | } |
| 2075 | |
| 2076 | /// isUndefOrEqual - Op is either an undef node or a ConstantSDNode. Return |
| 2077 | /// true if Op is undef or if its value equal to the specified value. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2078 | static bool isUndefOrEqual(SDValue Op, unsigned Val) { |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2079 | if (Op.getOpcode() == ISD::UNDEF) |
| 2080 | return true; |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2081 | return cast<ConstantSDNode>(Op)->getZExtValue() == Val; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2082 | } |
| 2083 | |
Evan Cheng | 0188ecb | 2006-03-22 18:59:22 +0000 | [diff] [blame] | 2084 | /// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2085 | /// specifies a shuffle of elements that is suitable for input to PSHUFD. |
| 2086 | bool X86::isPSHUFDMask(SDNode *N) { |
| 2087 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2088 | |
Dan Gohman | 7f55fcb | 2007-08-02 21:17:01 +0000 | [diff] [blame] | 2089 | if (N->getNumOperands() != 2 && N->getNumOperands() != 4) |
Evan Cheng | 0188ecb | 2006-03-22 18:59:22 +0000 | [diff] [blame] | 2090 | return false; |
| 2091 | |
| 2092 | // Check if the value doesn't reference the second vector. |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2093 | for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2094 | SDValue Arg = N->getOperand(i); |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2095 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2096 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2097 | if (cast<ConstantSDNode>(Arg)->getZExtValue() >= e) |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2098 | return false; |
| 2099 | } |
| 2100 | |
| 2101 | return true; |
| 2102 | } |
| 2103 | |
| 2104 | /// isPSHUFHWMask - Return true if the specified VECTOR_SHUFFLE operand |
Evan Cheng | c21a053 | 2006-04-05 01:47:37 +0000 | [diff] [blame] | 2105 | /// specifies a shuffle of elements that is suitable for input to PSHUFHW. |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2106 | bool X86::isPSHUFHWMask(SDNode *N) { |
| 2107 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2108 | |
| 2109 | if (N->getNumOperands() != 8) |
| 2110 | return false; |
| 2111 | |
| 2112 | // Lower quadword copied in order. |
| 2113 | for (unsigned i = 0; i != 4; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2114 | SDValue Arg = N->getOperand(i); |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2115 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2116 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2117 | if (cast<ConstantSDNode>(Arg)->getZExtValue() != i) |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2118 | return false; |
| 2119 | } |
| 2120 | |
| 2121 | // Upper quadword shuffled. |
| 2122 | for (unsigned i = 4; i != 8; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2123 | SDValue Arg = N->getOperand(i); |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2124 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2125 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2126 | unsigned Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2127 | if (Val < 4 || Val > 7) |
| 2128 | return false; |
| 2129 | } |
| 2130 | |
| 2131 | return true; |
| 2132 | } |
| 2133 | |
| 2134 | /// isPSHUFLWMask - Return true if the specified VECTOR_SHUFFLE operand |
Evan Cheng | c21a053 | 2006-04-05 01:47:37 +0000 | [diff] [blame] | 2135 | /// specifies a shuffle of elements that is suitable for input to PSHUFLW. |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2136 | bool X86::isPSHUFLWMask(SDNode *N) { |
| 2137 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2138 | |
| 2139 | if (N->getNumOperands() != 8) |
| 2140 | return false; |
| 2141 | |
| 2142 | // Upper quadword copied in order. |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2143 | for (unsigned i = 4; i != 8; ++i) |
| 2144 | if (!isUndefOrEqual(N->getOperand(i), i)) |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2145 | return false; |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2146 | |
| 2147 | // Lower quadword shuffled. |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2148 | for (unsigned i = 0; i != 4; ++i) |
| 2149 | if (!isUndefOrInRange(N->getOperand(i), 0, 4)) |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2150 | return false; |
Evan Cheng | 0188ecb | 2006-03-22 18:59:22 +0000 | [diff] [blame] | 2151 | |
| 2152 | return true; |
| 2153 | } |
| 2154 | |
Evan Cheng | 14aed5e | 2006-03-24 01:18:28 +0000 | [diff] [blame] | 2155 | /// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2156 | /// specifies a shuffle of elements that is suitable for input to SHUFP*. |
Roman Levenstein | 9cac525 | 2008-04-16 16:15:27 +0000 | [diff] [blame] | 2157 | static bool isSHUFPMask(SDOperandPtr Elems, unsigned NumElems) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2158 | if (NumElems != 2 && NumElems != 4) return false; |
Evan Cheng | 14aed5e | 2006-03-24 01:18:28 +0000 | [diff] [blame] | 2159 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2160 | unsigned Half = NumElems / 2; |
| 2161 | for (unsigned i = 0; i < Half; ++i) |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2162 | if (!isUndefOrInRange(Elems[i], 0, NumElems)) |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2163 | return false; |
| 2164 | for (unsigned i = Half; i < NumElems; ++i) |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2165 | if (!isUndefOrInRange(Elems[i], NumElems, NumElems*2)) |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2166 | return false; |
Evan Cheng | 14aed5e | 2006-03-24 01:18:28 +0000 | [diff] [blame] | 2167 | |
| 2168 | return true; |
| 2169 | } |
| 2170 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2171 | bool X86::isSHUFPMask(SDNode *N) { |
| 2172 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2173 | return ::isSHUFPMask(N->op_begin(), N->getNumOperands()); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2174 | } |
| 2175 | |
Evan Cheng | 213d2cf | 2007-05-17 18:45:50 +0000 | [diff] [blame] | 2176 | /// isCommutedSHUFP - Returns true if the shuffle mask is exactly |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2177 | /// the reverse of what x86 shuffles want. x86 shuffles requires the lower |
| 2178 | /// half elements to come from vector 1 (which would equal the dest.) and |
| 2179 | /// the upper half to come from vector 2. |
Roman Levenstein | 9cac525 | 2008-04-16 16:15:27 +0000 | [diff] [blame] | 2180 | static bool isCommutedSHUFP(SDOperandPtr Ops, unsigned NumOps) { |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2181 | if (NumOps != 2 && NumOps != 4) return false; |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2182 | |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2183 | unsigned Half = NumOps / 2; |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2184 | for (unsigned i = 0; i < Half; ++i) |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2185 | if (!isUndefOrInRange(Ops[i], NumOps, NumOps*2)) |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2186 | return false; |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2187 | for (unsigned i = Half; i < NumOps; ++i) |
| 2188 | if (!isUndefOrInRange(Ops[i], 0, NumOps)) |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2189 | return false; |
| 2190 | return true; |
| 2191 | } |
| 2192 | |
| 2193 | static bool isCommutedSHUFP(SDNode *N) { |
| 2194 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2195 | return isCommutedSHUFP(N->op_begin(), N->getNumOperands()); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2196 | } |
| 2197 | |
Evan Cheng | 2c0dbd0 | 2006-03-24 02:58:06 +0000 | [diff] [blame] | 2198 | /// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2199 | /// specifies a shuffle of elements that is suitable for input to MOVHLPS. |
| 2200 | bool X86::isMOVHLPSMask(SDNode *N) { |
| 2201 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2202 | |
Evan Cheng | 2064a2b | 2006-03-28 06:50:32 +0000 | [diff] [blame] | 2203 | if (N->getNumOperands() != 4) |
Evan Cheng | 2c0dbd0 | 2006-03-24 02:58:06 +0000 | [diff] [blame] | 2204 | return false; |
| 2205 | |
Evan Cheng | 2064a2b | 2006-03-28 06:50:32 +0000 | [diff] [blame] | 2206 | // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2207 | return isUndefOrEqual(N->getOperand(0), 6) && |
| 2208 | isUndefOrEqual(N->getOperand(1), 7) && |
| 2209 | isUndefOrEqual(N->getOperand(2), 2) && |
| 2210 | isUndefOrEqual(N->getOperand(3), 3); |
Evan Cheng | 2064a2b | 2006-03-28 06:50:32 +0000 | [diff] [blame] | 2211 | } |
| 2212 | |
Evan Cheng | 6e56e2c | 2006-11-07 22:14:24 +0000 | [diff] [blame] | 2213 | /// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form |
| 2214 | /// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, |
| 2215 | /// <2, 3, 2, 3> |
| 2216 | bool X86::isMOVHLPS_v_undef_Mask(SDNode *N) { |
| 2217 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2218 | |
| 2219 | if (N->getNumOperands() != 4) |
| 2220 | return false; |
| 2221 | |
| 2222 | // Expect bit0 == 2, bit1 == 3, bit2 == 2, bit3 == 3 |
| 2223 | return isUndefOrEqual(N->getOperand(0), 2) && |
| 2224 | isUndefOrEqual(N->getOperand(1), 3) && |
| 2225 | isUndefOrEqual(N->getOperand(2), 2) && |
| 2226 | isUndefOrEqual(N->getOperand(3), 3); |
| 2227 | } |
| 2228 | |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2229 | /// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2230 | /// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. |
| 2231 | bool X86::isMOVLPMask(SDNode *N) { |
| 2232 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2233 | |
| 2234 | unsigned NumElems = N->getNumOperands(); |
| 2235 | if (NumElems != 2 && NumElems != 4) |
| 2236 | return false; |
| 2237 | |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2238 | for (unsigned i = 0; i < NumElems/2; ++i) |
| 2239 | if (!isUndefOrEqual(N->getOperand(i), i + NumElems)) |
| 2240 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2241 | |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2242 | for (unsigned i = NumElems/2; i < NumElems; ++i) |
| 2243 | if (!isUndefOrEqual(N->getOperand(i), i)) |
| 2244 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2245 | |
| 2246 | return true; |
| 2247 | } |
| 2248 | |
| 2249 | /// isMOVHPMask - Return true if the specified VECTOR_SHUFFLE operand |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 2250 | /// specifies a shuffle of elements that is suitable for input to MOVHP{S|D} |
| 2251 | /// and MOVLHPS. |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2252 | bool X86::isMOVHPMask(SDNode *N) { |
| 2253 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2254 | |
| 2255 | unsigned NumElems = N->getNumOperands(); |
| 2256 | if (NumElems != 2 && NumElems != 4) |
| 2257 | return false; |
| 2258 | |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2259 | for (unsigned i = 0; i < NumElems/2; ++i) |
| 2260 | if (!isUndefOrEqual(N->getOperand(i), i)) |
| 2261 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2262 | |
| 2263 | for (unsigned i = 0; i < NumElems/2; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2264 | SDValue Arg = N->getOperand(i + NumElems/2); |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2265 | if (!isUndefOrEqual(Arg, i + NumElems)) |
| 2266 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2267 | } |
| 2268 | |
| 2269 | return true; |
| 2270 | } |
| 2271 | |
Evan Cheng | 0038e59 | 2006-03-28 00:39:58 +0000 | [diff] [blame] | 2272 | /// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2273 | /// specifies a shuffle of elements that is suitable for input to UNPCKL. |
Roman Levenstein | 9cac525 | 2008-04-16 16:15:27 +0000 | [diff] [blame] | 2274 | bool static isUNPCKLMask(SDOperandPtr Elts, unsigned NumElts, |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2275 | bool V2IsSplat = false) { |
| 2276 | if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) |
Evan Cheng | 0038e59 | 2006-03-28 00:39:58 +0000 | [diff] [blame] | 2277 | return false; |
| 2278 | |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2279 | for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2280 | SDValue BitI = Elts[i]; |
| 2281 | SDValue BitI1 = Elts[i+1]; |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2282 | if (!isUndefOrEqual(BitI, j)) |
| 2283 | return false; |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2284 | if (V2IsSplat) { |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2285 | if (isUndefOrEqual(BitI1, NumElts)) |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2286 | return false; |
| 2287 | } else { |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2288 | if (!isUndefOrEqual(BitI1, j + NumElts)) |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2289 | return false; |
| 2290 | } |
Evan Cheng | 0038e59 | 2006-03-28 00:39:58 +0000 | [diff] [blame] | 2291 | } |
| 2292 | |
| 2293 | return true; |
| 2294 | } |
| 2295 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2296 | bool X86::isUNPCKLMask(SDNode *N, bool V2IsSplat) { |
| 2297 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2298 | return ::isUNPCKLMask(N->op_begin(), N->getNumOperands(), V2IsSplat); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2299 | } |
| 2300 | |
Evan Cheng | 4fcb922 | 2006-03-28 02:43:26 +0000 | [diff] [blame] | 2301 | /// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2302 | /// specifies a shuffle of elements that is suitable for input to UNPCKH. |
Roman Levenstein | 9cac525 | 2008-04-16 16:15:27 +0000 | [diff] [blame] | 2303 | bool static isUNPCKHMask(SDOperandPtr Elts, unsigned NumElts, |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2304 | bool V2IsSplat = false) { |
| 2305 | if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) |
Evan Cheng | 4fcb922 | 2006-03-28 02:43:26 +0000 | [diff] [blame] | 2306 | return false; |
| 2307 | |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2308 | for (unsigned i = 0, j = 0; i != NumElts; i += 2, ++j) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2309 | SDValue BitI = Elts[i]; |
| 2310 | SDValue BitI1 = Elts[i+1]; |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2311 | if (!isUndefOrEqual(BitI, j + NumElts/2)) |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2312 | return false; |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2313 | if (V2IsSplat) { |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2314 | if (isUndefOrEqual(BitI1, NumElts)) |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2315 | return false; |
| 2316 | } else { |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2317 | if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts)) |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2318 | return false; |
| 2319 | } |
Evan Cheng | 4fcb922 | 2006-03-28 02:43:26 +0000 | [diff] [blame] | 2320 | } |
| 2321 | |
| 2322 | return true; |
| 2323 | } |
| 2324 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2325 | bool X86::isUNPCKHMask(SDNode *N, bool V2IsSplat) { |
| 2326 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2327 | return ::isUNPCKHMask(N->op_begin(), N->getNumOperands(), V2IsSplat); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2328 | } |
| 2329 | |
Evan Cheng | 1d5a8cc | 2006-04-05 07:20:06 +0000 | [diff] [blame] | 2330 | /// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form |
| 2331 | /// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, |
| 2332 | /// <0, 0, 1, 1> |
| 2333 | bool X86::isUNPCKL_v_undef_Mask(SDNode *N) { |
| 2334 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2335 | |
| 2336 | unsigned NumElems = N->getNumOperands(); |
Bill Wendling | 2f9bb1a | 2007-04-24 21:16:55 +0000 | [diff] [blame] | 2337 | if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) |
Evan Cheng | 1d5a8cc | 2006-04-05 07:20:06 +0000 | [diff] [blame] | 2338 | return false; |
| 2339 | |
| 2340 | for (unsigned i = 0, j = 0; i != NumElems; i += 2, ++j) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2341 | SDValue BitI = N->getOperand(i); |
| 2342 | SDValue BitI1 = N->getOperand(i+1); |
Evan Cheng | 1d5a8cc | 2006-04-05 07:20:06 +0000 | [diff] [blame] | 2343 | |
Evan Cheng | c5cdff2 | 2006-04-07 21:53:05 +0000 | [diff] [blame] | 2344 | if (!isUndefOrEqual(BitI, j)) |
| 2345 | return false; |
| 2346 | if (!isUndefOrEqual(BitI1, j)) |
| 2347 | return false; |
Evan Cheng | 1d5a8cc | 2006-04-05 07:20:06 +0000 | [diff] [blame] | 2348 | } |
| 2349 | |
| 2350 | return true; |
| 2351 | } |
| 2352 | |
Bill Wendling | 2f9bb1a | 2007-04-24 21:16:55 +0000 | [diff] [blame] | 2353 | /// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form |
| 2354 | /// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, |
| 2355 | /// <2, 2, 3, 3> |
| 2356 | bool X86::isUNPCKH_v_undef_Mask(SDNode *N) { |
| 2357 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2358 | |
| 2359 | unsigned NumElems = N->getNumOperands(); |
| 2360 | if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16) |
| 2361 | return false; |
| 2362 | |
| 2363 | for (unsigned i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2364 | SDValue BitI = N->getOperand(i); |
| 2365 | SDValue BitI1 = N->getOperand(i + 1); |
Bill Wendling | 2f9bb1a | 2007-04-24 21:16:55 +0000 | [diff] [blame] | 2366 | |
| 2367 | if (!isUndefOrEqual(BitI, j)) |
| 2368 | return false; |
| 2369 | if (!isUndefOrEqual(BitI1, j)) |
| 2370 | return false; |
| 2371 | } |
| 2372 | |
| 2373 | return true; |
| 2374 | } |
| 2375 | |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 2376 | /// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2377 | /// specifies a shuffle of elements that is suitable for input to MOVSS, |
| 2378 | /// MOVSD, and MOVD, i.e. setting the lowest element. |
Roman Levenstein | 9cac525 | 2008-04-16 16:15:27 +0000 | [diff] [blame] | 2379 | static bool isMOVLMask(SDOperandPtr Elts, unsigned NumElts) { |
Evan Cheng | 1076210 | 2007-12-06 22:14:22 +0000 | [diff] [blame] | 2380 | if (NumElts != 2 && NumElts != 4) |
Evan Cheng | d6d1cbd | 2006-04-11 00:19:04 +0000 | [diff] [blame] | 2381 | return false; |
| 2382 | |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2383 | if (!isUndefOrEqual(Elts[0], NumElts)) |
Evan Cheng | d6d1cbd | 2006-04-11 00:19:04 +0000 | [diff] [blame] | 2384 | return false; |
| 2385 | |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2386 | for (unsigned i = 1; i < NumElts; ++i) { |
| 2387 | if (!isUndefOrEqual(Elts[i], i)) |
Evan Cheng | d6d1cbd | 2006-04-11 00:19:04 +0000 | [diff] [blame] | 2388 | return false; |
| 2389 | } |
| 2390 | |
| 2391 | return true; |
| 2392 | } |
Evan Cheng | 1d5a8cc | 2006-04-05 07:20:06 +0000 | [diff] [blame] | 2393 | |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 2394 | bool X86::isMOVLMask(SDNode *N) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2395 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2396 | return ::isMOVLMask(N->op_begin(), N->getNumOperands()); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2397 | } |
| 2398 | |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 2399 | /// isCommutedMOVL - Returns true if the shuffle mask is except the reverse |
| 2400 | /// of what x86 movss want. X86 movs requires the lowest element to be lowest |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2401 | /// element of vector 2 and the other elements to come from vector 1 in order. |
Roman Levenstein | 9cac525 | 2008-04-16 16:15:27 +0000 | [diff] [blame] | 2402 | static bool isCommutedMOVL(SDOperandPtr Ops, unsigned NumOps, |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2403 | bool V2IsSplat = false, |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2404 | bool V2IsUndef = false) { |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2405 | if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16) |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2406 | return false; |
| 2407 | |
| 2408 | if (!isUndefOrEqual(Ops[0], 0)) |
| 2409 | return false; |
| 2410 | |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2411 | for (unsigned i = 1; i < NumOps; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2412 | SDValue Arg = Ops[i]; |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2413 | if (!(isUndefOrEqual(Arg, i+NumOps) || |
| 2414 | (V2IsUndef && isUndefOrInRange(Arg, NumOps, NumOps*2)) || |
| 2415 | (V2IsSplat && isUndefOrEqual(Arg, NumOps)))) |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2416 | return false; |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2417 | } |
| 2418 | |
| 2419 | return true; |
| 2420 | } |
| 2421 | |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2422 | static bool isCommutedMOVL(SDNode *N, bool V2IsSplat = false, |
| 2423 | bool V2IsUndef = false) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2424 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 2425 | return isCommutedMOVL(N->op_begin(), N->getNumOperands(), |
| 2426 | V2IsSplat, V2IsUndef); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2427 | } |
| 2428 | |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2429 | /// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2430 | /// specifies a shuffle of elements that is suitable for input to MOVSHDUP. |
| 2431 | bool X86::isMOVSHDUPMask(SDNode *N) { |
| 2432 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2433 | |
| 2434 | if (N->getNumOperands() != 4) |
| 2435 | return false; |
| 2436 | |
| 2437 | // Expect 1, 1, 3, 3 |
| 2438 | for (unsigned i = 0; i < 2; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2439 | SDValue Arg = N->getOperand(i); |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2440 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2441 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2442 | unsigned Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2443 | if (Val != 1) return false; |
| 2444 | } |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2445 | |
| 2446 | bool HasHi = false; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2447 | for (unsigned i = 2; i < 4; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2448 | SDValue Arg = N->getOperand(i); |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2449 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2450 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2451 | unsigned Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2452 | if (Val != 3) return false; |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2453 | HasHi = true; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2454 | } |
Evan Cheng | 39fc145 | 2006-04-15 03:13:24 +0000 | [diff] [blame] | 2455 | |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2456 | // Don't use movshdup if it can be done with a shufps. |
| 2457 | return HasHi; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2458 | } |
| 2459 | |
| 2460 | /// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2461 | /// specifies a shuffle of elements that is suitable for input to MOVSLDUP. |
| 2462 | bool X86::isMOVSLDUPMask(SDNode *N) { |
| 2463 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2464 | |
| 2465 | if (N->getNumOperands() != 4) |
| 2466 | return false; |
| 2467 | |
| 2468 | // Expect 0, 0, 2, 2 |
| 2469 | for (unsigned i = 0; i < 2; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2470 | SDValue Arg = N->getOperand(i); |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2471 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2472 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2473 | unsigned Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2474 | if (Val != 0) return false; |
| 2475 | } |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2476 | |
| 2477 | bool HasHi = false; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2478 | for (unsigned i = 2; i < 4; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2479 | SDValue Arg = N->getOperand(i); |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2480 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2481 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2482 | unsigned Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2483 | if (Val != 2) return false; |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2484 | HasHi = true; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2485 | } |
Evan Cheng | 39fc145 | 2006-04-15 03:13:24 +0000 | [diff] [blame] | 2486 | |
Evan Cheng | 57ebe9f | 2006-04-15 05:37:34 +0000 | [diff] [blame] | 2487 | // Don't use movshdup if it can be done with a shufps. |
| 2488 | return HasHi; |
Evan Cheng | d953947 | 2006-04-14 21:59:03 +0000 | [diff] [blame] | 2489 | } |
| 2490 | |
Evan Cheng | 49892af | 2007-06-19 00:02:56 +0000 | [diff] [blame] | 2491 | /// isIdentityMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2492 | /// specifies a identity operation on the LHS or RHS. |
| 2493 | static bool isIdentityMask(SDNode *N, bool RHS = false) { |
| 2494 | unsigned NumElems = N->getNumOperands(); |
| 2495 | for (unsigned i = 0; i < NumElems; ++i) |
| 2496 | if (!isUndefOrEqual(N->getOperand(i), i + (RHS ? NumElems : 0))) |
| 2497 | return false; |
| 2498 | return true; |
| 2499 | } |
| 2500 | |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2501 | /// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies |
| 2502 | /// a splat of a single element. |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 2503 | static bool isSplatMask(SDNode *N) { |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2504 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2505 | |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2506 | // This is a splat operation if each element of the permute is the same, and |
| 2507 | // if the value doesn't reference the second vector. |
Evan Cheng | 94fe5eb | 2006-04-19 23:28:59 +0000 | [diff] [blame] | 2508 | unsigned NumElems = N->getNumOperands(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2509 | SDValue ElementBase; |
Evan Cheng | 94fe5eb | 2006-04-19 23:28:59 +0000 | [diff] [blame] | 2510 | unsigned i = 0; |
| 2511 | for (; i != NumElems; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2512 | SDValue Elt = N->getOperand(i); |
Reid Spencer | 3ed469c | 2006-11-02 20:25:50 +0000 | [diff] [blame] | 2513 | if (isa<ConstantSDNode>(Elt)) { |
Evan Cheng | 94fe5eb | 2006-04-19 23:28:59 +0000 | [diff] [blame] | 2514 | ElementBase = Elt; |
| 2515 | break; |
| 2516 | } |
| 2517 | } |
| 2518 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 2519 | if (!ElementBase.getNode()) |
Evan Cheng | 94fe5eb | 2006-04-19 23:28:59 +0000 | [diff] [blame] | 2520 | return false; |
| 2521 | |
| 2522 | for (; i != NumElems; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2523 | SDValue Arg = N->getOperand(i); |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2524 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2525 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Evan Cheng | 94fe5eb | 2006-04-19 23:28:59 +0000 | [diff] [blame] | 2526 | if (Arg != ElementBase) return false; |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2527 | } |
| 2528 | |
| 2529 | // Make sure it is a splat of the first vector operand. |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2530 | return cast<ConstantSDNode>(ElementBase)->getZExtValue() < NumElems; |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2531 | } |
| 2532 | |
Mon P Wang | 62c75ea | 2008-12-23 04:03:27 +0000 | [diff] [blame] | 2533 | /// getSplatMaskEltNo - Given a splat mask, return the index to the element |
| 2534 | /// we want to splat. |
| 2535 | static SDValue getSplatMaskEltNo(SDNode *N) { |
| 2536 | assert(isSplatMask(N) && "Not a splat mask"); |
| 2537 | unsigned NumElems = N->getNumOperands(); |
| 2538 | SDValue ElementBase; |
| 2539 | unsigned i = 0; |
| 2540 | for (; i != NumElems; ++i) { |
| 2541 | SDValue Elt = N->getOperand(i); |
| 2542 | if (isa<ConstantSDNode>(Elt)) |
| 2543 | return Elt; |
| 2544 | } |
| 2545 | assert(0 && " No splat value found!"); |
| 2546 | return SDValue(); |
| 2547 | } |
| 2548 | |
| 2549 | |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 2550 | /// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies |
| 2551 | /// a splat of a single element and it's a 2 or 4 element mask. |
| 2552 | bool X86::isSplatMask(SDNode *N) { |
| 2553 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2554 | |
Evan Cheng | 94fe5eb | 2006-04-19 23:28:59 +0000 | [diff] [blame] | 2555 | // We can only splat 64-bit, and 32-bit quantities with a single instruction. |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 2556 | if (N->getNumOperands() != 4 && N->getNumOperands() != 2) |
| 2557 | return false; |
| 2558 | return ::isSplatMask(N); |
| 2559 | } |
| 2560 | |
Evan Cheng | f686d9b | 2006-10-27 21:08:32 +0000 | [diff] [blame] | 2561 | /// isSplatLoMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2562 | /// specifies a splat of zero element. |
| 2563 | bool X86::isSplatLoMask(SDNode *N) { |
| 2564 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2565 | |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 2566 | for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) |
Evan Cheng | f686d9b | 2006-10-27 21:08:32 +0000 | [diff] [blame] | 2567 | if (!isUndefOrEqual(N->getOperand(i), 0)) |
| 2568 | return false; |
| 2569 | return true; |
| 2570 | } |
| 2571 | |
Evan Cheng | 0b457f0 | 2008-09-25 20:50:48 +0000 | [diff] [blame] | 2572 | /// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand |
| 2573 | /// specifies a shuffle of elements that is suitable for input to MOVDDUP. |
| 2574 | bool X86::isMOVDDUPMask(SDNode *N) { |
| 2575 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2576 | |
| 2577 | unsigned e = N->getNumOperands() / 2; |
| 2578 | for (unsigned i = 0; i < e; ++i) |
| 2579 | if (!isUndefOrEqual(N->getOperand(i), i)) |
| 2580 | return false; |
| 2581 | for (unsigned i = 0; i < e; ++i) |
| 2582 | if (!isUndefOrEqual(N->getOperand(e+i), i)) |
| 2583 | return false; |
| 2584 | return true; |
| 2585 | } |
| 2586 | |
Evan Cheng | 63d3300 | 2006-03-22 08:01:21 +0000 | [diff] [blame] | 2587 | /// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle |
| 2588 | /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP* |
| 2589 | /// instructions. |
| 2590 | unsigned X86::getShuffleSHUFImmediate(SDNode *N) { |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 2591 | unsigned NumOperands = N->getNumOperands(); |
| 2592 | unsigned Shift = (NumOperands == 4) ? 2 : 1; |
| 2593 | unsigned Mask = 0; |
Evan Cheng | 36b27f3 | 2006-03-28 23:41:33 +0000 | [diff] [blame] | 2594 | for (unsigned i = 0; i < NumOperands; ++i) { |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2595 | unsigned Val = 0; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2596 | SDValue Arg = N->getOperand(NumOperands-i-1); |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2597 | if (Arg.getOpcode() != ISD::UNDEF) |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2598 | Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | 14aed5e | 2006-03-24 01:18:28 +0000 | [diff] [blame] | 2599 | if (Val >= NumOperands) Val -= NumOperands; |
Evan Cheng | 63d3300 | 2006-03-22 08:01:21 +0000 | [diff] [blame] | 2600 | Mask |= Val; |
Evan Cheng | 36b27f3 | 2006-03-28 23:41:33 +0000 | [diff] [blame] | 2601 | if (i != NumOperands - 1) |
| 2602 | Mask <<= Shift; |
| 2603 | } |
Evan Cheng | 63d3300 | 2006-03-22 08:01:21 +0000 | [diff] [blame] | 2604 | |
| 2605 | return Mask; |
| 2606 | } |
| 2607 | |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2608 | /// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle |
| 2609 | /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFHW |
| 2610 | /// instructions. |
| 2611 | unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) { |
| 2612 | unsigned Mask = 0; |
| 2613 | // 8 nodes, but we only care about the last 4. |
| 2614 | for (unsigned i = 7; i >= 4; --i) { |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2615 | unsigned Val = 0; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2616 | SDValue Arg = N->getOperand(i); |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2617 | if (Arg.getOpcode() != ISD::UNDEF) |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2618 | Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2619 | Mask |= (Val - 4); |
| 2620 | if (i != 4) |
| 2621 | Mask <<= 2; |
| 2622 | } |
| 2623 | |
| 2624 | return Mask; |
| 2625 | } |
| 2626 | |
| 2627 | /// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle |
| 2628 | /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFLW |
| 2629 | /// instructions. |
| 2630 | unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) { |
| 2631 | unsigned Mask = 0; |
| 2632 | // 8 nodes, but we only care about the first 4. |
| 2633 | for (int i = 3; i >= 0; --i) { |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2634 | unsigned Val = 0; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2635 | SDValue Arg = N->getOperand(i); |
Evan Cheng | ef698ca | 2006-03-31 00:30:29 +0000 | [diff] [blame] | 2636 | if (Arg.getOpcode() != ISD::UNDEF) |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2637 | Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | 506d3df | 2006-03-29 23:07:14 +0000 | [diff] [blame] | 2638 | Mask |= Val; |
| 2639 | if (i != 0) |
| 2640 | Mask <<= 2; |
| 2641 | } |
| 2642 | |
| 2643 | return Mask; |
| 2644 | } |
| 2645 | |
Evan Cheng | c21a053 | 2006-04-05 01:47:37 +0000 | [diff] [blame] | 2646 | /// isPSHUFHW_PSHUFLWMask - true if the specified VECTOR_SHUFFLE operand |
| 2647 | /// specifies a 8 element shuffle that can be broken into a pair of |
| 2648 | /// PSHUFHW and PSHUFLW. |
| 2649 | static bool isPSHUFHW_PSHUFLWMask(SDNode *N) { |
| 2650 | assert(N->getOpcode() == ISD::BUILD_VECTOR); |
| 2651 | |
| 2652 | if (N->getNumOperands() != 8) |
| 2653 | return false; |
| 2654 | |
| 2655 | // Lower quadword shuffled. |
| 2656 | for (unsigned i = 0; i != 4; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2657 | SDValue Arg = N->getOperand(i); |
Evan Cheng | c21a053 | 2006-04-05 01:47:37 +0000 | [diff] [blame] | 2658 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2659 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2660 | unsigned Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 2661 | if (Val >= 4) |
Evan Cheng | c21a053 | 2006-04-05 01:47:37 +0000 | [diff] [blame] | 2662 | return false; |
| 2663 | } |
| 2664 | |
| 2665 | // Upper quadword shuffled. |
| 2666 | for (unsigned i = 4; i != 8; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2667 | SDValue Arg = N->getOperand(i); |
Evan Cheng | c21a053 | 2006-04-05 01:47:37 +0000 | [diff] [blame] | 2668 | if (Arg.getOpcode() == ISD::UNDEF) continue; |
| 2669 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2670 | unsigned Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | c21a053 | 2006-04-05 01:47:37 +0000 | [diff] [blame] | 2671 | if (Val < 4 || Val > 7) |
| 2672 | return false; |
| 2673 | } |
| 2674 | |
| 2675 | return true; |
| 2676 | } |
| 2677 | |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2678 | /// CommuteVectorShuffle - Swap vector_shuffle operands as well as |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2679 | /// values in ther permute mask. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2680 | static SDValue CommuteVectorShuffle(SDValue Op, SDValue &V1, |
| 2681 | SDValue &V2, SDValue &Mask, |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 2682 | SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 2683 | MVT VT = Op.getValueType(); |
| 2684 | MVT MaskVT = Mask.getValueType(); |
| 2685 | MVT EltVT = MaskVT.getVectorElementType(); |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2686 | unsigned NumElems = Mask.getNumOperands(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2687 | SmallVector<SDValue, 8> MaskVec; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2688 | |
| 2689 | for (unsigned i = 0; i != NumElems; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2690 | SDValue Arg = Mask.getOperand(i); |
Evan Cheng | 80d428c | 2006-04-19 22:48:17 +0000 | [diff] [blame] | 2691 | if (Arg.getOpcode() == ISD::UNDEF) { |
| 2692 | MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); |
| 2693 | continue; |
| 2694 | } |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2695 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2696 | unsigned Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2697 | if (Val < NumElems) |
| 2698 | MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); |
| 2699 | else |
| 2700 | MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); |
| 2701 | } |
| 2702 | |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 2703 | std::swap(V1, V2); |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 2704 | Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 2705 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2706 | } |
| 2707 | |
Evan Cheng | 779ccea | 2007-12-07 21:30:01 +0000 | [diff] [blame] | 2708 | /// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming |
| 2709 | /// the two vector operands have swapped position. |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 2710 | static |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2711 | SDValue CommuteVectorShuffleMask(SDValue Mask, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 2712 | MVT MaskVT = Mask.getValueType(); |
| 2713 | MVT EltVT = MaskVT.getVectorElementType(); |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 2714 | unsigned NumElems = Mask.getNumOperands(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2715 | SmallVector<SDValue, 8> MaskVec; |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 2716 | for (unsigned i = 0; i != NumElems; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2717 | SDValue Arg = Mask.getOperand(i); |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 2718 | if (Arg.getOpcode() == ISD::UNDEF) { |
| 2719 | MaskVec.push_back(DAG.getNode(ISD::UNDEF, EltVT)); |
| 2720 | continue; |
| 2721 | } |
| 2722 | assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2723 | unsigned Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 2724 | if (Val < NumElems) |
| 2725 | MaskVec.push_back(DAG.getConstant(Val + NumElems, EltVT)); |
| 2726 | else |
| 2727 | MaskVec.push_back(DAG.getConstant(Val - NumElems, EltVT)); |
| 2728 | } |
| 2729 | return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], NumElems); |
| 2730 | } |
| 2731 | |
| 2732 | |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 2733 | /// ShouldXformToMOVHLPS - Return true if the node should be transformed to |
| 2734 | /// match movhlps. The lower half elements should come from upper half of |
| 2735 | /// V1 (and in order), and the upper half elements should come from the upper |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 2736 | /// half of V2 (and in order). |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 2737 | static bool ShouldXformToMOVHLPS(SDNode *Mask) { |
| 2738 | unsigned NumElems = Mask->getNumOperands(); |
| 2739 | if (NumElems != 4) |
| 2740 | return false; |
| 2741 | for (unsigned i = 0, e = 2; i != e; ++i) |
| 2742 | if (!isUndefOrEqual(Mask->getOperand(i), i+2)) |
| 2743 | return false; |
| 2744 | for (unsigned i = 2; i != 4; ++i) |
| 2745 | if (!isUndefOrEqual(Mask->getOperand(i), i+4)) |
| 2746 | return false; |
| 2747 | return true; |
| 2748 | } |
| 2749 | |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2750 | /// isScalarLoadToVector - Returns true if the node is a scalar load that |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 2751 | /// is promoted to a vector. It also returns the LoadSDNode by reference if |
| 2752 | /// required. |
| 2753 | static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) { |
Evan Cheng | 0b457f0 | 2008-09-25 20:50:48 +0000 | [diff] [blame] | 2754 | if (N->getOpcode() != ISD::SCALAR_TO_VECTOR) |
| 2755 | return false; |
| 2756 | N = N->getOperand(0).getNode(); |
| 2757 | if (!ISD::isNON_EXTLoad(N)) |
| 2758 | return false; |
| 2759 | if (LD) |
| 2760 | *LD = cast<LoadSDNode>(N); |
| 2761 | return true; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2762 | } |
| 2763 | |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 2764 | /// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to |
| 2765 | /// match movlp{s|d}. The lower half elements should come from lower half of |
| 2766 | /// V1 (and in order), and the upper half elements should come from the upper |
| 2767 | /// half of V2 (and in order). And since V1 will become the source of the |
| 2768 | /// MOVLP, it must be either a vector load or a scalar load to vector. |
Evan Cheng | 23425f5 | 2006-10-09 21:39:25 +0000 | [diff] [blame] | 2769 | static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, SDNode *Mask) { |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 2770 | if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 2771 | return false; |
Evan Cheng | 23425f5 | 2006-10-09 21:39:25 +0000 | [diff] [blame] | 2772 | // Is V2 is a vector load, don't do this transformation. We will try to use |
| 2773 | // load folding shufps op. |
| 2774 | if (ISD::isNON_EXTLoad(V2)) |
| 2775 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2776 | |
Evan Cheng | 533a0aa | 2006-04-19 20:35:22 +0000 | [diff] [blame] | 2777 | unsigned NumElems = Mask->getNumOperands(); |
| 2778 | if (NumElems != 2 && NumElems != 4) |
| 2779 | return false; |
| 2780 | for (unsigned i = 0, e = NumElems/2; i != e; ++i) |
| 2781 | if (!isUndefOrEqual(Mask->getOperand(i), i)) |
| 2782 | return false; |
| 2783 | for (unsigned i = NumElems/2; i != NumElems; ++i) |
| 2784 | if (!isUndefOrEqual(Mask->getOperand(i), i+NumElems)) |
| 2785 | return false; |
| 2786 | return true; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2787 | } |
| 2788 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2789 | /// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are |
| 2790 | /// all the same. |
| 2791 | static bool isSplatVector(SDNode *N) { |
| 2792 | if (N->getOpcode() != ISD::BUILD_VECTOR) |
| 2793 | return false; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2794 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2795 | SDValue SplatValue = N->getOperand(0); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2796 | for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) |
| 2797 | if (N->getOperand(i) != SplatValue) |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 2798 | return false; |
| 2799 | return true; |
| 2800 | } |
| 2801 | |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2802 | /// isUndefShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved |
| 2803 | /// to an undef. |
| 2804 | static bool isUndefShuffle(SDNode *N) { |
Evan Cheng | 213d2cf | 2007-05-17 18:45:50 +0000 | [diff] [blame] | 2805 | if (N->getOpcode() != ISD::VECTOR_SHUFFLE) |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2806 | return false; |
| 2807 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2808 | SDValue V1 = N->getOperand(0); |
| 2809 | SDValue V2 = N->getOperand(1); |
| 2810 | SDValue Mask = N->getOperand(2); |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2811 | unsigned NumElems = Mask.getNumOperands(); |
| 2812 | for (unsigned i = 0; i != NumElems; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2813 | SDValue Arg = Mask.getOperand(i); |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2814 | if (Arg.getOpcode() != ISD::UNDEF) { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2815 | unsigned Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 2816 | if (Val < NumElems && V1.getOpcode() != ISD::UNDEF) |
| 2817 | return false; |
| 2818 | else if (Val >= NumElems && V2.getOpcode() != ISD::UNDEF) |
| 2819 | return false; |
| 2820 | } |
| 2821 | } |
| 2822 | return true; |
| 2823 | } |
| 2824 | |
Evan Cheng | 213d2cf | 2007-05-17 18:45:50 +0000 | [diff] [blame] | 2825 | /// isZeroNode - Returns true if Elt is a constant zero or a floating point |
| 2826 | /// constant +0.0. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2827 | static inline bool isZeroNode(SDValue Elt) { |
Evan Cheng | 213d2cf | 2007-05-17 18:45:50 +0000 | [diff] [blame] | 2828 | return ((isa<ConstantSDNode>(Elt) && |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2829 | cast<ConstantSDNode>(Elt)->getZExtValue() == 0) || |
Evan Cheng | 213d2cf | 2007-05-17 18:45:50 +0000 | [diff] [blame] | 2830 | (isa<ConstantFPSDNode>(Elt) && |
Dale Johannesen | eaf0894 | 2007-08-31 04:03:46 +0000 | [diff] [blame] | 2831 | cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero())); |
Evan Cheng | 213d2cf | 2007-05-17 18:45:50 +0000 | [diff] [blame] | 2832 | } |
| 2833 | |
| 2834 | /// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved |
| 2835 | /// to an zero vector. |
| 2836 | static bool isZeroShuffle(SDNode *N) { |
| 2837 | if (N->getOpcode() != ISD::VECTOR_SHUFFLE) |
| 2838 | return false; |
| 2839 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2840 | SDValue V1 = N->getOperand(0); |
| 2841 | SDValue V2 = N->getOperand(1); |
| 2842 | SDValue Mask = N->getOperand(2); |
Evan Cheng | 213d2cf | 2007-05-17 18:45:50 +0000 | [diff] [blame] | 2843 | unsigned NumElems = Mask.getNumOperands(); |
| 2844 | for (unsigned i = 0; i != NumElems; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2845 | SDValue Arg = Mask.getOperand(i); |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2846 | if (Arg.getOpcode() == ISD::UNDEF) |
| 2847 | continue; |
| 2848 | |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2849 | unsigned Idx = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2850 | if (Idx < NumElems) { |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 2851 | unsigned Opc = V1.getNode()->getOpcode(); |
| 2852 | if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode())) |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2853 | continue; |
| 2854 | if (Opc != ISD::BUILD_VECTOR || |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 2855 | !isZeroNode(V1.getNode()->getOperand(Idx))) |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2856 | return false; |
| 2857 | } else if (Idx >= NumElems) { |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 2858 | unsigned Opc = V2.getNode()->getOpcode(); |
| 2859 | if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode())) |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2860 | continue; |
| 2861 | if (Opc != ISD::BUILD_VECTOR || |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 2862 | !isZeroNode(V2.getNode()->getOperand(Idx - NumElems))) |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2863 | return false; |
Evan Cheng | 213d2cf | 2007-05-17 18:45:50 +0000 | [diff] [blame] | 2864 | } |
| 2865 | } |
| 2866 | return true; |
| 2867 | } |
| 2868 | |
| 2869 | /// getZeroVector - Returns a vector of specified type with all zero elements. |
| 2870 | /// |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2871 | static SDValue getZeroVector(MVT VT, bool HasSSE2, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 2872 | assert(VT.isVector() && "Expected a vector type"); |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2873 | |
| 2874 | // Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest |
| 2875 | // type. This ensures they get CSE'd. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2876 | SDValue Vec; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 2877 | if (VT.getSizeInBits() == 64) { // MMX |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2878 | SDValue Cst = DAG.getTargetConstant(0, MVT::i32); |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2879 | Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 2880 | } else if (HasSSE2) { // SSE2 |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2881 | SDValue Cst = DAG.getTargetConstant(0, MVT::i32); |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2882 | Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 2883 | } else { // SSE1 |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2884 | SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32); |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 2885 | Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4f32, Cst, Cst, Cst, Cst); |
| 2886 | } |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2887 | return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); |
Evan Cheng | 213d2cf | 2007-05-17 18:45:50 +0000 | [diff] [blame] | 2888 | } |
| 2889 | |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2890 | /// getOnesVector - Returns a vector of specified type with all bits set. |
| 2891 | /// |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2892 | static SDValue getOnesVector(MVT VT, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 2893 | assert(VT.isVector() && "Expected a vector type"); |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2894 | |
| 2895 | // Always build ones vectors as <4 x i32> or <2 x i32> bitcasted to their dest |
| 2896 | // type. This ensures they get CSE'd. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2897 | SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32); |
| 2898 | SDValue Vec; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 2899 | if (VT.getSizeInBits() == 64) // MMX |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 2900 | Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); |
| 2901 | else // SSE |
| 2902 | Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); |
| 2903 | return DAG.getNode(ISD::BIT_CONVERT, VT, Vec); |
| 2904 | } |
| 2905 | |
| 2906 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2907 | /// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements |
| 2908 | /// that point to V2 points to its first element. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2909 | static SDValue NormalizeMask(SDValue Mask, SelectionDAG &DAG) { |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2910 | assert(Mask.getOpcode() == ISD::BUILD_VECTOR); |
| 2911 | |
| 2912 | bool Changed = false; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2913 | SmallVector<SDValue, 8> MaskVec; |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2914 | unsigned NumElems = Mask.getNumOperands(); |
| 2915 | for (unsigned i = 0; i != NumElems; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2916 | SDValue Arg = Mask.getOperand(i); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2917 | if (Arg.getOpcode() != ISD::UNDEF) { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 2918 | unsigned Val = cast<ConstantSDNode>(Arg)->getZExtValue(); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2919 | if (Val > NumElems) { |
| 2920 | Arg = DAG.getConstant(NumElems, Arg.getValueType()); |
| 2921 | Changed = true; |
| 2922 | } |
| 2923 | } |
| 2924 | MaskVec.push_back(Arg); |
| 2925 | } |
| 2926 | |
| 2927 | if (Changed) |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 2928 | Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getValueType(), |
| 2929 | &MaskVec[0], MaskVec.size()); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2930 | return Mask; |
| 2931 | } |
| 2932 | |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 2933 | /// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd |
| 2934 | /// operation of specified width. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2935 | static SDValue getMOVLMask(unsigned NumElems, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 2936 | MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 2937 | MVT BaseVT = MaskVT.getVectorElementType(); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2938 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2939 | SmallVector<SDValue, 8> MaskVec; |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2940 | MaskVec.push_back(DAG.getConstant(NumElems, BaseVT)); |
| 2941 | for (unsigned i = 1; i != NumElems; ++i) |
| 2942 | MaskVec.push_back(DAG.getConstant(i, BaseVT)); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 2943 | return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2944 | } |
| 2945 | |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 2946 | /// getUnpacklMask - Returns a vector_shuffle mask for an unpackl operation |
| 2947 | /// of specified width. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2948 | static SDValue getUnpacklMask(unsigned NumElems, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 2949 | MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 2950 | MVT BaseVT = MaskVT.getVectorElementType(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2951 | SmallVector<SDValue, 8> MaskVec; |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 2952 | for (unsigned i = 0, e = NumElems/2; i != e; ++i) { |
| 2953 | MaskVec.push_back(DAG.getConstant(i, BaseVT)); |
| 2954 | MaskVec.push_back(DAG.getConstant(i + NumElems, BaseVT)); |
| 2955 | } |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 2956 | return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 2957 | } |
| 2958 | |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2959 | /// getUnpackhMask - Returns a vector_shuffle mask for an unpackh operation |
| 2960 | /// of specified width. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2961 | static SDValue getUnpackhMask(unsigned NumElems, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 2962 | MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 2963 | MVT BaseVT = MaskVT.getVectorElementType(); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2964 | unsigned Half = NumElems/2; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2965 | SmallVector<SDValue, 8> MaskVec; |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2966 | for (unsigned i = 0; i != Half; ++i) { |
| 2967 | MaskVec.push_back(DAG.getConstant(i + Half, BaseVT)); |
| 2968 | MaskVec.push_back(DAG.getConstant(i + NumElems + Half, BaseVT)); |
| 2969 | } |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 2970 | return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); |
Evan Cheng | 39623da | 2006-04-20 08:58:49 +0000 | [diff] [blame] | 2971 | } |
| 2972 | |
Chris Lattner | 6209804 | 2008-03-09 01:05:04 +0000 | [diff] [blame] | 2973 | /// getSwapEltZeroMask - Returns a vector_shuffle mask for a shuffle that swaps |
| 2974 | /// element #0 of a vector with the specified index, leaving the rest of the |
| 2975 | /// elements in place. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2976 | static SDValue getSwapEltZeroMask(unsigned NumElems, unsigned DestElt, |
Chris Lattner | 6209804 | 2008-03-09 01:05:04 +0000 | [diff] [blame] | 2977 | SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 2978 | MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 2979 | MVT BaseVT = MaskVT.getVectorElementType(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2980 | SmallVector<SDValue, 8> MaskVec; |
Chris Lattner | 6209804 | 2008-03-09 01:05:04 +0000 | [diff] [blame] | 2981 | // Element #0 of the result gets the elt we are replacing. |
| 2982 | MaskVec.push_back(DAG.getConstant(DestElt, BaseVT)); |
| 2983 | for (unsigned i = 1; i != NumElems; ++i) |
| 2984 | MaskVec.push_back(DAG.getConstant(i == DestElt ? 0 : i, BaseVT)); |
| 2985 | return DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); |
| 2986 | } |
| 2987 | |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 2988 | /// PromoteSplat - Promote a splat of v4f32, v8i16 or v16i8 to v4i32. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2989 | static SDValue PromoteSplat(SDValue Op, SelectionDAG &DAG, bool HasSSE2) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 2990 | MVT PVT = HasSSE2 ? MVT::v4i32 : MVT::v4f32; |
| 2991 | MVT VT = Op.getValueType(); |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 2992 | if (PVT == VT) |
| 2993 | return Op; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 2994 | SDValue V1 = Op.getOperand(0); |
| 2995 | SDValue Mask = Op.getOperand(2); |
Mon P Wang | 62c75ea | 2008-12-23 04:03:27 +0000 | [diff] [blame] | 2996 | unsigned MaskNumElems = Mask.getNumOperands(); |
| 2997 | unsigned NumElems = MaskNumElems; |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 2998 | // Special handling of v4f32 -> v4i32. |
| 2999 | if (VT != MVT::v4f32) { |
Mon P Wang | 62c75ea | 2008-12-23 04:03:27 +0000 | [diff] [blame] | 3000 | // Find which element we want to splat. |
| 3001 | SDNode* EltNoNode = getSplatMaskEltNo(Mask.getNode()).getNode(); |
| 3002 | unsigned EltNo = cast<ConstantSDNode>(EltNoNode)->getZExtValue(); |
| 3003 | // unpack elements to the correct location |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 3004 | while (NumElems > 4) { |
Mon P Wang | 62c75ea | 2008-12-23 04:03:27 +0000 | [diff] [blame] | 3005 | if (EltNo < NumElems/2) { |
| 3006 | Mask = getUnpacklMask(MaskNumElems, DAG); |
| 3007 | } else { |
| 3008 | Mask = getUnpackhMask(MaskNumElems, DAG); |
| 3009 | EltNo -= NumElems/2; |
| 3010 | } |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 3011 | V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask); |
| 3012 | NumElems >>= 1; |
| 3013 | } |
Mon P Wang | 62c75ea | 2008-12-23 04:03:27 +0000 | [diff] [blame] | 3014 | SDValue Cst = DAG.getConstant(EltNo, MVT::i32); |
| 3015 | Mask = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst); |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 3016 | } |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 3017 | |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 3018 | V1 = DAG.getNode(ISD::BIT_CONVERT, PVT, V1); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3019 | SDValue Shuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, PVT, V1, |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 3020 | DAG.getNode(ISD::UNDEF, PVT), Mask); |
Evan Cheng | c575ca2 | 2006-04-17 20:43:08 +0000 | [diff] [blame] | 3021 | return DAG.getNode(ISD::BIT_CONVERT, VT, Shuffle); |
| 3022 | } |
| 3023 | |
Evan Cheng | 0b457f0 | 2008-09-25 20:50:48 +0000 | [diff] [blame] | 3024 | /// isVectorLoad - Returns true if the node is a vector load, a scalar |
| 3025 | /// load that's promoted to vector, or a load bitcasted. |
| 3026 | static bool isVectorLoad(SDValue Op) { |
| 3027 | assert(Op.getValueType().isVector() && "Expected a vector type"); |
| 3028 | if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR || |
| 3029 | Op.getOpcode() == ISD::BIT_CONVERT) { |
| 3030 | return isa<LoadSDNode>(Op.getOperand(0)); |
| 3031 | } |
| 3032 | return isa<LoadSDNode>(Op); |
| 3033 | } |
| 3034 | |
| 3035 | |
| 3036 | /// CanonicalizeMovddup - Cannonicalize movddup shuffle to v2f64. |
| 3037 | /// |
| 3038 | static SDValue CanonicalizeMovddup(SDValue Op, SDValue V1, SDValue Mask, |
| 3039 | SelectionDAG &DAG, bool HasSSE3) { |
| 3040 | // If we have sse3 and shuffle has more than one use or input is a load, then |
| 3041 | // use movddup. Otherwise, use movlhps. |
| 3042 | bool UseMovddup = HasSSE3 && (!Op.hasOneUse() || isVectorLoad(V1)); |
| 3043 | MVT PVT = UseMovddup ? MVT::v2f64 : MVT::v4f32; |
| 3044 | MVT VT = Op.getValueType(); |
| 3045 | if (VT == PVT) |
| 3046 | return Op; |
| 3047 | unsigned NumElems = PVT.getVectorNumElements(); |
| 3048 | if (NumElems == 2) { |
| 3049 | SDValue Cst = DAG.getTargetConstant(0, MVT::i32); |
| 3050 | Mask = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst); |
| 3051 | } else { |
| 3052 | assert(NumElems == 4); |
| 3053 | SDValue Cst0 = DAG.getTargetConstant(0, MVT::i32); |
| 3054 | SDValue Cst1 = DAG.getTargetConstant(1, MVT::i32); |
| 3055 | Mask = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst0, Cst1, Cst0, Cst1); |
| 3056 | } |
| 3057 | |
| 3058 | V1 = DAG.getNode(ISD::BIT_CONVERT, PVT, V1); |
| 3059 | SDValue Shuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, PVT, V1, |
| 3060 | DAG.getNode(ISD::UNDEF, PVT), Mask); |
| 3061 | return DAG.getNode(ISD::BIT_CONVERT, VT, Shuffle); |
| 3062 | } |
| 3063 | |
Evan Cheng | ba05f72 | 2006-04-21 23:03:30 +0000 | [diff] [blame] | 3064 | /// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 3065 | /// vector of zero or undef vector. This produces a shuffle where the low |
| 3066 | /// element of V2 is swizzled into the zero/undef vector, landing at element |
| 3067 | /// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3068 | static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx, |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 3069 | bool isZero, bool HasSSE2, |
| 3070 | SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3071 | MVT VT = V2.getValueType(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3072 | SDValue V1 = isZero |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 3073 | ? getZeroVector(VT, HasSSE2, DAG) : DAG.getNode(ISD::UNDEF, VT); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3074 | unsigned NumElems = V2.getValueType().getVectorNumElements(); |
| 3075 | MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 3076 | MVT EVT = MaskVT.getVectorElementType(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3077 | SmallVector<SDValue, 16> MaskVec; |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 3078 | for (unsigned i = 0; i != NumElems; ++i) |
| 3079 | if (i == Idx) // If this is the insertion idx, put the low elt of V2 here. |
| 3080 | MaskVec.push_back(DAG.getConstant(NumElems, EVT)); |
| 3081 | else |
| 3082 | MaskVec.push_back(DAG.getConstant(i, EVT)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3083 | SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 3084 | &MaskVec[0], MaskVec.size()); |
Evan Cheng | ba05f72 | 2006-04-21 23:03:30 +0000 | [diff] [blame] | 3085 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, Mask); |
Evan Cheng | 017dcc6 | 2006-04-21 01:05:10 +0000 | [diff] [blame] | 3086 | } |
| 3087 | |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3088 | /// getNumOfConsecutiveZeros - Return the number of elements in a result of |
| 3089 | /// a shuffle that is zero. |
| 3090 | static |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3091 | unsigned getNumOfConsecutiveZeros(SDValue Op, SDValue Mask, |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3092 | unsigned NumElems, bool Low, |
| 3093 | SelectionDAG &DAG) { |
| 3094 | unsigned NumZeros = 0; |
| 3095 | for (unsigned i = 0; i < NumElems; ++i) { |
Evan Cheng | ab26227 | 2008-06-25 20:52:59 +0000 | [diff] [blame] | 3096 | unsigned Index = Low ? i : NumElems-i-1; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3097 | SDValue Idx = Mask.getOperand(Index); |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3098 | if (Idx.getOpcode() == ISD::UNDEF) { |
| 3099 | ++NumZeros; |
| 3100 | continue; |
| 3101 | } |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 3102 | SDValue Elt = DAG.getShuffleScalarElt(Op.getNode(), Index); |
| 3103 | if (Elt.getNode() && isZeroNode(Elt)) |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3104 | ++NumZeros; |
| 3105 | else |
| 3106 | break; |
| 3107 | } |
| 3108 | return NumZeros; |
| 3109 | } |
| 3110 | |
| 3111 | /// isVectorShift - Returns true if the shuffle can be implemented as a |
| 3112 | /// logical left or right shift of a vector. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3113 | static bool isVectorShift(SDValue Op, SDValue Mask, SelectionDAG &DAG, |
| 3114 | bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3115 | unsigned NumElems = Mask.getNumOperands(); |
| 3116 | |
| 3117 | isLeft = true; |
| 3118 | unsigned NumZeros= getNumOfConsecutiveZeros(Op, Mask, NumElems, true, DAG); |
| 3119 | if (!NumZeros) { |
| 3120 | isLeft = false; |
| 3121 | NumZeros = getNumOfConsecutiveZeros(Op, Mask, NumElems, false, DAG); |
| 3122 | if (!NumZeros) |
| 3123 | return false; |
| 3124 | } |
| 3125 | |
| 3126 | bool SeenV1 = false; |
| 3127 | bool SeenV2 = false; |
| 3128 | for (unsigned i = NumZeros; i < NumElems; ++i) { |
| 3129 | unsigned Val = isLeft ? (i - NumZeros) : i; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3130 | SDValue Idx = Mask.getOperand(isLeft ? i : (i - NumZeros)); |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3131 | if (Idx.getOpcode() == ISD::UNDEF) |
| 3132 | continue; |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3133 | unsigned Index = cast<ConstantSDNode>(Idx)->getZExtValue(); |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3134 | if (Index < NumElems) |
| 3135 | SeenV1 = true; |
| 3136 | else { |
| 3137 | Index -= NumElems; |
| 3138 | SeenV2 = true; |
| 3139 | } |
| 3140 | if (Index != Val) |
| 3141 | return false; |
| 3142 | } |
| 3143 | if (SeenV1 && SeenV2) |
| 3144 | return false; |
| 3145 | |
| 3146 | ShVal = SeenV1 ? Op.getOperand(0) : Op.getOperand(1); |
| 3147 | ShAmt = NumZeros; |
| 3148 | return true; |
| 3149 | } |
| 3150 | |
| 3151 | |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3152 | /// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. |
| 3153 | /// |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3154 | static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3155 | unsigned NumNonZero, unsigned NumZero, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3156 | SelectionDAG &DAG, TargetLowering &TLI) { |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3157 | if (NumNonZero > 8) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3158 | return SDValue(); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3159 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3160 | SDValue V(0, 0); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3161 | bool First = true; |
| 3162 | for (unsigned i = 0; i < 16; ++i) { |
| 3163 | bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; |
| 3164 | if (ThisIsNonZero && First) { |
| 3165 | if (NumZero) |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 3166 | V = getZeroVector(MVT::v8i16, true, DAG); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3167 | else |
| 3168 | V = DAG.getNode(ISD::UNDEF, MVT::v8i16); |
| 3169 | First = false; |
| 3170 | } |
| 3171 | |
| 3172 | if ((i & 1) != 0) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3173 | SDValue ThisElt(0, 0), LastElt(0, 0); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3174 | bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; |
| 3175 | if (LastIsNonZero) { |
| 3176 | LastElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i-1)); |
| 3177 | } |
| 3178 | if (ThisIsNonZero) { |
| 3179 | ThisElt = DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, Op.getOperand(i)); |
| 3180 | ThisElt = DAG.getNode(ISD::SHL, MVT::i16, |
| 3181 | ThisElt, DAG.getConstant(8, MVT::i8)); |
| 3182 | if (LastIsNonZero) |
| 3183 | ThisElt = DAG.getNode(ISD::OR, MVT::i16, ThisElt, LastElt); |
| 3184 | } else |
| 3185 | ThisElt = LastElt; |
| 3186 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 3187 | if (ThisElt.getNode()) |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3188 | V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, ThisElt, |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 3189 | DAG.getIntPtrConstant(i/2)); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3190 | } |
| 3191 | } |
| 3192 | |
| 3193 | return DAG.getNode(ISD::BIT_CONVERT, MVT::v16i8, V); |
| 3194 | } |
| 3195 | |
Bill Wendling | a348c56 | 2007-03-22 18:42:45 +0000 | [diff] [blame] | 3196 | /// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3197 | /// |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3198 | static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros, |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3199 | unsigned NumNonZero, unsigned NumZero, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3200 | SelectionDAG &DAG, TargetLowering &TLI) { |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3201 | if (NumNonZero > 4) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3202 | return SDValue(); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3203 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3204 | SDValue V(0, 0); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3205 | bool First = true; |
| 3206 | for (unsigned i = 0; i < 8; ++i) { |
| 3207 | bool isNonZero = (NonZeros & (1 << i)) != 0; |
| 3208 | if (isNonZero) { |
| 3209 | if (First) { |
| 3210 | if (NumZero) |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 3211 | V = getZeroVector(MVT::v8i16, true, DAG); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3212 | else |
| 3213 | V = DAG.getNode(ISD::UNDEF, MVT::v8i16); |
| 3214 | First = false; |
| 3215 | } |
| 3216 | V = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, V, Op.getOperand(i), |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 3217 | DAG.getIntPtrConstant(i)); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 3218 | } |
| 3219 | } |
| 3220 | |
| 3221 | return V; |
| 3222 | } |
| 3223 | |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3224 | /// getVShift - Return a vector logical shift node. |
| 3225 | /// |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3226 | static SDValue getVShift(bool isLeft, MVT VT, SDValue SrcOp, |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3227 | unsigned NumBits, SelectionDAG &DAG, |
| 3228 | const TargetLowering &TLI) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3229 | bool isMMX = VT.getSizeInBits() == 64; |
| 3230 | MVT ShVT = isMMX ? MVT::v1i64 : MVT::v2i64; |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3231 | unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL; |
| 3232 | SrcOp = DAG.getNode(ISD::BIT_CONVERT, ShVT, SrcOp); |
| 3233 | return DAG.getNode(ISD::BIT_CONVERT, VT, |
| 3234 | DAG.getNode(Opc, ShVT, SrcOp, |
Gabor Greif | 327ef03 | 2008-08-28 23:19:51 +0000 | [diff] [blame] | 3235 | DAG.getConstant(NumBits, TLI.getShiftAmountTy()))); |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3236 | } |
| 3237 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3238 | SDValue |
| 3239 | X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) { |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 3240 | // All zero's are handled with pxor, all one's are handled with pcmpeqd. |
Gabor Greif | 327ef03 | 2008-08-28 23:19:51 +0000 | [diff] [blame] | 3241 | if (ISD::isBuildVectorAllZeros(Op.getNode()) |
| 3242 | || ISD::isBuildVectorAllOnes(Op.getNode())) { |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 3243 | // Canonicalize this to either <4 x i32> or <2 x i32> (SSE vs MMX) to |
| 3244 | // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are |
| 3245 | // eliminated on x86-32 hosts. |
| 3246 | if (Op.getValueType() == MVT::v4i32 || Op.getValueType() == MVT::v2i32) |
| 3247 | return Op; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3248 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 3249 | if (ISD::isBuildVectorAllOnes(Op.getNode())) |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 3250 | return getOnesVector(Op.getValueType(), DAG); |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 3251 | return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG); |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 3252 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3253 | |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3254 | MVT VT = Op.getValueType(); |
| 3255 | MVT EVT = VT.getVectorElementType(); |
| 3256 | unsigned EVTBits = EVT.getSizeInBits(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3257 | |
| 3258 | unsigned NumElems = Op.getNumOperands(); |
| 3259 | unsigned NumZero = 0; |
| 3260 | unsigned NumNonZero = 0; |
| 3261 | unsigned NonZeros = 0; |
Chris Lattner | c9517fb | 2008-03-08 22:48:29 +0000 | [diff] [blame] | 3262 | bool IsAllConstants = true; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3263 | SmallSet<SDValue, 8> Values; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3264 | for (unsigned i = 0; i < NumElems; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3265 | SDValue Elt = Op.getOperand(i); |
Evan Cheng | db2d524 | 2007-12-12 06:45:40 +0000 | [diff] [blame] | 3266 | if (Elt.getOpcode() == ISD::UNDEF) |
| 3267 | continue; |
| 3268 | Values.insert(Elt); |
| 3269 | if (Elt.getOpcode() != ISD::Constant && |
| 3270 | Elt.getOpcode() != ISD::ConstantFP) |
Chris Lattner | c9517fb | 2008-03-08 22:48:29 +0000 | [diff] [blame] | 3271 | IsAllConstants = false; |
Evan Cheng | db2d524 | 2007-12-12 06:45:40 +0000 | [diff] [blame] | 3272 | if (isZeroNode(Elt)) |
| 3273 | NumZero++; |
| 3274 | else { |
| 3275 | NonZeros |= (1 << i); |
| 3276 | NumNonZero++; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3277 | } |
| 3278 | } |
| 3279 | |
Dan Gohman | 7f32156 | 2007-06-25 16:23:39 +0000 | [diff] [blame] | 3280 | if (NumNonZero == 0) { |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 3281 | // All undef vector. Return an UNDEF. All zero vectors were handled above. |
| 3282 | return DAG.getNode(ISD::UNDEF, VT); |
Dan Gohman | 7f32156 | 2007-06-25 16:23:39 +0000 | [diff] [blame] | 3283 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3284 | |
Chris Lattner | 67f453a | 2008-03-09 05:42:06 +0000 | [diff] [blame] | 3285 | // Special case for single non-zero, non-undef, element. |
Evan Cheng | db2d524 | 2007-12-12 06:45:40 +0000 | [diff] [blame] | 3286 | if (NumNonZero == 1 && NumElems <= 4) { |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3287 | unsigned Idx = CountTrailingZeros_32(NonZeros); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3288 | SDValue Item = Op.getOperand(Idx); |
Chris Lattner | 19f7969 | 2008-03-08 22:59:52 +0000 | [diff] [blame] | 3289 | |
Chris Lattner | 6209804 | 2008-03-09 01:05:04 +0000 | [diff] [blame] | 3290 | // If this is an insertion of an i64 value on x86-32, and if the top bits of |
| 3291 | // the value are obviously zero, truncate the value to i32 and do the |
| 3292 | // insertion that way. Only do this if the value is non-constant or if the |
| 3293 | // value is a constant being inserted into element 0. It is cheaper to do |
| 3294 | // a constant pool load than it is to do a movd + shuffle. |
| 3295 | if (EVT == MVT::i64 && !Subtarget->is64Bit() && |
| 3296 | (!IsAllConstants || Idx == 0)) { |
| 3297 | if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { |
| 3298 | // Handle MMX and SSE both. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3299 | MVT VecVT = VT == MVT::v2i64 ? MVT::v4i32 : MVT::v2i32; |
| 3300 | unsigned VecElts = VT == MVT::v2i64 ? 4 : 2; |
Chris Lattner | 6209804 | 2008-03-09 01:05:04 +0000 | [diff] [blame] | 3301 | |
| 3302 | // Truncate the value (which may itself be a constant) to i32, and |
| 3303 | // convert it to a vector with movd (S2V+shuffle to zero extend). |
| 3304 | Item = DAG.getNode(ISD::TRUNCATE, MVT::i32, Item); |
| 3305 | Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecVT, Item); |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 3306 | Item = getShuffleVectorZeroOrUndef(Item, 0, true, |
| 3307 | Subtarget->hasSSE2(), DAG); |
Chris Lattner | 6209804 | 2008-03-09 01:05:04 +0000 | [diff] [blame] | 3308 | |
| 3309 | // Now we have our 32-bit value zero extended in the low element of |
| 3310 | // a vector. If Idx != 0, swizzle it into place. |
| 3311 | if (Idx != 0) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3312 | SDValue Ops[] = { |
Chris Lattner | 6209804 | 2008-03-09 01:05:04 +0000 | [diff] [blame] | 3313 | Item, DAG.getNode(ISD::UNDEF, Item.getValueType()), |
| 3314 | getSwapEltZeroMask(VecElts, Idx, DAG) |
| 3315 | }; |
| 3316 | Item = DAG.getNode(ISD::VECTOR_SHUFFLE, VecVT, Ops, 3); |
| 3317 | } |
| 3318 | return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), Item); |
| 3319 | } |
| 3320 | } |
| 3321 | |
Chris Lattner | 19f7969 | 2008-03-08 22:59:52 +0000 | [diff] [blame] | 3322 | // If we have a constant or non-constant insertion into the low element of |
| 3323 | // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into |
| 3324 | // the rest of the elements. This will be matched as movd/movq/movss/movsd |
| 3325 | // depending on what the source datatype is. Because we can only get here |
| 3326 | // when NumElems <= 4, this only needs to handle i32/f32/i64/f64. |
| 3327 | if (Idx == 0 && |
| 3328 | // Don't do this for i64 values on x86-32. |
| 3329 | (EVT != MVT::i64 || Subtarget->is64Bit())) { |
Chris Lattner | c9517fb | 2008-03-08 22:48:29 +0000 | [diff] [blame] | 3330 | Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3331 | // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 3332 | return getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, |
| 3333 | Subtarget->hasSSE2(), DAG); |
Chris Lattner | c9517fb | 2008-03-08 22:48:29 +0000 | [diff] [blame] | 3334 | } |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3335 | |
| 3336 | // Is it a vector logical left shift? |
| 3337 | if (NumElems == 2 && Idx == 1 && |
| 3338 | isZeroNode(Op.getOperand(0)) && !isZeroNode(Op.getOperand(1))) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3339 | unsigned NumBits = VT.getSizeInBits(); |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 3340 | return getVShift(true, VT, |
| 3341 | DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(1)), |
| 3342 | NumBits/2, DAG, *this); |
| 3343 | } |
Chris Lattner | c9517fb | 2008-03-08 22:48:29 +0000 | [diff] [blame] | 3344 | |
| 3345 | if (IsAllConstants) // Otherwise, it's better to do a constpool load. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3346 | return SDValue(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3347 | |
Chris Lattner | 19f7969 | 2008-03-08 22:59:52 +0000 | [diff] [blame] | 3348 | // Otherwise, if this is a vector with i32 or f32 elements, and the element |
| 3349 | // is a non-constant being inserted into an element other than the low one, |
| 3350 | // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka |
| 3351 | // movd/movss) to move this into the low element, then shuffle it into |
| 3352 | // place. |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3353 | if (EVTBits == 32) { |
Chris Lattner | c9517fb | 2008-03-08 22:48:29 +0000 | [diff] [blame] | 3354 | Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item); |
| 3355 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3356 | // Turn it into a shuffle of zero and zero-extended scalar to vector. |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 3357 | Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, |
| 3358 | Subtarget->hasSSE2(), DAG); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3359 | MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 3360 | MVT MaskEVT = MaskVT.getVectorElementType(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3361 | SmallVector<SDValue, 8> MaskVec; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3362 | for (unsigned i = 0; i < NumElems; i++) |
| 3363 | MaskVec.push_back(DAG.getConstant((i == Idx) ? 0 : 1, MaskEVT)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3364 | SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 3365 | &MaskVec[0], MaskVec.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3366 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Item, |
| 3367 | DAG.getNode(ISD::UNDEF, VT), Mask); |
| 3368 | } |
| 3369 | } |
| 3370 | |
Chris Lattner | 67f453a | 2008-03-09 05:42:06 +0000 | [diff] [blame] | 3371 | // Splat is obviously ok. Let legalizer expand it to a shuffle. |
| 3372 | if (Values.size() == 1) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3373 | return SDValue(); |
Chris Lattner | 67f453a | 2008-03-09 05:42:06 +0000 | [diff] [blame] | 3374 | |
Dan Gohman | a394117 | 2007-07-24 22:55:08 +0000 | [diff] [blame] | 3375 | // A vector full of immediates; various special cases are already |
| 3376 | // handled, so this is best done with a single constant-pool load. |
Chris Lattner | c9517fb | 2008-03-08 22:48:29 +0000 | [diff] [blame] | 3377 | if (IsAllConstants) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3378 | return SDValue(); |
Dan Gohman | a394117 | 2007-07-24 22:55:08 +0000 | [diff] [blame] | 3379 | |
Bill Wendling | 2f9bb1a | 2007-04-24 21:16:55 +0000 | [diff] [blame] | 3380 | // Let legalizer expand 2-wide build_vectors. |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 3381 | if (EVTBits == 64) { |
| 3382 | if (NumNonZero == 1) { |
| 3383 | // One half is zero or undef. |
| 3384 | unsigned Idx = CountTrailingZeros_32(NonZeros); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3385 | SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 3386 | Op.getOperand(Idx)); |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 3387 | return getShuffleVectorZeroOrUndef(V2, Idx, true, |
| 3388 | Subtarget->hasSSE2(), DAG); |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 3389 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3390 | return SDValue(); |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 3391 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3392 | |
| 3393 | // If element VT is < 32 bits, convert it to inserts into a zero vector. |
Bill Wendling | 826f36f | 2007-03-28 00:57:11 +0000 | [diff] [blame] | 3394 | if (EVTBits == 8 && NumElems == 16) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3395 | SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3396 | *this); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 3397 | if (V.getNode()) return V; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3398 | } |
| 3399 | |
Bill Wendling | 826f36f | 2007-03-28 00:57:11 +0000 | [diff] [blame] | 3400 | if (EVTBits == 16 && NumElems == 8) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3401 | SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 3402 | *this); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 3403 | if (V.getNode()) return V; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3404 | } |
| 3405 | |
| 3406 | // If element VT is == 32 bits, turn it into a number of shuffles. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3407 | SmallVector<SDValue, 8> V; |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 3408 | V.resize(NumElems); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3409 | if (NumElems == 4 && NumZero > 0) { |
| 3410 | for (unsigned i = 0; i < 4; ++i) { |
| 3411 | bool isZero = !(NonZeros & (1 << i)); |
| 3412 | if (isZero) |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 3413 | V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3414 | else |
| 3415 | V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); |
| 3416 | } |
| 3417 | |
| 3418 | for (unsigned i = 0; i < 2; ++i) { |
| 3419 | switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { |
| 3420 | default: break; |
| 3421 | case 0: |
| 3422 | V[i] = V[i*2]; // Must be a zero vector. |
| 3423 | break; |
| 3424 | case 1: |
| 3425 | V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2+1], V[i*2], |
| 3426 | getMOVLMask(NumElems, DAG)); |
| 3427 | break; |
| 3428 | case 2: |
| 3429 | V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], |
| 3430 | getMOVLMask(NumElems, DAG)); |
| 3431 | break; |
| 3432 | case 3: |
| 3433 | V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i*2], V[i*2+1], |
| 3434 | getUnpacklMask(NumElems, DAG)); |
| 3435 | break; |
| 3436 | } |
| 3437 | } |
| 3438 | |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3439 | MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); |
| 3440 | MVT EVT = MaskVT.getVectorElementType(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3441 | SmallVector<SDValue, 8> MaskVec; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3442 | bool Reverse = (NonZeros & 0x3) == 2; |
| 3443 | for (unsigned i = 0; i < 2; ++i) |
| 3444 | if (Reverse) |
| 3445 | MaskVec.push_back(DAG.getConstant(1-i, EVT)); |
| 3446 | else |
| 3447 | MaskVec.push_back(DAG.getConstant(i, EVT)); |
| 3448 | Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2; |
| 3449 | for (unsigned i = 0; i < 2; ++i) |
| 3450 | if (Reverse) |
| 3451 | MaskVec.push_back(DAG.getConstant(1-i+NumElems, EVT)); |
| 3452 | else |
| 3453 | MaskVec.push_back(DAG.getConstant(i+NumElems, EVT)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3454 | SDValue ShufMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 3455 | &MaskVec[0], MaskVec.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3456 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[0], V[1], ShufMask); |
| 3457 | } |
| 3458 | |
| 3459 | if (Values.size() > 2) { |
| 3460 | // Expand into a number of unpckl*. |
| 3461 | // e.g. for v4f32 |
| 3462 | // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0> |
| 3463 | // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1> |
| 3464 | // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3465 | SDValue UnpckMask = getUnpacklMask(NumElems, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3466 | for (unsigned i = 0; i < NumElems; ++i) |
| 3467 | V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i)); |
| 3468 | NumElems >>= 1; |
| 3469 | while (NumElems != 0) { |
| 3470 | for (unsigned i = 0; i < NumElems; ++i) |
| 3471 | V[i] = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V[i], V[i + NumElems], |
| 3472 | UnpckMask); |
| 3473 | NumElems >>= 1; |
| 3474 | } |
| 3475 | return V[0]; |
| 3476 | } |
| 3477 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3478 | return SDValue(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3479 | } |
| 3480 | |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 3481 | static |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3482 | SDValue LowerVECTOR_SHUFFLEv8i16(SDValue V1, SDValue V2, |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3483 | SDValue PermMask, SelectionDAG &DAG, |
| 3484 | TargetLowering &TLI) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3485 | SDValue NewV; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3486 | MVT MaskVT = MVT::getIntVectorWithNumElements(8); |
| 3487 | MVT MaskEVT = MaskVT.getVectorElementType(); |
| 3488 | MVT PtrVT = TLI.getPointerTy(); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 3489 | SmallVector<SDValue, 8> MaskElts(PermMask.getNode()->op_begin(), |
| 3490 | PermMask.getNode()->op_end()); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3491 | |
| 3492 | // First record which half of which vector the low elements come from. |
| 3493 | SmallVector<unsigned, 4> LowQuad(4); |
| 3494 | for (unsigned i = 0; i < 4; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3495 | SDValue Elt = MaskElts[i]; |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3496 | if (Elt.getOpcode() == ISD::UNDEF) |
| 3497 | continue; |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3498 | unsigned EltIdx = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3499 | int QuadIdx = EltIdx / 4; |
| 3500 | ++LowQuad[QuadIdx]; |
| 3501 | } |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3502 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3503 | int BestLowQuad = -1; |
| 3504 | unsigned MaxQuad = 1; |
| 3505 | for (unsigned i = 0; i < 4; ++i) { |
| 3506 | if (LowQuad[i] > MaxQuad) { |
| 3507 | BestLowQuad = i; |
| 3508 | MaxQuad = LowQuad[i]; |
| 3509 | } |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 3510 | } |
| 3511 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3512 | // Record which half of which vector the high elements come from. |
| 3513 | SmallVector<unsigned, 4> HighQuad(4); |
| 3514 | for (unsigned i = 4; i < 8; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3515 | SDValue Elt = MaskElts[i]; |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3516 | if (Elt.getOpcode() == ISD::UNDEF) |
| 3517 | continue; |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3518 | unsigned EltIdx = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3519 | int QuadIdx = EltIdx / 4; |
| 3520 | ++HighQuad[QuadIdx]; |
| 3521 | } |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3522 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3523 | int BestHighQuad = -1; |
| 3524 | MaxQuad = 1; |
| 3525 | for (unsigned i = 0; i < 4; ++i) { |
| 3526 | if (HighQuad[i] > MaxQuad) { |
| 3527 | BestHighQuad = i; |
| 3528 | MaxQuad = HighQuad[i]; |
| 3529 | } |
| 3530 | } |
| 3531 | |
| 3532 | // If it's possible to sort parts of either half with PSHUF{H|L}W, then do it. |
| 3533 | if (BestLowQuad != -1 || BestHighQuad != -1) { |
| 3534 | // First sort the 4 chunks in order using shufpd. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3535 | SmallVector<SDValue, 8> MaskVec; |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3536 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3537 | if (BestLowQuad != -1) |
| 3538 | MaskVec.push_back(DAG.getConstant(BestLowQuad, MVT::i32)); |
| 3539 | else |
| 3540 | MaskVec.push_back(DAG.getConstant(0, MVT::i32)); |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3541 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3542 | if (BestHighQuad != -1) |
| 3543 | MaskVec.push_back(DAG.getConstant(BestHighQuad, MVT::i32)); |
| 3544 | else |
| 3545 | MaskVec.push_back(DAG.getConstant(1, MVT::i32)); |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3546 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3547 | SDValue Mask= DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, &MaskVec[0],2); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3548 | NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2i64, |
| 3549 | DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V1), |
| 3550 | DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, V2), Mask); |
| 3551 | NewV = DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, NewV); |
| 3552 | |
| 3553 | // Now sort high and low parts separately. |
| 3554 | BitVector InOrder(8); |
| 3555 | if (BestLowQuad != -1) { |
| 3556 | // Sort lower half in order using PSHUFLW. |
| 3557 | MaskVec.clear(); |
| 3558 | bool AnyOutOrder = false; |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3559 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3560 | for (unsigned i = 0; i != 4; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3561 | SDValue Elt = MaskElts[i]; |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3562 | if (Elt.getOpcode() == ISD::UNDEF) { |
| 3563 | MaskVec.push_back(Elt); |
| 3564 | InOrder.set(i); |
| 3565 | } else { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3566 | unsigned EltIdx = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3567 | if (EltIdx != i) |
| 3568 | AnyOutOrder = true; |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3569 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3570 | MaskVec.push_back(DAG.getConstant(EltIdx % 4, MaskEVT)); |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3571 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3572 | // If this element is in the right place after this shuffle, then |
| 3573 | // remember it. |
| 3574 | if ((int)(EltIdx / 4) == BestLowQuad) |
| 3575 | InOrder.set(i); |
| 3576 | } |
| 3577 | } |
| 3578 | if (AnyOutOrder) { |
| 3579 | for (unsigned i = 4; i != 8; ++i) |
| 3580 | MaskVec.push_back(DAG.getConstant(i, MaskEVT)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3581 | SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3582 | NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); |
| 3583 | } |
| 3584 | } |
| 3585 | |
| 3586 | if (BestHighQuad != -1) { |
| 3587 | // Sort high half in order using PSHUFHW if possible. |
| 3588 | MaskVec.clear(); |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3589 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3590 | for (unsigned i = 0; i != 4; ++i) |
| 3591 | MaskVec.push_back(DAG.getConstant(i, MaskEVT)); |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3592 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3593 | bool AnyOutOrder = false; |
| 3594 | for (unsigned i = 4; i != 8; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3595 | SDValue Elt = MaskElts[i]; |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3596 | if (Elt.getOpcode() == ISD::UNDEF) { |
| 3597 | MaskVec.push_back(Elt); |
| 3598 | InOrder.set(i); |
| 3599 | } else { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3600 | unsigned EltIdx = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3601 | if (EltIdx != i) |
| 3602 | AnyOutOrder = true; |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3603 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3604 | MaskVec.push_back(DAG.getConstant((EltIdx % 4) + 4, MaskEVT)); |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3605 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3606 | // If this element is in the right place after this shuffle, then |
| 3607 | // remember it. |
| 3608 | if ((int)(EltIdx / 4) == BestHighQuad) |
| 3609 | InOrder.set(i); |
| 3610 | } |
| 3611 | } |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3612 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3613 | if (AnyOutOrder) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3614 | SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3615 | NewV = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, NewV, NewV, Mask); |
| 3616 | } |
| 3617 | } |
| 3618 | |
| 3619 | // The other elements are put in the right place using pextrw and pinsrw. |
| 3620 | for (unsigned i = 0; i != 8; ++i) { |
| 3621 | if (InOrder[i]) |
| 3622 | continue; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3623 | SDValue Elt = MaskElts[i]; |
Bill Wendling | ae0218c | 2008-08-21 22:36:36 +0000 | [diff] [blame] | 3624 | if (Elt.getOpcode() == ISD::UNDEF) |
| 3625 | continue; |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3626 | unsigned EltIdx = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3627 | SDValue ExtOp = (EltIdx < 8) |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3628 | ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, |
| 3629 | DAG.getConstant(EltIdx, PtrVT)) |
| 3630 | : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, |
| 3631 | DAG.getConstant(EltIdx - 8, PtrVT)); |
| 3632 | NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, |
| 3633 | DAG.getConstant(i, PtrVT)); |
| 3634 | } |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3635 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3636 | return NewV; |
| 3637 | } |
| 3638 | |
Bill Wendling | e85dc49 | 2008-08-21 22:35:37 +0000 | [diff] [blame] | 3639 | // PSHUF{H|L}W are not used. Lower into extracts and inserts but try to use as |
| 3640 | // few as possible. First, let's find out how many elements are already in the |
| 3641 | // right order. |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 3642 | unsigned V1InOrder = 0; |
| 3643 | unsigned V1FromV1 = 0; |
| 3644 | unsigned V2InOrder = 0; |
| 3645 | unsigned V2FromV2 = 0; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3646 | SmallVector<SDValue, 8> V1Elts; |
| 3647 | SmallVector<SDValue, 8> V2Elts; |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 3648 | for (unsigned i = 0; i < 8; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3649 | SDValue Elt = MaskElts[i]; |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 3650 | if (Elt.getOpcode() == ISD::UNDEF) { |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3651 | V1Elts.push_back(Elt); |
| 3652 | V2Elts.push_back(Elt); |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 3653 | ++V1InOrder; |
| 3654 | ++V2InOrder; |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3655 | continue; |
| 3656 | } |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3657 | unsigned EltIdx = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3658 | if (EltIdx == i) { |
| 3659 | V1Elts.push_back(Elt); |
| 3660 | V2Elts.push_back(DAG.getConstant(i+8, MaskEVT)); |
| 3661 | ++V1InOrder; |
| 3662 | } else if (EltIdx == i+8) { |
| 3663 | V1Elts.push_back(Elt); |
| 3664 | V2Elts.push_back(DAG.getConstant(i, MaskEVT)); |
| 3665 | ++V2InOrder; |
| 3666 | } else if (EltIdx < 8) { |
| 3667 | V1Elts.push_back(Elt); |
Mon P Wang | 62c75ea | 2008-12-23 04:03:27 +0000 | [diff] [blame] | 3668 | V2Elts.push_back(DAG.getConstant(i+8, MaskEVT)); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3669 | ++V1FromV1; |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 3670 | } else { |
Mon P Wang | 62c75ea | 2008-12-23 04:03:27 +0000 | [diff] [blame] | 3671 | V1Elts.push_back(Elt); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3672 | V2Elts.push_back(DAG.getConstant(EltIdx-8, MaskEVT)); |
| 3673 | ++V2FromV2; |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 3674 | } |
| 3675 | } |
| 3676 | |
| 3677 | if (V2InOrder > V1InOrder) { |
| 3678 | PermMask = CommuteVectorShuffleMask(PermMask, DAG); |
| 3679 | std::swap(V1, V2); |
| 3680 | std::swap(V1Elts, V2Elts); |
| 3681 | std::swap(V1FromV1, V2FromV2); |
| 3682 | } |
| 3683 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3684 | if ((V1FromV1 + V1InOrder) != 8) { |
| 3685 | // Some elements are from V2. |
| 3686 | if (V1FromV1) { |
| 3687 | // If there are elements that are from V1 but out of place, |
| 3688 | // then first sort them in place |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3689 | SmallVector<SDValue, 8> MaskVec; |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3690 | for (unsigned i = 0; i < 8; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3691 | SDValue Elt = V1Elts[i]; |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3692 | if (Elt.getOpcode() == ISD::UNDEF) { |
| 3693 | MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); |
| 3694 | continue; |
| 3695 | } |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3696 | unsigned EltIdx = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3697 | if (EltIdx >= 8) |
| 3698 | MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); |
| 3699 | else |
| 3700 | MaskVec.push_back(DAG.getConstant(EltIdx, MaskEVT)); |
| 3701 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3702 | SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], 8); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3703 | V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v8i16, V1, V1, Mask); |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 3704 | } |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3705 | |
| 3706 | NewV = V1; |
| 3707 | for (unsigned i = 0; i < 8; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3708 | SDValue Elt = V1Elts[i]; |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3709 | if (Elt.getOpcode() == ISD::UNDEF) |
| 3710 | continue; |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3711 | unsigned EltIdx = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3712 | if (EltIdx < 8) |
| 3713 | continue; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3714 | SDValue ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V2, |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3715 | DAG.getConstant(EltIdx - 8, PtrVT)); |
| 3716 | NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, |
| 3717 | DAG.getConstant(i, PtrVT)); |
| 3718 | } |
| 3719 | return NewV; |
| 3720 | } else { |
| 3721 | // All elements are from V1. |
| 3722 | NewV = V1; |
| 3723 | for (unsigned i = 0; i < 8; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3724 | SDValue Elt = V1Elts[i]; |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3725 | if (Elt.getOpcode() == ISD::UNDEF) |
| 3726 | continue; |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3727 | unsigned EltIdx = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3728 | SDValue ExtOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, V1, |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3729 | DAG.getConstant(EltIdx, PtrVT)); |
| 3730 | NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, MVT::v8i16, NewV, ExtOp, |
| 3731 | DAG.getConstant(i, PtrVT)); |
| 3732 | } |
| 3733 | return NewV; |
| 3734 | } |
| 3735 | } |
| 3736 | |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 3737 | /// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide |
| 3738 | /// ones, or rewriting v4i32 / v2f32 as 2 wide ones if possible. This can be |
| 3739 | /// done when every pair / quad of shuffle mask elements point to elements in |
| 3740 | /// the right sequence. e.g. |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3741 | /// vector_shuffle <>, <>, < 3, 4, | 10, 11, | 0, 1, | 14, 15> |
| 3742 | static |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3743 | SDValue RewriteAsNarrowerShuffle(SDValue V1, SDValue V2, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3744 | MVT VT, |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3745 | SDValue PermMask, SelectionDAG &DAG, |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3746 | TargetLowering &TLI) { |
| 3747 | unsigned NumElems = PermMask.getNumOperands(); |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 3748 | unsigned NewWidth = (NumElems == 4) ? 2 : 4; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3749 | MVT MaskVT = MVT::getIntVectorWithNumElements(NewWidth); |
Duncan Sands | d038e04 | 2008-07-21 10:20:31 +0000 | [diff] [blame] | 3750 | MVT MaskEltVT = MaskVT.getVectorElementType(); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3751 | MVT NewVT = MaskVT; |
| 3752 | switch (VT.getSimpleVT()) { |
| 3753 | default: assert(false && "Unexpected!"); |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 3754 | case MVT::v4f32: NewVT = MVT::v2f64; break; |
| 3755 | case MVT::v4i32: NewVT = MVT::v2i64; break; |
| 3756 | case MVT::v8i16: NewVT = MVT::v4i32; break; |
| 3757 | case MVT::v16i8: NewVT = MVT::v4i32; break; |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 3758 | } |
| 3759 | |
Anton Korobeynikov | 7c1c261 | 2008-02-20 11:22:39 +0000 | [diff] [blame] | 3760 | if (NewWidth == 2) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3761 | if (VT.isInteger()) |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 3762 | NewVT = MVT::v2i64; |
| 3763 | else |
| 3764 | NewVT = MVT::v2f64; |
Anton Korobeynikov | 7c1c261 | 2008-02-20 11:22:39 +0000 | [diff] [blame] | 3765 | } |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 3766 | unsigned Scale = NumElems / NewWidth; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3767 | SmallVector<SDValue, 8> MaskVec; |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3768 | for (unsigned i = 0; i < NumElems; i += Scale) { |
| 3769 | unsigned StartIdx = ~0U; |
| 3770 | for (unsigned j = 0; j < Scale; ++j) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3771 | SDValue Elt = PermMask.getOperand(i+j); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3772 | if (Elt.getOpcode() == ISD::UNDEF) |
| 3773 | continue; |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3774 | unsigned EltIdx = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3775 | if (StartIdx == ~0U) |
| 3776 | StartIdx = EltIdx - (EltIdx % Scale); |
| 3777 | if (EltIdx != StartIdx + j) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3778 | return SDValue(); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3779 | } |
| 3780 | if (StartIdx == ~0U) |
Duncan Sands | d038e04 | 2008-07-21 10:20:31 +0000 | [diff] [blame] | 3781 | MaskVec.push_back(DAG.getNode(ISD::UNDEF, MaskEltVT)); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 3782 | else |
Duncan Sands | d038e04 | 2008-07-21 10:20:31 +0000 | [diff] [blame] | 3783 | MaskVec.push_back(DAG.getConstant(StartIdx / Scale, MaskEltVT)); |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 3784 | } |
| 3785 | |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 3786 | V1 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V1); |
| 3787 | V2 = DAG.getNode(ISD::BIT_CONVERT, NewVT, V2); |
| 3788 | return DAG.getNode(ISD::VECTOR_SHUFFLE, NewVT, V1, V2, |
| 3789 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3790 | &MaskVec[0], MaskVec.size())); |
Evan Cheng | 8a86c3f | 2007-12-07 08:07:39 +0000 | [diff] [blame] | 3791 | } |
| 3792 | |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 3793 | /// getVZextMovL - Return a zero-extending vector move low node. |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 3794 | /// |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3795 | static SDValue getVZextMovL(MVT VT, MVT OpVT, |
| 3796 | SDValue SrcOp, SelectionDAG &DAG, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3797 | const X86Subtarget *Subtarget) { |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 3798 | if (VT == MVT::v2f64 || VT == MVT::v4f32) { |
| 3799 | LoadSDNode *LD = NULL; |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 3800 | if (!isScalarLoadToVector(SrcOp.getNode(), &LD)) |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 3801 | LD = dyn_cast<LoadSDNode>(SrcOp); |
| 3802 | if (!LD) { |
| 3803 | // movssrr and movsdrr do not clear top bits. Try to use movd, movq |
| 3804 | // instead. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3805 | MVT EVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 3806 | if ((EVT != MVT::i64 || Subtarget->is64Bit()) && |
| 3807 | SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR && |
| 3808 | SrcOp.getOperand(0).getOpcode() == ISD::BIT_CONVERT && |
| 3809 | SrcOp.getOperand(0).getOperand(0).getValueType() == EVT) { |
| 3810 | // PR2108 |
| 3811 | OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; |
| 3812 | return DAG.getNode(ISD::BIT_CONVERT, VT, |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 3813 | DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 3814 | DAG.getNode(ISD::SCALAR_TO_VECTOR, OpVT, |
Gabor Greif | 327ef03 | 2008-08-28 23:19:51 +0000 | [diff] [blame] | 3815 | SrcOp.getOperand(0) |
| 3816 | .getOperand(0)))); |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 3817 | } |
| 3818 | } |
| 3819 | } |
| 3820 | |
| 3821 | return DAG.getNode(ISD::BIT_CONVERT, VT, |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 3822 | DAG.getNode(X86ISD::VZEXT_MOVL, OpVT, |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 3823 | DAG.getNode(ISD::BIT_CONVERT, OpVT, SrcOp))); |
| 3824 | } |
| 3825 | |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3826 | /// LowerVECTOR_SHUFFLE_4wide - Handle all 4 wide cases with a number of |
| 3827 | /// shuffles. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3828 | static SDValue |
| 3829 | LowerVECTOR_SHUFFLE_4wide(SDValue V1, SDValue V2, |
| 3830 | SDValue PermMask, MVT VT, SelectionDAG &DAG) { |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3831 | MVT MaskVT = PermMask.getValueType(); |
| 3832 | MVT MaskEVT = MaskVT.getVectorElementType(); |
| 3833 | SmallVector<std::pair<int, int>, 8> Locs; |
Rafael Espindola | 833a990 | 2008-08-28 18:32:53 +0000 | [diff] [blame] | 3834 | Locs.resize(4); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3835 | SmallVector<SDValue, 8> Mask1(4, DAG.getNode(ISD::UNDEF, MaskEVT)); |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3836 | unsigned NumHi = 0; |
| 3837 | unsigned NumLo = 0; |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3838 | for (unsigned i = 0; i != 4; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3839 | SDValue Elt = PermMask.getOperand(i); |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3840 | if (Elt.getOpcode() == ISD::UNDEF) { |
| 3841 | Locs[i] = std::make_pair(-1, -1); |
| 3842 | } else { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3843 | unsigned Val = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Dan Gohman | d085994 | 2008-08-04 23:09:15 +0000 | [diff] [blame] | 3844 | assert(Val < 8 && "Invalid VECTOR_SHUFFLE index!"); |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3845 | if (Val < 4) { |
| 3846 | Locs[i] = std::make_pair(0, NumLo); |
| 3847 | Mask1[NumLo] = Elt; |
| 3848 | NumLo++; |
| 3849 | } else { |
| 3850 | Locs[i] = std::make_pair(1, NumHi); |
| 3851 | if (2+NumHi < 4) |
| 3852 | Mask1[2+NumHi] = Elt; |
| 3853 | NumHi++; |
| 3854 | } |
| 3855 | } |
| 3856 | } |
Evan Cheng | 5e6ebaf | 2008-07-23 00:22:17 +0000 | [diff] [blame] | 3857 | |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3858 | if (NumLo <= 2 && NumHi <= 2) { |
Evan Cheng | 5e6ebaf | 2008-07-23 00:22:17 +0000 | [diff] [blame] | 3859 | // If no more than two elements come from either vector. This can be |
| 3860 | // implemented with two shuffles. First shuffle gather the elements. |
| 3861 | // The second shuffle, which takes the first shuffle as both of its |
| 3862 | // vector operands, put the elements into the right order. |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3863 | V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, |
| 3864 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3865 | &Mask1[0], Mask1.size())); |
Evan Cheng | 5e6ebaf | 2008-07-23 00:22:17 +0000 | [diff] [blame] | 3866 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3867 | SmallVector<SDValue, 8> Mask2(4, DAG.getNode(ISD::UNDEF, MaskEVT)); |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3868 | for (unsigned i = 0; i != 4; ++i) { |
| 3869 | if (Locs[i].first == -1) |
| 3870 | continue; |
| 3871 | else { |
| 3872 | unsigned Idx = (i < 2) ? 0 : 4; |
| 3873 | Idx += Locs[i].first * 2 + Locs[i].second; |
| 3874 | Mask2[i] = DAG.getConstant(Idx, MaskEVT); |
| 3875 | } |
| 3876 | } |
| 3877 | |
| 3878 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, |
| 3879 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3880 | &Mask2[0], Mask2.size())); |
Evan Cheng | 5e6ebaf | 2008-07-23 00:22:17 +0000 | [diff] [blame] | 3881 | } else if (NumLo == 3 || NumHi == 3) { |
| 3882 | // Otherwise, we must have three elements from one vector, call it X, and |
| 3883 | // one element from the other, call it Y. First, use a shufps to build an |
| 3884 | // intermediate vector with the one element from Y and the element from X |
| 3885 | // that will be in the same half in the final destination (the indexes don't |
| 3886 | // matter). Then, use a shufps to build the final vector, taking the half |
| 3887 | // containing the element from Y from the intermediate, and the other half |
| 3888 | // from X. |
| 3889 | if (NumHi == 3) { |
| 3890 | // Normalize it so the 3 elements come from V1. |
| 3891 | PermMask = CommuteVectorShuffleMask(PermMask, DAG); |
| 3892 | std::swap(V1, V2); |
| 3893 | } |
| 3894 | |
| 3895 | // Find the element from V2. |
| 3896 | unsigned HiIndex; |
| 3897 | for (HiIndex = 0; HiIndex < 3; ++HiIndex) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3898 | SDValue Elt = PermMask.getOperand(HiIndex); |
Evan Cheng | 5e6ebaf | 2008-07-23 00:22:17 +0000 | [diff] [blame] | 3899 | if (Elt.getOpcode() == ISD::UNDEF) |
| 3900 | continue; |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3901 | unsigned Val = cast<ConstantSDNode>(Elt)->getZExtValue(); |
Evan Cheng | 5e6ebaf | 2008-07-23 00:22:17 +0000 | [diff] [blame] | 3902 | if (Val >= 4) |
| 3903 | break; |
| 3904 | } |
| 3905 | |
| 3906 | Mask1[0] = PermMask.getOperand(HiIndex); |
| 3907 | Mask1[1] = DAG.getNode(ISD::UNDEF, MaskEVT); |
| 3908 | Mask1[2] = PermMask.getOperand(HiIndex^1); |
| 3909 | Mask1[3] = DAG.getNode(ISD::UNDEF, MaskEVT); |
| 3910 | V2 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, |
| 3911 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); |
| 3912 | |
| 3913 | if (HiIndex >= 2) { |
| 3914 | Mask1[0] = PermMask.getOperand(0); |
| 3915 | Mask1[1] = PermMask.getOperand(1); |
| 3916 | Mask1[2] = DAG.getConstant(HiIndex & 1 ? 6 : 4, MaskEVT); |
| 3917 | Mask1[3] = DAG.getConstant(HiIndex & 1 ? 4 : 6, MaskEVT); |
| 3918 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, |
| 3919 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); |
| 3920 | } else { |
| 3921 | Mask1[0] = DAG.getConstant(HiIndex & 1 ? 2 : 0, MaskEVT); |
| 3922 | Mask1[1] = DAG.getConstant(HiIndex & 1 ? 0 : 2, MaskEVT); |
| 3923 | Mask1[2] = PermMask.getOperand(2); |
| 3924 | Mask1[3] = PermMask.getOperand(3); |
| 3925 | if (Mask1[2].getOpcode() != ISD::UNDEF) |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3926 | Mask1[2] = |
| 3927 | DAG.getConstant(cast<ConstantSDNode>(Mask1[2])->getZExtValue()+4, |
| 3928 | MaskEVT); |
Evan Cheng | 5e6ebaf | 2008-07-23 00:22:17 +0000 | [diff] [blame] | 3929 | if (Mask1[3].getOpcode() != ISD::UNDEF) |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3930 | Mask1[3] = |
| 3931 | DAG.getConstant(cast<ConstantSDNode>(Mask1[3])->getZExtValue()+4, |
| 3932 | MaskEVT); |
Evan Cheng | 5e6ebaf | 2008-07-23 00:22:17 +0000 | [diff] [blame] | 3933 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V2, V1, |
| 3934 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &Mask1[0], 4)); |
| 3935 | } |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3936 | } |
| 3937 | |
| 3938 | // Break it into (shuffle shuffle_hi, shuffle_lo). |
| 3939 | Locs.clear(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3940 | SmallVector<SDValue,8> LoMask(4, DAG.getNode(ISD::UNDEF, MaskEVT)); |
| 3941 | SmallVector<SDValue,8> HiMask(4, DAG.getNode(ISD::UNDEF, MaskEVT)); |
| 3942 | SmallVector<SDValue,8> *MaskPtr = &LoMask; |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3943 | unsigned MaskIdx = 0; |
| 3944 | unsigned LoIdx = 0; |
| 3945 | unsigned HiIdx = 2; |
| 3946 | for (unsigned i = 0; i != 4; ++i) { |
| 3947 | if (i == 2) { |
| 3948 | MaskPtr = &HiMask; |
| 3949 | MaskIdx = 1; |
| 3950 | LoIdx = 0; |
| 3951 | HiIdx = 2; |
| 3952 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3953 | SDValue Elt = PermMask.getOperand(i); |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3954 | if (Elt.getOpcode() == ISD::UNDEF) { |
| 3955 | Locs[i] = std::make_pair(-1, -1); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 3956 | } else if (cast<ConstantSDNode>(Elt)->getZExtValue() < 4) { |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3957 | Locs[i] = std::make_pair(MaskIdx, LoIdx); |
| 3958 | (*MaskPtr)[LoIdx] = Elt; |
| 3959 | LoIdx++; |
| 3960 | } else { |
| 3961 | Locs[i] = std::make_pair(MaskIdx, HiIdx); |
| 3962 | (*MaskPtr)[HiIdx] = Elt; |
| 3963 | HiIdx++; |
| 3964 | } |
| 3965 | } |
| 3966 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3967 | SDValue LoShuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3968 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3969 | &LoMask[0], LoMask.size())); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3970 | SDValue HiShuffle = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3971 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3972 | &HiMask[0], HiMask.size())); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3973 | SmallVector<SDValue, 8> MaskOps; |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 3974 | for (unsigned i = 0; i != 4; ++i) { |
| 3975 | if (Locs[i].first == -1) { |
| 3976 | MaskOps.push_back(DAG.getNode(ISD::UNDEF, MaskEVT)); |
| 3977 | } else { |
| 3978 | unsigned Idx = Locs[i].first * 4 + Locs[i].second; |
| 3979 | MaskOps.push_back(DAG.getConstant(Idx, MaskEVT)); |
| 3980 | } |
| 3981 | } |
| 3982 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, LoShuffle, HiShuffle, |
| 3983 | DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
| 3984 | &MaskOps[0], MaskOps.size())); |
| 3985 | } |
| 3986 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 3987 | SDValue |
| 3988 | X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) { |
| 3989 | SDValue V1 = Op.getOperand(0); |
| 3990 | SDValue V2 = Op.getOperand(1); |
| 3991 | SDValue PermMask = Op.getOperand(2); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3992 | MVT VT = Op.getValueType(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3993 | unsigned NumElems = PermMask.getNumOperands(); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 3994 | bool isMMX = VT.getSizeInBits() == 64; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3995 | bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; |
| 3996 | bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; |
Evan Cheng | d9b8e40 | 2006-10-16 06:36:00 +0000 | [diff] [blame] | 3997 | bool V1IsSplat = false; |
| 3998 | bool V2IsSplat = false; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 3999 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4000 | if (isUndefShuffle(Op.getNode())) |
Evan Cheng | 8cf723d | 2006-09-08 01:50:06 +0000 | [diff] [blame] | 4001 | return DAG.getNode(ISD::UNDEF, VT); |
| 4002 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4003 | if (isZeroShuffle(Op.getNode())) |
Evan Cheng | f0df031 | 2008-05-15 08:39:06 +0000 | [diff] [blame] | 4004 | return getZeroVector(VT, Subtarget->hasSSE2(), DAG); |
Evan Cheng | 213d2cf | 2007-05-17 18:45:50 +0000 | [diff] [blame] | 4005 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4006 | if (isIdentityMask(PermMask.getNode())) |
Evan Cheng | 49892af | 2007-06-19 00:02:56 +0000 | [diff] [blame] | 4007 | return V1; |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4008 | else if (isIdentityMask(PermMask.getNode(), true)) |
Evan Cheng | 49892af | 2007-06-19 00:02:56 +0000 | [diff] [blame] | 4009 | return V2; |
| 4010 | |
Evan Cheng | 4dcc8a3 | 2008-09-25 23:35:16 +0000 | [diff] [blame] | 4011 | // Canonicalize movddup shuffles. |
| 4012 | if (V2IsUndef && Subtarget->hasSSE2() && |
Evan Cheng | 882cdfd | 2008-10-06 21:13:08 +0000 | [diff] [blame] | 4013 | VT.getSizeInBits() == 128 && |
Evan Cheng | 4dcc8a3 | 2008-09-25 23:35:16 +0000 | [diff] [blame] | 4014 | X86::isMOVDDUPMask(PermMask.getNode())) |
| 4015 | return CanonicalizeMovddup(Op, V1, PermMask, DAG, Subtarget->hasSSE3()); |
| 4016 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4017 | if (isSplatMask(PermMask.getNode())) { |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 4018 | if (isMMX || NumElems < 4) return Op; |
| 4019 | // Promote it to a v4{if}32 splat. |
| 4020 | return PromoteSplat(Op, DAG, Subtarget->hasSSE2()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4021 | } |
| 4022 | |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 4023 | // If the shuffle can be profitably rewritten as a narrower shuffle, then |
| 4024 | // do it! |
| 4025 | if (VT == MVT::v8i16 || VT == MVT::v16i8) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4026 | SDValue NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, DAG, *this); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4027 | if (NewOp.getNode()) |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 4028 | return DAG.getNode(ISD::BIT_CONVERT, VT, LowerVECTOR_SHUFFLE(NewOp, DAG)); |
| 4029 | } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) { |
| 4030 | // FIXME: Figure out a cleaner way to do this. |
| 4031 | // Try to make use of movq to zero out the top part. |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4032 | if (ISD::isBuildVectorAllZeros(V2.getNode())) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4033 | SDValue NewOp = RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 4034 | DAG, *this); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4035 | if (NewOp.getNode()) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4036 | SDValue NewV1 = NewOp.getOperand(0); |
| 4037 | SDValue NewV2 = NewOp.getOperand(1); |
| 4038 | SDValue NewMask = NewOp.getOperand(2); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4039 | if (isCommutedMOVL(NewMask.getNode(), true, false)) { |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 4040 | NewOp = CommuteVectorShuffle(NewOp, NewV1, NewV2, NewMask, DAG); |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 4041 | return getVZextMovL(VT, NewOp.getValueType(), NewV2, DAG, Subtarget); |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 4042 | } |
| 4043 | } |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4044 | } else if (ISD::isBuildVectorAllZeros(V1.getNode())) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4045 | SDValue NewOp= RewriteAsNarrowerShuffle(V1, V2, VT, PermMask, |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 4046 | DAG, *this); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4047 | if (NewOp.getNode() && X86::isMOVLMask(NewOp.getOperand(2).getNode())) |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 4048 | return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(1), |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 4049 | DAG, Subtarget); |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 4050 | } |
| 4051 | } |
| 4052 | |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 4053 | // Check if this can be converted into a logical shift. |
| 4054 | bool isLeft = false; |
| 4055 | unsigned ShAmt = 0; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4056 | SDValue ShVal; |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 4057 | bool isShift = isVectorShift(Op, PermMask, DAG, isLeft, ShVal, ShAmt); |
| 4058 | if (isShift && ShVal.hasOneUse()) { |
| 4059 | // If the shifted value has multiple uses, it may be cheaper to use |
| 4060 | // v_set0 + movlhps or movhlps, etc. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4061 | MVT EVT = VT.getVectorElementType(); |
| 4062 | ShAmt *= EVT.getSizeInBits(); |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 4063 | return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); |
| 4064 | } |
| 4065 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4066 | if (X86::isMOVLMask(PermMask.getNode())) { |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 4067 | if (V1IsUndef) |
| 4068 | return V2; |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4069 | if (ISD::isBuildVectorAllZeros(V1.getNode())) |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 4070 | return getVZextMovL(VT, VT, V2, DAG, Subtarget); |
Nate Begeman | fb8ead0 | 2008-07-25 19:05:58 +0000 | [diff] [blame] | 4071 | if (!isMMX) |
| 4072 | return Op; |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 4073 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 4074 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4075 | if (!isMMX && (X86::isMOVSHDUPMask(PermMask.getNode()) || |
| 4076 | X86::isMOVSLDUPMask(PermMask.getNode()) || |
| 4077 | X86::isMOVHLPSMask(PermMask.getNode()) || |
| 4078 | X86::isMOVHPMask(PermMask.getNode()) || |
| 4079 | X86::isMOVLPMask(PermMask.getNode()))) |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4080 | return Op; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4081 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4082 | if (ShouldXformToMOVHLPS(PermMask.getNode()) || |
| 4083 | ShouldXformToMOVLP(V1.getNode(), V2.getNode(), PermMask.getNode())) |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 4084 | return CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4085 | |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 4086 | if (isShift) { |
| 4087 | // No better options. Use a vshl / vsrl. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4088 | MVT EVT = VT.getVectorElementType(); |
| 4089 | ShAmt *= EVT.getSizeInBits(); |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 4090 | return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this); |
| 4091 | } |
| 4092 | |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 4093 | bool Commuted = false; |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 4094 | // FIXME: This should also accept a bitcast of a splat? Be careful, not |
| 4095 | // 1,1,1,1 -> v8i16 though. |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4096 | V1IsSplat = isSplatVector(V1.getNode()); |
| 4097 | V2IsSplat = isSplatVector(V2.getNode()); |
Chris Lattner | 8a59448 | 2007-11-25 00:24:49 +0000 | [diff] [blame] | 4098 | |
| 4099 | // Canonicalize the splat or undef, if present, to be on the RHS. |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4100 | if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) { |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 4101 | Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4102 | std::swap(V1IsSplat, V2IsSplat); |
| 4103 | std::swap(V1IsUndef, V2IsUndef); |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 4104 | Commuted = true; |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4105 | } |
| 4106 | |
Evan Cheng | 7a831ce | 2007-12-15 03:00:47 +0000 | [diff] [blame] | 4107 | // FIXME: Figure out a cleaner way to do this. |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4108 | if (isCommutedMOVL(PermMask.getNode(), V2IsSplat, V2IsUndef)) { |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4109 | if (V2IsUndef) return V1; |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 4110 | Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4111 | if (V2IsSplat) { |
| 4112 | // V2 is a splat, so the mask may be malformed. That is, it may point |
| 4113 | // to any V2 element. The instruction selectior won't like this. Get |
| 4114 | // a corrected mask and commute to form a proper MOVS{S|D}. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4115 | SDValue NewMask = getMOVLMask(NumElems, DAG); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4116 | if (NewMask.getNode() != PermMask.getNode()) |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4117 | Op = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4118 | } |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4119 | return Op; |
Evan Cheng | d9b8e40 | 2006-10-16 06:36:00 +0000 | [diff] [blame] | 4120 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4121 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4122 | if (X86::isUNPCKL_v_undef_Mask(PermMask.getNode()) || |
| 4123 | X86::isUNPCKH_v_undef_Mask(PermMask.getNode()) || |
| 4124 | X86::isUNPCKLMask(PermMask.getNode()) || |
| 4125 | X86::isUNPCKHMask(PermMask.getNode())) |
Evan Cheng | d9b8e40 | 2006-10-16 06:36:00 +0000 | [diff] [blame] | 4126 | return Op; |
Evan Cheng | e111303 | 2006-10-04 18:33:38 +0000 | [diff] [blame] | 4127 | |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4128 | if (V2IsSplat) { |
| 4129 | // Normalize mask so all entries that point to V2 points to its first |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 4130 | // element then try to match unpck{h|l} again. If match, return a |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4131 | // new vector_shuffle with the corrected mask. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4132 | SDValue NewMask = NormalizeMask(PermMask, DAG); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4133 | if (NewMask.getNode() != PermMask.getNode()) { |
| 4134 | if (X86::isUNPCKLMask(PermMask.getNode(), true)) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4135 | SDValue NewMask = getUnpacklMask(NumElems, DAG); |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4136 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4137 | } else if (X86::isUNPCKHMask(PermMask.getNode(), true)) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4138 | SDValue NewMask = getUnpackhMask(NumElems, DAG); |
Evan Cheng | 9bbbb98 | 2006-10-25 20:48:19 +0000 | [diff] [blame] | 4139 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V2, NewMask); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4140 | } |
| 4141 | } |
| 4142 | } |
| 4143 | |
| 4144 | // Normalize the node to match x86 shuffle ops if needed |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4145 | if (V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(PermMask.getNode())) |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 4146 | Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); |
| 4147 | |
| 4148 | if (Commuted) { |
| 4149 | // Commute is back and try unpck* again. |
| 4150 | Op = CommuteVectorShuffle(Op, V1, V2, PermMask, DAG); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4151 | if (X86::isUNPCKL_v_undef_Mask(PermMask.getNode()) || |
| 4152 | X86::isUNPCKH_v_undef_Mask(PermMask.getNode()) || |
| 4153 | X86::isUNPCKLMask(PermMask.getNode()) || |
| 4154 | X86::isUNPCKHMask(PermMask.getNode())) |
Evan Cheng | 9eca5e8 | 2006-10-25 21:49:50 +0000 | [diff] [blame] | 4155 | return Op; |
| 4156 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4157 | |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 4158 | // Try PSHUF* first, then SHUFP*. |
| 4159 | // MMX doesn't have PSHUFD but it does have PSHUFW. While it's theoretically |
| 4160 | // possible to shuffle a v2i32 using PSHUFW, that's not yet implemented. |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4161 | if (isMMX && NumElems == 4 && X86::isPSHUFDMask(PermMask.getNode())) { |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 4162 | if (V2.getOpcode() != ISD::UNDEF) |
| 4163 | return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, |
| 4164 | DAG.getNode(ISD::UNDEF, VT), PermMask); |
| 4165 | return Op; |
| 4166 | } |
| 4167 | |
| 4168 | if (!isMMX) { |
| 4169 | if (Subtarget->hasSSE2() && |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4170 | (X86::isPSHUFDMask(PermMask.getNode()) || |
| 4171 | X86::isPSHUFHWMask(PermMask.getNode()) || |
| 4172 | X86::isPSHUFLWMask(PermMask.getNode()))) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4173 | MVT RVT = VT; |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 4174 | if (VT == MVT::v4f32) { |
| 4175 | RVT = MVT::v4i32; |
| 4176 | Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, |
| 4177 | DAG.getNode(ISD::BIT_CONVERT, RVT, V1), |
| 4178 | DAG.getNode(ISD::UNDEF, RVT), PermMask); |
| 4179 | } else if (V2.getOpcode() != ISD::UNDEF) |
| 4180 | Op = DAG.getNode(ISD::VECTOR_SHUFFLE, RVT, V1, |
| 4181 | DAG.getNode(ISD::UNDEF, RVT), PermMask); |
| 4182 | if (RVT != VT) |
| 4183 | Op = DAG.getNode(ISD::BIT_CONVERT, VT, Op); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4184 | return Op; |
| 4185 | } |
| 4186 | |
Evan Cheng | 0c0f83f | 2008-04-05 00:30:36 +0000 | [diff] [blame] | 4187 | // Binary or unary shufps. |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4188 | if (X86::isSHUFPMask(PermMask.getNode()) || |
| 4189 | (V2.getOpcode() == ISD::UNDEF && X86::isPSHUFDMask(PermMask.getNode()))) |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4190 | return Op; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4191 | } |
| 4192 | |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 4193 | // Handle v8i16 specifically since SSE can do byte extraction and insertion. |
| 4194 | if (VT == MVT::v8i16) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4195 | SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(V1, V2, PermMask, DAG, *this); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4196 | if (NewOp.getNode()) |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 4197 | return NewOp; |
| 4198 | } |
| 4199 | |
Evan Cheng | ace3c17 | 2008-07-22 21:13:36 +0000 | [diff] [blame] | 4200 | // Handle all 4 wide cases with a number of shuffles except for MMX. |
| 4201 | if (NumElems == 4 && !isMMX) |
| 4202 | return LowerVECTOR_SHUFFLE_4wide(V1, V2, PermMask, VT, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4203 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4204 | return SDValue(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4205 | } |
| 4206 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4207 | SDValue |
| 4208 | X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4209 | SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4210 | MVT VT = Op.getValueType(); |
| 4211 | if (VT.getSizeInBits() == 8) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4212 | SDValue Extract = DAG.getNode(X86ISD::PEXTRB, MVT::i32, |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4213 | Op.getOperand(0), Op.getOperand(1)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4214 | SDValue Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4215 | DAG.getValueType(VT)); |
| 4216 | return DAG.getNode(ISD::TRUNCATE, VT, Assert); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4217 | } else if (VT.getSizeInBits() == 16) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4218 | SDValue Extract = DAG.getNode(X86ISD::PEXTRW, MVT::i32, |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4219 | Op.getOperand(0), Op.getOperand(1)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4220 | SDValue Assert = DAG.getNode(ISD::AssertZext, MVT::i32, Extract, |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4221 | DAG.getValueType(VT)); |
| 4222 | return DAG.getNode(ISD::TRUNCATE, VT, Assert); |
Evan Cheng | 62a3f15 | 2008-03-24 21:52:23 +0000 | [diff] [blame] | 4223 | } else if (VT == MVT::f32) { |
| 4224 | // EXTRACTPS outputs to a GPR32 register which will require a movd to copy |
| 4225 | // the result back to FR32 register. It's only worth matching if the |
Dan Gohman | d17cfbe | 2008-10-31 00:57:24 +0000 | [diff] [blame] | 4226 | // result has a single use which is a store or a bitcast to i32. And in |
| 4227 | // the case of a store, it's not worth it if the index is a constant 0, |
| 4228 | // because a MOVSSmr can be used instead, which is smaller and faster. |
Evan Cheng | 62a3f15 | 2008-03-24 21:52:23 +0000 | [diff] [blame] | 4229 | if (!Op.hasOneUse()) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4230 | return SDValue(); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4231 | SDNode *User = *Op.getNode()->use_begin(); |
Dan Gohman | d17cfbe | 2008-10-31 00:57:24 +0000 | [diff] [blame] | 4232 | if ((User->getOpcode() != ISD::STORE || |
| 4233 | (isa<ConstantSDNode>(Op.getOperand(1)) && |
| 4234 | cast<ConstantSDNode>(Op.getOperand(1))->isNullValue())) && |
Dan Gohman | 171c11e | 2008-04-16 02:32:24 +0000 | [diff] [blame] | 4235 | (User->getOpcode() != ISD::BIT_CONVERT || |
| 4236 | User->getValueType(0) != MVT::i32)) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4237 | return SDValue(); |
| 4238 | SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, |
Evan Cheng | 62a3f15 | 2008-03-24 21:52:23 +0000 | [diff] [blame] | 4239 | DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Op.getOperand(0)), |
| 4240 | Op.getOperand(1)); |
| 4241 | return DAG.getNode(ISD::BIT_CONVERT, MVT::f32, Extract); |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4242 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4243 | return SDValue(); |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4244 | } |
| 4245 | |
| 4246 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4247 | SDValue |
| 4248 | X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4249 | if (!isa<ConstantSDNode>(Op.getOperand(1))) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4250 | return SDValue(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4251 | |
Evan Cheng | 62a3f15 | 2008-03-24 21:52:23 +0000 | [diff] [blame] | 4252 | if (Subtarget->hasSSE41()) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4253 | SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4254 | if (Res.getNode()) |
Evan Cheng | 62a3f15 | 2008-03-24 21:52:23 +0000 | [diff] [blame] | 4255 | return Res; |
| 4256 | } |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4257 | |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4258 | MVT VT = Op.getValueType(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4259 | // TODO: handle v16i8. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4260 | if (VT.getSizeInBits() == 16) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4261 | SDValue Vec = Op.getOperand(0); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 4262 | unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); |
Evan Cheng | 14b32e1 | 2007-12-11 01:46:18 +0000 | [diff] [blame] | 4263 | if (Idx == 0) |
| 4264 | return DAG.getNode(ISD::TRUNCATE, MVT::i16, |
| 4265 | DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, |
| 4266 | DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, Vec), |
| 4267 | Op.getOperand(1))); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4268 | // Transform it so it match pextrw which produces a 32-bit result. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4269 | MVT EVT = (MVT::SimpleValueType)(VT.getSimpleVT()+1); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4270 | SDValue Extract = DAG.getNode(X86ISD::PEXTRW, EVT, |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4271 | Op.getOperand(0), Op.getOperand(1)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4272 | SDValue Assert = DAG.getNode(ISD::AssertZext, EVT, Extract, |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4273 | DAG.getValueType(VT)); |
| 4274 | return DAG.getNode(ISD::TRUNCATE, VT, Assert); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4275 | } else if (VT.getSizeInBits() == 32) { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 4276 | unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4277 | if (Idx == 0) |
| 4278 | return Op; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4279 | // SHUFPS the element to the lowest double word, then movss. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4280 | MVT MaskVT = MVT::getIntVectorWithNumElements(4); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4281 | SmallVector<SDValue, 8> IdxVec; |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 4282 | IdxVec. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4283 | push_back(DAG.getConstant(Idx, MaskVT.getVectorElementType())); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 4284 | IdxVec. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4285 | push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 4286 | IdxVec. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4287 | push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 4288 | IdxVec. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4289 | push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4290 | SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 4291 | &IdxVec[0], IdxVec.size()); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4292 | SDValue Vec = Op.getOperand(0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4293 | Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), |
Evan Cheng | 6e56e2c | 2006-11-07 22:14:24 +0000 | [diff] [blame] | 4294 | Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4295 | return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 4296 | DAG.getIntPtrConstant(0)); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4297 | } else if (VT.getSizeInBits() == 64) { |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4298 | // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b |
| 4299 | // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught |
| 4300 | // to match extract_elt for f64. |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 4301 | unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4302 | if (Idx == 0) |
| 4303 | return Op; |
| 4304 | |
| 4305 | // UNPCKHPD the element to the lowest double word, then movsd. |
| 4306 | // Note if the lower 64 bits of the result of the UNPCKHPD is then stored |
| 4307 | // to a f64mem, the whole operation is folded into a single MOVHPDmr. |
Duncan Sands | d038e04 | 2008-07-21 10:20:31 +0000 | [diff] [blame] | 4308 | MVT MaskVT = MVT::getIntVectorWithNumElements(2); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4309 | SmallVector<SDValue, 8> IdxVec; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4310 | IdxVec.push_back(DAG.getConstant(1, MaskVT.getVectorElementType())); |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 4311 | IdxVec. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4312 | push_back(DAG.getNode(ISD::UNDEF, MaskVT.getVectorElementType())); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4313 | SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, |
Chris Lattner | e219945 | 2006-08-11 17:38:39 +0000 | [diff] [blame] | 4314 | &IdxVec[0], IdxVec.size()); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4315 | SDValue Vec = Op.getOperand(0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4316 | Vec = DAG.getNode(ISD::VECTOR_SHUFFLE, Vec.getValueType(), |
| 4317 | Vec, DAG.getNode(ISD::UNDEF, Vec.getValueType()), Mask); |
| 4318 | return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, VT, Vec, |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 4319 | DAG.getIntPtrConstant(0)); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4320 | } |
| 4321 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4322 | return SDValue(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4323 | } |
| 4324 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4325 | SDValue |
| 4326 | X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG){ |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4327 | MVT VT = Op.getValueType(); |
| 4328 | MVT EVT = VT.getVectorElementType(); |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4329 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4330 | SDValue N0 = Op.getOperand(0); |
| 4331 | SDValue N1 = Op.getOperand(1); |
| 4332 | SDValue N2 = Op.getOperand(2); |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4333 | |
Dan Gohman | ef521f1 | 2008-08-14 22:53:18 +0000 | [diff] [blame] | 4334 | if ((EVT.getSizeInBits() == 8 || EVT.getSizeInBits() == 16) && |
| 4335 | isa<ConstantSDNode>(N2)) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4336 | unsigned Opc = (EVT.getSizeInBits() == 8) ? X86ISD::PINSRB |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4337 | : X86ISD::PINSRW; |
| 4338 | // Transform it so it match pinsr{b,w} which expects a GR32 as its second |
| 4339 | // argument. |
| 4340 | if (N1.getValueType() != MVT::i32) |
| 4341 | N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); |
| 4342 | if (N2.getValueType() != MVT::i32) |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 4343 | N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue()); |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4344 | return DAG.getNode(Opc, VT, N0, N1, N2); |
Dan Gohman | c0573b1 | 2008-08-14 22:43:26 +0000 | [diff] [blame] | 4345 | } else if (EVT == MVT::f32 && isa<ConstantSDNode>(N2)) { |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4346 | // Bits [7:6] of the constant are the source select. This will always be |
| 4347 | // zero here. The DAG Combiner may combine an extract_elt index into these |
| 4348 | // bits. For example (insert (extract, 3), 2) could be matched by putting |
| 4349 | // the '3' into bits [7:6] of X86ISD::INSERTPS. |
| 4350 | // Bits [5:4] of the constant are the destination select. This is the |
| 4351 | // value of the incoming immediate. |
| 4352 | // Bits [3:0] of the constant are the zero mask. The DAG Combiner may |
| 4353 | // combine either bitwise AND or insert of float 0.0 to set these bits. |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 4354 | N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue() << 4); |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4355 | return DAG.getNode(X86ISD::INSERTPS, VT, N0, N1, N2); |
| 4356 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4357 | return SDValue(); |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4358 | } |
| 4359 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4360 | SDValue |
| 4361 | X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4362 | MVT VT = Op.getValueType(); |
| 4363 | MVT EVT = VT.getVectorElementType(); |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 4364 | |
| 4365 | if (Subtarget->hasSSE41()) |
| 4366 | return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG); |
| 4367 | |
Evan Cheng | 794405e | 2007-12-12 07:55:34 +0000 | [diff] [blame] | 4368 | if (EVT == MVT::i8) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4369 | return SDValue(); |
Evan Cheng | 794405e | 2007-12-12 07:55:34 +0000 | [diff] [blame] | 4370 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4371 | SDValue N0 = Op.getOperand(0); |
| 4372 | SDValue N1 = Op.getOperand(1); |
| 4373 | SDValue N2 = Op.getOperand(2); |
Evan Cheng | 794405e | 2007-12-12 07:55:34 +0000 | [diff] [blame] | 4374 | |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4375 | if (EVT.getSizeInBits() == 16) { |
Evan Cheng | 794405e | 2007-12-12 07:55:34 +0000 | [diff] [blame] | 4376 | // Transform it so it match pinsrw which expects a 16-bit value in a GR32 |
| 4377 | // as its second argument. |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4378 | if (N1.getValueType() != MVT::i32) |
| 4379 | N1 = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, N1); |
| 4380 | if (N2.getValueType() != MVT::i32) |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 4381 | N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4382 | return DAG.getNode(X86ISD::PINSRW, VT, N0, N1, N2); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4383 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4384 | return SDValue(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4385 | } |
| 4386 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4387 | SDValue |
| 4388 | X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) { |
Evan Cheng | 52672b8 | 2008-07-22 18:39:19 +0000 | [diff] [blame] | 4389 | if (Op.getValueType() == MVT::v2f32) |
| 4390 | return DAG.getNode(ISD::BIT_CONVERT, MVT::v2f32, |
| 4391 | DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2i32, |
| 4392 | DAG.getNode(ISD::BIT_CONVERT, MVT::i32, |
| 4393 | Op.getOperand(0)))); |
| 4394 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4395 | SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, Op.getOperand(0)); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4396 | MVT VT = MVT::v2i32; |
| 4397 | switch (Op.getValueType().getSimpleVT()) { |
Evan Cheng | efec751 | 2008-02-18 23:04:32 +0000 | [diff] [blame] | 4398 | default: break; |
| 4399 | case MVT::v16i8: |
| 4400 | case MVT::v8i16: |
| 4401 | VT = MVT::v4i32; |
| 4402 | break; |
| 4403 | } |
| 4404 | return DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(), |
| 4405 | DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, AnyExt)); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4406 | } |
| 4407 | |
Bill Wendling | 056292f | 2008-09-16 21:48:12 +0000 | [diff] [blame] | 4408 | // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as |
| 4409 | // their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is |
| 4410 | // one of the above mentioned nodes. It has to be wrapped because otherwise |
| 4411 | // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only |
| 4412 | // be used to form addressing mode. These wrapped nodes will be selected |
| 4413 | // into MOV32ri. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4414 | SDValue |
| 4415 | X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) { |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4416 | ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4417 | SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), |
Evan Cheng | d0ff02c | 2006-11-29 23:19:46 +0000 | [diff] [blame] | 4418 | getPointerTy(), |
| 4419 | CP->getAlignment()); |
Evan Cheng | 19f2ffc | 2006-12-05 04:01:03 +0000 | [diff] [blame] | 4420 | Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); |
Anton Korobeynikov | 7f70559 | 2007-01-12 19:20:47 +0000 | [diff] [blame] | 4421 | // With PIC, the address is actually $g + Offset. |
| 4422 | if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && |
| 4423 | !Subtarget->isPICStyleRIPRel()) { |
| 4424 | Result = DAG.getNode(ISD::ADD, getPointerTy(), |
| 4425 | DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), |
| 4426 | Result); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4427 | } |
| 4428 | |
| 4429 | return Result; |
| 4430 | } |
| 4431 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4432 | SDValue |
Evan Cheng | da43bcf | 2008-09-24 00:05:32 +0000 | [diff] [blame] | 4433 | X86TargetLowering::LowerGlobalAddress(const GlobalValue *GV, |
Dan Gohman | 6520e20 | 2008-10-18 02:06:02 +0000 | [diff] [blame] | 4434 | int64_t Offset, |
Evan Cheng | da43bcf | 2008-09-24 00:05:32 +0000 | [diff] [blame] | 4435 | SelectionDAG &DAG) const { |
Dan Gohman | 6520e20 | 2008-10-18 02:06:02 +0000 | [diff] [blame] | 4436 | bool IsPic = getTargetMachine().getRelocationModel() == Reloc::PIC_; |
| 4437 | bool ExtraLoadRequired = |
| 4438 | Subtarget->GVRequiresExtraLoad(GV, getTargetMachine(), false); |
| 4439 | |
| 4440 | // Create the TargetGlobalAddress node, folding in the constant |
| 4441 | // offset if it is legal. |
| 4442 | SDValue Result; |
Dan Gohman | 4401361 | 2008-10-21 03:38:42 +0000 | [diff] [blame] | 4443 | if (!IsPic && !ExtraLoadRequired && isInt32(Offset)) { |
Dan Gohman | 6520e20 | 2008-10-18 02:06:02 +0000 | [diff] [blame] | 4444 | Result = DAG.getTargetGlobalAddress(GV, getPointerTy(), Offset); |
| 4445 | Offset = 0; |
| 4446 | } else |
| 4447 | Result = DAG.getTargetGlobalAddress(GV, getPointerTy(), 0); |
Evan Cheng | 19f2ffc | 2006-12-05 04:01:03 +0000 | [diff] [blame] | 4448 | Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); |
Dan Gohman | 6520e20 | 2008-10-18 02:06:02 +0000 | [diff] [blame] | 4449 | |
Anton Korobeynikov | 7f70559 | 2007-01-12 19:20:47 +0000 | [diff] [blame] | 4450 | // With PIC, the address is actually $g + Offset. |
Dan Gohman | 6520e20 | 2008-10-18 02:06:02 +0000 | [diff] [blame] | 4451 | if (IsPic && !Subtarget->isPICStyleRIPRel()) { |
Anton Korobeynikov | 7f70559 | 2007-01-12 19:20:47 +0000 | [diff] [blame] | 4452 | Result = DAG.getNode(ISD::ADD, getPointerTy(), |
| 4453 | DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), |
| 4454 | Result); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4455 | } |
Anton Korobeynikov | 2b2bc68 | 2006-12-22 22:29:05 +0000 | [diff] [blame] | 4456 | |
| 4457 | // For Darwin & Mingw32, external and weak symbols are indirect, so we want to |
| 4458 | // load the value at address GV, not the value of GV itself. This means that |
| 4459 | // the GlobalAddress must be in the base or index register of the address, not |
| 4460 | // the GV offset field. Platform check is inside GVRequiresExtraLoad() call |
Anton Korobeynikov | 7f70559 | 2007-01-12 19:20:47 +0000 | [diff] [blame] | 4461 | // The same applies for external symbols during PIC codegen |
Dan Gohman | 6520e20 | 2008-10-18 02:06:02 +0000 | [diff] [blame] | 4462 | if (ExtraLoadRequired) |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 4463 | Result = DAG.getLoad(getPointerTy(), DAG.getEntryNode(), Result, |
Dan Gohman | 3069b87 | 2008-02-07 18:41:25 +0000 | [diff] [blame] | 4464 | PseudoSourceValue::getGOT(), 0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4465 | |
Dan Gohman | 6520e20 | 2008-10-18 02:06:02 +0000 | [diff] [blame] | 4466 | // If there was a non-zero offset that we didn't fold, create an explicit |
| 4467 | // addition for it. |
| 4468 | if (Offset != 0) |
| 4469 | Result = DAG.getNode(ISD::ADD, getPointerTy(), Result, |
| 4470 | DAG.getConstant(Offset, getPointerTy())); |
| 4471 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4472 | return Result; |
| 4473 | } |
| 4474 | |
Evan Cheng | da43bcf | 2008-09-24 00:05:32 +0000 | [diff] [blame] | 4475 | SDValue |
| 4476 | X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) { |
| 4477 | const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); |
Dan Gohman | 6520e20 | 2008-10-18 02:06:02 +0000 | [diff] [blame] | 4478 | int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset(); |
| 4479 | return LowerGlobalAddress(GV, Offset, DAG); |
Evan Cheng | da43bcf | 2008-09-24 00:05:32 +0000 | [diff] [blame] | 4480 | } |
| 4481 | |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 4482 | // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4483 | static SDValue |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 4484 | LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4485 | const MVT PtrVT) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4486 | SDValue InFlag; |
| 4487 | SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), X86::EBX, |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4488 | DAG.getNode(X86ISD::GlobalBaseReg, |
| 4489 | PtrVT), InFlag); |
| 4490 | InFlag = Chain.getValue(1); |
| 4491 | |
| 4492 | // emit leal symbol@TLSGD(,%ebx,1), %eax |
| 4493 | SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4494 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4495 | GA->getValueType(0), |
| 4496 | GA->getOffset()); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4497 | SDValue Ops[] = { Chain, TGA, InFlag }; |
| 4498 | SDValue Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 3); |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4499 | InFlag = Result.getValue(2); |
| 4500 | Chain = Result.getValue(1); |
| 4501 | |
| 4502 | // call ___tls_get_addr. This function receives its argument in |
| 4503 | // the register EAX. |
| 4504 | Chain = DAG.getCopyToReg(Chain, X86::EAX, Result, InFlag); |
| 4505 | InFlag = Chain.getValue(1); |
| 4506 | |
| 4507 | NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4508 | SDValue Ops1[] = { Chain, |
Bill Wendling | 056292f | 2008-09-16 21:48:12 +0000 | [diff] [blame] | 4509 | DAG.getTargetExternalSymbol("___tls_get_addr", |
| 4510 | PtrVT), |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4511 | DAG.getRegister(X86::EAX, PtrVT), |
| 4512 | DAG.getRegister(X86::EBX, PtrVT), |
| 4513 | InFlag }; |
| 4514 | Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 5); |
| 4515 | InFlag = Chain.getValue(1); |
| 4516 | |
| 4517 | return DAG.getCopyFromReg(Chain, X86::EAX, PtrVT, InFlag); |
| 4518 | } |
| 4519 | |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 4520 | // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4521 | static SDValue |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 4522 | LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4523 | const MVT PtrVT) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4524 | SDValue InFlag, Chain; |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 4525 | |
| 4526 | // emit leaq symbol@TLSGD(%rip), %rdi |
| 4527 | SDVTList NodeTys = DAG.getVTList(PtrVT, MVT::Other, MVT::Flag); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4528 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 4529 | GA->getValueType(0), |
| 4530 | GA->getOffset()); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4531 | SDValue Ops[] = { DAG.getEntryNode(), TGA}; |
| 4532 | SDValue Result = DAG.getNode(X86ISD::TLSADDR, NodeTys, Ops, 2); |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 4533 | Chain = Result.getValue(1); |
| 4534 | InFlag = Result.getValue(2); |
| 4535 | |
Anton Korobeynikov | d97f295 | 2008-08-16 12:58:29 +0000 | [diff] [blame] | 4536 | // call __tls_get_addr. This function receives its argument in |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 4537 | // the register RDI. |
| 4538 | Chain = DAG.getCopyToReg(Chain, X86::RDI, Result, InFlag); |
| 4539 | InFlag = Chain.getValue(1); |
| 4540 | |
| 4541 | NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4542 | SDValue Ops1[] = { Chain, |
Bill Wendling | 056292f | 2008-09-16 21:48:12 +0000 | [diff] [blame] | 4543 | DAG.getTargetExternalSymbol("__tls_get_addr", |
| 4544 | PtrVT), |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 4545 | DAG.getRegister(X86::RDI, PtrVT), |
| 4546 | InFlag }; |
| 4547 | Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops1, 4); |
| 4548 | InFlag = Chain.getValue(1); |
| 4549 | |
| 4550 | return DAG.getCopyFromReg(Chain, X86::RAX, PtrVT, InFlag); |
| 4551 | } |
| 4552 | |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4553 | // Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or |
| 4554 | // "local exec" model. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4555 | static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4556 | const MVT PtrVT) { |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4557 | // Get the Thread Pointer |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4558 | SDValue ThreadPointer = DAG.getNode(X86ISD::THREAD_POINTER, PtrVT); |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4559 | // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial |
| 4560 | // exec) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4561 | SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4562 | GA->getValueType(0), |
| 4563 | GA->getOffset()); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4564 | SDValue Offset = DAG.getNode(X86ISD::Wrapper, PtrVT, TGA); |
Lauro Ramos Venancio | 7d2cc2b | 2007-04-22 22:50:52 +0000 | [diff] [blame] | 4565 | |
| 4566 | if (GA->getGlobal()->isDeclaration()) // initial exec TLS model |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 4567 | Offset = DAG.getLoad(PtrVT, DAG.getEntryNode(), Offset, |
Dan Gohman | 3069b87 | 2008-02-07 18:41:25 +0000 | [diff] [blame] | 4568 | PseudoSourceValue::getGOT(), 0); |
Lauro Ramos Venancio | 7d2cc2b | 2007-04-22 22:50:52 +0000 | [diff] [blame] | 4569 | |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4570 | // The address of the thread local variable is the add of the thread |
| 4571 | // pointer with the offset of the variable. |
| 4572 | return DAG.getNode(ISD::ADD, PtrVT, ThreadPointer, Offset); |
| 4573 | } |
| 4574 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4575 | SDValue |
| 4576 | X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) { |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4577 | // TODO: implement the "local dynamic" model |
Lauro Ramos Venancio | 2c5c111 | 2007-04-21 20:56:26 +0000 | [diff] [blame] | 4578 | // TODO: implement the "initial exec"model for pic executables |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 4579 | assert(Subtarget->isTargetELF() && |
| 4580 | "TLS not implemented for non-ELF targets"); |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4581 | GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); |
| 4582 | // If the relocation model is PIC, use the "General Dynamic" TLS Model, |
| 4583 | // otherwise use the "Local Exec"TLS Model |
Anton Korobeynikov | 6625eff | 2008-05-04 21:36:32 +0000 | [diff] [blame] | 4584 | if (Subtarget->is64Bit()) { |
| 4585 | return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy()); |
| 4586 | } else { |
| 4587 | if (getTargetMachine().getRelocationModel() == Reloc::PIC_) |
| 4588 | return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy()); |
| 4589 | else |
| 4590 | return LowerToTLSExecModel(GA, DAG, getPointerTy()); |
| 4591 | } |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 4592 | } |
| 4593 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4594 | SDValue |
| 4595 | X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) { |
Bill Wendling | 056292f | 2008-09-16 21:48:12 +0000 | [diff] [blame] | 4596 | const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol(); |
| 4597 | SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy()); |
Evan Cheng | 19f2ffc | 2006-12-05 04:01:03 +0000 | [diff] [blame] | 4598 | Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); |
Anton Korobeynikov | 7f70559 | 2007-01-12 19:20:47 +0000 | [diff] [blame] | 4599 | // With PIC, the address is actually $g + Offset. |
| 4600 | if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && |
| 4601 | !Subtarget->isPICStyleRIPRel()) { |
| 4602 | Result = DAG.getNode(ISD::ADD, getPointerTy(), |
| 4603 | DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), |
| 4604 | Result); |
| 4605 | } |
| 4606 | |
| 4607 | return Result; |
| 4608 | } |
| 4609 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4610 | SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) { |
Anton Korobeynikov | 7f70559 | 2007-01-12 19:20:47 +0000 | [diff] [blame] | 4611 | JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4612 | SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy()); |
Anton Korobeynikov | 7f70559 | 2007-01-12 19:20:47 +0000 | [diff] [blame] | 4613 | Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(), Result); |
| 4614 | // With PIC, the address is actually $g + Offset. |
| 4615 | if (getTargetMachine().getRelocationModel() == Reloc::PIC_ && |
| 4616 | !Subtarget->isPICStyleRIPRel()) { |
| 4617 | Result = DAG.getNode(ISD::ADD, getPointerTy(), |
| 4618 | DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), |
| 4619 | Result); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4620 | } |
| 4621 | |
| 4622 | return Result; |
| 4623 | } |
| 4624 | |
Chris Lattner | 2ff75ee | 2007-10-17 06:02:13 +0000 | [diff] [blame] | 4625 | /// LowerShift - Lower SRA_PARTS and friends, which return two i32 values and |
| 4626 | /// take a 2 x i32 value to shift plus a shift amount. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4627 | SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) { |
Dan Gohman | 4c1fa61 | 2008-03-03 22:22:09 +0000 | [diff] [blame] | 4628 | assert(Op.getNumOperands() == 3 && "Not a double-shift!"); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4629 | MVT VT = Op.getValueType(); |
| 4630 | unsigned VTBits = VT.getSizeInBits(); |
Chris Lattner | 2ff75ee | 2007-10-17 06:02:13 +0000 | [diff] [blame] | 4631 | bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4632 | SDValue ShOpLo = Op.getOperand(0); |
| 4633 | SDValue ShOpHi = Op.getOperand(1); |
| 4634 | SDValue ShAmt = Op.getOperand(2); |
| 4635 | SDValue Tmp1 = isSRA ? |
Dan Gohman | 4c1fa61 | 2008-03-03 22:22:09 +0000 | [diff] [blame] | 4636 | DAG.getNode(ISD::SRA, VT, ShOpHi, DAG.getConstant(VTBits - 1, MVT::i8)) : |
| 4637 | DAG.getConstant(0, VT); |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 4638 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4639 | SDValue Tmp2, Tmp3; |
Chris Lattner | 2ff75ee | 2007-10-17 06:02:13 +0000 | [diff] [blame] | 4640 | if (Op.getOpcode() == ISD::SHL_PARTS) { |
Dan Gohman | 4c1fa61 | 2008-03-03 22:22:09 +0000 | [diff] [blame] | 4641 | Tmp2 = DAG.getNode(X86ISD::SHLD, VT, ShOpHi, ShOpLo, ShAmt); |
| 4642 | Tmp3 = DAG.getNode(ISD::SHL, VT, ShOpLo, ShAmt); |
Chris Lattner | 2ff75ee | 2007-10-17 06:02:13 +0000 | [diff] [blame] | 4643 | } else { |
Dan Gohman | 4c1fa61 | 2008-03-03 22:22:09 +0000 | [diff] [blame] | 4644 | Tmp2 = DAG.getNode(X86ISD::SHRD, VT, ShOpLo, ShOpHi, ShAmt); |
| 4645 | Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, VT, ShOpHi, ShAmt); |
Chris Lattner | 2ff75ee | 2007-10-17 06:02:13 +0000 | [diff] [blame] | 4646 | } |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 4647 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4648 | SDValue AndNode = DAG.getNode(ISD::AND, MVT::i8, ShAmt, |
Dan Gohman | 4c1fa61 | 2008-03-03 22:22:09 +0000 | [diff] [blame] | 4649 | DAG.getConstant(VTBits, MVT::i8)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4650 | SDValue Cond = DAG.getNode(X86ISD::CMP, VT, |
Chris Lattner | 2ff75ee | 2007-10-17 06:02:13 +0000 | [diff] [blame] | 4651 | AndNode, DAG.getConstant(0, MVT::i8)); |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 4652 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4653 | SDValue Hi, Lo; |
| 4654 | SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8); |
| 4655 | SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond }; |
| 4656 | SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond }; |
Duncan Sands | f951620 | 2008-06-30 10:19:09 +0000 | [diff] [blame] | 4657 | |
Chris Lattner | 2ff75ee | 2007-10-17 06:02:13 +0000 | [diff] [blame] | 4658 | if (Op.getOpcode() == ISD::SHL_PARTS) { |
Duncan Sands | f951620 | 2008-06-30 10:19:09 +0000 | [diff] [blame] | 4659 | Hi = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); |
| 4660 | Lo = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); |
Chris Lattner | 2ff75ee | 2007-10-17 06:02:13 +0000 | [diff] [blame] | 4661 | } else { |
Duncan Sands | f951620 | 2008-06-30 10:19:09 +0000 | [diff] [blame] | 4662 | Lo = DAG.getNode(X86ISD::CMOV, VT, Ops0, 4); |
| 4663 | Hi = DAG.getNode(X86ISD::CMOV, VT, Ops1, 4); |
Chris Lattner | 2ff75ee | 2007-10-17 06:02:13 +0000 | [diff] [blame] | 4664 | } |
| 4665 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4666 | SDValue Ops[2] = { Lo, Hi }; |
Duncan Sands | 4bdcb61 | 2008-07-02 17:40:58 +0000 | [diff] [blame] | 4667 | return DAG.getMergeValues(Ops, 2); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4668 | } |
Evan Cheng | a3195e8 | 2006-01-12 22:54:21 +0000 | [diff] [blame] | 4669 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4670 | SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4671 | MVT SrcVT = Op.getOperand(0).getValueType(); |
Duncan Sands | 8e4eb09 | 2008-06-08 20:54:56 +0000 | [diff] [blame] | 4672 | assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 && |
Chris Lattner | b09916b | 2008-02-27 05:57:41 +0000 | [diff] [blame] | 4673 | "Unknown SINT_TO_FP to lower!"); |
| 4674 | |
| 4675 | // These are really Legal; caller falls through into that case. |
| 4676 | if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4677 | return SDValue(); |
Chris Lattner | b09916b | 2008-02-27 05:57:41 +0000 | [diff] [blame] | 4678 | if (SrcVT == MVT::i64 && Op.getValueType() != MVT::f80 && |
| 4679 | Subtarget->is64Bit()) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4680 | return SDValue(); |
Chris Lattner | b09916b | 2008-02-27 05:57:41 +0000 | [diff] [blame] | 4681 | |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4682 | unsigned Size = SrcVT.getSizeInBits()/8; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4683 | MachineFunction &MF = DAG.getMachineFunction(); |
| 4684 | int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4685 | SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
| 4686 | SDValue Chain = DAG.getStore(DAG.getEntryNode(), Op.getOperand(0), |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 4687 | StackSlot, |
Dan Gohman | a54cf17 | 2008-07-11 22:44:52 +0000 | [diff] [blame] | 4688 | PseudoSourceValue::getFixedStack(SSFI), 0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4689 | |
| 4690 | // Build the FILD |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 4691 | SDVTList Tys; |
Chris Lattner | 7863116 | 2008-01-16 06:24:21 +0000 | [diff] [blame] | 4692 | bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType()); |
Dale Johannesen | 9e3d3ab | 2007-09-14 22:26:36 +0000 | [diff] [blame] | 4693 | if (useSSE) |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 4694 | Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag); |
| 4695 | else |
Dale Johannesen | 849f214 | 2007-07-03 00:53:03 +0000 | [diff] [blame] | 4696 | Tys = DAG.getVTList(Op.getValueType(), MVT::Other); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4697 | SmallVector<SDValue, 8> Ops; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4698 | Ops.push_back(Chain); |
| 4699 | Ops.push_back(StackSlot); |
| 4700 | Ops.push_back(DAG.getValueType(SrcVT)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4701 | SDValue Result = DAG.getNode(useSSE ? X86ISD::FILD_FLAG : X86ISD::FILD, |
Chris Lattner | b09916b | 2008-02-27 05:57:41 +0000 | [diff] [blame] | 4702 | Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4703 | |
Dale Johannesen | 9e3d3ab | 2007-09-14 22:26:36 +0000 | [diff] [blame] | 4704 | if (useSSE) { |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4705 | Chain = Result.getValue(1); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4706 | SDValue InFlag = Result.getValue(2); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4707 | |
| 4708 | // FIXME: Currently the FST is flagged to the FILD_FLAG. This |
| 4709 | // shouldn't be necessary except that RFP cannot be live across |
| 4710 | // multiple blocks. When stackifier is fixed, they can be uncoupled. |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4711 | MachineFunction &MF = DAG.getMachineFunction(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4712 | int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4713 | SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 4714 | Tys = DAG.getVTList(MVT::Other); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4715 | SmallVector<SDValue, 8> Ops; |
Evan Cheng | a3195e8 | 2006-01-12 22:54:21 +0000 | [diff] [blame] | 4716 | Ops.push_back(Chain); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4717 | Ops.push_back(Result); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4718 | Ops.push_back(StackSlot); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4719 | Ops.push_back(DAG.getValueType(Op.getValueType())); |
| 4720 | Ops.push_back(InFlag); |
Chris Lattner | bd564bf | 2006-08-08 02:23:42 +0000 | [diff] [blame] | 4721 | Chain = DAG.getNode(X86ISD::FST, Tys, &Ops[0], Ops.size()); |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 4722 | Result = DAG.getLoad(Op.getValueType(), Chain, StackSlot, |
Dan Gohman | a54cf17 | 2008-07-11 22:44:52 +0000 | [diff] [blame] | 4723 | PseudoSourceValue::getFixedStack(SSFI), 0); |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4724 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4725 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4726 | return Result; |
| 4727 | } |
| 4728 | |
Dale Johannesen | 1c15bf5 | 2008-10-21 20:50:01 +0000 | [diff] [blame] | 4729 | SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) { |
| 4730 | MVT SrcVT = Op.getOperand(0).getValueType(); |
| 4731 | assert(SrcVT.getSimpleVT() == MVT::i64 && "Unknown UINT_TO_FP to lower!"); |
| 4732 | |
| 4733 | // We only handle SSE2 f64 target here; caller can handle the rest. |
| 4734 | if (Op.getValueType() != MVT::f64 || !X86ScalarSSEf64) |
| 4735 | return SDValue(); |
| 4736 | |
Dale Johannesen | 040225f | 2008-10-21 23:07:49 +0000 | [diff] [blame] | 4737 | // This algorithm is not obvious. Here it is in C code, more or less: |
| 4738 | /* |
| 4739 | double uint64_to_double( uint32_t hi, uint32_t lo ) |
| 4740 | { |
| 4741 | static const __m128i exp = { 0x4330000045300000ULL, 0 }; |
| 4742 | static const __m128d bias = { 0x1.0p84, 0x1.0p52 }; |
| 4743 | |
Dale Johannesen | 9ae0d6d | 2008-10-22 00:02:32 +0000 | [diff] [blame] | 4744 | // copy ints to xmm registers |
Dale Johannesen | 040225f | 2008-10-21 23:07:49 +0000 | [diff] [blame] | 4745 | __m128i xh = _mm_cvtsi32_si128( hi ); |
| 4746 | __m128i xl = _mm_cvtsi32_si128( lo ); |
| 4747 | |
Dale Johannesen | 9ae0d6d | 2008-10-22 00:02:32 +0000 | [diff] [blame] | 4748 | // combine into low half of a single xmm register |
Dale Johannesen | 040225f | 2008-10-21 23:07:49 +0000 | [diff] [blame] | 4749 | __m128i x = _mm_unpacklo_epi32( xh, xl ); |
| 4750 | __m128d d; |
| 4751 | double sd; |
| 4752 | |
Dale Johannesen | 9ae0d6d | 2008-10-22 00:02:32 +0000 | [diff] [blame] | 4753 | // merge in appropriate exponents to give the integer bits the |
Dale Johannesen | 040225f | 2008-10-21 23:07:49 +0000 | [diff] [blame] | 4754 | // right magnitude |
| 4755 | x = _mm_unpacklo_epi32( x, exp ); |
| 4756 | |
Dale Johannesen | 9ae0d6d | 2008-10-22 00:02:32 +0000 | [diff] [blame] | 4757 | // subtract away the biases to deal with the IEEE-754 double precision |
| 4758 | // implicit 1 |
Dale Johannesen | 040225f | 2008-10-21 23:07:49 +0000 | [diff] [blame] | 4759 | d = _mm_sub_pd( (__m128d) x, bias ); |
| 4760 | |
Dale Johannesen | 9ae0d6d | 2008-10-22 00:02:32 +0000 | [diff] [blame] | 4761 | // All conversions up to here are exact. The correctly rounded result is |
Dale Johannesen | 040225f | 2008-10-21 23:07:49 +0000 | [diff] [blame] | 4762 | // calculated using the |
Dale Johannesen | 9ae0d6d | 2008-10-22 00:02:32 +0000 | [diff] [blame] | 4763 | // current rounding mode using the following horizontal add. |
Dale Johannesen | 040225f | 2008-10-21 23:07:49 +0000 | [diff] [blame] | 4764 | d = _mm_add_sd( d, _mm_unpackhi_pd( d, d ) ); |
| 4765 | _mm_store_sd( &sd, d ); //since we are returning doubles in XMM, this |
Dale Johannesen | 9ae0d6d | 2008-10-22 00:02:32 +0000 | [diff] [blame] | 4766 | // store doesn't really need to be here (except maybe to zero the other |
| 4767 | // double) |
Dale Johannesen | 040225f | 2008-10-21 23:07:49 +0000 | [diff] [blame] | 4768 | return sd; |
| 4769 | } |
| 4770 | */ |
| 4771 | |
Dale Johannesen | 1c15bf5 | 2008-10-21 20:50:01 +0000 | [diff] [blame] | 4772 | // Build some magic constants. |
| 4773 | std::vector<Constant*>CV0; |
| 4774 | CV0.push_back(ConstantInt::get(APInt(32, 0x45300000))); |
| 4775 | CV0.push_back(ConstantInt::get(APInt(32, 0x43300000))); |
| 4776 | CV0.push_back(ConstantInt::get(APInt(32, 0))); |
| 4777 | CV0.push_back(ConstantInt::get(APInt(32, 0))); |
| 4778 | Constant *C0 = ConstantVector::get(CV0); |
| 4779 | SDValue CPIdx0 = DAG.getConstantPool(C0, getPointerTy(), 4); |
| 4780 | |
| 4781 | std::vector<Constant*>CV1; |
| 4782 | CV1.push_back(ConstantFP::get(APFloat(APInt(64, 0x4530000000000000ULL)))); |
| 4783 | CV1.push_back(ConstantFP::get(APFloat(APInt(64, 0x4330000000000000ULL)))); |
| 4784 | Constant *C1 = ConstantVector::get(CV1); |
| 4785 | SDValue CPIdx1 = DAG.getConstantPool(C1, getPointerTy(), 4); |
| 4786 | |
| 4787 | SmallVector<SDValue, 4> MaskVec; |
| 4788 | MaskVec.push_back(DAG.getConstant(0, MVT::i32)); |
| 4789 | MaskVec.push_back(DAG.getConstant(4, MVT::i32)); |
| 4790 | MaskVec.push_back(DAG.getConstant(1, MVT::i32)); |
| 4791 | MaskVec.push_back(DAG.getConstant(5, MVT::i32)); |
| 4792 | SDValue UnpcklMask = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, &MaskVec[0], |
| 4793 | MaskVec.size()); |
| 4794 | SmallVector<SDValue, 4> MaskVec2; |
Duncan Sands | 6b6aeb3 | 2008-10-22 11:24:12 +0000 | [diff] [blame] | 4795 | MaskVec2.push_back(DAG.getConstant(1, MVT::i32)); |
| 4796 | MaskVec2.push_back(DAG.getConstant(0, MVT::i32)); |
| 4797 | SDValue ShufMask = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, &MaskVec2[0], |
Dale Johannesen | 1c15bf5 | 2008-10-21 20:50:01 +0000 | [diff] [blame] | 4798 | MaskVec2.size()); |
| 4799 | |
| 4800 | SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v4i32, |
Duncan Sands | 6b6aeb3 | 2008-10-22 11:24:12 +0000 | [diff] [blame] | 4801 | DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, |
| 4802 | Op.getOperand(0), |
| 4803 | DAG.getIntPtrConstant(1))); |
Dale Johannesen | 1c15bf5 | 2008-10-21 20:50:01 +0000 | [diff] [blame] | 4804 | SDValue XR2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v4i32, |
Duncan Sands | 6b6aeb3 | 2008-10-22 11:24:12 +0000 | [diff] [blame] | 4805 | DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, |
| 4806 | Op.getOperand(0), |
| 4807 | DAG.getIntPtrConstant(0))); |
Dale Johannesen | 1c15bf5 | 2008-10-21 20:50:01 +0000 | [diff] [blame] | 4808 | SDValue Unpck1 = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v4i32, |
| 4809 | XR1, XR2, UnpcklMask); |
| 4810 | SDValue CLod0 = DAG.getLoad(MVT::v4i32, DAG.getEntryNode(), CPIdx0, |
| 4811 | PseudoSourceValue::getConstantPool(), 0, false, 16); |
| 4812 | SDValue Unpck2 = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v4i32, |
| 4813 | Unpck1, CLod0, UnpcklMask); |
| 4814 | SDValue XR2F = DAG.getNode(ISD::BIT_CONVERT, MVT::v2f64, Unpck2); |
| 4815 | SDValue CLod1 = DAG.getLoad(MVT::v2f64, CLod0.getValue(1), CPIdx1, |
| 4816 | PseudoSourceValue::getConstantPool(), 0, false, 16); |
| 4817 | SDValue Sub = DAG.getNode(ISD::FSUB, MVT::v2f64, XR2F, CLod1); |
| 4818 | // Add the halves; easiest way is to swap them into another reg first. |
| 4819 | SDValue Shuf = DAG.getNode(ISD::VECTOR_SHUFFLE, MVT::v2f64, |
| 4820 | Sub, Sub, ShufMask); |
| 4821 | SDValue Add = DAG.getNode(ISD::FADD, MVT::v2f64, Shuf, Sub); |
| 4822 | return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::f64, Add, |
| 4823 | DAG.getIntPtrConstant(0)); |
| 4824 | } |
| 4825 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4826 | std::pair<SDValue,SDValue> X86TargetLowering:: |
| 4827 | FP_TO_SINTHelper(SDValue Op, SelectionDAG &DAG) { |
Duncan Sands | 8e4eb09 | 2008-06-08 20:54:56 +0000 | [diff] [blame] | 4828 | assert(Op.getValueType().getSimpleVT() <= MVT::i64 && |
| 4829 | Op.getValueType().getSimpleVT() >= MVT::i16 && |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4830 | "Unknown FP_TO_SINT to lower!"); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4831 | |
Dale Johannesen | 9e3d3ab | 2007-09-14 22:26:36 +0000 | [diff] [blame] | 4832 | // These are really Legal. |
Dale Johannesen | f1fc3a8 | 2007-09-23 14:52:20 +0000 | [diff] [blame] | 4833 | if (Op.getValueType() == MVT::i32 && |
Chris Lattner | 7863116 | 2008-01-16 06:24:21 +0000 | [diff] [blame] | 4834 | isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4835 | return std::make_pair(SDValue(), SDValue()); |
Dale Johannesen | 73328d1 | 2007-09-19 23:55:34 +0000 | [diff] [blame] | 4836 | if (Subtarget->is64Bit() && |
| 4837 | Op.getValueType() == MVT::i64 && |
| 4838 | Op.getOperand(0).getValueType() != MVT::f80) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4839 | return std::make_pair(SDValue(), SDValue()); |
Dale Johannesen | 9e3d3ab | 2007-09-14 22:26:36 +0000 | [diff] [blame] | 4840 | |
Evan Cheng | 87c8935 | 2007-10-15 20:11:21 +0000 | [diff] [blame] | 4841 | // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary |
| 4842 | // stack slot. |
| 4843 | MachineFunction &MF = DAG.getMachineFunction(); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4844 | unsigned MemSize = Op.getValueType().getSizeInBits()/8; |
Evan Cheng | 87c8935 | 2007-10-15 20:11:21 +0000 | [diff] [blame] | 4845 | int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4846 | SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4847 | unsigned Opc; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4848 | switch (Op.getValueType().getSimpleVT()) { |
Chris Lattner | 27a6c73 | 2007-11-24 07:07:01 +0000 | [diff] [blame] | 4849 | default: assert(0 && "Invalid FP_TO_SINT to lower!"); |
| 4850 | case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; |
| 4851 | case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; |
| 4852 | case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4853 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4854 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4855 | SDValue Chain = DAG.getEntryNode(); |
| 4856 | SDValue Value = Op.getOperand(0); |
Chris Lattner | 7863116 | 2008-01-16 06:24:21 +0000 | [diff] [blame] | 4857 | if (isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) { |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4858 | assert(Op.getValueType() == MVT::i64 && "Invalid FP_TO_SINT to lower!"); |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 4859 | Chain = DAG.getStore(Chain, Value, StackSlot, |
Dan Gohman | a54cf17 | 2008-07-11 22:44:52 +0000 | [diff] [blame] | 4860 | PseudoSourceValue::getFixedStack(SSFI), 0); |
Dale Johannesen | 849f214 | 2007-07-03 00:53:03 +0000 | [diff] [blame] | 4861 | SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4862 | SDValue Ops[] = { |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 4863 | Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType()) |
| 4864 | }; |
| 4865 | Value = DAG.getNode(X86ISD::FLD, Tys, Ops, 3); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4866 | Chain = Value.getValue(1); |
| 4867 | SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize); |
| 4868 | StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
| 4869 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 4870 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4871 | // Build the FP_TO_INT*_IN_MEM |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4872 | SDValue Ops[] = { Chain, Value, StackSlot }; |
| 4873 | SDValue FIST = DAG.getNode(Opc, MVT::Other, Ops, 3); |
Evan Cheng | d9558e0 | 2006-01-06 00:43:03 +0000 | [diff] [blame] | 4874 | |
Chris Lattner | 27a6c73 | 2007-11-24 07:07:01 +0000 | [diff] [blame] | 4875 | return std::make_pair(FIST, StackSlot); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4876 | } |
| 4877 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4878 | SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) { |
| 4879 | std::pair<SDValue,SDValue> Vals = FP_TO_SINTHelper(Op, DAG); |
| 4880 | SDValue FIST = Vals.first, StackSlot = Vals.second; |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 4881 | if (FIST.getNode() == 0) return SDValue(); |
Chris Lattner | 27a6c73 | 2007-11-24 07:07:01 +0000 | [diff] [blame] | 4882 | |
| 4883 | // Load the result. |
| 4884 | return DAG.getLoad(Op.getValueType(), FIST, StackSlot, NULL, 0); |
| 4885 | } |
| 4886 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4887 | SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4888 | MVT VT = Op.getValueType(); |
| 4889 | MVT EltVT = VT; |
| 4890 | if (VT.isVector()) |
| 4891 | EltVT = VT.getVectorElementType(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4892 | std::vector<Constant*> CV; |
Dan Gohman | 2038252 | 2007-07-10 00:05:58 +0000 | [diff] [blame] | 4893 | if (EltVT == MVT::f64) { |
Chris Lattner | 02a260a | 2008-04-20 00:41:09 +0000 | [diff] [blame] | 4894 | Constant *C = ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63)))); |
Dan Gohman | 2038252 | 2007-07-10 00:05:58 +0000 | [diff] [blame] | 4895 | CV.push_back(C); |
| 4896 | CV.push_back(C); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4897 | } else { |
Chris Lattner | 02a260a | 2008-04-20 00:41:09 +0000 | [diff] [blame] | 4898 | Constant *C = ConstantFP::get(APFloat(APInt(32, ~(1U << 31)))); |
Dan Gohman | 2038252 | 2007-07-10 00:05:58 +0000 | [diff] [blame] | 4899 | CV.push_back(C); |
| 4900 | CV.push_back(C); |
| 4901 | CV.push_back(C); |
| 4902 | CV.push_back(C); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4903 | } |
Dan Gohman | d300622 | 2007-07-27 17:16:43 +0000 | [diff] [blame] | 4904 | Constant *C = ConstantVector::get(CV); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4905 | SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); |
| 4906 | SDValue Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, |
Dan Gohman | 3069b87 | 2008-02-07 18:41:25 +0000 | [diff] [blame] | 4907 | PseudoSourceValue::getConstantPool(), 0, |
Dan Gohman | d300622 | 2007-07-27 17:16:43 +0000 | [diff] [blame] | 4908 | false, 16); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4909 | return DAG.getNode(X86ISD::FAND, VT, Op.getOperand(0), Mask); |
| 4910 | } |
| 4911 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4912 | SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4913 | MVT VT = Op.getValueType(); |
| 4914 | MVT EltVT = VT; |
Evan Cheng | d4d01b7 | 2007-07-19 23:36:01 +0000 | [diff] [blame] | 4915 | unsigned EltNum = 1; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4916 | if (VT.isVector()) { |
| 4917 | EltVT = VT.getVectorElementType(); |
| 4918 | EltNum = VT.getVectorNumElements(); |
Evan Cheng | d4d01b7 | 2007-07-19 23:36:01 +0000 | [diff] [blame] | 4919 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4920 | std::vector<Constant*> CV; |
Dan Gohman | 2038252 | 2007-07-10 00:05:58 +0000 | [diff] [blame] | 4921 | if (EltVT == MVT::f64) { |
Chris Lattner | 02a260a | 2008-04-20 00:41:09 +0000 | [diff] [blame] | 4922 | Constant *C = ConstantFP::get(APFloat(APInt(64, 1ULL << 63))); |
Dan Gohman | 2038252 | 2007-07-10 00:05:58 +0000 | [diff] [blame] | 4923 | CV.push_back(C); |
| 4924 | CV.push_back(C); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4925 | } else { |
Chris Lattner | 02a260a | 2008-04-20 00:41:09 +0000 | [diff] [blame] | 4926 | Constant *C = ConstantFP::get(APFloat(APInt(32, 1U << 31))); |
Dan Gohman | 2038252 | 2007-07-10 00:05:58 +0000 | [diff] [blame] | 4927 | CV.push_back(C); |
| 4928 | CV.push_back(C); |
| 4929 | CV.push_back(C); |
| 4930 | CV.push_back(C); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4931 | } |
Dan Gohman | d300622 | 2007-07-27 17:16:43 +0000 | [diff] [blame] | 4932 | Constant *C = ConstantVector::get(CV); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4933 | SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); |
| 4934 | SDValue Mask = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, |
Dan Gohman | 3069b87 | 2008-02-07 18:41:25 +0000 | [diff] [blame] | 4935 | PseudoSourceValue::getConstantPool(), 0, |
Dan Gohman | d300622 | 2007-07-27 17:16:43 +0000 | [diff] [blame] | 4936 | false, 16); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4937 | if (VT.isVector()) { |
Evan Cheng | d4d01b7 | 2007-07-19 23:36:01 +0000 | [diff] [blame] | 4938 | return DAG.getNode(ISD::BIT_CONVERT, VT, |
| 4939 | DAG.getNode(ISD::XOR, MVT::v2i64, |
| 4940 | DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Op.getOperand(0)), |
| 4941 | DAG.getNode(ISD::BIT_CONVERT, MVT::v2i64, Mask))); |
| 4942 | } else { |
Evan Cheng | d4d01b7 | 2007-07-19 23:36:01 +0000 | [diff] [blame] | 4943 | return DAG.getNode(X86ISD::FXOR, VT, Op.getOperand(0), Mask); |
| 4944 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 4945 | } |
| 4946 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4947 | SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) { |
| 4948 | SDValue Op0 = Op.getOperand(0); |
| 4949 | SDValue Op1 = Op.getOperand(1); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 4950 | MVT VT = Op.getValueType(); |
| 4951 | MVT SrcVT = Op1.getValueType(); |
Evan Cheng | 73d6cf1 | 2007-01-05 21:37:56 +0000 | [diff] [blame] | 4952 | |
| 4953 | // If second operand is smaller, extend it first. |
Duncan Sands | 8e4eb09 | 2008-06-08 20:54:56 +0000 | [diff] [blame] | 4954 | if (SrcVT.bitsLT(VT)) { |
Evan Cheng | 73d6cf1 | 2007-01-05 21:37:56 +0000 | [diff] [blame] | 4955 | Op1 = DAG.getNode(ISD::FP_EXTEND, VT, Op1); |
| 4956 | SrcVT = VT; |
| 4957 | } |
Dale Johannesen | 61c7ef3 | 2007-10-21 01:07:44 +0000 | [diff] [blame] | 4958 | // And if it is bigger, shrink it first. |
Duncan Sands | 8e4eb09 | 2008-06-08 20:54:56 +0000 | [diff] [blame] | 4959 | if (SrcVT.bitsGT(VT)) { |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 4960 | Op1 = DAG.getNode(ISD::FP_ROUND, VT, Op1, DAG.getIntPtrConstant(1)); |
Dale Johannesen | 61c7ef3 | 2007-10-21 01:07:44 +0000 | [diff] [blame] | 4961 | SrcVT = VT; |
Dale Johannesen | 61c7ef3 | 2007-10-21 01:07:44 +0000 | [diff] [blame] | 4962 | } |
| 4963 | |
| 4964 | // At this point the operands and the result should have the same |
| 4965 | // type, and that won't be f80 since that is not custom lowered. |
Evan Cheng | 73d6cf1 | 2007-01-05 21:37:56 +0000 | [diff] [blame] | 4966 | |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 4967 | // First get the sign bit of second operand. |
| 4968 | std::vector<Constant*> CV; |
| 4969 | if (SrcVT == MVT::f64) { |
Chris Lattner | 02a260a | 2008-04-20 00:41:09 +0000 | [diff] [blame] | 4970 | CV.push_back(ConstantFP::get(APFloat(APInt(64, 1ULL << 63)))); |
| 4971 | CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 4972 | } else { |
Chris Lattner | 02a260a | 2008-04-20 00:41:09 +0000 | [diff] [blame] | 4973 | CV.push_back(ConstantFP::get(APFloat(APInt(32, 1U << 31)))); |
| 4974 | CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); |
| 4975 | CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); |
| 4976 | CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 4977 | } |
Dan Gohman | d300622 | 2007-07-27 17:16:43 +0000 | [diff] [blame] | 4978 | Constant *C = ConstantVector::get(CV); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4979 | SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); |
| 4980 | SDValue Mask1 = DAG.getLoad(SrcVT, DAG.getEntryNode(), CPIdx, |
Dan Gohman | 3069b87 | 2008-02-07 18:41:25 +0000 | [diff] [blame] | 4981 | PseudoSourceValue::getConstantPool(), 0, |
Dan Gohman | d300622 | 2007-07-27 17:16:43 +0000 | [diff] [blame] | 4982 | false, 16); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 4983 | SDValue SignBit = DAG.getNode(X86ISD::FAND, SrcVT, Op1, Mask1); |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 4984 | |
| 4985 | // Shift sign bit right or left if the two operands have different types. |
Duncan Sands | 8e4eb09 | 2008-06-08 20:54:56 +0000 | [diff] [blame] | 4986 | if (SrcVT.bitsGT(VT)) { |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 4987 | // Op0 is MVT::f32, Op1 is MVT::f64. |
| 4988 | SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, MVT::v2f64, SignBit); |
| 4989 | SignBit = DAG.getNode(X86ISD::FSRL, MVT::v2f64, SignBit, |
| 4990 | DAG.getConstant(32, MVT::i32)); |
| 4991 | SignBit = DAG.getNode(ISD::BIT_CONVERT, MVT::v4f32, SignBit); |
| 4992 | SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::f32, SignBit, |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 4993 | DAG.getIntPtrConstant(0)); |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 4994 | } |
| 4995 | |
Evan Cheng | 73d6cf1 | 2007-01-05 21:37:56 +0000 | [diff] [blame] | 4996 | // Clear first operand sign bit. |
| 4997 | CV.clear(); |
| 4998 | if (VT == MVT::f64) { |
Chris Lattner | 02a260a | 2008-04-20 00:41:09 +0000 | [diff] [blame] | 4999 | CV.push_back(ConstantFP::get(APFloat(APInt(64, ~(1ULL << 63))))); |
| 5000 | CV.push_back(ConstantFP::get(APFloat(APInt(64, 0)))); |
Evan Cheng | 73d6cf1 | 2007-01-05 21:37:56 +0000 | [diff] [blame] | 5001 | } else { |
Chris Lattner | 02a260a | 2008-04-20 00:41:09 +0000 | [diff] [blame] | 5002 | CV.push_back(ConstantFP::get(APFloat(APInt(32, ~(1U << 31))))); |
| 5003 | CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); |
| 5004 | CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); |
| 5005 | CV.push_back(ConstantFP::get(APFloat(APInt(32, 0)))); |
Evan Cheng | 73d6cf1 | 2007-01-05 21:37:56 +0000 | [diff] [blame] | 5006 | } |
Dan Gohman | d300622 | 2007-07-27 17:16:43 +0000 | [diff] [blame] | 5007 | C = ConstantVector::get(CV); |
| 5008 | CPIdx = DAG.getConstantPool(C, getPointerTy(), 4); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5009 | SDValue Mask2 = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, |
Dan Gohman | 3069b87 | 2008-02-07 18:41:25 +0000 | [diff] [blame] | 5010 | PseudoSourceValue::getConstantPool(), 0, |
Dan Gohman | d300622 | 2007-07-27 17:16:43 +0000 | [diff] [blame] | 5011 | false, 16); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5012 | SDValue Val = DAG.getNode(X86ISD::FAND, VT, Op0, Mask2); |
Evan Cheng | 73d6cf1 | 2007-01-05 21:37:56 +0000 | [diff] [blame] | 5013 | |
| 5014 | // Or the value with the sign bit. |
| 5015 | return DAG.getNode(X86ISD::FOR, VT, Val, SignBit); |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 5016 | } |
| 5017 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5018 | SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) { |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5019 | assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer"); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5020 | SDValue Op0 = Op.getOperand(0); |
| 5021 | SDValue Op1 = Op.getOperand(1); |
Chris Lattner | e55484e | 2008-12-25 05:34:37 +0000 | [diff] [blame] | 5022 | ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); |
| 5023 | |
| 5024 | // Lower (X & (1 << N)) == 0 to BT. |
| 5025 | // Lower ((X >>u N) & 1) != 0 to BT. |
| 5026 | // Lower ((X >>s N) & 1) != 0 to BT. |
| 5027 | // FIXME: Is i386 or later or available only on some chips? |
| 5028 | if (Op0.getOpcode() == ISD::AND && Op1.getOpcode() == ISD::Constant && |
| 5029 | Op0.getOperand(1).getOpcode() == ISD::Constant && |
| 5030 | (CC == ISD::SETEQ || CC == ISD::SETNE)) { |
| 5031 | ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op0.getOperand(1)); |
| 5032 | ConstantSDNode *CmpRHS = cast<ConstantSDNode>(Op1); |
| 5033 | SDValue AndLHS = Op0.getOperand(0); |
| 5034 | if (CmpRHS->getZExtValue() == 0 && AndRHS->getZExtValue() == 1 && |
| 5035 | AndLHS.getOpcode() == ISD::SRL) { |
| 5036 | SDValue LHS = AndLHS.getOperand(0); |
| 5037 | SDValue RHS = AndLHS.getOperand(1); |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5038 | |
Chris Lattner | e55484e | 2008-12-25 05:34:37 +0000 | [diff] [blame] | 5039 | // If LHS is i8, promote it to i16 with any_extend. There is no i8 BT |
| 5040 | // instruction. Since the shift amount is in-range-or-undefined, we know |
| 5041 | // that doing a bittest on the i16 value is ok. We extend to i32 because |
| 5042 | // the encoding for the i16 version is larger than the i32 version. |
| 5043 | if (LHS.getValueType() == MVT::i8) |
| 5044 | LHS = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, LHS); |
| 5045 | |
| 5046 | // If the operand types disagree, extend the shift amount to match. Since |
| 5047 | // BT ignores high bits (like shifts) we can use anyextend. |
| 5048 | if (LHS.getValueType() != RHS.getValueType()) |
| 5049 | RHS = DAG.getNode(ISD::ANY_EXTEND, LHS.getValueType(), RHS); |
| 5050 | |
| 5051 | SDValue BT = DAG.getNode(X86ISD::BT, MVT::i32, LHS, RHS); |
| 5052 | unsigned Cond = CC == ISD::SETEQ ? X86::COND_NC : X86::COND_C; |
| 5053 | return DAG.getNode(X86ISD::SETCC, MVT::i8, |
| 5054 | DAG.getConstant(Cond, MVT::i8), BT); |
| 5055 | } |
| 5056 | } |
| 5057 | |
| 5058 | bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); |
| 5059 | unsigned X86CC = TranslateX86CC(CC, isFP, Op0, Op1, DAG); |
Chris Lattner | 4328708 | 2008-12-24 00:11:37 +0000 | [diff] [blame] | 5060 | |
Chris Lattner | e55484e | 2008-12-25 05:34:37 +0000 | [diff] [blame] | 5061 | SDValue Cond = DAG.getNode(X86ISD::CMP, MVT::i32, Op0, Op1); |
Chris Lattner | 4328708 | 2008-12-24 00:11:37 +0000 | [diff] [blame] | 5062 | return DAG.getNode(X86ISD::SETCC, MVT::i8, |
| 5063 | DAG.getConstant(X86CC, MVT::i8), Cond); |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5064 | } |
| 5065 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5066 | SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) { |
| 5067 | SDValue Cond; |
| 5068 | SDValue Op0 = Op.getOperand(0); |
| 5069 | SDValue Op1 = Op.getOperand(1); |
| 5070 | SDValue CC = Op.getOperand(2); |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 5071 | MVT VT = Op.getValueType(); |
| 5072 | ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); |
| 5073 | bool isFP = Op.getOperand(1).getValueType().isFloatingPoint(); |
| 5074 | |
| 5075 | if (isFP) { |
| 5076 | unsigned SSECC = 8; |
Evan Cheng | e9d5035 | 2008-08-05 22:19:15 +0000 | [diff] [blame] | 5077 | MVT VT0 = Op0.getValueType(); |
| 5078 | assert(VT0 == MVT::v4f32 || VT0 == MVT::v2f64); |
| 5079 | unsigned Opc = VT0 == MVT::v4f32 ? X86ISD::CMPPS : X86ISD::CMPPD; |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 5080 | bool Swap = false; |
| 5081 | |
| 5082 | switch (SetCCOpcode) { |
| 5083 | default: break; |
Nate Begeman | fb8ead0 | 2008-07-25 19:05:58 +0000 | [diff] [blame] | 5084 | case ISD::SETOEQ: |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 5085 | case ISD::SETEQ: SSECC = 0; break; |
| 5086 | case ISD::SETOGT: |
| 5087 | case ISD::SETGT: Swap = true; // Fallthrough |
| 5088 | case ISD::SETLT: |
| 5089 | case ISD::SETOLT: SSECC = 1; break; |
| 5090 | case ISD::SETOGE: |
| 5091 | case ISD::SETGE: Swap = true; // Fallthrough |
| 5092 | case ISD::SETLE: |
| 5093 | case ISD::SETOLE: SSECC = 2; break; |
| 5094 | case ISD::SETUO: SSECC = 3; break; |
Nate Begeman | fb8ead0 | 2008-07-25 19:05:58 +0000 | [diff] [blame] | 5095 | case ISD::SETUNE: |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 5096 | case ISD::SETNE: SSECC = 4; break; |
| 5097 | case ISD::SETULE: Swap = true; |
| 5098 | case ISD::SETUGE: SSECC = 5; break; |
| 5099 | case ISD::SETULT: Swap = true; |
| 5100 | case ISD::SETUGT: SSECC = 6; break; |
| 5101 | case ISD::SETO: SSECC = 7; break; |
| 5102 | } |
| 5103 | if (Swap) |
| 5104 | std::swap(Op0, Op1); |
| 5105 | |
Nate Begeman | fb8ead0 | 2008-07-25 19:05:58 +0000 | [diff] [blame] | 5106 | // In the two special cases we can't handle, emit two comparisons. |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 5107 | if (SSECC == 8) { |
Nate Begeman | fb8ead0 | 2008-07-25 19:05:58 +0000 | [diff] [blame] | 5108 | if (SetCCOpcode == ISD::SETUEQ) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5109 | SDValue UNORD, EQ; |
Nate Begeman | fb8ead0 | 2008-07-25 19:05:58 +0000 | [diff] [blame] | 5110 | UNORD = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(3, MVT::i8)); |
| 5111 | EQ = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(0, MVT::i8)); |
| 5112 | return DAG.getNode(ISD::OR, VT, UNORD, EQ); |
| 5113 | } |
| 5114 | else if (SetCCOpcode == ISD::SETONE) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5115 | SDValue ORD, NEQ; |
Nate Begeman | fb8ead0 | 2008-07-25 19:05:58 +0000 | [diff] [blame] | 5116 | ORD = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(7, MVT::i8)); |
| 5117 | NEQ = DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(4, MVT::i8)); |
| 5118 | return DAG.getNode(ISD::AND, VT, ORD, NEQ); |
| 5119 | } |
| 5120 | assert(0 && "Illegal FP comparison"); |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 5121 | } |
| 5122 | // Handle all other FP comparisons here. |
| 5123 | return DAG.getNode(Opc, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8)); |
| 5124 | } |
| 5125 | |
| 5126 | // We are handling one of the integer comparisons here. Since SSE only has |
| 5127 | // GT and EQ comparisons for integer, swapping operands and multiple |
| 5128 | // operations may be required for some comparisons. |
| 5129 | unsigned Opc = 0, EQOpc = 0, GTOpc = 0; |
| 5130 | bool Swap = false, Invert = false, FlipSigns = false; |
| 5131 | |
| 5132 | switch (VT.getSimpleVT()) { |
| 5133 | default: break; |
| 5134 | case MVT::v16i8: EQOpc = X86ISD::PCMPEQB; GTOpc = X86ISD::PCMPGTB; break; |
| 5135 | case MVT::v8i16: EQOpc = X86ISD::PCMPEQW; GTOpc = X86ISD::PCMPGTW; break; |
| 5136 | case MVT::v4i32: EQOpc = X86ISD::PCMPEQD; GTOpc = X86ISD::PCMPGTD; break; |
| 5137 | case MVT::v2i64: EQOpc = X86ISD::PCMPEQQ; GTOpc = X86ISD::PCMPGTQ; break; |
| 5138 | } |
| 5139 | |
| 5140 | switch (SetCCOpcode) { |
| 5141 | default: break; |
| 5142 | case ISD::SETNE: Invert = true; |
| 5143 | case ISD::SETEQ: Opc = EQOpc; break; |
| 5144 | case ISD::SETLT: Swap = true; |
| 5145 | case ISD::SETGT: Opc = GTOpc; break; |
| 5146 | case ISD::SETGE: Swap = true; |
| 5147 | case ISD::SETLE: Opc = GTOpc; Invert = true; break; |
| 5148 | case ISD::SETULT: Swap = true; |
| 5149 | case ISD::SETUGT: Opc = GTOpc; FlipSigns = true; break; |
| 5150 | case ISD::SETUGE: Swap = true; |
| 5151 | case ISD::SETULE: Opc = GTOpc; FlipSigns = true; Invert = true; break; |
| 5152 | } |
| 5153 | if (Swap) |
| 5154 | std::swap(Op0, Op1); |
| 5155 | |
| 5156 | // Since SSE has no unsigned integer comparisons, we need to flip the sign |
| 5157 | // bits of the inputs before performing those operations. |
| 5158 | if (FlipSigns) { |
| 5159 | MVT EltVT = VT.getVectorElementType(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5160 | SDValue SignBit = DAG.getConstant(EltVT.getIntegerVTSignBit(), EltVT); |
| 5161 | std::vector<SDValue> SignBits(VT.getVectorNumElements(), SignBit); |
| 5162 | SDValue SignVec = DAG.getNode(ISD::BUILD_VECTOR, VT, &SignBits[0], |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 5163 | SignBits.size()); |
| 5164 | Op0 = DAG.getNode(ISD::XOR, VT, Op0, SignVec); |
| 5165 | Op1 = DAG.getNode(ISD::XOR, VT, Op1, SignVec); |
| 5166 | } |
| 5167 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5168 | SDValue Result = DAG.getNode(Opc, VT, Op0, Op1); |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 5169 | |
| 5170 | // If the logical-not of the result is required, perform that now. |
| 5171 | if (Invert) { |
| 5172 | MVT EltVT = VT.getVectorElementType(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5173 | SDValue NegOne = DAG.getConstant(EltVT.getIntegerVTBitMask(), EltVT); |
| 5174 | std::vector<SDValue> NegOnes(VT.getVectorNumElements(), NegOne); |
| 5175 | SDValue NegOneV = DAG.getNode(ISD::BUILD_VECTOR, VT, &NegOnes[0], |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 5176 | NegOnes.size()); |
| 5177 | Result = DAG.getNode(ISD::XOR, VT, Result, NegOneV); |
| 5178 | } |
| 5179 | return Result; |
| 5180 | } |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5181 | |
Evan Cheng | 370e534 | 2008-12-03 08:38:43 +0000 | [diff] [blame] | 5182 | // isX86LogicalCmp - Return true if opcode is a X86 logical comparison. |
| 5183 | static bool isX86LogicalCmp(unsigned Opc) { |
| 5184 | return Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI; |
| 5185 | } |
| 5186 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5187 | SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) { |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 5188 | bool addTest = true; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5189 | SDValue Cond = Op.getOperand(0); |
| 5190 | SDValue CC; |
Evan Cheng | 9bba894 | 2006-01-26 02:13:10 +0000 | [diff] [blame] | 5191 | |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 5192 | if (Cond.getOpcode() == ISD::SETCC) |
Evan Cheng | e5f6204 | 2007-09-29 00:00:36 +0000 | [diff] [blame] | 5193 | Cond = LowerSETCC(Cond, DAG); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 5194 | |
Evan Cheng | 3f41d66 | 2007-10-08 22:16:29 +0000 | [diff] [blame] | 5195 | // If condition flag is set by a X86ISD::CMP, then use it as the condition |
| 5196 | // setting operand in place of the X86ISD::SETCC. |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 5197 | if (Cond.getOpcode() == X86ISD::SETCC) { |
| 5198 | CC = Cond.getOperand(0); |
| 5199 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5200 | SDValue Cmp = Cond.getOperand(1); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 5201 | unsigned Opc = Cmp.getOpcode(); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5202 | MVT VT = Op.getValueType(); |
Chris Lattner | 1956d15 | 2008-01-16 06:19:45 +0000 | [diff] [blame] | 5203 | |
Evan Cheng | 3f41d66 | 2007-10-08 22:16:29 +0000 | [diff] [blame] | 5204 | bool IllegalFPCMov = false; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5205 | if (VT.isFloatingPoint() && !VT.isVector() && |
Chris Lattner | 7863116 | 2008-01-16 06:24:21 +0000 | [diff] [blame] | 5206 | !isScalarFPTypeInSSEReg(VT)) // FPStack? |
Dan Gohman | 7810bfe | 2008-09-26 21:54:37 +0000 | [diff] [blame] | 5207 | IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue()); |
Chris Lattner | 1956d15 | 2008-01-16 06:19:45 +0000 | [diff] [blame] | 5208 | |
Evan Cheng | 370e534 | 2008-12-03 08:38:43 +0000 | [diff] [blame] | 5209 | if (isX86LogicalCmp(Opc) && !IllegalFPCMov) { |
Evan Cheng | 3f41d66 | 2007-10-08 22:16:29 +0000 | [diff] [blame] | 5210 | Cond = Cmp; |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5211 | addTest = false; |
| 5212 | } |
| 5213 | } |
| 5214 | |
| 5215 | if (addTest) { |
| 5216 | CC = DAG.getConstant(X86::COND_NE, MVT::i8); |
Evan Cheng | 3f41d66 | 2007-10-08 22:16:29 +0000 | [diff] [blame] | 5217 | Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5218 | } |
| 5219 | |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5220 | const MVT *VTs = DAG.getNodeValueTypes(Op.getValueType(), |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5221 | MVT::Flag); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5222 | SmallVector<SDValue, 4> Ops; |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5223 | // X86ISD::CMOV means set the result (which is operand 1) to the RHS if |
| 5224 | // condition is true. |
| 5225 | Ops.push_back(Op.getOperand(2)); |
| 5226 | Ops.push_back(Op.getOperand(1)); |
| 5227 | Ops.push_back(CC); |
| 5228 | Ops.push_back(Cond); |
Evan Cheng | e5f6204 | 2007-09-29 00:00:36 +0000 | [diff] [blame] | 5229 | return DAG.getNode(X86ISD::CMOV, VTs, 2, &Ops[0], Ops.size()); |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5230 | } |
| 5231 | |
Evan Cheng | 370e534 | 2008-12-03 08:38:43 +0000 | [diff] [blame] | 5232 | // isAndOrOfSingleUseSetCCs - Return true if node is an ISD::AND or |
| 5233 | // ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart |
| 5234 | // from the AND / OR. |
| 5235 | static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) { |
| 5236 | Opc = Op.getOpcode(); |
| 5237 | if (Opc != ISD::OR && Opc != ISD::AND) |
| 5238 | return false; |
| 5239 | return (Op.getOperand(0).getOpcode() == X86ISD::SETCC && |
| 5240 | Op.getOperand(0).hasOneUse() && |
| 5241 | Op.getOperand(1).getOpcode() == X86ISD::SETCC && |
| 5242 | Op.getOperand(1).hasOneUse()); |
| 5243 | } |
| 5244 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5245 | SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) { |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 5246 | bool addTest = true; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5247 | SDValue Chain = Op.getOperand(0); |
| 5248 | SDValue Cond = Op.getOperand(1); |
| 5249 | SDValue Dest = Op.getOperand(2); |
| 5250 | SDValue CC; |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 5251 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5252 | if (Cond.getOpcode() == ISD::SETCC) |
Evan Cheng | e5f6204 | 2007-09-29 00:00:36 +0000 | [diff] [blame] | 5253 | Cond = LowerSETCC(Cond, DAG); |
Chris Lattner | e55484e | 2008-12-25 05:34:37 +0000 | [diff] [blame] | 5254 | #if 0 |
| 5255 | // FIXME: LowerXALUO doesn't handle these!! |
Bill Wendling | d350e02 | 2008-12-12 21:15:41 +0000 | [diff] [blame] | 5256 | else if (Cond.getOpcode() == X86ISD::ADD || |
| 5257 | Cond.getOpcode() == X86ISD::SUB || |
| 5258 | Cond.getOpcode() == X86ISD::SMUL || |
| 5259 | Cond.getOpcode() == X86ISD::UMUL) |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 5260 | Cond = LowerXALUO(Cond, DAG); |
Chris Lattner | e55484e | 2008-12-25 05:34:37 +0000 | [diff] [blame] | 5261 | #endif |
| 5262 | |
Evan Cheng | 3f41d66 | 2007-10-08 22:16:29 +0000 | [diff] [blame] | 5263 | // If condition flag is set by a X86ISD::CMP, then use it as the condition |
| 5264 | // setting operand in place of the X86ISD::SETCC. |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5265 | if (Cond.getOpcode() == X86ISD::SETCC) { |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 5266 | CC = Cond.getOperand(0); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5267 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5268 | SDValue Cmp = Cond.getOperand(1); |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 5269 | unsigned Opc = Cmp.getOpcode(); |
Chris Lattner | e55484e | 2008-12-25 05:34:37 +0000 | [diff] [blame] | 5270 | // FIXME: WHY THE SPECIAL CASING OF LogicalCmp?? |
| 5271 | if (isX86LogicalCmp(Opc) || Opc == X86ISD::BT) { |
Evan Cheng | 3f41d66 | 2007-10-08 22:16:29 +0000 | [diff] [blame] | 5272 | Cond = Cmp; |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5273 | addTest = false; |
Bill Wendling | 61edeb5 | 2008-12-02 01:06:39 +0000 | [diff] [blame] | 5274 | } else { |
Evan Cheng | 370e534 | 2008-12-03 08:38:43 +0000 | [diff] [blame] | 5275 | switch (cast<ConstantSDNode>(CC)->getZExtValue()) { |
Bill Wendling | 0ea25cb | 2008-12-03 08:32:02 +0000 | [diff] [blame] | 5276 | default: break; |
| 5277 | case X86::COND_O: |
| 5278 | case X86::COND_C: |
Chris Lattner | e55484e | 2008-12-25 05:34:37 +0000 | [diff] [blame] | 5279 | // These can only come from an arithmetic instruction with overflow, |
| 5280 | // e.g. SADDO, UADDO. |
Bill Wendling | 0ea25cb | 2008-12-03 08:32:02 +0000 | [diff] [blame] | 5281 | Cond = Cond.getNode()->getOperand(1); |
| 5282 | addTest = false; |
| 5283 | break; |
Bill Wendling | 61edeb5 | 2008-12-02 01:06:39 +0000 | [diff] [blame] | 5284 | } |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5285 | } |
Evan Cheng | 370e534 | 2008-12-03 08:38:43 +0000 | [diff] [blame] | 5286 | } else { |
| 5287 | unsigned CondOpc; |
| 5288 | if (Cond.hasOneUse() && isAndOrOfSetCCs(Cond, CondOpc)) { |
| 5289 | SDValue Cmp = Cond.getOperand(0).getOperand(1); |
| 5290 | unsigned Opc = Cmp.getOpcode(); |
| 5291 | if (CondOpc == ISD::OR) { |
| 5292 | // Also, recognize the pattern generated by an FCMP_UNE. We can emit |
| 5293 | // two branches instead of an explicit OR instruction with a |
| 5294 | // separate test. |
| 5295 | if (Cmp == Cond.getOperand(1).getOperand(1) && |
| 5296 | isX86LogicalCmp(Opc)) { |
| 5297 | CC = Cond.getOperand(0).getOperand(0); |
| 5298 | Chain = DAG.getNode(X86ISD::BRCOND, Op.getValueType(), |
| 5299 | Chain, Dest, CC, Cmp); |
| 5300 | CC = Cond.getOperand(1).getOperand(0); |
| 5301 | Cond = Cmp; |
| 5302 | addTest = false; |
| 5303 | } |
| 5304 | } else { // ISD::AND |
| 5305 | // Also, recognize the pattern generated by an FCMP_OEQ. We can emit |
| 5306 | // two branches instead of an explicit AND instruction with a |
| 5307 | // separate test. However, we only do this if this block doesn't |
| 5308 | // have a fall-through edge, because this requires an explicit |
| 5309 | // jmp when the condition is false. |
| 5310 | if (Cmp == Cond.getOperand(1).getOperand(1) && |
| 5311 | isX86LogicalCmp(Opc) && |
| 5312 | Op.getNode()->hasOneUse()) { |
| 5313 | X86::CondCode CCode = |
| 5314 | (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0); |
| 5315 | CCode = X86::GetOppositeBranchCondition(CCode); |
| 5316 | CC = DAG.getConstant(CCode, MVT::i8); |
| 5317 | SDValue User = SDValue(*Op.getNode()->use_begin(), 0); |
| 5318 | // Look for an unconditional branch following this conditional branch. |
| 5319 | // We need this because we need to reverse the successors in order |
| 5320 | // to implement FCMP_OEQ. |
| 5321 | if (User.getOpcode() == ISD::BR) { |
| 5322 | SDValue FalseBB = User.getOperand(1); |
| 5323 | SDValue NewBR = |
| 5324 | DAG.UpdateNodeOperands(User, User.getOperand(0), Dest); |
| 5325 | assert(NewBR == User); |
| 5326 | Dest = FalseBB; |
Dan Gohman | 279c22e | 2008-10-21 03:29:32 +0000 | [diff] [blame] | 5327 | |
Evan Cheng | 370e534 | 2008-12-03 08:38:43 +0000 | [diff] [blame] | 5328 | Chain = DAG.getNode(X86ISD::BRCOND, Op.getValueType(), |
| 5329 | Chain, Dest, CC, Cmp); |
| 5330 | X86::CondCode CCode = |
| 5331 | (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0); |
| 5332 | CCode = X86::GetOppositeBranchCondition(CCode); |
| 5333 | CC = DAG.getConstant(CCode, MVT::i8); |
| 5334 | Cond = Cmp; |
| 5335 | addTest = false; |
| 5336 | } |
| 5337 | } |
Dan Gohman | 279c22e | 2008-10-21 03:29:32 +0000 | [diff] [blame] | 5338 | } |
| 5339 | } |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5340 | } |
| 5341 | |
| 5342 | if (addTest) { |
| 5343 | CC = DAG.getConstant(X86::COND_NE, MVT::i8); |
Evan Cheng | e5f6204 | 2007-09-29 00:00:36 +0000 | [diff] [blame] | 5344 | Cond= DAG.getNode(X86ISD::CMP, MVT::i32, Cond, DAG.getConstant(0, MVT::i8)); |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5345 | } |
Evan Cheng | e5f6204 | 2007-09-29 00:00:36 +0000 | [diff] [blame] | 5346 | return DAG.getNode(X86ISD::BRCOND, Op.getValueType(), |
Dan Gohman | 279c22e | 2008-10-21 03:29:32 +0000 | [diff] [blame] | 5347 | Chain, Dest, CC, Cond); |
Evan Cheng | 0488db9 | 2007-09-25 01:57:46 +0000 | [diff] [blame] | 5348 | } |
| 5349 | |
Anton Korobeynikov | e060b53 | 2007-04-17 19:34:00 +0000 | [diff] [blame] | 5350 | |
| 5351 | // Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets. |
| 5352 | // Calls to _alloca is needed to probe the stack when allocating more than 4k |
| 5353 | // bytes in one go. Touching the stack at 4K increments is necessary to ensure |
| 5354 | // that the guard pages used by the OS virtual memory manager are allocated in |
| 5355 | // correct sequence. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5356 | SDValue |
| 5357 | X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, |
Anton Korobeynikov | 4304bcc | 2007-07-05 20:36:08 +0000 | [diff] [blame] | 5358 | SelectionDAG &DAG) { |
Anton Korobeynikov | e060b53 | 2007-04-17 19:34:00 +0000 | [diff] [blame] | 5359 | assert(Subtarget->isTargetCygMing() && |
| 5360 | "This should be used only on Cygwin/Mingw targets"); |
Anton Korobeynikov | 096b461 | 2008-06-11 20:16:42 +0000 | [diff] [blame] | 5361 | |
Anton Korobeynikov | 57fc00d | 2007-04-17 09:20:00 +0000 | [diff] [blame] | 5362 | // Get the inputs. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5363 | SDValue Chain = Op.getOperand(0); |
| 5364 | SDValue Size = Op.getOperand(1); |
Anton Korobeynikov | 57fc00d | 2007-04-17 09:20:00 +0000 | [diff] [blame] | 5365 | // FIXME: Ensure alignment here |
| 5366 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5367 | SDValue Flag; |
Anton Korobeynikov | 096b461 | 2008-06-11 20:16:42 +0000 | [diff] [blame] | 5368 | |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5369 | MVT IntPtr = getPointerTy(); |
| 5370 | MVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32; |
Anton Korobeynikov | 57fc00d | 2007-04-17 09:20:00 +0000 | [diff] [blame] | 5371 | |
Chris Lattner | e563bbc | 2008-10-11 22:08:30 +0000 | [diff] [blame] | 5372 | Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true)); |
Anton Korobeynikov | 096b461 | 2008-06-11 20:16:42 +0000 | [diff] [blame] | 5373 | |
Anton Korobeynikov | 4304bcc | 2007-07-05 20:36:08 +0000 | [diff] [blame] | 5374 | Chain = DAG.getCopyToReg(Chain, X86::EAX, Size, Flag); |
| 5375 | Flag = Chain.getValue(1); |
| 5376 | |
| 5377 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5378 | SDValue Ops[] = { Chain, |
Bill Wendling | 056292f | 2008-09-16 21:48:12 +0000 | [diff] [blame] | 5379 | DAG.getTargetExternalSymbol("_alloca", IntPtr), |
Anton Korobeynikov | 4304bcc | 2007-07-05 20:36:08 +0000 | [diff] [blame] | 5380 | DAG.getRegister(X86::EAX, IntPtr), |
Anton Korobeynikov | 096b461 | 2008-06-11 20:16:42 +0000 | [diff] [blame] | 5381 | DAG.getRegister(X86StackPtr, SPTy), |
Anton Korobeynikov | 4304bcc | 2007-07-05 20:36:08 +0000 | [diff] [blame] | 5382 | Flag }; |
Anton Korobeynikov | 096b461 | 2008-06-11 20:16:42 +0000 | [diff] [blame] | 5383 | Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops, 5); |
Anton Korobeynikov | 4304bcc | 2007-07-05 20:36:08 +0000 | [diff] [blame] | 5384 | Flag = Chain.getValue(1); |
| 5385 | |
Anton Korobeynikov | 096b461 | 2008-06-11 20:16:42 +0000 | [diff] [blame] | 5386 | Chain = DAG.getCALLSEQ_END(Chain, |
Chris Lattner | e563bbc | 2008-10-11 22:08:30 +0000 | [diff] [blame] | 5387 | DAG.getIntPtrConstant(0, true), |
| 5388 | DAG.getIntPtrConstant(0, true), |
Anton Korobeynikov | 096b461 | 2008-06-11 20:16:42 +0000 | [diff] [blame] | 5389 | Flag); |
| 5390 | |
Anton Korobeynikov | 4304bcc | 2007-07-05 20:36:08 +0000 | [diff] [blame] | 5391 | Chain = DAG.getCopyFromReg(Chain, X86StackPtr, SPTy).getValue(1); |
Anton Korobeynikov | 096b461 | 2008-06-11 20:16:42 +0000 | [diff] [blame] | 5392 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5393 | SDValue Ops1[2] = { Chain.getValue(0), Chain }; |
Duncan Sands | 4bdcb61 | 2008-07-02 17:40:58 +0000 | [diff] [blame] | 5394 | return DAG.getMergeValues(Ops1, 2); |
Anton Korobeynikov | 57fc00d | 2007-04-17 09:20:00 +0000 | [diff] [blame] | 5395 | } |
| 5396 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5397 | SDValue |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5398 | X86TargetLowering::EmitTargetCodeForMemset(SelectionDAG &DAG, |
Bill Wendling | 6f287b2 | 2008-09-30 21:22:07 +0000 | [diff] [blame] | 5399 | SDValue Chain, |
| 5400 | SDValue Dst, SDValue Src, |
| 5401 | SDValue Size, unsigned Align, |
| 5402 | const Value *DstSV, |
Bill Wendling | 6158d84 | 2008-10-01 00:59:58 +0000 | [diff] [blame] | 5403 | uint64_t DstSVOff) { |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5404 | ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5405 | |
Bill Wendling | 6f287b2 | 2008-09-30 21:22:07 +0000 | [diff] [blame] | 5406 | // If not DWORD aligned or size is more than the threshold, call the library. |
| 5407 | // The libc version is likely to be faster for these cases. It can use the |
| 5408 | // address value and run time information about the CPU. |
Evan Cheng | 1887c1c | 2008-08-21 21:00:15 +0000 | [diff] [blame] | 5409 | if ((Align & 3) != 0 || |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5410 | !ConstantSize || |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 5411 | ConstantSize->getZExtValue() > |
| 5412 | getSubtarget()->getMaxInlineSizeThreshold()) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5413 | SDValue InFlag(0, 0); |
Dan Gohman | 68d599d | 2008-04-01 20:38:36 +0000 | [diff] [blame] | 5414 | |
| 5415 | // Check to see if there is a specialized entry-point for memory zeroing. |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5416 | ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src); |
Bill Wendling | 6f287b2 | 2008-09-30 21:22:07 +0000 | [diff] [blame] | 5417 | |
Bill Wendling | 6158d84 | 2008-10-01 00:59:58 +0000 | [diff] [blame] | 5418 | if (const char *bzeroEntry = V && |
| 5419 | V->isNullValue() ? Subtarget->getBZeroEntry() : 0) { |
| 5420 | MVT IntPtr = getPointerTy(); |
| 5421 | const Type *IntPtrTy = TD->getIntPtrType(); |
| 5422 | TargetLowering::ArgListTy Args; |
| 5423 | TargetLowering::ArgListEntry Entry; |
| 5424 | Entry.Node = Dst; |
| 5425 | Entry.Ty = IntPtrTy; |
| 5426 | Args.push_back(Entry); |
| 5427 | Entry.Node = Size; |
| 5428 | Args.push_back(Entry); |
| 5429 | std::pair<SDValue,SDValue> CallResult = |
| 5430 | LowerCallTo(Chain, Type::VoidTy, false, false, false, false, |
| 5431 | CallingConv::C, false, |
| 5432 | DAG.getExternalSymbol(bzeroEntry, IntPtr), Args, DAG); |
| 5433 | return CallResult.second; |
Dan Gohman | 68d599d | 2008-04-01 20:38:36 +0000 | [diff] [blame] | 5434 | } |
| 5435 | |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5436 | // Otherwise have the target-independent code call memset. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5437 | return SDValue(); |
Evan Cheng | 48090aa | 2006-03-21 23:01:21 +0000 | [diff] [blame] | 5438 | } |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 5439 | |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 5440 | uint64_t SizeVal = ConstantSize->getZExtValue(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5441 | SDValue InFlag(0, 0); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5442 | MVT AVT; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5443 | SDValue Count; |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5444 | ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Src); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5445 | unsigned BytesLeft = 0; |
| 5446 | bool TwoRepStos = false; |
| 5447 | if (ValC) { |
| 5448 | unsigned ValReg; |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 5449 | uint64_t Val = ValC->getZExtValue() & 255; |
Evan Cheng | 5ced1d8 | 2006-04-06 23:23:56 +0000 | [diff] [blame] | 5450 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5451 | // If the value is a constant, then we can potentially use larger sets. |
| 5452 | switch (Align & 3) { |
Evan Cheng | 1887c1c | 2008-08-21 21:00:15 +0000 | [diff] [blame] | 5453 | case 2: // WORD aligned |
| 5454 | AVT = MVT::i16; |
| 5455 | ValReg = X86::AX; |
| 5456 | Val = (Val << 8) | Val; |
| 5457 | break; |
| 5458 | case 0: // DWORD aligned |
| 5459 | AVT = MVT::i32; |
| 5460 | ValReg = X86::EAX; |
| 5461 | Val = (Val << 8) | Val; |
| 5462 | Val = (Val << 16) | Val; |
| 5463 | if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) { // QWORD aligned |
| 5464 | AVT = MVT::i64; |
| 5465 | ValReg = X86::RAX; |
| 5466 | Val = (Val << 32) | Val; |
| 5467 | } |
| 5468 | break; |
| 5469 | default: // Byte aligned |
| 5470 | AVT = MVT::i8; |
| 5471 | ValReg = X86::AL; |
| 5472 | Count = DAG.getIntPtrConstant(SizeVal); |
| 5473 | break; |
Evan Cheng | 80d428c | 2006-04-19 22:48:17 +0000 | [diff] [blame] | 5474 | } |
| 5475 | |
Duncan Sands | 8e4eb09 | 2008-06-08 20:54:56 +0000 | [diff] [blame] | 5476 | if (AVT.bitsGT(MVT::i8)) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5477 | unsigned UBytes = AVT.getSizeInBits() / 8; |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5478 | Count = DAG.getIntPtrConstant(SizeVal / UBytes); |
| 5479 | BytesLeft = SizeVal % UBytes; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5480 | } |
| 5481 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5482 | Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT), |
| 5483 | InFlag); |
| 5484 | InFlag = Chain.getValue(1); |
| 5485 | } else { |
| 5486 | AVT = MVT::i8; |
Dan Gohman | bcda285 | 2008-04-16 01:32:32 +0000 | [diff] [blame] | 5487 | Count = DAG.getIntPtrConstant(SizeVal); |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5488 | Chain = DAG.getCopyToReg(Chain, X86::AL, Src, InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5489 | InFlag = Chain.getValue(1); |
Evan Cheng | b9df0ca | 2006-03-22 02:53:00 +0000 | [diff] [blame] | 5490 | } |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 5491 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5492 | Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, |
| 5493 | Count, InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5494 | InFlag = Chain.getValue(1); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5495 | Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5496 | Dst, InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5497 | InFlag = Chain.getValue(1); |
Evan Cheng | a0b3afb | 2006-03-27 07:00:16 +0000 | [diff] [blame] | 5498 | |
Chris Lattner | d96d072 | 2007-02-25 06:40:16 +0000 | [diff] [blame] | 5499 | SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5500 | SmallVector<SDValue, 8> Ops; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5501 | Ops.push_back(Chain); |
| 5502 | Ops.push_back(DAG.getValueType(AVT)); |
| 5503 | Ops.push_back(InFlag); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 5504 | Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); |
Evan Cheng | c78d3b4 | 2006-04-24 18:01:45 +0000 | [diff] [blame] | 5505 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5506 | if (TwoRepStos) { |
| 5507 | InFlag = Chain.getValue(1); |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5508 | Count = Size; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5509 | MVT CVT = Count.getValueType(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5510 | SDValue Left = DAG.getNode(ISD::AND, CVT, Count, |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5511 | DAG.getConstant((AVT == MVT::i64) ? 7 : 3, CVT)); |
| 5512 | Chain = DAG.getCopyToReg(Chain, (CVT == MVT::i64) ? X86::RCX : X86::ECX, |
| 5513 | Left, InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5514 | InFlag = Chain.getValue(1); |
Chris Lattner | d96d072 | 2007-02-25 06:40:16 +0000 | [diff] [blame] | 5515 | Tys = DAG.getVTList(MVT::Other, MVT::Flag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5516 | Ops.clear(); |
| 5517 | Ops.push_back(Chain); |
| 5518 | Ops.push_back(DAG.getValueType(MVT::i8)); |
| 5519 | Ops.push_back(InFlag); |
Evan Cheng | 311ace0 | 2006-08-11 07:35:45 +0000 | [diff] [blame] | 5520 | Chain = DAG.getNode(X86ISD::REP_STOS, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5521 | } else if (BytesLeft) { |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5522 | // Handle the last 1 - 7 bytes. |
| 5523 | unsigned Offset = SizeVal - BytesLeft; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5524 | MVT AddrVT = Dst.getValueType(); |
| 5525 | MVT SizeVT = Size.getValueType(); |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5526 | |
| 5527 | Chain = DAG.getMemset(Chain, |
| 5528 | DAG.getNode(ISD::ADD, AddrVT, Dst, |
| 5529 | DAG.getConstant(Offset, AddrVT)), |
| 5530 | Src, |
| 5531 | DAG.getConstant(BytesLeft, SizeVT), |
Dan Gohman | 1f13c68 | 2008-04-28 17:15:20 +0000 | [diff] [blame] | 5532 | Align, DstSV, DstSVOff + Offset); |
Evan Cheng | 386031a | 2006-03-24 07:29:27 +0000 | [diff] [blame] | 5533 | } |
Evan Cheng | 11e15b3 | 2006-04-03 20:53:28 +0000 | [diff] [blame] | 5534 | |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5535 | // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain. |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5536 | return Chain; |
| 5537 | } |
Evan Cheng | 11e15b3 | 2006-04-03 20:53:28 +0000 | [diff] [blame] | 5538 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5539 | SDValue |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5540 | X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, |
Evan Cheng | 1887c1c | 2008-08-21 21:00:15 +0000 | [diff] [blame] | 5541 | SDValue Chain, SDValue Dst, SDValue Src, |
| 5542 | SDValue Size, unsigned Align, |
| 5543 | bool AlwaysInline, |
| 5544 | const Value *DstSV, uint64_t DstSVOff, |
| 5545 | const Value *SrcSV, uint64_t SrcSVOff) { |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5546 | // This requires the copy size to be a constant, preferrably |
| 5547 | // within a subtarget-specific limit. |
| 5548 | ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); |
| 5549 | if (!ConstantSize) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5550 | return SDValue(); |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 5551 | uint64_t SizeVal = ConstantSize->getZExtValue(); |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5552 | if (!AlwaysInline && SizeVal > getSubtarget()->getMaxInlineSizeThreshold()) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5553 | return SDValue(); |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5554 | |
Evan Cheng | 1887c1c | 2008-08-21 21:00:15 +0000 | [diff] [blame] | 5555 | /// If not DWORD aligned, call the library. |
| 5556 | if ((Align & 3) != 0) |
| 5557 | return SDValue(); |
| 5558 | |
| 5559 | // DWORD aligned |
| 5560 | MVT AVT = MVT::i32; |
| 5561 | if (Subtarget->is64Bit() && ((Align & 0x7) == 0)) // QWORD aligned |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5562 | AVT = MVT::i64; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5563 | |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5564 | unsigned UBytes = AVT.getSizeInBits() / 8; |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5565 | unsigned CountVal = SizeVal / UBytes; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5566 | SDValue Count = DAG.getIntPtrConstant(CountVal); |
Evan Cheng | 1887c1c | 2008-08-21 21:00:15 +0000 | [diff] [blame] | 5567 | unsigned BytesLeft = SizeVal % UBytes; |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5568 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5569 | SDValue InFlag(0, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5570 | Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RCX : X86::ECX, |
| 5571 | Count, InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5572 | InFlag = Chain.getValue(1); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5573 | Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RDI : X86::EDI, |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5574 | Dst, InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5575 | InFlag = Chain.getValue(1); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5576 | Chain = DAG.getCopyToReg(Chain, Subtarget->is64Bit() ? X86::RSI : X86::ESI, |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5577 | Src, InFlag); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5578 | InFlag = Chain.getValue(1); |
| 5579 | |
Chris Lattner | d96d072 | 2007-02-25 06:40:16 +0000 | [diff] [blame] | 5580 | SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5581 | SmallVector<SDValue, 8> Ops; |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5582 | Ops.push_back(Chain); |
| 5583 | Ops.push_back(DAG.getValueType(AVT)); |
| 5584 | Ops.push_back(InFlag); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5585 | SDValue RepMovs = DAG.getNode(X86ISD::REP_MOVS, Tys, &Ops[0], Ops.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5586 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5587 | SmallVector<SDValue, 4> Results; |
Evan Cheng | 2749c72 | 2008-04-25 00:26:43 +0000 | [diff] [blame] | 5588 | Results.push_back(RepMovs); |
Rafael Espindola | 068317b | 2007-09-28 12:53:01 +0000 | [diff] [blame] | 5589 | if (BytesLeft) { |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5590 | // Handle the last 1 - 7 bytes. |
| 5591 | unsigned Offset = SizeVal - BytesLeft; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5592 | MVT DstVT = Dst.getValueType(); |
| 5593 | MVT SrcVT = Src.getValueType(); |
| 5594 | MVT SizeVT = Size.getValueType(); |
Evan Cheng | 2749c72 | 2008-04-25 00:26:43 +0000 | [diff] [blame] | 5595 | Results.push_back(DAG.getMemcpy(Chain, |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5596 | DAG.getNode(ISD::ADD, DstVT, Dst, |
Evan Cheng | 2749c72 | 2008-04-25 00:26:43 +0000 | [diff] [blame] | 5597 | DAG.getConstant(Offset, DstVT)), |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5598 | DAG.getNode(ISD::ADD, SrcVT, Src, |
Evan Cheng | 2749c72 | 2008-04-25 00:26:43 +0000 | [diff] [blame] | 5599 | DAG.getConstant(Offset, SrcVT)), |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5600 | DAG.getConstant(BytesLeft, SizeVT), |
| 5601 | Align, AlwaysInline, |
Dan Gohman | 1f13c68 | 2008-04-28 17:15:20 +0000 | [diff] [blame] | 5602 | DstSV, DstSVOff + Offset, |
| 5603 | SrcSV, SrcSVOff + Offset)); |
Evan Cheng | b067a1e | 2006-03-31 19:22:53 +0000 | [diff] [blame] | 5604 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5605 | |
Dan Gohman | 707e018 | 2008-04-12 04:36:06 +0000 | [diff] [blame] | 5606 | return DAG.getNode(ISD::TokenFactor, MVT::Other, &Results[0], Results.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5607 | } |
| 5608 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5609 | SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) { |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5610 | const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); |
Evan Cheng | 8b2794a | 2006-10-13 21:14:26 +0000 | [diff] [blame] | 5611 | |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5612 | if (!Subtarget->is64Bit()) { |
| 5613 | // vastart just stores the address of the VarArgsFrameIndex slot into the |
| 5614 | // memory location argument. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5615 | SDValue FR = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5616 | return DAG.getStore(Op.getOperand(0), FR,Op.getOperand(1), SV, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5617 | } |
| 5618 | |
| 5619 | // __va_list_tag: |
| 5620 | // gp_offset (0 - 6 * 8) |
| 5621 | // fp_offset (48 - 48 + 8 * 16) |
| 5622 | // overflow_arg_area (point to parameters coming in memory). |
| 5623 | // reg_save_area |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5624 | SmallVector<SDValue, 8> MemOps; |
| 5625 | SDValue FIN = Op.getOperand(1); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5626 | // Store gp_offset |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5627 | SDValue Store = DAG.getStore(Op.getOperand(0), |
Evan Cheng | 786225a | 2006-10-05 23:01:46 +0000 | [diff] [blame] | 5628 | DAG.getConstant(VarArgsGPOffset, MVT::i32), |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5629 | FIN, SV, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5630 | MemOps.push_back(Store); |
| 5631 | |
| 5632 | // Store fp_offset |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 5633 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); |
Evan Cheng | 786225a | 2006-10-05 23:01:46 +0000 | [diff] [blame] | 5634 | Store = DAG.getStore(Op.getOperand(0), |
| 5635 | DAG.getConstant(VarArgsFPOffset, MVT::i32), |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5636 | FIN, SV, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5637 | MemOps.push_back(Store); |
| 5638 | |
| 5639 | // Store ptr to overflow_arg_area |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 5640 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(4)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5641 | SDValue OVFIN = DAG.getFrameIndex(VarArgsFrameIndex, getPointerTy()); |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5642 | Store = DAG.getStore(Op.getOperand(0), OVFIN, FIN, SV, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5643 | MemOps.push_back(Store); |
| 5644 | |
| 5645 | // Store ptr to reg_save_area. |
Chris Lattner | 0bd4893 | 2008-01-17 07:00:52 +0000 | [diff] [blame] | 5646 | FIN = DAG.getNode(ISD::ADD, getPointerTy(), FIN, DAG.getIntPtrConstant(8)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5647 | SDValue RSFIN = DAG.getFrameIndex(RegSaveFrameIndex, getPointerTy()); |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5648 | Store = DAG.getStore(Op.getOperand(0), RSFIN, FIN, SV, 0); |
Evan Cheng | 25ab690 | 2006-09-08 06:48:29 +0000 | [diff] [blame] | 5649 | MemOps.push_back(Store); |
| 5650 | return DAG.getNode(ISD::TokenFactor, MVT::Other, &MemOps[0], MemOps.size()); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5651 | } |
| 5652 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5653 | SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) { |
Dan Gohman | 9018e83 | 2008-05-10 01:26:14 +0000 | [diff] [blame] | 5654 | // X86-64 va_list is a struct { i32, i32, i8*, i8* }. |
| 5655 | assert(Subtarget->is64Bit() && "This code only handles 64-bit va_arg!"); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5656 | SDValue Chain = Op.getOperand(0); |
| 5657 | SDValue SrcPtr = Op.getOperand(1); |
| 5658 | SDValue SrcSV = Op.getOperand(2); |
Dan Gohman | 9018e83 | 2008-05-10 01:26:14 +0000 | [diff] [blame] | 5659 | |
| 5660 | assert(0 && "VAArgInst is not yet implemented for x86-64!"); |
| 5661 | abort(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5662 | return SDValue(); |
Dan Gohman | 9018e83 | 2008-05-10 01:26:14 +0000 | [diff] [blame] | 5663 | } |
| 5664 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5665 | SDValue X86TargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) { |
Evan Cheng | ae64219 | 2007-03-02 23:16:35 +0000 | [diff] [blame] | 5666 | // X86-64 va_list is a struct { i32, i32, i8*, i8* }. |
Dan Gohman | 2826913 | 2008-04-18 20:55:41 +0000 | [diff] [blame] | 5667 | assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!"); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5668 | SDValue Chain = Op.getOperand(0); |
| 5669 | SDValue DstPtr = Op.getOperand(1); |
| 5670 | SDValue SrcPtr = Op.getOperand(2); |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5671 | const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); |
| 5672 | const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); |
Evan Cheng | ae64219 | 2007-03-02 23:16:35 +0000 | [diff] [blame] | 5673 | |
Dan Gohman | 2826913 | 2008-04-18 20:55:41 +0000 | [diff] [blame] | 5674 | return DAG.getMemcpy(Chain, DstPtr, SrcPtr, |
| 5675 | DAG.getIntPtrConstant(24), 8, false, |
| 5676 | DstSV, 0, SrcSV, 0); |
Evan Cheng | ae64219 | 2007-03-02 23:16:35 +0000 | [diff] [blame] | 5677 | } |
| 5678 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5679 | SDValue |
| 5680 | X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 5681 | unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5682 | switch (IntNo) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5683 | default: return SDValue(); // Don't custom lower most intrinsics. |
Evan Cheng | 5759f97 | 2008-05-04 09:15:50 +0000 | [diff] [blame] | 5684 | // Comparison intrinsics. |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5685 | case Intrinsic::x86_sse_comieq_ss: |
| 5686 | case Intrinsic::x86_sse_comilt_ss: |
| 5687 | case Intrinsic::x86_sse_comile_ss: |
| 5688 | case Intrinsic::x86_sse_comigt_ss: |
| 5689 | case Intrinsic::x86_sse_comige_ss: |
| 5690 | case Intrinsic::x86_sse_comineq_ss: |
| 5691 | case Intrinsic::x86_sse_ucomieq_ss: |
| 5692 | case Intrinsic::x86_sse_ucomilt_ss: |
| 5693 | case Intrinsic::x86_sse_ucomile_ss: |
| 5694 | case Intrinsic::x86_sse_ucomigt_ss: |
| 5695 | case Intrinsic::x86_sse_ucomige_ss: |
| 5696 | case Intrinsic::x86_sse_ucomineq_ss: |
| 5697 | case Intrinsic::x86_sse2_comieq_sd: |
| 5698 | case Intrinsic::x86_sse2_comilt_sd: |
| 5699 | case Intrinsic::x86_sse2_comile_sd: |
| 5700 | case Intrinsic::x86_sse2_comigt_sd: |
| 5701 | case Intrinsic::x86_sse2_comige_sd: |
| 5702 | case Intrinsic::x86_sse2_comineq_sd: |
| 5703 | case Intrinsic::x86_sse2_ucomieq_sd: |
| 5704 | case Intrinsic::x86_sse2_ucomilt_sd: |
| 5705 | case Intrinsic::x86_sse2_ucomile_sd: |
| 5706 | case Intrinsic::x86_sse2_ucomigt_sd: |
| 5707 | case Intrinsic::x86_sse2_ucomige_sd: |
| 5708 | case Intrinsic::x86_sse2_ucomineq_sd: { |
| 5709 | unsigned Opc = 0; |
| 5710 | ISD::CondCode CC = ISD::SETCC_INVALID; |
| 5711 | switch (IntNo) { |
| 5712 | default: break; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 5713 | case Intrinsic::x86_sse_comieq_ss: |
| 5714 | case Intrinsic::x86_sse2_comieq_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5715 | Opc = X86ISD::COMI; |
| 5716 | CC = ISD::SETEQ; |
| 5717 | break; |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5718 | case Intrinsic::x86_sse_comilt_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5719 | case Intrinsic::x86_sse2_comilt_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5720 | Opc = X86ISD::COMI; |
| 5721 | CC = ISD::SETLT; |
| 5722 | break; |
| 5723 | case Intrinsic::x86_sse_comile_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5724 | case Intrinsic::x86_sse2_comile_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5725 | Opc = X86ISD::COMI; |
| 5726 | CC = ISD::SETLE; |
| 5727 | break; |
| 5728 | case Intrinsic::x86_sse_comigt_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5729 | case Intrinsic::x86_sse2_comigt_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5730 | Opc = X86ISD::COMI; |
| 5731 | CC = ISD::SETGT; |
| 5732 | break; |
| 5733 | case Intrinsic::x86_sse_comige_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5734 | case Intrinsic::x86_sse2_comige_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5735 | Opc = X86ISD::COMI; |
| 5736 | CC = ISD::SETGE; |
| 5737 | break; |
| 5738 | case Intrinsic::x86_sse_comineq_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5739 | case Intrinsic::x86_sse2_comineq_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5740 | Opc = X86ISD::COMI; |
| 5741 | CC = ISD::SETNE; |
| 5742 | break; |
| 5743 | case Intrinsic::x86_sse_ucomieq_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5744 | case Intrinsic::x86_sse2_ucomieq_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5745 | Opc = X86ISD::UCOMI; |
| 5746 | CC = ISD::SETEQ; |
| 5747 | break; |
| 5748 | case Intrinsic::x86_sse_ucomilt_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5749 | case Intrinsic::x86_sse2_ucomilt_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5750 | Opc = X86ISD::UCOMI; |
| 5751 | CC = ISD::SETLT; |
| 5752 | break; |
| 5753 | case Intrinsic::x86_sse_ucomile_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5754 | case Intrinsic::x86_sse2_ucomile_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5755 | Opc = X86ISD::UCOMI; |
| 5756 | CC = ISD::SETLE; |
| 5757 | break; |
| 5758 | case Intrinsic::x86_sse_ucomigt_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5759 | case Intrinsic::x86_sse2_ucomigt_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5760 | Opc = X86ISD::UCOMI; |
| 5761 | CC = ISD::SETGT; |
| 5762 | break; |
| 5763 | case Intrinsic::x86_sse_ucomige_ss: |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5764 | case Intrinsic::x86_sse2_ucomige_sd: |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 5765 | Opc = X86ISD::UCOMI; |
| 5766 | CC = ISD::SETGE; |
| 5767 | break; |
| 5768 | case Intrinsic::x86_sse_ucomineq_ss: |
| 5769 | case Intrinsic::x86_sse2_ucomineq_sd: |
| 5770 | Opc = X86ISD::UCOMI; |
| 5771 | CC = ISD::SETNE; |
| 5772 | break; |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5773 | } |
Evan Cheng | 734503b | 2006-09-11 02:19:56 +0000 | [diff] [blame] | 5774 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5775 | SDValue LHS = Op.getOperand(1); |
| 5776 | SDValue RHS = Op.getOperand(2); |
Chris Lattner | 1c39d4c | 2008-12-24 23:53:05 +0000 | [diff] [blame] | 5777 | unsigned X86CC = TranslateX86CC(CC, true, LHS, RHS, DAG); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5778 | SDValue Cond = DAG.getNode(Opc, MVT::i32, LHS, RHS); |
| 5779 | SDValue SetCC = DAG.getNode(X86ISD::SETCC, MVT::i8, |
Evan Cheng | 0ac3fc2 | 2008-08-17 19:22:34 +0000 | [diff] [blame] | 5780 | DAG.getConstant(X86CC, MVT::i8), Cond); |
| 5781 | return DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, SetCC); |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 5782 | } |
Evan Cheng | 5759f97 | 2008-05-04 09:15:50 +0000 | [diff] [blame] | 5783 | |
| 5784 | // Fix vector shift instructions where the last operand is a non-immediate |
| 5785 | // i32 value. |
| 5786 | case Intrinsic::x86_sse2_pslli_w: |
| 5787 | case Intrinsic::x86_sse2_pslli_d: |
| 5788 | case Intrinsic::x86_sse2_pslli_q: |
| 5789 | case Intrinsic::x86_sse2_psrli_w: |
| 5790 | case Intrinsic::x86_sse2_psrli_d: |
| 5791 | case Intrinsic::x86_sse2_psrli_q: |
| 5792 | case Intrinsic::x86_sse2_psrai_w: |
| 5793 | case Intrinsic::x86_sse2_psrai_d: |
| 5794 | case Intrinsic::x86_mmx_pslli_w: |
| 5795 | case Intrinsic::x86_mmx_pslli_d: |
| 5796 | case Intrinsic::x86_mmx_pslli_q: |
| 5797 | case Intrinsic::x86_mmx_psrli_w: |
| 5798 | case Intrinsic::x86_mmx_psrli_d: |
| 5799 | case Intrinsic::x86_mmx_psrli_q: |
| 5800 | case Intrinsic::x86_mmx_psrai_w: |
| 5801 | case Intrinsic::x86_mmx_psrai_d: { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5802 | SDValue ShAmt = Op.getOperand(2); |
Evan Cheng | 5759f97 | 2008-05-04 09:15:50 +0000 | [diff] [blame] | 5803 | if (isa<ConstantSDNode>(ShAmt)) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5804 | return SDValue(); |
Evan Cheng | 5759f97 | 2008-05-04 09:15:50 +0000 | [diff] [blame] | 5805 | |
| 5806 | unsigned NewIntNo = 0; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5807 | MVT ShAmtVT = MVT::v4i32; |
Evan Cheng | 5759f97 | 2008-05-04 09:15:50 +0000 | [diff] [blame] | 5808 | switch (IntNo) { |
| 5809 | case Intrinsic::x86_sse2_pslli_w: |
| 5810 | NewIntNo = Intrinsic::x86_sse2_psll_w; |
| 5811 | break; |
| 5812 | case Intrinsic::x86_sse2_pslli_d: |
| 5813 | NewIntNo = Intrinsic::x86_sse2_psll_d; |
| 5814 | break; |
| 5815 | case Intrinsic::x86_sse2_pslli_q: |
| 5816 | NewIntNo = Intrinsic::x86_sse2_psll_q; |
| 5817 | break; |
| 5818 | case Intrinsic::x86_sse2_psrli_w: |
| 5819 | NewIntNo = Intrinsic::x86_sse2_psrl_w; |
| 5820 | break; |
| 5821 | case Intrinsic::x86_sse2_psrli_d: |
| 5822 | NewIntNo = Intrinsic::x86_sse2_psrl_d; |
| 5823 | break; |
| 5824 | case Intrinsic::x86_sse2_psrli_q: |
| 5825 | NewIntNo = Intrinsic::x86_sse2_psrl_q; |
| 5826 | break; |
| 5827 | case Intrinsic::x86_sse2_psrai_w: |
| 5828 | NewIntNo = Intrinsic::x86_sse2_psra_w; |
| 5829 | break; |
| 5830 | case Intrinsic::x86_sse2_psrai_d: |
| 5831 | NewIntNo = Intrinsic::x86_sse2_psra_d; |
| 5832 | break; |
| 5833 | default: { |
| 5834 | ShAmtVT = MVT::v2i32; |
| 5835 | switch (IntNo) { |
| 5836 | case Intrinsic::x86_mmx_pslli_w: |
| 5837 | NewIntNo = Intrinsic::x86_mmx_psll_w; |
| 5838 | break; |
| 5839 | case Intrinsic::x86_mmx_pslli_d: |
| 5840 | NewIntNo = Intrinsic::x86_mmx_psll_d; |
| 5841 | break; |
| 5842 | case Intrinsic::x86_mmx_pslli_q: |
| 5843 | NewIntNo = Intrinsic::x86_mmx_psll_q; |
| 5844 | break; |
| 5845 | case Intrinsic::x86_mmx_psrli_w: |
| 5846 | NewIntNo = Intrinsic::x86_mmx_psrl_w; |
| 5847 | break; |
| 5848 | case Intrinsic::x86_mmx_psrli_d: |
| 5849 | NewIntNo = Intrinsic::x86_mmx_psrl_d; |
| 5850 | break; |
| 5851 | case Intrinsic::x86_mmx_psrli_q: |
| 5852 | NewIntNo = Intrinsic::x86_mmx_psrl_q; |
| 5853 | break; |
| 5854 | case Intrinsic::x86_mmx_psrai_w: |
| 5855 | NewIntNo = Intrinsic::x86_mmx_psra_w; |
| 5856 | break; |
| 5857 | case Intrinsic::x86_mmx_psrai_d: |
| 5858 | NewIntNo = Intrinsic::x86_mmx_psra_d; |
| 5859 | break; |
| 5860 | default: abort(); // Can't reach here. |
| 5861 | } |
| 5862 | break; |
| 5863 | } |
| 5864 | } |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 5865 | MVT VT = Op.getValueType(); |
Evan Cheng | 5759f97 | 2008-05-04 09:15:50 +0000 | [diff] [blame] | 5866 | ShAmt = DAG.getNode(ISD::BIT_CONVERT, VT, |
| 5867 | DAG.getNode(ISD::SCALAR_TO_VECTOR, ShAmtVT, ShAmt)); |
| 5868 | return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, |
| 5869 | DAG.getConstant(NewIntNo, MVT::i32), |
| 5870 | Op.getOperand(1), ShAmt); |
| 5871 | } |
Evan Cheng | 38bcbaf | 2005-12-23 07:31:11 +0000 | [diff] [blame] | 5872 | } |
Chris Lattner | dbdbf0c | 2005-11-15 00:40:23 +0000 | [diff] [blame] | 5873 | } |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 5874 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5875 | SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) { |
Nate Begeman | bcc5f36 | 2007-01-29 22:58:52 +0000 | [diff] [blame] | 5876 | // Depths > 0 not supported yet! |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 5877 | if (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue() > 0) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5878 | return SDValue(); |
Nate Begeman | bcc5f36 | 2007-01-29 22:58:52 +0000 | [diff] [blame] | 5879 | |
| 5880 | // Just load the return address |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5881 | SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); |
Nate Begeman | bcc5f36 | 2007-01-29 22:58:52 +0000 | [diff] [blame] | 5882 | return DAG.getLoad(getPointerTy(), DAG.getEntryNode(), RetAddrFI, NULL, 0); |
| 5883 | } |
| 5884 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5885 | SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) { |
Evan Cheng | 184793f | 2008-09-27 01:56:22 +0000 | [diff] [blame] | 5886 | MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); |
| 5887 | MFI->setFrameAddressIsTaken(true); |
| 5888 | MVT VT = Op.getValueType(); |
| 5889 | unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); |
| 5890 | unsigned FrameReg = Subtarget->is64Bit() ? X86::RBP : X86::EBP; |
| 5891 | SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), FrameReg, VT); |
| 5892 | while (Depth--) |
| 5893 | FrameAddr = DAG.getLoad(VT, DAG.getEntryNode(), FrameAddr, NULL, 0); |
| 5894 | return FrameAddr; |
Nate Begeman | bcc5f36 | 2007-01-29 22:58:52 +0000 | [diff] [blame] | 5895 | } |
| 5896 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5897 | SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op, |
Anton Korobeynikov | 260a6b8 | 2008-09-08 21:12:11 +0000 | [diff] [blame] | 5898 | SelectionDAG &DAG) { |
Anton Korobeynikov | bff66b0 | 2008-09-09 18:22:57 +0000 | [diff] [blame] | 5899 | return DAG.getIntPtrConstant(2*TD->getPointerSize()); |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 5900 | } |
| 5901 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5902 | SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 5903 | { |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 5904 | MachineFunction &MF = DAG.getMachineFunction(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5905 | SDValue Chain = Op.getOperand(0); |
| 5906 | SDValue Offset = Op.getOperand(1); |
| 5907 | SDValue Handler = Op.getOperand(2); |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 5908 | |
Anton Korobeynikov | b84c167 | 2008-09-08 21:12:47 +0000 | [diff] [blame] | 5909 | SDValue Frame = DAG.getRegister(Subtarget->is64Bit() ? X86::RBP : X86::EBP, |
| 5910 | getPointerTy()); |
| 5911 | unsigned StoreAddrReg = (Subtarget->is64Bit() ? X86::RCX : X86::ECX); |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 5912 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5913 | SDValue StoreAddr = DAG.getNode(ISD::SUB, getPointerTy(), Frame, |
Anton Korobeynikov | bff66b0 | 2008-09-09 18:22:57 +0000 | [diff] [blame] | 5914 | DAG.getIntPtrConstant(-TD->getPointerSize())); |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 5915 | StoreAddr = DAG.getNode(ISD::ADD, getPointerTy(), StoreAddr, Offset); |
| 5916 | Chain = DAG.getStore(Chain, Handler, StoreAddr, NULL, 0); |
Anton Korobeynikov | b84c167 | 2008-09-08 21:12:47 +0000 | [diff] [blame] | 5917 | Chain = DAG.getCopyToReg(Chain, StoreAddrReg, StoreAddr); |
| 5918 | MF.getRegInfo().addLiveOut(StoreAddrReg); |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 5919 | |
Anton Korobeynikov | b84c167 | 2008-09-08 21:12:47 +0000 | [diff] [blame] | 5920 | return DAG.getNode(X86ISD::EH_RETURN, |
| 5921 | MVT::Other, |
| 5922 | Chain, DAG.getRegister(StoreAddrReg, getPointerTy())); |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 5923 | } |
| 5924 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5925 | SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op, |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 5926 | SelectionDAG &DAG) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5927 | SDValue Root = Op.getOperand(0); |
| 5928 | SDValue Trmp = Op.getOperand(1); // trampoline |
| 5929 | SDValue FPtr = Op.getOperand(2); // nested function |
| 5930 | SDValue Nest = Op.getOperand(3); // 'nest' parameter value |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 5931 | |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5932 | const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 5933 | |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 5934 | const X86InstrInfo *TII = |
| 5935 | ((X86TargetMachine&)getTargetMachine()).getInstrInfo(); |
| 5936 | |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 5937 | if (Subtarget->is64Bit()) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5938 | SDValue OutChains[6]; |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 5939 | |
| 5940 | // Large code-model. |
| 5941 | |
| 5942 | const unsigned char JMP64r = TII->getBaseOpcodeFor(X86::JMP64r); |
| 5943 | const unsigned char MOV64ri = TII->getBaseOpcodeFor(X86::MOV64ri); |
| 5944 | |
Dan Gohman | c9f5f3f | 2008-05-14 01:58:56 +0000 | [diff] [blame] | 5945 | const unsigned char N86R10 = RegInfo->getX86RegNum(X86::R10); |
| 5946 | const unsigned char N86R11 = RegInfo->getX86RegNum(X86::R11); |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 5947 | |
| 5948 | const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix |
| 5949 | |
| 5950 | // Load the pointer to the nested function into R11. |
| 5951 | unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11 |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5952 | SDValue Addr = Trmp; |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 5953 | OutChains[0] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5954 | TrmpAddr, 0); |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 5955 | |
| 5956 | Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(2, MVT::i64)); |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5957 | OutChains[1] = DAG.getStore(Root, FPtr, Addr, TrmpAddr, 2, false, 2); |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 5958 | |
| 5959 | // Load the 'nest' parameter value into R10. |
| 5960 | // R10 is specified in X86CallingConv.td |
| 5961 | OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10 |
| 5962 | Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(10, MVT::i64)); |
| 5963 | OutChains[2] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5964 | TrmpAddr, 10); |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 5965 | |
| 5966 | Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(12, MVT::i64)); |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5967 | OutChains[3] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 12, false, 2); |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 5968 | |
| 5969 | // Jump to the nested function. |
| 5970 | OpCode = (JMP64r << 8) | REX_WB; // jmpq *... |
| 5971 | Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(20, MVT::i64)); |
| 5972 | OutChains[4] = DAG.getStore(Root, DAG.getConstant(OpCode, MVT::i16), Addr, |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5973 | TrmpAddr, 20); |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 5974 | |
| 5975 | unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11 |
| 5976 | Addr = DAG.getNode(ISD::ADD, MVT::i64, Trmp, DAG.getConstant(22, MVT::i64)); |
| 5977 | OutChains[5] = DAG.getStore(Root, DAG.getConstant(ModRM, MVT::i8), Addr, |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 5978 | TrmpAddr, 22); |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 5979 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 5980 | SDValue Ops[] = |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 5981 | { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 6) }; |
Duncan Sands | 4bdcb61 | 2008-07-02 17:40:58 +0000 | [diff] [blame] | 5982 | return DAG.getMergeValues(Ops, 2); |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 5983 | } else { |
Dan Gohman | bbfb9c5 | 2008-01-31 01:01:48 +0000 | [diff] [blame] | 5984 | const Function *Func = |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 5985 | cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue()); |
| 5986 | unsigned CC = Func->getCallingConv(); |
Duncan Sands | ee46574 | 2007-08-29 19:01:20 +0000 | [diff] [blame] | 5987 | unsigned NestReg; |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 5988 | |
| 5989 | switch (CC) { |
| 5990 | default: |
| 5991 | assert(0 && "Unsupported calling convention"); |
| 5992 | case CallingConv::C: |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 5993 | case CallingConv::X86_StdCall: { |
| 5994 | // Pass 'nest' parameter in ECX. |
| 5995 | // Must be kept in sync with X86CallingConv.td |
Duncan Sands | ee46574 | 2007-08-29 19:01:20 +0000 | [diff] [blame] | 5996 | NestReg = X86::ECX; |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 5997 | |
| 5998 | // Check that ECX wasn't needed by an 'inreg' parameter. |
| 5999 | const FunctionType *FTy = Func->getFunctionType(); |
Devang Patel | 0598866 | 2008-09-25 21:00:45 +0000 | [diff] [blame] | 6000 | const AttrListPtr &Attrs = Func->getAttributes(); |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6001 | |
Chris Lattner | 58d7491 | 2008-03-12 17:45:29 +0000 | [diff] [blame] | 6002 | if (!Attrs.isEmpty() && !Func->isVarArg()) { |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6003 | unsigned InRegCount = 0; |
| 6004 | unsigned Idx = 1; |
| 6005 | |
| 6006 | for (FunctionType::param_iterator I = FTy->param_begin(), |
| 6007 | E = FTy->param_end(); I != E; ++I, ++Idx) |
Devang Patel | 0598866 | 2008-09-25 21:00:45 +0000 | [diff] [blame] | 6008 | if (Attrs.paramHasAttr(Idx, Attribute::InReg)) |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6009 | // FIXME: should only count parameters that are lowered to integers. |
Anton Korobeynikov | bff66b0 | 2008-09-09 18:22:57 +0000 | [diff] [blame] | 6010 | InRegCount += (TD->getTypeSizeInBits(*I) + 31) / 32; |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6011 | |
| 6012 | if (InRegCount > 2) { |
| 6013 | cerr << "Nest register in use - reduce number of inreg parameters!\n"; |
| 6014 | abort(); |
| 6015 | } |
| 6016 | } |
| 6017 | break; |
| 6018 | } |
| 6019 | case CallingConv::X86_FastCall: |
Duncan Sands | bf53c29 | 2008-09-10 13:22:10 +0000 | [diff] [blame] | 6020 | case CallingConv::Fast: |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6021 | // Pass 'nest' parameter in EAX. |
| 6022 | // Must be kept in sync with X86CallingConv.td |
Duncan Sands | ee46574 | 2007-08-29 19:01:20 +0000 | [diff] [blame] | 6023 | NestReg = X86::EAX; |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6024 | break; |
| 6025 | } |
| 6026 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6027 | SDValue OutChains[4]; |
| 6028 | SDValue Addr, Disp; |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6029 | |
| 6030 | Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(10, MVT::i32)); |
| 6031 | Disp = DAG.getNode(ISD::SUB, MVT::i32, FPtr, Addr); |
| 6032 | |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 6033 | const unsigned char MOV32ri = TII->getBaseOpcodeFor(X86::MOV32ri); |
Dan Gohman | c9f5f3f | 2008-05-14 01:58:56 +0000 | [diff] [blame] | 6034 | const unsigned char N86Reg = RegInfo->getX86RegNum(NestReg); |
Duncan Sands | ee46574 | 2007-08-29 19:01:20 +0000 | [diff] [blame] | 6035 | OutChains[0] = DAG.getStore(Root, DAG.getConstant(MOV32ri|N86Reg, MVT::i8), |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 6036 | Trmp, TrmpAddr, 0); |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6037 | |
| 6038 | Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(1, MVT::i32)); |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 6039 | OutChains[1] = DAG.getStore(Root, Nest, Addr, TrmpAddr, 1, false, 1); |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6040 | |
Duncan Sands | 339e14f | 2008-01-16 22:55:25 +0000 | [diff] [blame] | 6041 | const unsigned char JMP = TII->getBaseOpcodeFor(X86::JMP); |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6042 | Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(5, MVT::i32)); |
| 6043 | OutChains[2] = DAG.getStore(Root, DAG.getConstant(JMP, MVT::i8), Addr, |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 6044 | TrmpAddr, 5, false, 1); |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6045 | |
| 6046 | Addr = DAG.getNode(ISD::ADD, MVT::i32, Trmp, DAG.getConstant(6, MVT::i32)); |
Dan Gohman | 69de193 | 2008-02-06 22:27:42 +0000 | [diff] [blame] | 6047 | OutChains[3] = DAG.getStore(Root, Disp, Addr, TrmpAddr, 6, false, 1); |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6048 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6049 | SDValue Ops[] = |
Duncan Sands | f7331b3 | 2007-09-11 14:10:23 +0000 | [diff] [blame] | 6050 | { Trmp, DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains, 4) }; |
Duncan Sands | 4bdcb61 | 2008-07-02 17:40:58 +0000 | [diff] [blame] | 6051 | return DAG.getMergeValues(Ops, 2); |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6052 | } |
| 6053 | } |
| 6054 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6055 | SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) { |
Anton Korobeynikov | 45b22fa | 2007-11-16 01:31:51 +0000 | [diff] [blame] | 6056 | /* |
| 6057 | The rounding mode is in bits 11:10 of FPSR, and has the following |
| 6058 | settings: |
| 6059 | 00 Round to nearest |
| 6060 | 01 Round to -inf |
| 6061 | 10 Round to +inf |
| 6062 | 11 Round to 0 |
| 6063 | |
| 6064 | FLT_ROUNDS, on the other hand, expects the following: |
| 6065 | -1 Undefined |
| 6066 | 0 Round to 0 |
| 6067 | 1 Round to nearest |
| 6068 | 2 Round to +inf |
| 6069 | 3 Round to -inf |
| 6070 | |
| 6071 | To perform the conversion, we do: |
| 6072 | (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3) |
| 6073 | */ |
| 6074 | |
| 6075 | MachineFunction &MF = DAG.getMachineFunction(); |
| 6076 | const TargetMachine &TM = MF.getTarget(); |
| 6077 | const TargetFrameInfo &TFI = *TM.getFrameInfo(); |
| 6078 | unsigned StackAlignment = TFI.getStackAlignment(); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 6079 | MVT VT = Op.getValueType(); |
Anton Korobeynikov | 45b22fa | 2007-11-16 01:31:51 +0000 | [diff] [blame] | 6080 | |
| 6081 | // Save FP Control Word to stack slot |
| 6082 | int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6083 | SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy()); |
Anton Korobeynikov | 45b22fa | 2007-11-16 01:31:51 +0000 | [diff] [blame] | 6084 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6085 | SDValue Chain = DAG.getNode(X86ISD::FNSTCW16m, MVT::Other, |
Evan Cheng | 8a186ae | 2008-09-24 23:26:36 +0000 | [diff] [blame] | 6086 | DAG.getEntryNode(), StackSlot); |
Anton Korobeynikov | 45b22fa | 2007-11-16 01:31:51 +0000 | [diff] [blame] | 6087 | |
| 6088 | // Load FP Control Word from stack slot |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6089 | SDValue CWD = DAG.getLoad(MVT::i16, Chain, StackSlot, NULL, 0); |
Anton Korobeynikov | 45b22fa | 2007-11-16 01:31:51 +0000 | [diff] [blame] | 6090 | |
| 6091 | // Transform as necessary |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6092 | SDValue CWD1 = |
Anton Korobeynikov | 45b22fa | 2007-11-16 01:31:51 +0000 | [diff] [blame] | 6093 | DAG.getNode(ISD::SRL, MVT::i16, |
| 6094 | DAG.getNode(ISD::AND, MVT::i16, |
| 6095 | CWD, DAG.getConstant(0x800, MVT::i16)), |
| 6096 | DAG.getConstant(11, MVT::i8)); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6097 | SDValue CWD2 = |
Anton Korobeynikov | 45b22fa | 2007-11-16 01:31:51 +0000 | [diff] [blame] | 6098 | DAG.getNode(ISD::SRL, MVT::i16, |
| 6099 | DAG.getNode(ISD::AND, MVT::i16, |
| 6100 | CWD, DAG.getConstant(0x400, MVT::i16)), |
| 6101 | DAG.getConstant(9, MVT::i8)); |
| 6102 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6103 | SDValue RetVal = |
Anton Korobeynikov | 45b22fa | 2007-11-16 01:31:51 +0000 | [diff] [blame] | 6104 | DAG.getNode(ISD::AND, MVT::i16, |
| 6105 | DAG.getNode(ISD::ADD, MVT::i16, |
| 6106 | DAG.getNode(ISD::OR, MVT::i16, CWD1, CWD2), |
| 6107 | DAG.getConstant(1, MVT::i16)), |
| 6108 | DAG.getConstant(3, MVT::i16)); |
| 6109 | |
| 6110 | |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 6111 | return DAG.getNode((VT.getSizeInBits() < 16 ? |
Anton Korobeynikov | 45b22fa | 2007-11-16 01:31:51 +0000 | [diff] [blame] | 6112 | ISD::TRUNCATE : ISD::ZERO_EXTEND), VT, RetVal); |
| 6113 | } |
| 6114 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6115 | SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 6116 | MVT VT = Op.getValueType(); |
| 6117 | MVT OpVT = VT; |
| 6118 | unsigned NumBits = VT.getSizeInBits(); |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 6119 | |
| 6120 | Op = Op.getOperand(0); |
| 6121 | if (VT == MVT::i8) { |
Evan Cheng | 152804e | 2007-12-14 08:30:15 +0000 | [diff] [blame] | 6122 | // Zero extend to i32 since there is not an i8 bsr. |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 6123 | OpVT = MVT::i32; |
| 6124 | Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); |
| 6125 | } |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 6126 | |
Evan Cheng | 152804e | 2007-12-14 08:30:15 +0000 | [diff] [blame] | 6127 | // Issue a bsr (scan bits in reverse) which also sets EFLAGS. |
| 6128 | SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); |
| 6129 | Op = DAG.getNode(X86ISD::BSR, VTs, Op); |
| 6130 | |
| 6131 | // If src is zero (i.e. bsr sets ZF), returns NumBits. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6132 | SmallVector<SDValue, 4> Ops; |
Evan Cheng | 152804e | 2007-12-14 08:30:15 +0000 | [diff] [blame] | 6133 | Ops.push_back(Op); |
| 6134 | Ops.push_back(DAG.getConstant(NumBits+NumBits-1, OpVT)); |
| 6135 | Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); |
| 6136 | Ops.push_back(Op.getValue(1)); |
| 6137 | Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); |
| 6138 | |
| 6139 | // Finally xor with NumBits-1. |
| 6140 | Op = DAG.getNode(ISD::XOR, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); |
| 6141 | |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 6142 | if (VT == MVT::i8) |
| 6143 | Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); |
| 6144 | return Op; |
| 6145 | } |
| 6146 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6147 | SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) { |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 6148 | MVT VT = Op.getValueType(); |
| 6149 | MVT OpVT = VT; |
| 6150 | unsigned NumBits = VT.getSizeInBits(); |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 6151 | |
| 6152 | Op = Op.getOperand(0); |
| 6153 | if (VT == MVT::i8) { |
| 6154 | OpVT = MVT::i32; |
| 6155 | Op = DAG.getNode(ISD::ZERO_EXTEND, OpVT, Op); |
| 6156 | } |
Evan Cheng | 152804e | 2007-12-14 08:30:15 +0000 | [diff] [blame] | 6157 | |
| 6158 | // Issue a bsf (scan bits forward) which also sets EFLAGS. |
| 6159 | SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); |
| 6160 | Op = DAG.getNode(X86ISD::BSF, VTs, Op); |
| 6161 | |
| 6162 | // If src is zero (i.e. bsf sets ZF), returns NumBits. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6163 | SmallVector<SDValue, 4> Ops; |
Evan Cheng | 152804e | 2007-12-14 08:30:15 +0000 | [diff] [blame] | 6164 | Ops.push_back(Op); |
| 6165 | Ops.push_back(DAG.getConstant(NumBits, OpVT)); |
| 6166 | Ops.push_back(DAG.getConstant(X86::COND_E, MVT::i8)); |
| 6167 | Ops.push_back(Op.getValue(1)); |
| 6168 | Op = DAG.getNode(X86ISD::CMOV, OpVT, &Ops[0], 4); |
| 6169 | |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 6170 | if (VT == MVT::i8) |
| 6171 | Op = DAG.getNode(ISD::TRUNCATE, MVT::i8, Op); |
| 6172 | return Op; |
| 6173 | } |
| 6174 | |
Mon P Wang | af9b952 | 2008-12-18 21:42:19 +0000 | [diff] [blame] | 6175 | SDValue X86TargetLowering::LowerMUL_V2I64(SDValue Op, SelectionDAG &DAG) { |
| 6176 | MVT VT = Op.getValueType(); |
| 6177 | assert(VT == MVT::v2i64 && "Only know how to lower V2I64 multiply"); |
| 6178 | |
| 6179 | // ulong2 Ahi = __builtin_ia32_psrlqi128( a, 32); |
| 6180 | // ulong2 Bhi = __builtin_ia32_psrlqi128( b, 32); |
| 6181 | // ulong2 AloBlo = __builtin_ia32_pmuludq128( a, b ); |
| 6182 | // ulong2 AloBhi = __builtin_ia32_pmuludq128( a, Bhi ); |
| 6183 | // ulong2 AhiBlo = __builtin_ia32_pmuludq128( Ahi, b ); |
| 6184 | // |
| 6185 | // AloBhi = __builtin_ia32_psllqi128( AloBhi, 32 ); |
| 6186 | // AhiBlo = __builtin_ia32_psllqi128( AhiBlo, 32 ); |
| 6187 | // return AloBlo + AloBhi + AhiBlo; |
| 6188 | |
| 6189 | SDValue A = Op.getOperand(0); |
| 6190 | SDValue B = Op.getOperand(1); |
| 6191 | |
| 6192 | SDValue Ahi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, |
| 6193 | DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), |
| 6194 | A, DAG.getConstant(32, MVT::i32)); |
| 6195 | SDValue Bhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, |
| 6196 | DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32), |
| 6197 | B, DAG.getConstant(32, MVT::i32)); |
| 6198 | SDValue AloBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, |
| 6199 | DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), |
| 6200 | A, B); |
| 6201 | SDValue AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, |
| 6202 | DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), |
| 6203 | A, Bhi); |
| 6204 | SDValue AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, |
| 6205 | DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32), |
| 6206 | Ahi, B); |
| 6207 | AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, |
| 6208 | DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), |
| 6209 | AloBhi, DAG.getConstant(32, MVT::i32)); |
| 6210 | AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, VT, |
| 6211 | DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32), |
| 6212 | AhiBlo, DAG.getConstant(32, MVT::i32)); |
| 6213 | SDValue Res = DAG.getNode(ISD::ADD, VT, AloBlo, AloBhi); |
| 6214 | Res = DAG.getNode(ISD::ADD, VT, Res, AhiBlo); |
| 6215 | return Res; |
| 6216 | } |
| 6217 | |
| 6218 | |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 6219 | SDValue X86TargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) { |
| 6220 | // Lower the "add/sub/mul with overflow" instruction into a regular ins plus |
| 6221 | // a "setcc" instruction that checks the overflow flag. The "brcond" lowering |
Bill Wendling | 61edeb5 | 2008-12-02 01:06:39 +0000 | [diff] [blame] | 6222 | // looks for this combo and may remove the "setcc" instruction if the "setcc" |
| 6223 | // has only one use. |
Bill Wendling | 3fafd93 | 2008-11-26 22:37:40 +0000 | [diff] [blame] | 6224 | SDNode *N = Op.getNode(); |
Bill Wendling | 61edeb5 | 2008-12-02 01:06:39 +0000 | [diff] [blame] | 6225 | SDValue LHS = N->getOperand(0); |
| 6226 | SDValue RHS = N->getOperand(1); |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 6227 | unsigned BaseOp = 0; |
| 6228 | unsigned Cond = 0; |
| 6229 | |
| 6230 | switch (Op.getOpcode()) { |
| 6231 | default: assert(0 && "Unknown ovf instruction!"); |
| 6232 | case ISD::SADDO: |
Bill Wendling | ab55ebd | 2008-12-12 00:56:36 +0000 | [diff] [blame] | 6233 | BaseOp = X86ISD::ADD; |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 6234 | Cond = X86::COND_O; |
| 6235 | break; |
| 6236 | case ISD::UADDO: |
Bill Wendling | ab55ebd | 2008-12-12 00:56:36 +0000 | [diff] [blame] | 6237 | BaseOp = X86ISD::ADD; |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 6238 | Cond = X86::COND_C; |
| 6239 | break; |
| 6240 | case ISD::SSUBO: |
Bill Wendling | ab55ebd | 2008-12-12 00:56:36 +0000 | [diff] [blame] | 6241 | BaseOp = X86ISD::SUB; |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 6242 | Cond = X86::COND_O; |
| 6243 | break; |
| 6244 | case ISD::USUBO: |
Bill Wendling | ab55ebd | 2008-12-12 00:56:36 +0000 | [diff] [blame] | 6245 | BaseOp = X86ISD::SUB; |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 6246 | Cond = X86::COND_C; |
| 6247 | break; |
| 6248 | case ISD::SMULO: |
Bill Wendling | d350e02 | 2008-12-12 21:15:41 +0000 | [diff] [blame] | 6249 | BaseOp = X86ISD::SMUL; |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 6250 | Cond = X86::COND_O; |
| 6251 | break; |
| 6252 | case ISD::UMULO: |
Bill Wendling | d350e02 | 2008-12-12 21:15:41 +0000 | [diff] [blame] | 6253 | BaseOp = X86ISD::UMUL; |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 6254 | Cond = X86::COND_C; |
| 6255 | break; |
| 6256 | } |
Bill Wendling | 3fafd93 | 2008-11-26 22:37:40 +0000 | [diff] [blame] | 6257 | |
Bill Wendling | 61edeb5 | 2008-12-02 01:06:39 +0000 | [diff] [blame] | 6258 | // Also sets EFLAGS. |
| 6259 | SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32); |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 6260 | SDValue Sum = DAG.getNode(BaseOp, VTs, LHS, RHS); |
Bill Wendling | 3fafd93 | 2008-11-26 22:37:40 +0000 | [diff] [blame] | 6261 | |
Bill Wendling | 61edeb5 | 2008-12-02 01:06:39 +0000 | [diff] [blame] | 6262 | SDValue SetCC = |
| 6263 | DAG.getNode(X86ISD::SETCC, N->getValueType(1), |
Bill Wendling | bc5e15e | 2008-12-10 02:01:32 +0000 | [diff] [blame] | 6264 | DAG.getConstant(Cond, MVT::i32), SDValue(Sum.getNode(), 1)); |
Bill Wendling | 3fafd93 | 2008-11-26 22:37:40 +0000 | [diff] [blame] | 6265 | |
Bill Wendling | 61edeb5 | 2008-12-02 01:06:39 +0000 | [diff] [blame] | 6266 | DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SetCC); |
| 6267 | return Sum; |
Bill Wendling | 41ea7e7 | 2008-11-24 19:21:46 +0000 | [diff] [blame] | 6268 | } |
| 6269 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6270 | SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) { |
Dan Gohman | fd4418f | 2008-06-25 16:07:49 +0000 | [diff] [blame] | 6271 | MVT T = Op.getValueType(); |
Andrew Lenharth | a76e2f0 | 2008-03-04 21:13:33 +0000 | [diff] [blame] | 6272 | unsigned Reg = 0; |
| 6273 | unsigned size = 0; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 6274 | switch(T.getSimpleVT()) { |
| 6275 | default: |
| 6276 | assert(false && "Invalid value type!"); |
Andrew Lenharth | 26ed869 | 2008-03-01 21:52:34 +0000 | [diff] [blame] | 6277 | case MVT::i8: Reg = X86::AL; size = 1; break; |
| 6278 | case MVT::i16: Reg = X86::AX; size = 2; break; |
| 6279 | case MVT::i32: Reg = X86::EAX; size = 4; break; |
Andrew Lenharth | d19189e | 2008-03-05 01:15:49 +0000 | [diff] [blame] | 6280 | case MVT::i64: |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6281 | assert(Subtarget->is64Bit() && "Node not type legal!"); |
| 6282 | Reg = X86::RAX; size = 8; |
Andrew Lenharth | d19189e | 2008-03-05 01:15:49 +0000 | [diff] [blame] | 6283 | break; |
Bill Wendling | 61edeb5 | 2008-12-02 01:06:39 +0000 | [diff] [blame] | 6284 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6285 | SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), Reg, |
Dale Johannesen | d18a462 | 2008-09-11 03:12:59 +0000 | [diff] [blame] | 6286 | Op.getOperand(2), SDValue()); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6287 | SDValue Ops[] = { cpIn.getValue(0), |
Evan Cheng | 8a186ae | 2008-09-24 23:26:36 +0000 | [diff] [blame] | 6288 | Op.getOperand(1), |
| 6289 | Op.getOperand(3), |
| 6290 | DAG.getTargetConstant(size, MVT::i8), |
| 6291 | cpIn.getValue(1) }; |
Andrew Lenharth | 26ed869 | 2008-03-01 21:52:34 +0000 | [diff] [blame] | 6292 | SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6293 | SDValue Result = DAG.getNode(X86ISD::LCMPXCHG_DAG, Tys, Ops, 5); |
| 6294 | SDValue cpOut = |
Andrew Lenharth | 26ed869 | 2008-03-01 21:52:34 +0000 | [diff] [blame] | 6295 | DAG.getCopyFromReg(Result.getValue(0), Reg, T, Result.getValue(1)); |
| 6296 | return cpOut; |
| 6297 | } |
| 6298 | |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6299 | SDValue X86TargetLowering::LowerREADCYCLECOUNTER(SDValue Op, |
Gabor Greif | 327ef03 | 2008-08-28 23:19:51 +0000 | [diff] [blame] | 6300 | SelectionDAG &DAG) { |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6301 | assert(Subtarget->is64Bit() && "Result not type legalized?"); |
Andrew Lenharth | d19189e | 2008-03-05 01:15:49 +0000 | [diff] [blame] | 6302 | SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6303 | SDValue TheChain = Op.getOperand(0); |
| 6304 | SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1); |
| 6305 | SDValue rax = DAG.getCopyFromReg(rd, X86::RAX, MVT::i64, rd.getValue(1)); |
| 6306 | SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), X86::RDX, MVT::i64, |
| 6307 | rax.getValue(2)); |
| 6308 | SDValue Tmp = DAG.getNode(ISD::SHL, MVT::i64, rdx, |
| 6309 | DAG.getConstant(32, MVT::i8)); |
| 6310 | SDValue Ops[] = { |
| 6311 | DAG.getNode(ISD::OR, MVT::i64, rax, Tmp), |
| 6312 | rdx.getValue(1) |
| 6313 | }; |
| 6314 | return DAG.getMergeValues(Ops, 2); |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 6315 | } |
| 6316 | |
Dale Johannesen | 71d1bf5 | 2008-09-29 22:25:26 +0000 | [diff] [blame] | 6317 | SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) { |
| 6318 | SDNode *Node = Op.getNode(); |
| 6319 | MVT T = Node->getValueType(0); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6320 | SDValue negOp = DAG.getNode(ISD::SUB, T, |
Dale Johannesen | 71d1bf5 | 2008-09-29 22:25:26 +0000 | [diff] [blame] | 6321 | DAG.getConstant(0, T), Node->getOperand(2)); |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 6322 | return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, |
| 6323 | cast<AtomicSDNode>(Node)->getMemoryVT(), |
Dale Johannesen | 71d1bf5 | 2008-09-29 22:25:26 +0000 | [diff] [blame] | 6324 | Node->getOperand(0), |
| 6325 | Node->getOperand(1), negOp, |
| 6326 | cast<AtomicSDNode>(Node)->getSrcValue(), |
| 6327 | cast<AtomicSDNode>(Node)->getAlignment()); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6328 | } |
| 6329 | |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 6330 | /// LowerOperation - Provide custom lowering hooks for some operations. |
| 6331 | /// |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6332 | SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) { |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 6333 | switch (Op.getOpcode()) { |
| 6334 | default: assert(0 && "Should not custom lower this!"); |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 6335 | case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG); |
| 6336 | case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 6337 | case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); |
| 6338 | case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); |
| 6339 | case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); |
| 6340 | case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); |
| 6341 | case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); |
| 6342 | case ISD::ConstantPool: return LowerConstantPool(Op, DAG); |
| 6343 | case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 6344 | case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); |
Bill Wendling | 056292f | 2008-09-16 21:48:12 +0000 | [diff] [blame] | 6345 | case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 6346 | case ISD::SHL_PARTS: |
| 6347 | case ISD::SRA_PARTS: |
| 6348 | case ISD::SRL_PARTS: return LowerShift(Op, DAG); |
| 6349 | case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); |
Dale Johannesen | 1c15bf5 | 2008-10-21 20:50:01 +0000 | [diff] [blame] | 6350 | case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 6351 | case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); |
| 6352 | case ISD::FABS: return LowerFABS(Op, DAG); |
| 6353 | case ISD::FNEG: return LowerFNEG(Op, DAG); |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 6354 | case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); |
Evan Cheng | e5f6204 | 2007-09-29 00:00:36 +0000 | [diff] [blame] | 6355 | case ISD::SETCC: return LowerSETCC(Op, DAG); |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 6356 | case ISD::VSETCC: return LowerVSETCC(Op, DAG); |
Evan Cheng | e5f6204 | 2007-09-29 00:00:36 +0000 | [diff] [blame] | 6357 | case ISD::SELECT: return LowerSELECT(Op, DAG); |
| 6358 | case ISD::BRCOND: return LowerBRCOND(Op, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 6359 | case ISD::JumpTable: return LowerJumpTable(Op, DAG); |
Evan Cheng | 32fe103 | 2006-05-25 00:59:30 +0000 | [diff] [blame] | 6360 | case ISD::CALL: return LowerCALL(Op, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 6361 | case ISD::RET: return LowerRET(Op, DAG); |
Evan Cheng | 1bc7804 | 2006-04-26 01:20:17 +0000 | [diff] [blame] | 6362 | case ISD::FORMAL_ARGUMENTS: return LowerFORMAL_ARGUMENTS(Op, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 6363 | case ISD::VASTART: return LowerVASTART(Op, DAG); |
Dan Gohman | 9018e83 | 2008-05-10 01:26:14 +0000 | [diff] [blame] | 6364 | case ISD::VAARG: return LowerVAARG(Op, DAG); |
Evan Cheng | ae64219 | 2007-03-02 23:16:35 +0000 | [diff] [blame] | 6365 | case ISD::VACOPY: return LowerVACOPY(Op, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 6366 | case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); |
Nate Begeman | bcc5f36 | 2007-01-29 22:58:52 +0000 | [diff] [blame] | 6367 | case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); |
| 6368 | case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 6369 | case ISD::FRAME_TO_ARGS_OFFSET: |
| 6370 | return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); |
Anton Korobeynikov | 57fc00d | 2007-04-17 09:20:00 +0000 | [diff] [blame] | 6371 | case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 6372 | case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); |
Duncan Sands | b116fac | 2007-07-27 20:02:49 +0000 | [diff] [blame] | 6373 | case ISD::TRAMPOLINE: return LowerTRAMPOLINE(Op, DAG); |
Dan Gohman | 1a02486 | 2008-01-31 00:41:03 +0000 | [diff] [blame] | 6374 | case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 6375 | case ISD::CTLZ: return LowerCTLZ(Op, DAG); |
| 6376 | case ISD::CTTZ: return LowerCTTZ(Op, DAG); |
Mon P Wang | af9b952 | 2008-12-18 21:42:19 +0000 | [diff] [blame] | 6377 | case ISD::MUL: return LowerMUL_V2I64(Op, DAG); |
Bill Wendling | 74c3765 | 2008-12-09 22:08:41 +0000 | [diff] [blame] | 6378 | case ISD::SADDO: |
| 6379 | case ISD::UADDO: |
| 6380 | case ISD::SSUBO: |
| 6381 | case ISD::USUBO: |
| 6382 | case ISD::SMULO: |
| 6383 | case ISD::UMULO: return LowerXALUO(Op, DAG); |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6384 | case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, DAG); |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 6385 | } |
Chris Lattner | 27a6c73 | 2007-11-24 07:07:01 +0000 | [diff] [blame] | 6386 | } |
| 6387 | |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6388 | void X86TargetLowering:: |
| 6389 | ReplaceATOMIC_BINARY_64(SDNode *Node, SmallVectorImpl<SDValue>&Results, |
| 6390 | SelectionDAG &DAG, unsigned NewOp) { |
| 6391 | MVT T = Node->getValueType(0); |
| 6392 | assert (T == MVT::i64 && "Only know how to expand i64 atomics"); |
| 6393 | |
| 6394 | SDValue Chain = Node->getOperand(0); |
| 6395 | SDValue In1 = Node->getOperand(1); |
| 6396 | SDValue In2L = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, |
| 6397 | Node->getOperand(2), DAG.getIntPtrConstant(0)); |
| 6398 | SDValue In2H = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, |
| 6399 | Node->getOperand(2), DAG.getIntPtrConstant(1)); |
| 6400 | // This is a generalized SDNode, not an AtomicSDNode, so it doesn't |
| 6401 | // have a MemOperand. Pass the info through as a normal operand. |
| 6402 | SDValue LSI = DAG.getMemOperand(cast<MemSDNode>(Node)->getMemOperand()); |
| 6403 | SDValue Ops[] = { Chain, In1, In2L, In2H, LSI }; |
| 6404 | SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); |
| 6405 | SDValue Result = DAG.getNode(NewOp, Tys, Ops, 5); |
| 6406 | SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)}; |
| 6407 | Results.push_back(DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2)); |
| 6408 | Results.push_back(Result.getValue(2)); |
| 6409 | } |
| 6410 | |
Duncan Sands | 126d907 | 2008-07-04 11:47:58 +0000 | [diff] [blame] | 6411 | /// ReplaceNodeResults - Replace a node with an illegal result type |
| 6412 | /// with a new node built out of custom code. |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6413 | void X86TargetLowering::ReplaceNodeResults(SDNode *N, |
| 6414 | SmallVectorImpl<SDValue>&Results, |
| 6415 | SelectionDAG &DAG) { |
Chris Lattner | 27a6c73 | 2007-11-24 07:07:01 +0000 | [diff] [blame] | 6416 | switch (N->getOpcode()) { |
Duncan Sands | ed294c4 | 2008-10-20 15:56:33 +0000 | [diff] [blame] | 6417 | default: |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6418 | assert(false && "Do not know how to custom type legalize this operation!"); |
| 6419 | return; |
| 6420 | case ISD::FP_TO_SINT: { |
| 6421 | std::pair<SDValue,SDValue> Vals = FP_TO_SINTHelper(SDValue(N, 0), DAG); |
| 6422 | SDValue FIST = Vals.first, StackSlot = Vals.second; |
| 6423 | if (FIST.getNode() != 0) { |
| 6424 | MVT VT = N->getValueType(0); |
| 6425 | // Return a load from the stack slot. |
| 6426 | Results.push_back(DAG.getLoad(VT, FIST, StackSlot, NULL, 0)); |
| 6427 | } |
| 6428 | return; |
| 6429 | } |
| 6430 | case ISD::READCYCLECOUNTER: { |
| 6431 | SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); |
| 6432 | SDValue TheChain = N->getOperand(0); |
| 6433 | SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, &TheChain, 1); |
| 6434 | SDValue eax = DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)); |
| 6435 | SDValue edx = DAG.getCopyFromReg(eax.getValue(1), X86::EDX, MVT::i32, |
| 6436 | eax.getValue(2)); |
| 6437 | // Use a buildpair to merge the two 32-bit values into a 64-bit one. |
| 6438 | SDValue Ops[] = { eax, edx }; |
| 6439 | Results.push_back(DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Ops, 2)); |
| 6440 | Results.push_back(edx.getValue(1)); |
| 6441 | return; |
| 6442 | } |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 6443 | case ISD::ATOMIC_CMP_SWAP: { |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6444 | MVT T = N->getValueType(0); |
| 6445 | assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap"); |
| 6446 | SDValue cpInL, cpInH; |
| 6447 | cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, N->getOperand(2), |
| 6448 | DAG.getConstant(0, MVT::i32)); |
| 6449 | cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, N->getOperand(2), |
| 6450 | DAG.getConstant(1, MVT::i32)); |
| 6451 | cpInL = DAG.getCopyToReg(N->getOperand(0), X86::EAX, cpInL, SDValue()); |
| 6452 | cpInH = DAG.getCopyToReg(cpInL.getValue(0), X86::EDX, cpInH, |
| 6453 | cpInL.getValue(1)); |
| 6454 | SDValue swapInL, swapInH; |
| 6455 | swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, N->getOperand(3), |
| 6456 | DAG.getConstant(0, MVT::i32)); |
| 6457 | swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, N->getOperand(3), |
| 6458 | DAG.getConstant(1, MVT::i32)); |
| 6459 | swapInL = DAG.getCopyToReg(cpInH.getValue(0), X86::EBX, swapInL, |
| 6460 | cpInH.getValue(1)); |
| 6461 | swapInH = DAG.getCopyToReg(swapInL.getValue(0), X86::ECX, swapInH, |
| 6462 | swapInL.getValue(1)); |
| 6463 | SDValue Ops[] = { swapInH.getValue(0), |
| 6464 | N->getOperand(1), |
| 6465 | swapInH.getValue(1) }; |
| 6466 | SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag); |
| 6467 | SDValue Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, Tys, Ops, 3); |
| 6468 | SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), X86::EAX, MVT::i32, |
| 6469 | Result.getValue(1)); |
| 6470 | SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), X86::EDX, MVT::i32, |
| 6471 | cpOutL.getValue(2)); |
| 6472 | SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)}; |
| 6473 | Results.push_back(DAG.getNode(ISD::BUILD_PAIR, MVT::i64, OpsF, 2)); |
| 6474 | Results.push_back(cpOutH.getValue(1)); |
| 6475 | return; |
| 6476 | } |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 6477 | case ISD::ATOMIC_LOAD_ADD: |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6478 | ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMADD64_DAG); |
| 6479 | return; |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 6480 | case ISD::ATOMIC_LOAD_AND: |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6481 | ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMAND64_DAG); |
| 6482 | return; |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 6483 | case ISD::ATOMIC_LOAD_NAND: |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6484 | ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMNAND64_DAG); |
| 6485 | return; |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 6486 | case ISD::ATOMIC_LOAD_OR: |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6487 | ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMOR64_DAG); |
| 6488 | return; |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 6489 | case ISD::ATOMIC_LOAD_SUB: |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6490 | ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSUB64_DAG); |
| 6491 | return; |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 6492 | case ISD::ATOMIC_LOAD_XOR: |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6493 | ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMXOR64_DAG); |
| 6494 | return; |
Dan Gohman | 0b1d4a7 | 2008-12-23 21:37:04 +0000 | [diff] [blame] | 6495 | case ISD::ATOMIC_SWAP: |
Duncan Sands | 1607f05 | 2008-12-01 11:39:25 +0000 | [diff] [blame] | 6496 | ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSWAP64_DAG); |
| 6497 | return; |
Chris Lattner | 27a6c73 | 2007-11-24 07:07:01 +0000 | [diff] [blame] | 6498 | } |
Evan Cheng | 0db9fe6 | 2006-04-25 20:13:52 +0000 | [diff] [blame] | 6499 | } |
| 6500 | |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 6501 | const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { |
| 6502 | switch (Opcode) { |
| 6503 | default: return NULL; |
Evan Cheng | 18efe26 | 2007-12-14 02:13:44 +0000 | [diff] [blame] | 6504 | case X86ISD::BSF: return "X86ISD::BSF"; |
| 6505 | case X86ISD::BSR: return "X86ISD::BSR"; |
Evan Cheng | e341316 | 2006-01-09 18:33:28 +0000 | [diff] [blame] | 6506 | case X86ISD::SHLD: return "X86ISD::SHLD"; |
| 6507 | case X86ISD::SHRD: return "X86ISD::SHRD"; |
Evan Cheng | ef6ffb1 | 2006-01-31 03:14:29 +0000 | [diff] [blame] | 6508 | case X86ISD::FAND: return "X86ISD::FAND"; |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 6509 | case X86ISD::FOR: return "X86ISD::FOR"; |
Evan Cheng | 223547a | 2006-01-31 22:28:30 +0000 | [diff] [blame] | 6510 | case X86ISD::FXOR: return "X86ISD::FXOR"; |
Evan Cheng | 68c47cb | 2007-01-05 07:55:56 +0000 | [diff] [blame] | 6511 | case X86ISD::FSRL: return "X86ISD::FSRL"; |
Evan Cheng | a3195e8 | 2006-01-12 22:54:21 +0000 | [diff] [blame] | 6512 | case X86ISD::FILD: return "X86ISD::FILD"; |
Evan Cheng | e3de85b | 2006-02-04 02:20:30 +0000 | [diff] [blame] | 6513 | case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG"; |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 6514 | case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM"; |
| 6515 | case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM"; |
| 6516 | case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM"; |
Evan Cheng | b077b84 | 2005-12-21 02:39:21 +0000 | [diff] [blame] | 6517 | case X86ISD::FLD: return "X86ISD::FLD"; |
Evan Cheng | d90eb7f | 2006-01-05 00:27:02 +0000 | [diff] [blame] | 6518 | case X86ISD::FST: return "X86ISD::FST"; |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 6519 | case X86ISD::CALL: return "X86ISD::CALL"; |
| 6520 | case X86ISD::TAILCALL: return "X86ISD::TAILCALL"; |
| 6521 | case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG"; |
Dan Gohman | c7a37d4 | 2008-12-23 22:45:23 +0000 | [diff] [blame] | 6522 | case X86ISD::BT: return "X86ISD::BT"; |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 6523 | case X86ISD::CMP: return "X86ISD::CMP"; |
Evan Cheng | 6be2c58 | 2006-04-05 23:38:46 +0000 | [diff] [blame] | 6524 | case X86ISD::COMI: return "X86ISD::COMI"; |
| 6525 | case X86ISD::UCOMI: return "X86ISD::UCOMI"; |
Evan Cheng | d5781fc | 2005-12-21 20:21:51 +0000 | [diff] [blame] | 6526 | case X86ISD::SETCC: return "X86ISD::SETCC"; |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 6527 | case X86ISD::CMOV: return "X86ISD::CMOV"; |
| 6528 | case X86ISD::BRCOND: return "X86ISD::BRCOND"; |
Evan Cheng | b077b84 | 2005-12-21 02:39:21 +0000 | [diff] [blame] | 6529 | case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG"; |
Evan Cheng | 8df346b | 2006-03-04 01:12:00 +0000 | [diff] [blame] | 6530 | case X86ISD::REP_STOS: return "X86ISD::REP_STOS"; |
| 6531 | case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS"; |
Evan Cheng | 7ccced6 | 2006-02-18 00:15:05 +0000 | [diff] [blame] | 6532 | case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg"; |
Evan Cheng | 020d2e8 | 2006-02-23 20:41:18 +0000 | [diff] [blame] | 6533 | case X86ISD::Wrapper: return "X86ISD::Wrapper"; |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 6534 | case X86ISD::PEXTRB: return "X86ISD::PEXTRB"; |
Evan Cheng | b067a1e | 2006-03-31 19:22:53 +0000 | [diff] [blame] | 6535 | case X86ISD::PEXTRW: return "X86ISD::PEXTRW"; |
Nate Begeman | 14d12ca | 2008-02-11 04:19:36 +0000 | [diff] [blame] | 6536 | case X86ISD::INSERTPS: return "X86ISD::INSERTPS"; |
| 6537 | case X86ISD::PINSRB: return "X86ISD::PINSRB"; |
Evan Cheng | 653159f | 2006-03-31 21:55:24 +0000 | [diff] [blame] | 6538 | case X86ISD::PINSRW: return "X86ISD::PINSRW"; |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 6539 | case X86ISD::FMAX: return "X86ISD::FMAX"; |
| 6540 | case X86ISD::FMIN: return "X86ISD::FMIN"; |
Dan Gohman | 2038252 | 2007-07-10 00:05:58 +0000 | [diff] [blame] | 6541 | case X86ISD::FRSQRT: return "X86ISD::FRSQRT"; |
| 6542 | case X86ISD::FRCP: return "X86ISD::FRCP"; |
Lauro Ramos Venancio | b3a0417 | 2007-04-20 21:38:10 +0000 | [diff] [blame] | 6543 | case X86ISD::TLSADDR: return "X86ISD::TLSADDR"; |
| 6544 | case X86ISD::THREAD_POINTER: return "X86ISD::THREAD_POINTER"; |
Anton Korobeynikov | 2365f51 | 2007-07-14 14:06:15 +0000 | [diff] [blame] | 6545 | case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN"; |
Arnold Schwaighofer | c85e171 | 2007-10-11 19:40:01 +0000 | [diff] [blame] | 6546 | case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN"; |
Anton Korobeynikov | 45b22fa | 2007-11-16 01:31:51 +0000 | [diff] [blame] | 6547 | case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m"; |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 6548 | case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG"; |
| 6549 | case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG"; |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 6550 | case X86ISD::ATOMADD64_DAG: return "X86ISD::ATOMADD64_DAG"; |
| 6551 | case X86ISD::ATOMSUB64_DAG: return "X86ISD::ATOMSUB64_DAG"; |
| 6552 | case X86ISD::ATOMOR64_DAG: return "X86ISD::ATOMOR64_DAG"; |
| 6553 | case X86ISD::ATOMXOR64_DAG: return "X86ISD::ATOMXOR64_DAG"; |
| 6554 | case X86ISD::ATOMAND64_DAG: return "X86ISD::ATOMAND64_DAG"; |
| 6555 | case X86ISD::ATOMNAND64_DAG: return "X86ISD::ATOMNAND64_DAG"; |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 6556 | case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL"; |
| 6557 | case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD"; |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 6558 | case X86ISD::VSHL: return "X86ISD::VSHL"; |
| 6559 | case X86ISD::VSRL: return "X86ISD::VSRL"; |
Nate Begeman | 30a0de9 | 2008-07-17 16:51:19 +0000 | [diff] [blame] | 6560 | case X86ISD::CMPPD: return "X86ISD::CMPPD"; |
| 6561 | case X86ISD::CMPPS: return "X86ISD::CMPPS"; |
| 6562 | case X86ISD::PCMPEQB: return "X86ISD::PCMPEQB"; |
| 6563 | case X86ISD::PCMPEQW: return "X86ISD::PCMPEQW"; |
| 6564 | case X86ISD::PCMPEQD: return "X86ISD::PCMPEQD"; |
| 6565 | case X86ISD::PCMPEQQ: return "X86ISD::PCMPEQQ"; |
| 6566 | case X86ISD::PCMPGTB: return "X86ISD::PCMPGTB"; |
| 6567 | case X86ISD::PCMPGTW: return "X86ISD::PCMPGTW"; |
| 6568 | case X86ISD::PCMPGTD: return "X86ISD::PCMPGTD"; |
| 6569 | case X86ISD::PCMPGTQ: return "X86ISD::PCMPGTQ"; |
Bill Wendling | ab55ebd | 2008-12-12 00:56:36 +0000 | [diff] [blame] | 6570 | case X86ISD::ADD: return "X86ISD::ADD"; |
| 6571 | case X86ISD::SUB: return "X86ISD::SUB"; |
Bill Wendling | d350e02 | 2008-12-12 21:15:41 +0000 | [diff] [blame] | 6572 | case X86ISD::SMUL: return "X86ISD::SMUL"; |
| 6573 | case X86ISD::UMUL: return "X86ISD::UMUL"; |
Evan Cheng | 7226158 | 2005-12-20 06:22:03 +0000 | [diff] [blame] | 6574 | } |
| 6575 | } |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 6576 | |
Chris Lattner | c9addb7 | 2007-03-30 23:15:24 +0000 | [diff] [blame] | 6577 | // isLegalAddressingMode - Return true if the addressing mode represented |
| 6578 | // by AM is legal for this target, for a load/store of the specified type. |
| 6579 | bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM, |
| 6580 | const Type *Ty) const { |
| 6581 | // X86 supports extremely general addressing modes. |
| 6582 | |
| 6583 | // X86 allows a sign-extended 32-bit immediate field as a displacement. |
| 6584 | if (AM.BaseOffs <= -(1LL << 32) || AM.BaseOffs >= (1LL << 32)-1) |
| 6585 | return false; |
| 6586 | |
| 6587 | if (AM.BaseGV) { |
Evan Cheng | 5278784 | 2007-08-01 23:46:47 +0000 | [diff] [blame] | 6588 | // We can only fold this if we don't need an extra load. |
Chris Lattner | c9addb7 | 2007-03-30 23:15:24 +0000 | [diff] [blame] | 6589 | if (Subtarget->GVRequiresExtraLoad(AM.BaseGV, getTargetMachine(), false)) |
| 6590 | return false; |
Dale Johannesen | 203af58 | 2008-12-05 21:47:27 +0000 | [diff] [blame] | 6591 | // If BaseGV requires a register, we cannot also have a BaseReg. |
| 6592 | if (Subtarget->GVRequiresRegister(AM.BaseGV, getTargetMachine(), false) && |
| 6593 | AM.HasBaseReg) |
| 6594 | return false; |
Evan Cheng | 5278784 | 2007-08-01 23:46:47 +0000 | [diff] [blame] | 6595 | |
| 6596 | // X86-64 only supports addr of globals in small code model. |
| 6597 | if (Subtarget->is64Bit()) { |
| 6598 | if (getTargetMachine().getCodeModel() != CodeModel::Small) |
| 6599 | return false; |
| 6600 | // If lower 4G is not available, then we must use rip-relative addressing. |
| 6601 | if (AM.BaseOffs || AM.Scale > 1) |
| 6602 | return false; |
| 6603 | } |
Chris Lattner | c9addb7 | 2007-03-30 23:15:24 +0000 | [diff] [blame] | 6604 | } |
| 6605 | |
| 6606 | switch (AM.Scale) { |
| 6607 | case 0: |
| 6608 | case 1: |
| 6609 | case 2: |
| 6610 | case 4: |
| 6611 | case 8: |
| 6612 | // These scales always work. |
| 6613 | break; |
| 6614 | case 3: |
| 6615 | case 5: |
| 6616 | case 9: |
| 6617 | // These scales are formed with basereg+scalereg. Only accept if there is |
| 6618 | // no basereg yet. |
| 6619 | if (AM.HasBaseReg) |
| 6620 | return false; |
| 6621 | break; |
| 6622 | default: // Other stuff never works. |
| 6623 | return false; |
| 6624 | } |
| 6625 | |
| 6626 | return true; |
| 6627 | } |
| 6628 | |
| 6629 | |
Evan Cheng | 2bd122c | 2007-10-26 01:56:11 +0000 | [diff] [blame] | 6630 | bool X86TargetLowering::isTruncateFree(const Type *Ty1, const Type *Ty2) const { |
| 6631 | if (!Ty1->isInteger() || !Ty2->isInteger()) |
| 6632 | return false; |
Evan Cheng | e127a73 | 2007-10-29 07:57:50 +0000 | [diff] [blame] | 6633 | unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); |
| 6634 | unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); |
Evan Cheng | 260e07e | 2008-03-20 02:18:41 +0000 | [diff] [blame] | 6635 | if (NumBits1 <= NumBits2) |
Evan Cheng | e127a73 | 2007-10-29 07:57:50 +0000 | [diff] [blame] | 6636 | return false; |
| 6637 | return Subtarget->is64Bit() || NumBits1 < 64; |
Evan Cheng | 2bd122c | 2007-10-26 01:56:11 +0000 | [diff] [blame] | 6638 | } |
| 6639 | |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 6640 | bool X86TargetLowering::isTruncateFree(MVT VT1, MVT VT2) const { |
| 6641 | if (!VT1.isInteger() || !VT2.isInteger()) |
Evan Cheng | 3c3ddb3 | 2007-10-29 19:58:20 +0000 | [diff] [blame] | 6642 | return false; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 6643 | unsigned NumBits1 = VT1.getSizeInBits(); |
| 6644 | unsigned NumBits2 = VT2.getSizeInBits(); |
Evan Cheng | 260e07e | 2008-03-20 02:18:41 +0000 | [diff] [blame] | 6645 | if (NumBits1 <= NumBits2) |
Evan Cheng | 3c3ddb3 | 2007-10-29 19:58:20 +0000 | [diff] [blame] | 6646 | return false; |
| 6647 | return Subtarget->is64Bit() || NumBits1 < 64; |
| 6648 | } |
Evan Cheng | 2bd122c | 2007-10-26 01:56:11 +0000 | [diff] [blame] | 6649 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 6650 | /// isShuffleMaskLegal - Targets can use this to indicate that they only |
| 6651 | /// support *some* VECTOR_SHUFFLE operations, those with specific masks. |
| 6652 | /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values |
| 6653 | /// are assumed to be legal. |
| 6654 | bool |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6655 | X86TargetLowering::isShuffleMaskLegal(SDValue Mask, MVT VT) const { |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 6656 | // Only do shuffles on 128-bit vector types for now. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 6657 | if (VT.getSizeInBits() == 64) return false; |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 6658 | return (Mask.getNode()->getNumOperands() <= 4 || |
| 6659 | isIdentityMask(Mask.getNode()) || |
| 6660 | isIdentityMask(Mask.getNode(), true) || |
| 6661 | isSplatMask(Mask.getNode()) || |
| 6662 | isPSHUFHW_PSHUFLWMask(Mask.getNode()) || |
| 6663 | X86::isUNPCKLMask(Mask.getNode()) || |
| 6664 | X86::isUNPCKHMask(Mask.getNode()) || |
| 6665 | X86::isUNPCKL_v_undef_Mask(Mask.getNode()) || |
| 6666 | X86::isUNPCKH_v_undef_Mask(Mask.getNode())); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 6667 | } |
| 6668 | |
Dan Gohman | 7d8143f | 2008-04-09 20:09:42 +0000 | [diff] [blame] | 6669 | bool |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 6670 | X86TargetLowering::isVectorClearMaskLegal(const std::vector<SDValue> &BVOps, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 6671 | MVT EVT, SelectionDAG &DAG) const { |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 6672 | unsigned NumElts = BVOps.size(); |
| 6673 | // Only do shuffles on 128-bit vector types for now. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 6674 | if (EVT.getSizeInBits() * NumElts == 64) return false; |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 6675 | if (NumElts == 2) return true; |
| 6676 | if (NumElts == 4) { |
Chris Lattner | 5a88b83 | 2007-02-25 07:10:00 +0000 | [diff] [blame] | 6677 | return (isMOVLMask(&BVOps[0], 4) || |
| 6678 | isCommutedMOVL(&BVOps[0], 4, true) || |
| 6679 | isSHUFPMask(&BVOps[0], 4) || |
| 6680 | isCommutedSHUFP(&BVOps[0], 4)); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 6681 | } |
| 6682 | return false; |
| 6683 | } |
| 6684 | |
| 6685 | //===----------------------------------------------------------------------===// |
| 6686 | // X86 Scheduler Hooks |
| 6687 | //===----------------------------------------------------------------------===// |
| 6688 | |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6689 | // private utility function |
| 6690 | MachineBasicBlock * |
| 6691 | X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr, |
| 6692 | MachineBasicBlock *MBB, |
| 6693 | unsigned regOpc, |
Andrew Lenharth | 507a58a | 2008-06-14 05:48:15 +0000 | [diff] [blame] | 6694 | unsigned immOpc, |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 6695 | unsigned LoadOpc, |
| 6696 | unsigned CXchgOpc, |
| 6697 | unsigned copyOpc, |
| 6698 | unsigned notOpc, |
| 6699 | unsigned EAXreg, |
| 6700 | TargetRegisterClass *RC, |
Andrew Lenharth | 507a58a | 2008-06-14 05:48:15 +0000 | [diff] [blame] | 6701 | bool invSrc) { |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6702 | // For the atomic bitwise operator, we generate |
| 6703 | // thisMBB: |
| 6704 | // newMBB: |
Mon P Wang | ab3e747 | 2008-05-05 22:56:23 +0000 | [diff] [blame] | 6705 | // ld t1 = [bitinstr.addr] |
| 6706 | // op t2 = t1, [bitinstr.val] |
| 6707 | // mov EAX = t1 |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6708 | // lcs dest = [bitinstr.addr], t2 [EAX is implicit] |
| 6709 | // bz newMBB |
| 6710 | // fallthrough -->nextMBB |
| 6711 | const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); |
| 6712 | const BasicBlock *LLVM_BB = MBB->getBasicBlock(); |
Dan Gohman | 8e5f2c6 | 2008-07-07 23:14:23 +0000 | [diff] [blame] | 6713 | MachineFunction::iterator MBBIter = MBB; |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6714 | ++MBBIter; |
| 6715 | |
| 6716 | /// First build the CFG |
| 6717 | MachineFunction *F = MBB->getParent(); |
| 6718 | MachineBasicBlock *thisMBB = MBB; |
Dan Gohman | 8e5f2c6 | 2008-07-07 23:14:23 +0000 | [diff] [blame] | 6719 | MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); |
| 6720 | MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); |
| 6721 | F->insert(MBBIter, newMBB); |
| 6722 | F->insert(MBBIter, nextMBB); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6723 | |
| 6724 | // Move all successors to thisMBB to nextMBB |
| 6725 | nextMBB->transferSuccessors(thisMBB); |
| 6726 | |
| 6727 | // Update thisMBB to fall through to newMBB |
| 6728 | thisMBB->addSuccessor(newMBB); |
| 6729 | |
| 6730 | // newMBB jumps to itself and fall through to nextMBB |
| 6731 | newMBB->addSuccessor(nextMBB); |
| 6732 | newMBB->addSuccessor(newMBB); |
| 6733 | |
| 6734 | // Insert instructions into newMBB based on incoming instruction |
| 6735 | assert(bInstr->getNumOperands() < 8 && "unexpected number of operands"); |
| 6736 | MachineOperand& destOper = bInstr->getOperand(0); |
| 6737 | MachineOperand* argOpers[6]; |
| 6738 | int numArgs = bInstr->getNumOperands() - 1; |
| 6739 | for (int i=0; i < numArgs; ++i) |
| 6740 | argOpers[i] = &bInstr->getOperand(i+1); |
| 6741 | |
| 6742 | // x86 address has 4 operands: base, index, scale, and displacement |
| 6743 | int lastAddrIndx = 3; // [0,3] |
| 6744 | int valArgIndx = 4; |
| 6745 | |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 6746 | unsigned t1 = F->getRegInfo().createVirtualRegister(RC); |
| 6747 | MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(LoadOpc), t1); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6748 | for (int i=0; i <= lastAddrIndx; ++i) |
| 6749 | (*MIB).addOperand(*argOpers[i]); |
Andrew Lenharth | 507a58a | 2008-06-14 05:48:15 +0000 | [diff] [blame] | 6750 | |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 6751 | unsigned tt = F->getRegInfo().createVirtualRegister(RC); |
Andrew Lenharth | 507a58a | 2008-06-14 05:48:15 +0000 | [diff] [blame] | 6752 | if (invSrc) { |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 6753 | MIB = BuildMI(newMBB, TII->get(notOpc), tt).addReg(t1); |
Andrew Lenharth | 507a58a | 2008-06-14 05:48:15 +0000 | [diff] [blame] | 6754 | } |
| 6755 | else |
| 6756 | tt = t1; |
| 6757 | |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 6758 | unsigned t2 = F->getRegInfo().createVirtualRegister(RC); |
Dan Gohman | d735b80 | 2008-10-03 15:45:36 +0000 | [diff] [blame] | 6759 | assert((argOpers[valArgIndx]->isReg() || |
| 6760 | argOpers[valArgIndx]->isImm()) && |
Dan Gohman | 014278e | 2008-09-13 17:58:21 +0000 | [diff] [blame] | 6761 | "invalid operand"); |
Dan Gohman | d735b80 | 2008-10-03 15:45:36 +0000 | [diff] [blame] | 6762 | if (argOpers[valArgIndx]->isReg()) |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6763 | MIB = BuildMI(newMBB, TII->get(regOpc), t2); |
| 6764 | else |
| 6765 | MIB = BuildMI(newMBB, TII->get(immOpc), t2); |
Andrew Lenharth | 507a58a | 2008-06-14 05:48:15 +0000 | [diff] [blame] | 6766 | MIB.addReg(tt); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6767 | (*MIB).addOperand(*argOpers[valArgIndx]); |
Andrew Lenharth | 507a58a | 2008-06-14 05:48:15 +0000 | [diff] [blame] | 6768 | |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 6769 | MIB = BuildMI(newMBB, TII->get(copyOpc), EAXreg); |
Mon P Wang | ab3e747 | 2008-05-05 22:56:23 +0000 | [diff] [blame] | 6770 | MIB.addReg(t1); |
| 6771 | |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 6772 | MIB = BuildMI(newMBB, TII->get(CXchgOpc)); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6773 | for (int i=0; i <= lastAddrIndx; ++i) |
| 6774 | (*MIB).addOperand(*argOpers[i]); |
| 6775 | MIB.addReg(t2); |
Mon P Wang | f595266 | 2008-07-17 04:54:06 +0000 | [diff] [blame] | 6776 | assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); |
| 6777 | (*MIB).addMemOperand(*F, *bInstr->memoperands_begin()); |
| 6778 | |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 6779 | MIB = BuildMI(newMBB, TII->get(copyOpc), destOper.getReg()); |
| 6780 | MIB.addReg(EAXreg); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6781 | |
| 6782 | // insert branch |
| 6783 | BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); |
| 6784 | |
Dan Gohman | 8e5f2c6 | 2008-07-07 23:14:23 +0000 | [diff] [blame] | 6785 | F->DeleteMachineInstr(bInstr); // The pseudo instruction is gone now. |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6786 | return nextMBB; |
| 6787 | } |
| 6788 | |
Dale Johannesen | 1b54c7f | 2008-10-03 19:41:08 +0000 | [diff] [blame] | 6789 | // private utility function: 64 bit atomics on 32 bit host. |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6790 | MachineBasicBlock * |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 6791 | X86TargetLowering::EmitAtomicBit6432WithCustomInserter(MachineInstr *bInstr, |
| 6792 | MachineBasicBlock *MBB, |
| 6793 | unsigned regOpcL, |
| 6794 | unsigned regOpcH, |
| 6795 | unsigned immOpcL, |
| 6796 | unsigned immOpcH, |
| 6797 | bool invSrc) { |
| 6798 | // For the atomic bitwise operator, we generate |
| 6799 | // thisMBB (instructions are in pairs, except cmpxchg8b) |
| 6800 | // ld t1,t2 = [bitinstr.addr] |
| 6801 | // newMBB: |
| 6802 | // out1, out2 = phi (thisMBB, t1/t2) (newMBB, t3/t4) |
| 6803 | // op t5, t6 <- out1, out2, [bitinstr.val] |
Dale Johannesen | 880ae36 | 2008-10-03 22:25:52 +0000 | [diff] [blame] | 6804 | // (for SWAP, substitute: mov t5, t6 <- [bitinstr.val]) |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 6805 | // mov ECX, EBX <- t5, t6 |
| 6806 | // mov EAX, EDX <- t1, t2 |
| 6807 | // cmpxchg8b [bitinstr.addr] [EAX, EDX, EBX, ECX implicit] |
| 6808 | // mov t3, t4 <- EAX, EDX |
| 6809 | // bz newMBB |
| 6810 | // result in out1, out2 |
| 6811 | // fallthrough -->nextMBB |
| 6812 | |
| 6813 | const TargetRegisterClass *RC = X86::GR32RegisterClass; |
| 6814 | const unsigned LoadOpc = X86::MOV32rm; |
| 6815 | const unsigned copyOpc = X86::MOV32rr; |
| 6816 | const unsigned NotOpc = X86::NOT32r; |
| 6817 | const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); |
| 6818 | const BasicBlock *LLVM_BB = MBB->getBasicBlock(); |
| 6819 | MachineFunction::iterator MBBIter = MBB; |
| 6820 | ++MBBIter; |
| 6821 | |
| 6822 | /// First build the CFG |
| 6823 | MachineFunction *F = MBB->getParent(); |
| 6824 | MachineBasicBlock *thisMBB = MBB; |
| 6825 | MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); |
| 6826 | MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); |
| 6827 | F->insert(MBBIter, newMBB); |
| 6828 | F->insert(MBBIter, nextMBB); |
| 6829 | |
| 6830 | // Move all successors to thisMBB to nextMBB |
| 6831 | nextMBB->transferSuccessors(thisMBB); |
| 6832 | |
| 6833 | // Update thisMBB to fall through to newMBB |
| 6834 | thisMBB->addSuccessor(newMBB); |
| 6835 | |
| 6836 | // newMBB jumps to itself and fall through to nextMBB |
| 6837 | newMBB->addSuccessor(nextMBB); |
| 6838 | newMBB->addSuccessor(newMBB); |
| 6839 | |
| 6840 | // Insert instructions into newMBB based on incoming instruction |
| 6841 | // There are 8 "real" operands plus 9 implicit def/uses, ignored here. |
| 6842 | assert(bInstr->getNumOperands() < 18 && "unexpected number of operands"); |
| 6843 | MachineOperand& dest1Oper = bInstr->getOperand(0); |
| 6844 | MachineOperand& dest2Oper = bInstr->getOperand(1); |
| 6845 | MachineOperand* argOpers[6]; |
| 6846 | for (int i=0; i < 6; ++i) |
| 6847 | argOpers[i] = &bInstr->getOperand(i+2); |
| 6848 | |
| 6849 | // x86 address has 4 operands: base, index, scale, and displacement |
| 6850 | int lastAddrIndx = 3; // [0,3] |
| 6851 | |
| 6852 | unsigned t1 = F->getRegInfo().createVirtualRegister(RC); |
| 6853 | MachineInstrBuilder MIB = BuildMI(thisMBB, TII->get(LoadOpc), t1); |
| 6854 | for (int i=0; i <= lastAddrIndx; ++i) |
| 6855 | (*MIB).addOperand(*argOpers[i]); |
| 6856 | unsigned t2 = F->getRegInfo().createVirtualRegister(RC); |
| 6857 | MIB = BuildMI(thisMBB, TII->get(LoadOpc), t2); |
Dale Johannesen | 880ae36 | 2008-10-03 22:25:52 +0000 | [diff] [blame] | 6858 | // add 4 to displacement. |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 6859 | for (int i=0; i <= lastAddrIndx-1; ++i) |
| 6860 | (*MIB).addOperand(*argOpers[i]); |
Dale Johannesen | 880ae36 | 2008-10-03 22:25:52 +0000 | [diff] [blame] | 6861 | MachineOperand newOp3 = *(argOpers[3]); |
| 6862 | if (newOp3.isImm()) |
| 6863 | newOp3.setImm(newOp3.getImm()+4); |
| 6864 | else |
| 6865 | newOp3.setOffset(newOp3.getOffset()+4); |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 6866 | (*MIB).addOperand(newOp3); |
| 6867 | |
| 6868 | // t3/4 are defined later, at the bottom of the loop |
| 6869 | unsigned t3 = F->getRegInfo().createVirtualRegister(RC); |
| 6870 | unsigned t4 = F->getRegInfo().createVirtualRegister(RC); |
| 6871 | BuildMI(newMBB, TII->get(X86::PHI), dest1Oper.getReg()) |
| 6872 | .addReg(t1).addMBB(thisMBB).addReg(t3).addMBB(newMBB); |
| 6873 | BuildMI(newMBB, TII->get(X86::PHI), dest2Oper.getReg()) |
| 6874 | .addReg(t2).addMBB(thisMBB).addReg(t4).addMBB(newMBB); |
| 6875 | |
| 6876 | unsigned tt1 = F->getRegInfo().createVirtualRegister(RC); |
| 6877 | unsigned tt2 = F->getRegInfo().createVirtualRegister(RC); |
| 6878 | if (invSrc) { |
| 6879 | MIB = BuildMI(newMBB, TII->get(NotOpc), tt1).addReg(t1); |
| 6880 | MIB = BuildMI(newMBB, TII->get(NotOpc), tt2).addReg(t2); |
| 6881 | } else { |
| 6882 | tt1 = t1; |
| 6883 | tt2 = t2; |
| 6884 | } |
| 6885 | |
Dan Gohman | d735b80 | 2008-10-03 15:45:36 +0000 | [diff] [blame] | 6886 | assert((argOpers[4]->isReg() || argOpers[4]->isImm()) && |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 6887 | "invalid operand"); |
| 6888 | unsigned t5 = F->getRegInfo().createVirtualRegister(RC); |
| 6889 | unsigned t6 = F->getRegInfo().createVirtualRegister(RC); |
Dan Gohman | d735b80 | 2008-10-03 15:45:36 +0000 | [diff] [blame] | 6890 | if (argOpers[4]->isReg()) |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 6891 | MIB = BuildMI(newMBB, TII->get(regOpcL), t5); |
| 6892 | else |
| 6893 | MIB = BuildMI(newMBB, TII->get(immOpcL), t5); |
Dale Johannesen | 880ae36 | 2008-10-03 22:25:52 +0000 | [diff] [blame] | 6894 | if (regOpcL != X86::MOV32rr) |
| 6895 | MIB.addReg(tt1); |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 6896 | (*MIB).addOperand(*argOpers[4]); |
Dan Gohman | d735b80 | 2008-10-03 15:45:36 +0000 | [diff] [blame] | 6897 | assert(argOpers[5]->isReg() == argOpers[4]->isReg()); |
| 6898 | assert(argOpers[5]->isImm() == argOpers[4]->isImm()); |
| 6899 | if (argOpers[5]->isReg()) |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 6900 | MIB = BuildMI(newMBB, TII->get(regOpcH), t6); |
| 6901 | else |
| 6902 | MIB = BuildMI(newMBB, TII->get(immOpcH), t6); |
Dale Johannesen | 880ae36 | 2008-10-03 22:25:52 +0000 | [diff] [blame] | 6903 | if (regOpcH != X86::MOV32rr) |
| 6904 | MIB.addReg(tt2); |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 6905 | (*MIB).addOperand(*argOpers[5]); |
| 6906 | |
| 6907 | MIB = BuildMI(newMBB, TII->get(copyOpc), X86::EAX); |
| 6908 | MIB.addReg(t1); |
| 6909 | MIB = BuildMI(newMBB, TII->get(copyOpc), X86::EDX); |
| 6910 | MIB.addReg(t2); |
| 6911 | |
| 6912 | MIB = BuildMI(newMBB, TII->get(copyOpc), X86::EBX); |
| 6913 | MIB.addReg(t5); |
| 6914 | MIB = BuildMI(newMBB, TII->get(copyOpc), X86::ECX); |
| 6915 | MIB.addReg(t6); |
| 6916 | |
| 6917 | MIB = BuildMI(newMBB, TII->get(X86::LCMPXCHG8B)); |
| 6918 | for (int i=0; i <= lastAddrIndx; ++i) |
| 6919 | (*MIB).addOperand(*argOpers[i]); |
| 6920 | |
| 6921 | assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand"); |
| 6922 | (*MIB).addMemOperand(*F, *bInstr->memoperands_begin()); |
| 6923 | |
| 6924 | MIB = BuildMI(newMBB, TII->get(copyOpc), t3); |
| 6925 | MIB.addReg(X86::EAX); |
| 6926 | MIB = BuildMI(newMBB, TII->get(copyOpc), t4); |
| 6927 | MIB.addReg(X86::EDX); |
| 6928 | |
| 6929 | // insert branch |
| 6930 | BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); |
| 6931 | |
| 6932 | F->DeleteMachineInstr(bInstr); // The pseudo instruction is gone now. |
| 6933 | return nextMBB; |
| 6934 | } |
| 6935 | |
| 6936 | // private utility function |
| 6937 | MachineBasicBlock * |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6938 | X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr, |
| 6939 | MachineBasicBlock *MBB, |
| 6940 | unsigned cmovOpc) { |
| 6941 | // For the atomic min/max operator, we generate |
| 6942 | // thisMBB: |
| 6943 | // newMBB: |
Mon P Wang | ab3e747 | 2008-05-05 22:56:23 +0000 | [diff] [blame] | 6944 | // ld t1 = [min/max.addr] |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6945 | // mov t2 = [min/max.val] |
| 6946 | // cmp t1, t2 |
| 6947 | // cmov[cond] t2 = t1 |
Mon P Wang | ab3e747 | 2008-05-05 22:56:23 +0000 | [diff] [blame] | 6948 | // mov EAX = t1 |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6949 | // lcs dest = [bitinstr.addr], t2 [EAX is implicit] |
| 6950 | // bz newMBB |
| 6951 | // fallthrough -->nextMBB |
| 6952 | // |
| 6953 | const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); |
| 6954 | const BasicBlock *LLVM_BB = MBB->getBasicBlock(); |
Dan Gohman | 8e5f2c6 | 2008-07-07 23:14:23 +0000 | [diff] [blame] | 6955 | MachineFunction::iterator MBBIter = MBB; |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6956 | ++MBBIter; |
| 6957 | |
| 6958 | /// First build the CFG |
| 6959 | MachineFunction *F = MBB->getParent(); |
| 6960 | MachineBasicBlock *thisMBB = MBB; |
Dan Gohman | 8e5f2c6 | 2008-07-07 23:14:23 +0000 | [diff] [blame] | 6961 | MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB); |
| 6962 | MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB); |
| 6963 | F->insert(MBBIter, newMBB); |
| 6964 | F->insert(MBBIter, nextMBB); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6965 | |
| 6966 | // Move all successors to thisMBB to nextMBB |
| 6967 | nextMBB->transferSuccessors(thisMBB); |
| 6968 | |
| 6969 | // Update thisMBB to fall through to newMBB |
| 6970 | thisMBB->addSuccessor(newMBB); |
| 6971 | |
| 6972 | // newMBB jumps to newMBB and fall through to nextMBB |
| 6973 | newMBB->addSuccessor(nextMBB); |
| 6974 | newMBB->addSuccessor(newMBB); |
| 6975 | |
| 6976 | // Insert instructions into newMBB based on incoming instruction |
| 6977 | assert(mInstr->getNumOperands() < 8 && "unexpected number of operands"); |
| 6978 | MachineOperand& destOper = mInstr->getOperand(0); |
| 6979 | MachineOperand* argOpers[6]; |
| 6980 | int numArgs = mInstr->getNumOperands() - 1; |
| 6981 | for (int i=0; i < numArgs; ++i) |
| 6982 | argOpers[i] = &mInstr->getOperand(i+1); |
| 6983 | |
| 6984 | // x86 address has 4 operands: base, index, scale, and displacement |
| 6985 | int lastAddrIndx = 3; // [0,3] |
| 6986 | int valArgIndx = 4; |
| 6987 | |
Mon P Wang | ab3e747 | 2008-05-05 22:56:23 +0000 | [diff] [blame] | 6988 | unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); |
| 6989 | MachineInstrBuilder MIB = BuildMI(newMBB, TII->get(X86::MOV32rm), t1); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6990 | for (int i=0; i <= lastAddrIndx; ++i) |
| 6991 | (*MIB).addOperand(*argOpers[i]); |
Mon P Wang | ab3e747 | 2008-05-05 22:56:23 +0000 | [diff] [blame] | 6992 | |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6993 | // We only support register and immediate values |
Dan Gohman | d735b80 | 2008-10-03 15:45:36 +0000 | [diff] [blame] | 6994 | assert((argOpers[valArgIndx]->isReg() || |
| 6995 | argOpers[valArgIndx]->isImm()) && |
Dan Gohman | 014278e | 2008-09-13 17:58:21 +0000 | [diff] [blame] | 6996 | "invalid operand"); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 6997 | |
| 6998 | unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); |
Dan Gohman | d735b80 | 2008-10-03 15:45:36 +0000 | [diff] [blame] | 6999 | if (argOpers[valArgIndx]->isReg()) |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 7000 | MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); |
| 7001 | else |
| 7002 | MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), t2); |
| 7003 | (*MIB).addOperand(*argOpers[valArgIndx]); |
| 7004 | |
Mon P Wang | ab3e747 | 2008-05-05 22:56:23 +0000 | [diff] [blame] | 7005 | MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), X86::EAX); |
| 7006 | MIB.addReg(t1); |
| 7007 | |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 7008 | MIB = BuildMI(newMBB, TII->get(X86::CMP32rr)); |
| 7009 | MIB.addReg(t1); |
| 7010 | MIB.addReg(t2); |
| 7011 | |
| 7012 | // Generate movc |
| 7013 | unsigned t3 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass); |
| 7014 | MIB = BuildMI(newMBB, TII->get(cmovOpc),t3); |
| 7015 | MIB.addReg(t2); |
| 7016 | MIB.addReg(t1); |
| 7017 | |
| 7018 | // Cmp and exchange if none has modified the memory location |
| 7019 | MIB = BuildMI(newMBB, TII->get(X86::LCMPXCHG32)); |
| 7020 | for (int i=0; i <= lastAddrIndx; ++i) |
| 7021 | (*MIB).addOperand(*argOpers[i]); |
| 7022 | MIB.addReg(t3); |
Mon P Wang | f595266 | 2008-07-17 04:54:06 +0000 | [diff] [blame] | 7023 | assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand"); |
| 7024 | (*MIB).addMemOperand(*F, *mInstr->memoperands_begin()); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 7025 | |
| 7026 | MIB = BuildMI(newMBB, TII->get(X86::MOV32rr), destOper.getReg()); |
| 7027 | MIB.addReg(X86::EAX); |
| 7028 | |
| 7029 | // insert branch |
| 7030 | BuildMI(newMBB, TII->get(X86::JNE)).addMBB(newMBB); |
| 7031 | |
Dan Gohman | 8e5f2c6 | 2008-07-07 23:14:23 +0000 | [diff] [blame] | 7032 | F->DeleteMachineInstr(mInstr); // The pseudo instruction is gone now. |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 7033 | return nextMBB; |
| 7034 | } |
| 7035 | |
| 7036 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7037 | MachineBasicBlock * |
Evan Cheng | ff9b373 | 2008-01-30 18:18:23 +0000 | [diff] [blame] | 7038 | X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, |
| 7039 | MachineBasicBlock *BB) { |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 7040 | const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7041 | switch (MI->getOpcode()) { |
| 7042 | default: assert(false && "Unexpected instr type to insert"); |
Mon P Wang | 9e5ecb8 | 2008-12-12 01:25:51 +0000 | [diff] [blame] | 7043 | case X86::CMOV_V1I64: |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7044 | case X86::CMOV_FR32: |
| 7045 | case X86::CMOV_FR64: |
| 7046 | case X86::CMOV_V4F32: |
| 7047 | case X86::CMOV_V2F64: |
Evan Cheng | e5f6204 | 2007-09-29 00:00:36 +0000 | [diff] [blame] | 7048 | case X86::CMOV_V2I64: { |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7049 | // To "insert" a SELECT_CC instruction, we actually have to insert the |
| 7050 | // diamond control-flow pattern. The incoming instruction knows the |
| 7051 | // destination vreg to set, the condition code register to branch on, the |
| 7052 | // true/false values to select between, and a branch opcode to use. |
| 7053 | const BasicBlock *LLVM_BB = BB->getBasicBlock(); |
Dan Gohman | 8e5f2c6 | 2008-07-07 23:14:23 +0000 | [diff] [blame] | 7054 | MachineFunction::iterator It = BB; |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7055 | ++It; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7056 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7057 | // thisMBB: |
| 7058 | // ... |
| 7059 | // TrueVal = ... |
| 7060 | // cmpTY ccX, r1, r2 |
| 7061 | // bCC copy1MBB |
| 7062 | // fallthrough --> copy0MBB |
| 7063 | MachineBasicBlock *thisMBB = BB; |
Dan Gohman | 8e5f2c6 | 2008-07-07 23:14:23 +0000 | [diff] [blame] | 7064 | MachineFunction *F = BB->getParent(); |
| 7065 | MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); |
| 7066 | MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7067 | unsigned Opc = |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 7068 | X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm()); |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 7069 | BuildMI(BB, TII->get(Opc)).addMBB(sinkMBB); |
Dan Gohman | 8e5f2c6 | 2008-07-07 23:14:23 +0000 | [diff] [blame] | 7070 | F->insert(It, copy0MBB); |
| 7071 | F->insert(It, sinkMBB); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 7072 | // Update machine-CFG edges by transferring all successors of the current |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7073 | // block to the new block which will contain the Phi node for the select. |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 7074 | sinkMBB->transferSuccessors(BB); |
| 7075 | |
| 7076 | // Add the true and fallthrough blocks as its successors. |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7077 | BB->addSuccessor(copy0MBB); |
| 7078 | BB->addSuccessor(sinkMBB); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7079 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7080 | // copy0MBB: |
| 7081 | // %FalseValue = ... |
| 7082 | // # fallthrough to sinkMBB |
| 7083 | BB = copy0MBB; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7084 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7085 | // Update machine-CFG edges |
| 7086 | BB->addSuccessor(sinkMBB); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7087 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7088 | // sinkMBB: |
| 7089 | // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] |
| 7090 | // ... |
| 7091 | BB = sinkMBB; |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 7092 | BuildMI(BB, TII->get(X86::PHI), MI->getOperand(0).getReg()) |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7093 | .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB) |
| 7094 | .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB); |
| 7095 | |
Dan Gohman | 8e5f2c6 | 2008-07-07 23:14:23 +0000 | [diff] [blame] | 7096 | F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7097 | return BB; |
| 7098 | } |
| 7099 | |
Dale Johannesen | 849f214 | 2007-07-03 00:53:03 +0000 | [diff] [blame] | 7100 | case X86::FP32_TO_INT16_IN_MEM: |
| 7101 | case X86::FP32_TO_INT32_IN_MEM: |
| 7102 | case X86::FP32_TO_INT64_IN_MEM: |
| 7103 | case X86::FP64_TO_INT16_IN_MEM: |
| 7104 | case X86::FP64_TO_INT32_IN_MEM: |
Dale Johannesen | a996d52 | 2007-08-07 01:17:37 +0000 | [diff] [blame] | 7105 | case X86::FP64_TO_INT64_IN_MEM: |
| 7106 | case X86::FP80_TO_INT16_IN_MEM: |
| 7107 | case X86::FP80_TO_INT32_IN_MEM: |
| 7108 | case X86::FP80_TO_INT64_IN_MEM: { |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7109 | // Change the floating point control register to use "round towards zero" |
| 7110 | // mode when truncating to an integer value. |
| 7111 | MachineFunction *F = BB->getParent(); |
| 7112 | int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2); |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 7113 | addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7114 | |
| 7115 | // Load the old value of the high byte of the control word... |
| 7116 | unsigned OldCW = |
Chris Lattner | 84bc542 | 2007-12-31 04:13:23 +0000 | [diff] [blame] | 7117 | F->getRegInfo().createVirtualRegister(X86::GR16RegisterClass); |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 7118 | addFrameReference(BuildMI(BB, TII->get(X86::MOV16rm), OldCW), CWFrameIdx); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7119 | |
| 7120 | // Set the high part to be round to zero... |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 7121 | addFrameReference(BuildMI(BB, TII->get(X86::MOV16mi)), CWFrameIdx) |
| 7122 | .addImm(0xC7F); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7123 | |
| 7124 | // Reload the modified control word now... |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 7125 | addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7126 | |
| 7127 | // Restore the memory image of control word to original value |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 7128 | addFrameReference(BuildMI(BB, TII->get(X86::MOV16mr)), CWFrameIdx) |
| 7129 | .addReg(OldCW); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7130 | |
| 7131 | // Get the X86 opcode to use. |
| 7132 | unsigned Opc; |
| 7133 | switch (MI->getOpcode()) { |
| 7134 | default: assert(0 && "illegal opcode!"); |
Dale Johannesen | e377d4d | 2007-07-04 21:07:47 +0000 | [diff] [blame] | 7135 | case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break; |
| 7136 | case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break; |
| 7137 | case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break; |
| 7138 | case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break; |
| 7139 | case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break; |
| 7140 | case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break; |
Dale Johannesen | a996d52 | 2007-08-07 01:17:37 +0000 | [diff] [blame] | 7141 | case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break; |
| 7142 | case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break; |
| 7143 | case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break; |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7144 | } |
| 7145 | |
| 7146 | X86AddressMode AM; |
| 7147 | MachineOperand &Op = MI->getOperand(0); |
Dan Gohman | d735b80 | 2008-10-03 15:45:36 +0000 | [diff] [blame] | 7148 | if (Op.isReg()) { |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7149 | AM.BaseType = X86AddressMode::RegBase; |
| 7150 | AM.Base.Reg = Op.getReg(); |
| 7151 | } else { |
| 7152 | AM.BaseType = X86AddressMode::FrameIndexBase; |
Chris Lattner | 8aa797a | 2007-12-30 23:10:15 +0000 | [diff] [blame] | 7153 | AM.Base.FrameIndex = Op.getIndex(); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7154 | } |
| 7155 | Op = MI->getOperand(1); |
Dan Gohman | d735b80 | 2008-10-03 15:45:36 +0000 | [diff] [blame] | 7156 | if (Op.isImm()) |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 7157 | AM.Scale = Op.getImm(); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7158 | Op = MI->getOperand(2); |
Dan Gohman | d735b80 | 2008-10-03 15:45:36 +0000 | [diff] [blame] | 7159 | if (Op.isImm()) |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 7160 | AM.IndexReg = Op.getImm(); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7161 | Op = MI->getOperand(3); |
Dan Gohman | d735b80 | 2008-10-03 15:45:36 +0000 | [diff] [blame] | 7162 | if (Op.isGlobal()) { |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7163 | AM.GV = Op.getGlobal(); |
| 7164 | } else { |
Chris Lattner | 7fbe972 | 2006-10-20 17:42:20 +0000 | [diff] [blame] | 7165 | AM.Disp = Op.getImm(); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7166 | } |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 7167 | addFullAddress(BuildMI(BB, TII->get(Opc)), AM) |
| 7168 | .addReg(MI->getOperand(4).getReg()); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7169 | |
| 7170 | // Reload the original control word now. |
Evan Cheng | c0f64ff | 2006-11-27 23:37:22 +0000 | [diff] [blame] | 7171 | addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7172 | |
Dan Gohman | 8e5f2c6 | 2008-07-07 23:14:23 +0000 | [diff] [blame] | 7173 | F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7174 | return BB; |
| 7175 | } |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 7176 | case X86::ATOMAND32: |
| 7177 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 7178 | X86::AND32ri, X86::MOV32rm, |
| 7179 | X86::LCMPXCHG32, X86::MOV32rr, |
| 7180 | X86::NOT32r, X86::EAX, |
| 7181 | X86::GR32RegisterClass); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 7182 | case X86::ATOMOR32: |
| 7183 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr, |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 7184 | X86::OR32ri, X86::MOV32rm, |
| 7185 | X86::LCMPXCHG32, X86::MOV32rr, |
| 7186 | X86::NOT32r, X86::EAX, |
| 7187 | X86::GR32RegisterClass); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 7188 | case X86::ATOMXOR32: |
| 7189 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr, |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 7190 | X86::XOR32ri, X86::MOV32rm, |
| 7191 | X86::LCMPXCHG32, X86::MOV32rr, |
| 7192 | X86::NOT32r, X86::EAX, |
| 7193 | X86::GR32RegisterClass); |
Andrew Lenharth | 507a58a | 2008-06-14 05:48:15 +0000 | [diff] [blame] | 7194 | case X86::ATOMNAND32: |
| 7195 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr, |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 7196 | X86::AND32ri, X86::MOV32rm, |
| 7197 | X86::LCMPXCHG32, X86::MOV32rr, |
| 7198 | X86::NOT32r, X86::EAX, |
| 7199 | X86::GR32RegisterClass, true); |
Mon P Wang | 63307c3 | 2008-05-05 19:05:59 +0000 | [diff] [blame] | 7200 | case X86::ATOMMIN32: |
| 7201 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr); |
| 7202 | case X86::ATOMMAX32: |
| 7203 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr); |
| 7204 | case X86::ATOMUMIN32: |
| 7205 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr); |
| 7206 | case X86::ATOMUMAX32: |
| 7207 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr); |
Dale Johannesen | 140be2d | 2008-08-19 18:47:28 +0000 | [diff] [blame] | 7208 | |
| 7209 | case X86::ATOMAND16: |
| 7210 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, |
| 7211 | X86::AND16ri, X86::MOV16rm, |
| 7212 | X86::LCMPXCHG16, X86::MOV16rr, |
| 7213 | X86::NOT16r, X86::AX, |
| 7214 | X86::GR16RegisterClass); |
| 7215 | case X86::ATOMOR16: |
| 7216 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR16rr, |
| 7217 | X86::OR16ri, X86::MOV16rm, |
| 7218 | X86::LCMPXCHG16, X86::MOV16rr, |
| 7219 | X86::NOT16r, X86::AX, |
| 7220 | X86::GR16RegisterClass); |
| 7221 | case X86::ATOMXOR16: |
| 7222 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR16rr, |
| 7223 | X86::XOR16ri, X86::MOV16rm, |
| 7224 | X86::LCMPXCHG16, X86::MOV16rr, |
| 7225 | X86::NOT16r, X86::AX, |
| 7226 | X86::GR16RegisterClass); |
| 7227 | case X86::ATOMNAND16: |
| 7228 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr, |
| 7229 | X86::AND16ri, X86::MOV16rm, |
| 7230 | X86::LCMPXCHG16, X86::MOV16rr, |
| 7231 | X86::NOT16r, X86::AX, |
| 7232 | X86::GR16RegisterClass, true); |
| 7233 | case X86::ATOMMIN16: |
| 7234 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL16rr); |
| 7235 | case X86::ATOMMAX16: |
| 7236 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG16rr); |
| 7237 | case X86::ATOMUMIN16: |
| 7238 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB16rr); |
| 7239 | case X86::ATOMUMAX16: |
| 7240 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA16rr); |
| 7241 | |
| 7242 | case X86::ATOMAND8: |
| 7243 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, |
| 7244 | X86::AND8ri, X86::MOV8rm, |
| 7245 | X86::LCMPXCHG8, X86::MOV8rr, |
| 7246 | X86::NOT8r, X86::AL, |
| 7247 | X86::GR8RegisterClass); |
| 7248 | case X86::ATOMOR8: |
| 7249 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR8rr, |
| 7250 | X86::OR8ri, X86::MOV8rm, |
| 7251 | X86::LCMPXCHG8, X86::MOV8rr, |
| 7252 | X86::NOT8r, X86::AL, |
| 7253 | X86::GR8RegisterClass); |
| 7254 | case X86::ATOMXOR8: |
| 7255 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR8rr, |
| 7256 | X86::XOR8ri, X86::MOV8rm, |
| 7257 | X86::LCMPXCHG8, X86::MOV8rr, |
| 7258 | X86::NOT8r, X86::AL, |
| 7259 | X86::GR8RegisterClass); |
| 7260 | case X86::ATOMNAND8: |
| 7261 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr, |
| 7262 | X86::AND8ri, X86::MOV8rm, |
| 7263 | X86::LCMPXCHG8, X86::MOV8rr, |
| 7264 | X86::NOT8r, X86::AL, |
| 7265 | X86::GR8RegisterClass, true); |
| 7266 | // FIXME: There are no CMOV8 instructions; MIN/MAX need some other way. |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 7267 | // This group is for 64-bit host. |
Dale Johannesen | a99e384 | 2008-08-20 00:48:50 +0000 | [diff] [blame] | 7268 | case X86::ATOMAND64: |
| 7269 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, |
| 7270 | X86::AND64ri32, X86::MOV64rm, |
| 7271 | X86::LCMPXCHG64, X86::MOV64rr, |
| 7272 | X86::NOT64r, X86::RAX, |
| 7273 | X86::GR64RegisterClass); |
| 7274 | case X86::ATOMOR64: |
| 7275 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR64rr, |
| 7276 | X86::OR64ri32, X86::MOV64rm, |
| 7277 | X86::LCMPXCHG64, X86::MOV64rr, |
| 7278 | X86::NOT64r, X86::RAX, |
| 7279 | X86::GR64RegisterClass); |
| 7280 | case X86::ATOMXOR64: |
| 7281 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR64rr, |
| 7282 | X86::XOR64ri32, X86::MOV64rm, |
| 7283 | X86::LCMPXCHG64, X86::MOV64rr, |
| 7284 | X86::NOT64r, X86::RAX, |
| 7285 | X86::GR64RegisterClass); |
| 7286 | case X86::ATOMNAND64: |
| 7287 | return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr, |
| 7288 | X86::AND64ri32, X86::MOV64rm, |
| 7289 | X86::LCMPXCHG64, X86::MOV64rr, |
| 7290 | X86::NOT64r, X86::RAX, |
| 7291 | X86::GR64RegisterClass, true); |
| 7292 | case X86::ATOMMIN64: |
| 7293 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL64rr); |
| 7294 | case X86::ATOMMAX64: |
| 7295 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG64rr); |
| 7296 | case X86::ATOMUMIN64: |
| 7297 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB64rr); |
| 7298 | case X86::ATOMUMAX64: |
| 7299 | return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA64rr); |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 7300 | |
| 7301 | // This group does 64-bit operations on a 32-bit host. |
| 7302 | case X86::ATOMAND6432: |
| 7303 | return EmitAtomicBit6432WithCustomInserter(MI, BB, |
| 7304 | X86::AND32rr, X86::AND32rr, |
| 7305 | X86::AND32ri, X86::AND32ri, |
| 7306 | false); |
| 7307 | case X86::ATOMOR6432: |
| 7308 | return EmitAtomicBit6432WithCustomInserter(MI, BB, |
| 7309 | X86::OR32rr, X86::OR32rr, |
| 7310 | X86::OR32ri, X86::OR32ri, |
| 7311 | false); |
| 7312 | case X86::ATOMXOR6432: |
| 7313 | return EmitAtomicBit6432WithCustomInserter(MI, BB, |
| 7314 | X86::XOR32rr, X86::XOR32rr, |
| 7315 | X86::XOR32ri, X86::XOR32ri, |
| 7316 | false); |
| 7317 | case X86::ATOMNAND6432: |
| 7318 | return EmitAtomicBit6432WithCustomInserter(MI, BB, |
| 7319 | X86::AND32rr, X86::AND32rr, |
| 7320 | X86::AND32ri, X86::AND32ri, |
| 7321 | true); |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 7322 | case X86::ATOMADD6432: |
| 7323 | return EmitAtomicBit6432WithCustomInserter(MI, BB, |
| 7324 | X86::ADD32rr, X86::ADC32rr, |
| 7325 | X86::ADD32ri, X86::ADC32ri, |
| 7326 | false); |
Dale Johannesen | 48c1bc2 | 2008-10-02 18:53:47 +0000 | [diff] [blame] | 7327 | case X86::ATOMSUB6432: |
| 7328 | return EmitAtomicBit6432WithCustomInserter(MI, BB, |
| 7329 | X86::SUB32rr, X86::SBB32rr, |
| 7330 | X86::SUB32ri, X86::SBB32ri, |
| 7331 | false); |
Dale Johannesen | 880ae36 | 2008-10-03 22:25:52 +0000 | [diff] [blame] | 7332 | case X86::ATOMSWAP6432: |
| 7333 | return EmitAtomicBit6432WithCustomInserter(MI, BB, |
| 7334 | X86::MOV32rr, X86::MOV32rr, |
| 7335 | X86::MOV32ri, X86::MOV32ri, |
| 7336 | false); |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7337 | } |
| 7338 | } |
| 7339 | |
| 7340 | //===----------------------------------------------------------------------===// |
| 7341 | // X86 Optimization Hooks |
| 7342 | //===----------------------------------------------------------------------===// |
| 7343 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7344 | void X86TargetLowering::computeMaskedBitsForTargetNode(const SDValue Op, |
Dan Gohman | 977a76f | 2008-02-13 22:28:48 +0000 | [diff] [blame] | 7345 | const APInt &Mask, |
Dan Gohman | fd29e0e | 2008-02-13 00:35:47 +0000 | [diff] [blame] | 7346 | APInt &KnownZero, |
| 7347 | APInt &KnownOne, |
Dan Gohman | ea859be | 2007-06-22 14:59:07 +0000 | [diff] [blame] | 7348 | const SelectionDAG &DAG, |
Nate Begeman | 368e18d | 2006-02-16 21:11:51 +0000 | [diff] [blame] | 7349 | unsigned Depth) const { |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 7350 | unsigned Opc = Op.getOpcode(); |
Evan Cheng | 865f060 | 2006-04-05 06:11:20 +0000 | [diff] [blame] | 7351 | assert((Opc >= ISD::BUILTIN_OP_END || |
| 7352 | Opc == ISD::INTRINSIC_WO_CHAIN || |
| 7353 | Opc == ISD::INTRINSIC_W_CHAIN || |
| 7354 | Opc == ISD::INTRINSIC_VOID) && |
| 7355 | "Should use MaskedValueIsZero if you don't know whether Op" |
| 7356 | " is a target node!"); |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 7357 | |
Dan Gohman | f4f92f5 | 2008-02-13 23:07:24 +0000 | [diff] [blame] | 7358 | KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0); // Don't know anything. |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 7359 | switch (Opc) { |
Evan Cheng | 865f060 | 2006-04-05 06:11:20 +0000 | [diff] [blame] | 7360 | default: break; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7361 | case X86ISD::SETCC: |
Dan Gohman | fd29e0e | 2008-02-13 00:35:47 +0000 | [diff] [blame] | 7362 | KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(), |
| 7363 | Mask.getBitWidth() - 1); |
Nate Begeman | 368e18d | 2006-02-16 21:11:51 +0000 | [diff] [blame] | 7364 | break; |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 7365 | } |
Evan Cheng | 3a03ebb | 2005-12-21 23:05:39 +0000 | [diff] [blame] | 7366 | } |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 7367 | |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7368 | /// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the |
Evan Cheng | ad4196b | 2008-05-12 19:56:52 +0000 | [diff] [blame] | 7369 | /// node is a GlobalAddress + offset. |
| 7370 | bool X86TargetLowering::isGAPlusOffset(SDNode *N, |
| 7371 | GlobalValue* &GA, int64_t &Offset) const{ |
| 7372 | if (N->getOpcode() == X86ISD::Wrapper) { |
| 7373 | if (isa<GlobalAddressSDNode>(N->getOperand(0))) { |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7374 | GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal(); |
Dan Gohman | 6520e20 | 2008-10-18 02:06:02 +0000 | [diff] [blame] | 7375 | Offset = cast<GlobalAddressSDNode>(N->getOperand(0))->getOffset(); |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7376 | return true; |
| 7377 | } |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7378 | } |
Evan Cheng | ad4196b | 2008-05-12 19:56:52 +0000 | [diff] [blame] | 7379 | return TargetLowering::isGAPlusOffset(N, GA, Offset); |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7380 | } |
| 7381 | |
Evan Cheng | ad4196b | 2008-05-12 19:56:52 +0000 | [diff] [blame] | 7382 | static bool isBaseAlignmentOfN(unsigned N, SDNode *Base, |
| 7383 | const TargetLowering &TLI) { |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7384 | GlobalValue *GV; |
Nick Lewycky | 916a9f0 | 2008-02-02 08:29:58 +0000 | [diff] [blame] | 7385 | int64_t Offset = 0; |
Evan Cheng | ad4196b | 2008-05-12 19:56:52 +0000 | [diff] [blame] | 7386 | if (TLI.isGAPlusOffset(Base, GV, Offset)) |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 7387 | return (GV->getAlignment() >= N && (Offset % N) == 0); |
Chris Lattner | ba96fbc | 2008-01-26 20:07:42 +0000 | [diff] [blame] | 7388 | // DAG combine handles the stack object case. |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7389 | return false; |
| 7390 | } |
| 7391 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7392 | static bool EltsFromConsecutiveLoads(SDNode *N, SDValue PermMask, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7393 | unsigned NumElems, MVT EVT, |
Evan Cheng | ad4196b | 2008-05-12 19:56:52 +0000 | [diff] [blame] | 7394 | SDNode *&Base, |
| 7395 | SelectionDAG &DAG, MachineFrameInfo *MFI, |
| 7396 | const TargetLowering &TLI) { |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 7397 | Base = NULL; |
| 7398 | for (unsigned i = 0; i < NumElems; ++i) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7399 | SDValue Idx = PermMask.getOperand(i); |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 7400 | if (Idx.getOpcode() == ISD::UNDEF) { |
| 7401 | if (!Base) |
| 7402 | return false; |
| 7403 | continue; |
| 7404 | } |
| 7405 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7406 | SDValue Elt = DAG.getShuffleScalarElt(N, i); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 7407 | if (!Elt.getNode() || |
| 7408 | (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode()))) |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 7409 | return false; |
| 7410 | if (!Base) { |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 7411 | Base = Elt.getNode(); |
Evan Cheng | 50d9e72 | 2008-05-10 06:46:49 +0000 | [diff] [blame] | 7412 | if (Base->getOpcode() == ISD::UNDEF) |
| 7413 | return false; |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 7414 | continue; |
| 7415 | } |
| 7416 | if (Elt.getOpcode() == ISD::UNDEF) |
| 7417 | continue; |
| 7418 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 7419 | if (!TLI.isConsecutiveLoad(Elt.getNode(), Base, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7420 | EVT.getSizeInBits()/8, i, MFI)) |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 7421 | return false; |
| 7422 | } |
| 7423 | return true; |
| 7424 | } |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7425 | |
| 7426 | /// PerformShuffleCombine - Combine a vector_shuffle that is equal to |
| 7427 | /// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load |
| 7428 | /// if the load addresses are consecutive, non-overlapping, and in the right |
| 7429 | /// order. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7430 | static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG, |
Evan Cheng | ad4196b | 2008-05-12 19:56:52 +0000 | [diff] [blame] | 7431 | const TargetLowering &TLI) { |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 7432 | MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7433 | MVT VT = N->getValueType(0); |
| 7434 | MVT EVT = VT.getVectorElementType(); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7435 | SDValue PermMask = N->getOperand(2); |
Evan Cheng | 71f489d | 2008-05-05 22:12:23 +0000 | [diff] [blame] | 7436 | unsigned NumElems = PermMask.getNumOperands(); |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7437 | SDNode *Base = NULL; |
Evan Cheng | ad4196b | 2008-05-12 19:56:52 +0000 | [diff] [blame] | 7438 | if (!EltsFromConsecutiveLoads(N, PermMask, NumElems, EVT, Base, |
| 7439 | DAG, MFI, TLI)) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7440 | return SDValue(); |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7441 | |
Dan Gohman | d300622 | 2007-07-27 17:16:43 +0000 | [diff] [blame] | 7442 | LoadSDNode *LD = cast<LoadSDNode>(Base); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 7443 | if (isBaseAlignmentOfN(16, Base->getOperand(1).getNode(), TLI)) |
Evan Cheng | 466685d | 2006-10-09 20:57:25 +0000 | [diff] [blame] | 7444 | return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), |
Dan Gohman | d300622 | 2007-07-27 17:16:43 +0000 | [diff] [blame] | 7445 | LD->getSrcValueOffset(), LD->isVolatile()); |
Evan Cheng | 71f489d | 2008-05-05 22:12:23 +0000 | [diff] [blame] | 7446 | return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), LD->getSrcValue(), |
| 7447 | LD->getSrcValueOffset(), LD->isVolatile(), |
| 7448 | LD->getAlignment()); |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7449 | } |
| 7450 | |
Evan Cheng | 9bfa03c | 2008-05-12 23:04:07 +0000 | [diff] [blame] | 7451 | /// PerformBuildVectorCombine - build_vector 0,(load i64 / f64) -> movq / movsd. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7452 | static SDValue PerformBuildVectorCombine(SDNode *N, SelectionDAG &DAG, |
Evan Cheng | 8a186ae | 2008-09-24 23:26:36 +0000 | [diff] [blame] | 7453 | const X86Subtarget *Subtarget, |
| 7454 | const TargetLowering &TLI) { |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 7455 | unsigned NumOps = N->getNumOperands(); |
| 7456 | |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 7457 | // Ignore single operand BUILD_VECTOR. |
Evan Cheng | f26ffe9 | 2008-05-29 08:22:04 +0000 | [diff] [blame] | 7458 | if (NumOps == 1) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7459 | return SDValue(); |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 7460 | |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7461 | MVT VT = N->getValueType(0); |
| 7462 | MVT EVT = VT.getVectorElementType(); |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 7463 | if ((EVT != MVT::i64 && EVT != MVT::f64) || Subtarget->is64Bit()) |
| 7464 | // We are looking for load i64 and zero extend. We want to transform |
| 7465 | // it before legalizer has a chance to expand it. Also look for i64 |
| 7466 | // BUILD_PAIR bit casted to f64. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7467 | return SDValue(); |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 7468 | // This must be an insertion into a zero vector. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7469 | SDValue HighElt = N->getOperand(1); |
Evan Cheng | 25210da | 2008-05-10 00:58:41 +0000 | [diff] [blame] | 7470 | if (!isZeroNode(HighElt)) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7471 | return SDValue(); |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 7472 | |
| 7473 | // Value must be a load. |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 7474 | SDNode *Base = N->getOperand(0).getNode(); |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 7475 | if (!isa<LoadSDNode>(Base)) { |
Evan Cheng | 9bfa03c | 2008-05-12 23:04:07 +0000 | [diff] [blame] | 7476 | if (Base->getOpcode() != ISD::BIT_CONVERT) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7477 | return SDValue(); |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 7478 | Base = Base->getOperand(0).getNode(); |
Evan Cheng | 9bfa03c | 2008-05-12 23:04:07 +0000 | [diff] [blame] | 7479 | if (!isa<LoadSDNode>(Base)) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7480 | return SDValue(); |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 7481 | } |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 7482 | |
| 7483 | // Transform it into VZEXT_LOAD addr. |
Evan Cheng | 9bfa03c | 2008-05-12 23:04:07 +0000 | [diff] [blame] | 7484 | LoadSDNode *LD = cast<LoadSDNode>(Base); |
Nate Begeman | f7333bf | 2008-05-28 00:24:25 +0000 | [diff] [blame] | 7485 | |
| 7486 | // Load must not be an extload. |
| 7487 | if (LD->getExtensionType() != ISD::NON_EXTLOAD) |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7488 | return SDValue(); |
Nate Begeman | f7333bf | 2008-05-28 00:24:25 +0000 | [diff] [blame] | 7489 | |
Evan Cheng | 8a186ae | 2008-09-24 23:26:36 +0000 | [diff] [blame] | 7490 | SDVTList Tys = DAG.getVTList(VT, MVT::Other); |
| 7491 | SDValue Ops[] = { LD->getChain(), LD->getBasePtr() }; |
| 7492 | SDValue ResNode = DAG.getNode(X86ISD::VZEXT_LOAD, Tys, Ops, 2); |
| 7493 | DAG.ReplaceAllUsesOfValueWith(SDValue(Base, 1), ResNode.getValue(1)); |
| 7494 | return ResNode; |
Evan Cheng | d880b97 | 2008-05-09 21:53:03 +0000 | [diff] [blame] | 7495 | } |
| 7496 | |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 7497 | /// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7498 | static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG, |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 7499 | const X86Subtarget *Subtarget) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7500 | SDValue Cond = N->getOperand(0); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7501 | |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 7502 | // If we have SSE[12] support, try to form min/max nodes. |
| 7503 | if (Subtarget->hasSSE2() && |
| 7504 | (N->getValueType(0) == MVT::f32 || N->getValueType(0) == MVT::f64)) { |
| 7505 | if (Cond.getOpcode() == ISD::SETCC) { |
| 7506 | // Get the LHS/RHS of the select. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7507 | SDValue LHS = N->getOperand(1); |
| 7508 | SDValue RHS = N->getOperand(2); |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 7509 | ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7510 | |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 7511 | unsigned Opcode = 0; |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 7512 | if (LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) { |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 7513 | switch (CC) { |
| 7514 | default: break; |
| 7515 | case ISD::SETOLE: // (X <= Y) ? X : Y -> min |
| 7516 | case ISD::SETULE: |
| 7517 | case ISD::SETLE: |
| 7518 | if (!UnsafeFPMath) break; |
| 7519 | // FALL THROUGH. |
| 7520 | case ISD::SETOLT: // (X olt/lt Y) ? X : Y -> min |
| 7521 | case ISD::SETLT: |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 7522 | Opcode = X86ISD::FMIN; |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 7523 | break; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7524 | |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 7525 | case ISD::SETOGT: // (X > Y) ? X : Y -> max |
| 7526 | case ISD::SETUGT: |
| 7527 | case ISD::SETGT: |
| 7528 | if (!UnsafeFPMath) break; |
| 7529 | // FALL THROUGH. |
| 7530 | case ISD::SETUGE: // (X uge/ge Y) ? X : Y -> max |
| 7531 | case ISD::SETGE: |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 7532 | Opcode = X86ISD::FMAX; |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 7533 | break; |
| 7534 | } |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 7535 | } else if (LHS == Cond.getOperand(1) && RHS == Cond.getOperand(0)) { |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 7536 | switch (CC) { |
| 7537 | default: break; |
| 7538 | case ISD::SETOGT: // (X > Y) ? Y : X -> min |
| 7539 | case ISD::SETUGT: |
| 7540 | case ISD::SETGT: |
| 7541 | if (!UnsafeFPMath) break; |
| 7542 | // FALL THROUGH. |
| 7543 | case ISD::SETUGE: // (X uge/ge Y) ? Y : X -> min |
| 7544 | case ISD::SETGE: |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 7545 | Opcode = X86ISD::FMIN; |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 7546 | break; |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7547 | |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 7548 | case ISD::SETOLE: // (X <= Y) ? Y : X -> max |
| 7549 | case ISD::SETULE: |
| 7550 | case ISD::SETLE: |
| 7551 | if (!UnsafeFPMath) break; |
| 7552 | // FALL THROUGH. |
| 7553 | case ISD::SETOLT: // (X olt/lt Y) ? Y : X -> max |
| 7554 | case ISD::SETLT: |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 7555 | Opcode = X86ISD::FMAX; |
Chris Lattner | 1907a7b | 2006-10-05 04:11:26 +0000 | [diff] [blame] | 7556 | break; |
| 7557 | } |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 7558 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7559 | |
Evan Cheng | 8ca2932 | 2006-11-10 21:43:37 +0000 | [diff] [blame] | 7560 | if (Opcode) |
| 7561 | return DAG.getNode(Opcode, N->getValueType(0), LHS, RHS); |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 7562 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7563 | |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 7564 | } |
| 7565 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7566 | return SDValue(); |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 7567 | } |
| 7568 | |
Chris Lattner | 149a4e5 | 2008-02-22 02:09:43 +0000 | [diff] [blame] | 7569 | /// PerformSTORECombine - Do target-specific dag combines on STORE nodes. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7570 | static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG, |
Chris Lattner | 149a4e5 | 2008-02-22 02:09:43 +0000 | [diff] [blame] | 7571 | const X86Subtarget *Subtarget) { |
| 7572 | // Turn load->store of MMX types into GPR load/stores. This avoids clobbering |
| 7573 | // the FP state in cases where an emms may be missing. |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7574 | // A preferable solution to the general problem is to figure out the right |
| 7575 | // places to insert EMMS. This qualifies as a quick hack. |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 7576 | StoreSDNode *St = cast<StoreSDNode>(N); |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7577 | if (St->getValue().getValueType().isVector() && |
| 7578 | St->getValue().getValueType().getSizeInBits() == 64 && |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7579 | isa<LoadSDNode>(St->getValue()) && |
| 7580 | !cast<LoadSDNode>(St->getValue())->isVolatile() && |
| 7581 | St->getChain().hasOneUse() && !St->isVolatile()) { |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 7582 | SDNode* LdVal = St->getValue().getNode(); |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7583 | LoadSDNode *Ld = 0; |
| 7584 | int TokenFactorIndex = -1; |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7585 | SmallVector<SDValue, 8> Ops; |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 7586 | SDNode* ChainVal = St->getChain().getNode(); |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7587 | // Must be a store of a load. We currently handle two cases: the load |
| 7588 | // is a direct child, and it's under an intervening TokenFactor. It is |
| 7589 | // possible to dig deeper under nested TokenFactors. |
Dale Johannesen | 14e2ea9 | 2008-02-25 22:29:22 +0000 | [diff] [blame] | 7590 | if (ChainVal == LdVal) |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7591 | Ld = cast<LoadSDNode>(St->getChain()); |
| 7592 | else if (St->getValue().hasOneUse() && |
| 7593 | ChainVal->getOpcode() == ISD::TokenFactor) { |
| 7594 | for (unsigned i=0, e = ChainVal->getNumOperands(); i != e; ++i) { |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 7595 | if (ChainVal->getOperand(i).getNode() == LdVal) { |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7596 | TokenFactorIndex = i; |
| 7597 | Ld = cast<LoadSDNode>(St->getValue()); |
| 7598 | } else |
| 7599 | Ops.push_back(ChainVal->getOperand(i)); |
| 7600 | } |
| 7601 | } |
| 7602 | if (Ld) { |
| 7603 | // If we are a 64-bit capable x86, lower to a single movq load/store pair. |
| 7604 | if (Subtarget->is64Bit()) { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7605 | SDValue NewLd = DAG.getLoad(MVT::i64, Ld->getChain(), |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7606 | Ld->getBasePtr(), Ld->getSrcValue(), |
| 7607 | Ld->getSrcValueOffset(), Ld->isVolatile(), |
| 7608 | Ld->getAlignment()); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7609 | SDValue NewChain = NewLd.getValue(1); |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7610 | if (TokenFactorIndex != -1) { |
Dan Gohman | d4a2ad3 | 2008-03-28 23:45:16 +0000 | [diff] [blame] | 7611 | Ops.push_back(NewChain); |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7612 | NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], |
| 7613 | Ops.size()); |
| 7614 | } |
| 7615 | return DAG.getStore(NewChain, NewLd, St->getBasePtr(), |
| 7616 | St->getSrcValue(), St->getSrcValueOffset(), |
| 7617 | St->isVolatile(), St->getAlignment()); |
| 7618 | } |
| 7619 | |
| 7620 | // Otherwise, lower to two 32-bit copies. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7621 | SDValue LoAddr = Ld->getBasePtr(); |
| 7622 | SDValue HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7623 | DAG.getConstant(4, MVT::i32)); |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7624 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7625 | SDValue LoLd = DAG.getLoad(MVT::i32, Ld->getChain(), LoAddr, |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7626 | Ld->getSrcValue(), Ld->getSrcValueOffset(), |
| 7627 | Ld->isVolatile(), Ld->getAlignment()); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7628 | SDValue HiLd = DAG.getLoad(MVT::i32, Ld->getChain(), HiAddr, |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7629 | Ld->getSrcValue(), Ld->getSrcValueOffset()+4, |
| 7630 | Ld->isVolatile(), |
| 7631 | MinAlign(Ld->getAlignment(), 4)); |
| 7632 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7633 | SDValue NewChain = LoLd.getValue(1); |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7634 | if (TokenFactorIndex != -1) { |
| 7635 | Ops.push_back(LoLd); |
| 7636 | Ops.push_back(HiLd); |
| 7637 | NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], |
| 7638 | Ops.size()); |
| 7639 | } |
| 7640 | |
| 7641 | LoAddr = St->getBasePtr(); |
| 7642 | HiAddr = DAG.getNode(ISD::ADD, MVT::i32, LoAddr, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7643 | DAG.getConstant(4, MVT::i32)); |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7644 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7645 | SDValue LoSt = DAG.getStore(NewChain, LoLd, LoAddr, |
Chris Lattner | 149a4e5 | 2008-02-22 02:09:43 +0000 | [diff] [blame] | 7646 | St->getSrcValue(), St->getSrcValueOffset(), |
| 7647 | St->isVolatile(), St->getAlignment()); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7648 | SDValue HiSt = DAG.getStore(NewChain, HiLd, HiAddr, |
Gabor Greif | 327ef03 | 2008-08-28 23:19:51 +0000 | [diff] [blame] | 7649 | St->getSrcValue(), |
| 7650 | St->getSrcValueOffset() + 4, |
Dale Johannesen | 079f2a6 | 2008-02-25 19:20:14 +0000 | [diff] [blame] | 7651 | St->isVolatile(), |
| 7652 | MinAlign(St->getAlignment(), 4)); |
| 7653 | return DAG.getNode(ISD::TokenFactor, MVT::Other, LoSt, HiSt); |
Chris Lattner | 149a4e5 | 2008-02-22 02:09:43 +0000 | [diff] [blame] | 7654 | } |
Chris Lattner | 149a4e5 | 2008-02-22 02:09:43 +0000 | [diff] [blame] | 7655 | } |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7656 | return SDValue(); |
Chris Lattner | 149a4e5 | 2008-02-22 02:09:43 +0000 | [diff] [blame] | 7657 | } |
| 7658 | |
Chris Lattner | 6cf7326 | 2008-01-25 06:14:17 +0000 | [diff] [blame] | 7659 | /// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and |
| 7660 | /// X86ISD::FXOR nodes. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7661 | static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) { |
Chris Lattner | 6cf7326 | 2008-01-25 06:14:17 +0000 | [diff] [blame] | 7662 | assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR); |
| 7663 | // F[X]OR(0.0, x) -> x |
| 7664 | // F[X]OR(x, 0.0) -> x |
Chris Lattner | af723b9 | 2008-01-25 05:46:26 +0000 | [diff] [blame] | 7665 | if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) |
| 7666 | if (C->getValueAPF().isPosZero()) |
| 7667 | return N->getOperand(1); |
| 7668 | if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) |
| 7669 | if (C->getValueAPF().isPosZero()) |
| 7670 | return N->getOperand(0); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7671 | return SDValue(); |
Chris Lattner | af723b9 | 2008-01-25 05:46:26 +0000 | [diff] [blame] | 7672 | } |
| 7673 | |
| 7674 | /// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7675 | static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) { |
Chris Lattner | af723b9 | 2008-01-25 05:46:26 +0000 | [diff] [blame] | 7676 | // FAND(0.0, x) -> 0.0 |
| 7677 | // FAND(x, 0.0) -> 0.0 |
| 7678 | if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) |
| 7679 | if (C->getValueAPF().isPosZero()) |
| 7680 | return N->getOperand(0); |
| 7681 | if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1))) |
| 7682 | if (C->getValueAPF().isPosZero()) |
| 7683 | return N->getOperand(1); |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7684 | return SDValue(); |
Chris Lattner | af723b9 | 2008-01-25 05:46:26 +0000 | [diff] [blame] | 7685 | } |
| 7686 | |
Chris Lattner | 83e6c99 | 2006-10-04 06:57:07 +0000 | [diff] [blame] | 7687 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7688 | SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, |
Evan Cheng | 9dd93b3 | 2008-11-05 06:03:38 +0000 | [diff] [blame] | 7689 | DAGCombinerInfo &DCI) const { |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7690 | SelectionDAG &DAG = DCI.DAG; |
| 7691 | switch (N->getOpcode()) { |
| 7692 | default: break; |
Evan Cheng | ad4196b | 2008-05-12 19:56:52 +0000 | [diff] [blame] | 7693 | case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, *this); |
| 7694 | case ISD::BUILD_VECTOR: |
| 7695 | return PerformBuildVectorCombine(N, DAG, Subtarget, *this); |
Chris Lattner | af723b9 | 2008-01-25 05:46:26 +0000 | [diff] [blame] | 7696 | case ISD::SELECT: return PerformSELECTCombine(N, DAG, Subtarget); |
Evan Cheng | 7e2ff77 | 2008-05-08 00:57:18 +0000 | [diff] [blame] | 7697 | case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); |
Chris Lattner | 6cf7326 | 2008-01-25 06:14:17 +0000 | [diff] [blame] | 7698 | case X86ISD::FXOR: |
Chris Lattner | af723b9 | 2008-01-25 05:46:26 +0000 | [diff] [blame] | 7699 | case X86ISD::FOR: return PerformFORCombine(N, DAG); |
| 7700 | case X86ISD::FAND: return PerformFANDCombine(N, DAG); |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7701 | } |
| 7702 | |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7703 | return SDValue(); |
Evan Cheng | 206ee9d | 2006-07-07 08:33:52 +0000 | [diff] [blame] | 7704 | } |
| 7705 | |
Evan Cheng | 60c07e1 | 2006-07-05 22:17:51 +0000 | [diff] [blame] | 7706 | //===----------------------------------------------------------------------===// |
| 7707 | // X86 Inline Assembly Support |
| 7708 | //===----------------------------------------------------------------------===// |
| 7709 | |
Chris Lattner | f4dff84 | 2006-07-11 02:54:03 +0000 | [diff] [blame] | 7710 | /// getConstraintType - Given a constraint letter, return the type of |
| 7711 | /// constraint it is for this target. |
| 7712 | X86TargetLowering::ConstraintType |
Chris Lattner | 4234f57 | 2007-03-25 02:14:49 +0000 | [diff] [blame] | 7713 | X86TargetLowering::getConstraintType(const std::string &Constraint) const { |
| 7714 | if (Constraint.size() == 1) { |
| 7715 | switch (Constraint[0]) { |
| 7716 | case 'A': |
Dale Johannesen | 330169f | 2008-11-13 21:52:36 +0000 | [diff] [blame] | 7717 | return C_Register; |
Chris Lattner | fce84ac | 2008-03-11 19:06:29 +0000 | [diff] [blame] | 7718 | case 'f': |
Chris Lattner | 4234f57 | 2007-03-25 02:14:49 +0000 | [diff] [blame] | 7719 | case 'r': |
| 7720 | case 'R': |
| 7721 | case 'l': |
| 7722 | case 'q': |
| 7723 | case 'Q': |
| 7724 | case 'x': |
Dale Johannesen | 2ffbcac | 2008-04-01 00:57:48 +0000 | [diff] [blame] | 7725 | case 'y': |
Chris Lattner | 4234f57 | 2007-03-25 02:14:49 +0000 | [diff] [blame] | 7726 | case 'Y': |
| 7727 | return C_RegisterClass; |
| 7728 | default: |
| 7729 | break; |
| 7730 | } |
Chris Lattner | f4dff84 | 2006-07-11 02:54:03 +0000 | [diff] [blame] | 7731 | } |
Chris Lattner | 4234f57 | 2007-03-25 02:14:49 +0000 | [diff] [blame] | 7732 | return TargetLowering::getConstraintType(Constraint); |
Chris Lattner | f4dff84 | 2006-07-11 02:54:03 +0000 | [diff] [blame] | 7733 | } |
| 7734 | |
Dale Johannesen | ba2a0b9 | 2008-01-29 02:21:21 +0000 | [diff] [blame] | 7735 | /// LowerXConstraint - try to replace an X constraint, which matches anything, |
| 7736 | /// with another that has more specific requirements based on the type of the |
| 7737 | /// corresponding operand. |
Chris Lattner | 5e76423 | 2008-04-26 23:02:14 +0000 | [diff] [blame] | 7738 | const char *X86TargetLowering:: |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7739 | LowerXConstraint(MVT ConstraintVT) const { |
Chris Lattner | 5e76423 | 2008-04-26 23:02:14 +0000 | [diff] [blame] | 7740 | // FP X constraints get lowered to SSE1/2 registers if available, otherwise |
| 7741 | // 'f' like normal targets. |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7742 | if (ConstraintVT.isFloatingPoint()) { |
Dale Johannesen | ba2a0b9 | 2008-01-29 02:21:21 +0000 | [diff] [blame] | 7743 | if (Subtarget->hasSSE2()) |
Chris Lattner | 5e76423 | 2008-04-26 23:02:14 +0000 | [diff] [blame] | 7744 | return "Y"; |
| 7745 | if (Subtarget->hasSSE1()) |
| 7746 | return "x"; |
| 7747 | } |
| 7748 | |
| 7749 | return TargetLowering::LowerXConstraint(ConstraintVT); |
Dale Johannesen | ba2a0b9 | 2008-01-29 02:21:21 +0000 | [diff] [blame] | 7750 | } |
| 7751 | |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7752 | /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops |
| 7753 | /// vector. If it is invalid, don't add anything to Ops. |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7754 | void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op, |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7755 | char Constraint, |
Evan Cheng | da43bcf | 2008-09-24 00:05:32 +0000 | [diff] [blame] | 7756 | bool hasMemory, |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7757 | std::vector<SDValue>&Ops, |
Chris Lattner | 5e76423 | 2008-04-26 23:02:14 +0000 | [diff] [blame] | 7758 | SelectionDAG &DAG) const { |
Dan Gohman | 475871a | 2008-07-27 21:46:04 +0000 | [diff] [blame] | 7759 | SDValue Result(0, 0); |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7760 | |
Chris Lattner | 22aaf1d | 2006-10-31 20:13:11 +0000 | [diff] [blame] | 7761 | switch (Constraint) { |
| 7762 | default: break; |
Devang Patel | 84f7fd2 | 2007-03-17 00:13:28 +0000 | [diff] [blame] | 7763 | case 'I': |
Chris Lattner | 188b9fe | 2007-03-25 01:57:35 +0000 | [diff] [blame] | 7764 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 7765 | if (C->getZExtValue() <= 31) { |
| 7766 | Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7767 | break; |
| 7768 | } |
Devang Patel | 84f7fd2 | 2007-03-17 00:13:28 +0000 | [diff] [blame] | 7769 | } |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7770 | return; |
Evan Cheng | 364091e | 2008-09-22 23:57:37 +0000 | [diff] [blame] | 7771 | case 'J': |
| 7772 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { |
| 7773 | if (C->getZExtValue() <= 63) { |
| 7774 | Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); |
| 7775 | break; |
| 7776 | } |
| 7777 | } |
| 7778 | return; |
Chris Lattner | 188b9fe | 2007-03-25 01:57:35 +0000 | [diff] [blame] | 7779 | case 'N': |
| 7780 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 7781 | if (C->getZExtValue() <= 255) { |
| 7782 | Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType()); |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7783 | break; |
| 7784 | } |
Chris Lattner | 188b9fe | 2007-03-25 01:57:35 +0000 | [diff] [blame] | 7785 | } |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7786 | return; |
Chris Lattner | dc43a88 | 2007-05-03 16:52:29 +0000 | [diff] [blame] | 7787 | case 'i': { |
Chris Lattner | 22aaf1d | 2006-10-31 20:13:11 +0000 | [diff] [blame] | 7788 | // Literal immediates are always ok. |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7789 | if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 7790 | Result = DAG.getTargetConstant(CST->getZExtValue(), Op.getValueType()); |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7791 | break; |
| 7792 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7793 | |
Chris Lattner | dc43a88 | 2007-05-03 16:52:29 +0000 | [diff] [blame] | 7794 | // If we are in non-pic codegen mode, we allow the address of a global (with |
| 7795 | // an optional displacement) to be used with 'i'. |
| 7796 | GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op); |
| 7797 | int64_t Offset = 0; |
| 7798 | |
| 7799 | // Match either (GA) or (GA+C) |
| 7800 | if (GA) { |
| 7801 | Offset = GA->getOffset(); |
| 7802 | } else if (Op.getOpcode() == ISD::ADD) { |
| 7803 | ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); |
| 7804 | GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); |
| 7805 | if (C && GA) { |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 7806 | Offset = GA->getOffset()+C->getZExtValue(); |
Chris Lattner | dc43a88 | 2007-05-03 16:52:29 +0000 | [diff] [blame] | 7807 | } else { |
| 7808 | C = dyn_cast<ConstantSDNode>(Op.getOperand(1)); |
| 7809 | GA = dyn_cast<GlobalAddressSDNode>(Op.getOperand(0)); |
| 7810 | if (C && GA) |
Dan Gohman | f5aeb1a | 2008-09-12 16:56:44 +0000 | [diff] [blame] | 7811 | Offset = GA->getOffset()+C->getZExtValue(); |
Chris Lattner | dc43a88 | 2007-05-03 16:52:29 +0000 | [diff] [blame] | 7812 | else |
| 7813 | C = 0, GA = 0; |
| 7814 | } |
| 7815 | } |
| 7816 | |
| 7817 | if (GA) { |
Evan Cheng | da43bcf | 2008-09-24 00:05:32 +0000 | [diff] [blame] | 7818 | if (hasMemory) |
Dan Gohman | 6520e20 | 2008-10-18 02:06:02 +0000 | [diff] [blame] | 7819 | Op = LowerGlobalAddress(GA->getGlobal(), Offset, DAG); |
Evan Cheng | da43bcf | 2008-09-24 00:05:32 +0000 | [diff] [blame] | 7820 | else |
| 7821 | Op = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0), |
| 7822 | Offset); |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7823 | Result = Op; |
| 7824 | break; |
Chris Lattner | 22aaf1d | 2006-10-31 20:13:11 +0000 | [diff] [blame] | 7825 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7826 | |
Chris Lattner | 22aaf1d | 2006-10-31 20:13:11 +0000 | [diff] [blame] | 7827 | // Otherwise, not valid for this mode. |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7828 | return; |
Chris Lattner | 22aaf1d | 2006-10-31 20:13:11 +0000 | [diff] [blame] | 7829 | } |
Chris Lattner | dc43a88 | 2007-05-03 16:52:29 +0000 | [diff] [blame] | 7830 | } |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7831 | |
Gabor Greif | ba36cb5 | 2008-08-28 21:40:38 +0000 | [diff] [blame] | 7832 | if (Result.getNode()) { |
Chris Lattner | 48884cd | 2007-08-25 00:47:38 +0000 | [diff] [blame] | 7833 | Ops.push_back(Result); |
| 7834 | return; |
| 7835 | } |
Evan Cheng | da43bcf | 2008-09-24 00:05:32 +0000 | [diff] [blame] | 7836 | return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, hasMemory, |
| 7837 | Ops, DAG); |
Chris Lattner | 22aaf1d | 2006-10-31 20:13:11 +0000 | [diff] [blame] | 7838 | } |
| 7839 | |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 7840 | std::vector<unsigned> X86TargetLowering:: |
Chris Lattner | 1efa40f | 2006-02-22 00:56:39 +0000 | [diff] [blame] | 7841 | getRegClassForInlineAsmConstraint(const std::string &Constraint, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7842 | MVT VT) const { |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 7843 | if (Constraint.size() == 1) { |
| 7844 | // FIXME: not handling fp-stack yet! |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 7845 | switch (Constraint[0]) { // GCC X86 Constraint Letters |
Chris Lattner | f4dff84 | 2006-07-11 02:54:03 +0000 | [diff] [blame] | 7846 | default: break; // Unknown constraint letter |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 7847 | case 'q': // Q_REGS (GENERAL_REGS in 64-bit mode) |
| 7848 | case 'Q': // Q_REGS |
Chris Lattner | 80a7ecc | 2006-05-06 00:29:37 +0000 | [diff] [blame] | 7849 | if (VT == MVT::i32) |
| 7850 | return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0); |
| 7851 | else if (VT == MVT::i16) |
| 7852 | return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0); |
| 7853 | else if (VT == MVT::i8) |
Evan Cheng | 1291438 | 2007-08-13 23:27:11 +0000 | [diff] [blame] | 7854 | return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0); |
Chris Lattner | 03e6c70 | 2007-11-04 06:51:12 +0000 | [diff] [blame] | 7855 | else if (VT == MVT::i64) |
| 7856 | return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, 0); |
| 7857 | break; |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 7858 | } |
| 7859 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7860 | |
Chris Lattner | 1efa40f | 2006-02-22 00:56:39 +0000 | [diff] [blame] | 7861 | return std::vector<unsigned>(); |
Chris Lattner | 259e97c | 2006-01-31 19:43:35 +0000 | [diff] [blame] | 7862 | } |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 7863 | |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7864 | std::pair<unsigned, const TargetRegisterClass*> |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 7865 | X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint, |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7866 | MVT VT) const { |
Chris Lattner | ad043e8 | 2007-04-09 05:11:28 +0000 | [diff] [blame] | 7867 | // First, see if this is a constraint that directly corresponds to an LLVM |
| 7868 | // register class. |
| 7869 | if (Constraint.size() == 1) { |
| 7870 | // GCC Constraint Letters |
| 7871 | switch (Constraint[0]) { |
| 7872 | default: break; |
Chris Lattner | 0f65cad | 2007-04-09 05:49:22 +0000 | [diff] [blame] | 7873 | case 'r': // GENERAL_REGS |
| 7874 | case 'R': // LEGACY_REGS |
| 7875 | case 'l': // INDEX_REGS |
Chris Lattner | 1fa7198 | 2008-10-17 18:15:05 +0000 | [diff] [blame] | 7876 | if (VT == MVT::i8) |
Chris Lattner | 0f65cad | 2007-04-09 05:49:22 +0000 | [diff] [blame] | 7877 | return std::make_pair(0U, X86::GR8RegisterClass); |
Chris Lattner | 1fa7198 | 2008-10-17 18:15:05 +0000 | [diff] [blame] | 7878 | if (VT == MVT::i16) |
| 7879 | return std::make_pair(0U, X86::GR16RegisterClass); |
| 7880 | if (VT == MVT::i32 || !Subtarget->is64Bit()) |
| 7881 | return std::make_pair(0U, X86::GR32RegisterClass); |
| 7882 | return std::make_pair(0U, X86::GR64RegisterClass); |
Chris Lattner | fce84ac | 2008-03-11 19:06:29 +0000 | [diff] [blame] | 7883 | case 'f': // FP Stack registers. |
| 7884 | // If SSE is enabled for this VT, use f80 to ensure the isel moves the |
| 7885 | // value to the correct fpstack register class. |
| 7886 | if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) |
| 7887 | return std::make_pair(0U, X86::RFP32RegisterClass); |
| 7888 | if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) |
| 7889 | return std::make_pair(0U, X86::RFP64RegisterClass); |
| 7890 | return std::make_pair(0U, X86::RFP80RegisterClass); |
Chris Lattner | 6c284d7 | 2007-04-12 04:14:49 +0000 | [diff] [blame] | 7891 | case 'y': // MMX_REGS if MMX allowed. |
| 7892 | if (!Subtarget->hasMMX()) break; |
| 7893 | return std::make_pair(0U, X86::VR64RegisterClass); |
Chris Lattner | 0f65cad | 2007-04-09 05:49:22 +0000 | [diff] [blame] | 7894 | case 'Y': // SSE_REGS if SSE2 allowed |
| 7895 | if (!Subtarget->hasSSE2()) break; |
| 7896 | // FALL THROUGH. |
| 7897 | case 'x': // SSE_REGS if SSE1 allowed |
| 7898 | if (!Subtarget->hasSSE1()) break; |
Duncan Sands | 83ec4b6 | 2008-06-06 12:08:01 +0000 | [diff] [blame] | 7899 | |
| 7900 | switch (VT.getSimpleVT()) { |
Chris Lattner | 0f65cad | 2007-04-09 05:49:22 +0000 | [diff] [blame] | 7901 | default: break; |
| 7902 | // Scalar SSE types. |
| 7903 | case MVT::f32: |
| 7904 | case MVT::i32: |
Chris Lattner | ad043e8 | 2007-04-09 05:11:28 +0000 | [diff] [blame] | 7905 | return std::make_pair(0U, X86::FR32RegisterClass); |
Chris Lattner | 0f65cad | 2007-04-09 05:49:22 +0000 | [diff] [blame] | 7906 | case MVT::f64: |
| 7907 | case MVT::i64: |
Chris Lattner | ad043e8 | 2007-04-09 05:11:28 +0000 | [diff] [blame] | 7908 | return std::make_pair(0U, X86::FR64RegisterClass); |
Chris Lattner | 0f65cad | 2007-04-09 05:49:22 +0000 | [diff] [blame] | 7909 | // Vector types. |
Chris Lattner | 0f65cad | 2007-04-09 05:49:22 +0000 | [diff] [blame] | 7910 | case MVT::v16i8: |
| 7911 | case MVT::v8i16: |
| 7912 | case MVT::v4i32: |
| 7913 | case MVT::v2i64: |
| 7914 | case MVT::v4f32: |
| 7915 | case MVT::v2f64: |
| 7916 | return std::make_pair(0U, X86::VR128RegisterClass); |
| 7917 | } |
Chris Lattner | ad043e8 | 2007-04-09 05:11:28 +0000 | [diff] [blame] | 7918 | break; |
| 7919 | } |
| 7920 | } |
| 7921 | |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 7922 | // Use the default implementation in TargetLowering to convert the register |
| 7923 | // constraint into a member of a register class. |
| 7924 | std::pair<unsigned, const TargetRegisterClass*> Res; |
| 7925 | Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); |
Chris Lattner | 1a60aa7 | 2006-10-31 19:42:44 +0000 | [diff] [blame] | 7926 | |
| 7927 | // Not found as a standard register? |
| 7928 | if (Res.second == 0) { |
| 7929 | // GCC calls "st(0)" just plain "st". |
| 7930 | if (StringsEqualNoCase("{st}", Constraint)) { |
| 7931 | Res.first = X86::ST0; |
Chris Lattner | 9b4baf1 | 2007-09-24 05:27:37 +0000 | [diff] [blame] | 7932 | Res.second = X86::RFP80RegisterClass; |
Chris Lattner | 1a60aa7 | 2006-10-31 19:42:44 +0000 | [diff] [blame] | 7933 | } |
Dale Johannesen | 330169f | 2008-11-13 21:52:36 +0000 | [diff] [blame] | 7934 | // 'A' means EAX + EDX. |
| 7935 | if (Constraint == "A") { |
| 7936 | Res.first = X86::EAX; |
| 7937 | Res.second = X86::GRADRegisterClass; |
| 7938 | } |
Chris Lattner | 1a60aa7 | 2006-10-31 19:42:44 +0000 | [diff] [blame] | 7939 | return Res; |
| 7940 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7941 | |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 7942 | // Otherwise, check to see if this is a register class of the wrong value |
| 7943 | // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to |
| 7944 | // turn into {ax},{dx}. |
| 7945 | if (Res.second->hasType(VT)) |
| 7946 | return Res; // Correct type already, nothing to do. |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 7947 | |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 7948 | // All of the single-register GCC register classes map their values onto |
| 7949 | // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we |
| 7950 | // really want an 8-bit or 32-bit register, map to the appropriate register |
| 7951 | // class and return the appropriate register. |
Chris Lattner | 6ba50a9 | 2008-08-26 06:19:02 +0000 | [diff] [blame] | 7952 | if (Res.second == X86::GR16RegisterClass) { |
| 7953 | if (VT == MVT::i8) { |
| 7954 | unsigned DestReg = 0; |
| 7955 | switch (Res.first) { |
| 7956 | default: break; |
| 7957 | case X86::AX: DestReg = X86::AL; break; |
| 7958 | case X86::DX: DestReg = X86::DL; break; |
| 7959 | case X86::CX: DestReg = X86::CL; break; |
| 7960 | case X86::BX: DestReg = X86::BL; break; |
| 7961 | } |
| 7962 | if (DestReg) { |
| 7963 | Res.first = DestReg; |
| 7964 | Res.second = Res.second = X86::GR8RegisterClass; |
| 7965 | } |
| 7966 | } else if (VT == MVT::i32) { |
| 7967 | unsigned DestReg = 0; |
| 7968 | switch (Res.first) { |
| 7969 | default: break; |
| 7970 | case X86::AX: DestReg = X86::EAX; break; |
| 7971 | case X86::DX: DestReg = X86::EDX; break; |
| 7972 | case X86::CX: DestReg = X86::ECX; break; |
| 7973 | case X86::BX: DestReg = X86::EBX; break; |
| 7974 | case X86::SI: DestReg = X86::ESI; break; |
| 7975 | case X86::DI: DestReg = X86::EDI; break; |
| 7976 | case X86::BP: DestReg = X86::EBP; break; |
| 7977 | case X86::SP: DestReg = X86::ESP; break; |
| 7978 | } |
| 7979 | if (DestReg) { |
| 7980 | Res.first = DestReg; |
| 7981 | Res.second = Res.second = X86::GR32RegisterClass; |
| 7982 | } |
| 7983 | } else if (VT == MVT::i64) { |
| 7984 | unsigned DestReg = 0; |
| 7985 | switch (Res.first) { |
| 7986 | default: break; |
| 7987 | case X86::AX: DestReg = X86::RAX; break; |
| 7988 | case X86::DX: DestReg = X86::RDX; break; |
| 7989 | case X86::CX: DestReg = X86::RCX; break; |
| 7990 | case X86::BX: DestReg = X86::RBX; break; |
| 7991 | case X86::SI: DestReg = X86::RSI; break; |
| 7992 | case X86::DI: DestReg = X86::RDI; break; |
| 7993 | case X86::BP: DestReg = X86::RBP; break; |
| 7994 | case X86::SP: DestReg = X86::RSP; break; |
| 7995 | } |
| 7996 | if (DestReg) { |
| 7997 | Res.first = DestReg; |
| 7998 | Res.second = Res.second = X86::GR64RegisterClass; |
| 7999 | } |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 8000 | } |
Chris Lattner | 6ba50a9 | 2008-08-26 06:19:02 +0000 | [diff] [blame] | 8001 | } else if (Res.second == X86::FR32RegisterClass || |
| 8002 | Res.second == X86::FR64RegisterClass || |
| 8003 | Res.second == X86::VR128RegisterClass) { |
| 8004 | // Handle references to XMM physical registers that got mapped into the |
| 8005 | // wrong class. This can happen with constraints like {xmm0} where the |
| 8006 | // target independent register mapper will just pick the first match it can |
| 8007 | // find, ignoring the required type. |
| 8008 | if (VT == MVT::f32) |
| 8009 | Res.second = X86::FR32RegisterClass; |
| 8010 | else if (VT == MVT::f64) |
| 8011 | Res.second = X86::FR64RegisterClass; |
| 8012 | else if (X86::VR128RegisterClass->hasType(VT)) |
| 8013 | Res.second = X86::VR128RegisterClass; |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 8014 | } |
Anton Korobeynikov | 12c49af | 2006-11-21 00:01:06 +0000 | [diff] [blame] | 8015 | |
Chris Lattner | f76d180 | 2006-07-31 23:26:50 +0000 | [diff] [blame] | 8016 | return Res; |
| 8017 | } |
Mon P Wang | 0c39719 | 2008-10-30 08:01:45 +0000 | [diff] [blame] | 8018 | |
| 8019 | //===----------------------------------------------------------------------===// |
| 8020 | // X86 Widen vector type |
| 8021 | //===----------------------------------------------------------------------===// |
| 8022 | |
| 8023 | /// getWidenVectorType: given a vector type, returns the type to widen |
| 8024 | /// to (e.g., v7i8 to v8i8). If the vector type is legal, it returns itself. |
| 8025 | /// If there is no vector type that we want to widen to, returns MVT::Other |
Mon P Wang | f007a8b | 2008-11-06 05:31:54 +0000 | [diff] [blame] | 8026 | /// When and where to widen is target dependent based on the cost of |
Mon P Wang | 0c39719 | 2008-10-30 08:01:45 +0000 | [diff] [blame] | 8027 | /// scalarizing vs using the wider vector type. |
| 8028 | |
| 8029 | MVT X86TargetLowering::getWidenVectorType(MVT VT) { |
| 8030 | assert(VT.isVector()); |
| 8031 | if (isTypeLegal(VT)) |
| 8032 | return VT; |
| 8033 | |
| 8034 | // TODO: In computeRegisterProperty, we can compute the list of legal vector |
| 8035 | // type based on element type. This would speed up our search (though |
| 8036 | // it may not be worth it since the size of the list is relatively |
| 8037 | // small). |
| 8038 | MVT EltVT = VT.getVectorElementType(); |
| 8039 | unsigned NElts = VT.getVectorNumElements(); |
| 8040 | |
| 8041 | // On X86, it make sense to widen any vector wider than 1 |
| 8042 | if (NElts <= 1) |
| 8043 | return MVT::Other; |
| 8044 | |
| 8045 | for (unsigned nVT = MVT::FIRST_VECTOR_VALUETYPE; |
| 8046 | nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) { |
| 8047 | MVT SVT = (MVT::SimpleValueType)nVT; |
| 8048 | |
| 8049 | if (isTypeLegal(SVT) && |
| 8050 | SVT.getVectorElementType() == EltVT && |
| 8051 | SVT.getVectorNumElements() > NElts) |
| 8052 | return SVT; |
| 8053 | } |
| 8054 | return MVT::Other; |
| 8055 | } |