Anders Carlsson | 022012e | 2007-08-20 18:05:56 +0000 | [diff] [blame] | 1 | //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
Chris Lattner | 0bc735f | 2007-12-29 19:59:25 +0000 | [diff] [blame] | 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
Anders Carlsson | 022012e | 2007-08-20 18:05:56 +0000 | [diff] [blame] | 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This contains code to emit Builtin calls as LLVM code. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "CodeGenFunction.h" |
Fariborz Jahanian | 55bcace | 2010-06-15 22:44:06 +0000 | [diff] [blame] | 15 | #include "CGObjCRuntime.h" |
Chandler Carruth | 55fc873 | 2012-12-04 09:13:33 +0000 | [diff] [blame] | 16 | #include "CodeGenModule.h" |
| 17 | #include "TargetInfo.h" |
Chris Lattner | bef20ac | 2007-08-31 04:31:45 +0000 | [diff] [blame] | 18 | #include "clang/AST/ASTContext.h" |
Daniel Dunbar | c4a1dea | 2008-08-11 05:35:13 +0000 | [diff] [blame] | 19 | #include "clang/AST/Decl.h" |
Chris Lattner | 6b15cdc | 2009-06-14 01:05:48 +0000 | [diff] [blame] | 20 | #include "clang/Basic/TargetBuiltins.h" |
Chandler Carruth | 55fc873 | 2012-12-04 09:13:33 +0000 | [diff] [blame] | 21 | #include "clang/Basic/TargetInfo.h" |
Mark Lacey | 8b54999 | 2013-10-30 21:53:58 +0000 | [diff] [blame] | 22 | #include "clang/CodeGen/CGFunctionInfo.h" |
Chandler Carruth | 3b844ba | 2013-01-02 11:45:17 +0000 | [diff] [blame] | 23 | #include "llvm/IR/DataLayout.h" |
| 24 | #include "llvm/IR/Intrinsics.h" |
Jakub Staszak | 558229f | 2011-07-08 22:45:14 +0000 | [diff] [blame] | 25 | |
Anders Carlsson | 022012e | 2007-08-20 18:05:56 +0000 | [diff] [blame] | 26 | using namespace clang; |
| 27 | using namespace CodeGen; |
Anders Carlsson | ca6fcfa | 2007-12-09 21:20:04 +0000 | [diff] [blame] | 28 | using namespace llvm; |
| 29 | |
John McCall | a45680b | 2011-09-13 23:05:03 +0000 | [diff] [blame] | 30 | /// getBuiltinLibFunction - Given a builtin id for a function like |
| 31 | /// "__builtin_fabsf", return a Function* for "fabsf". |
| 32 | llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD, |
| 33 | unsigned BuiltinID) { |
| 34 | assert(Context.BuiltinInfo.isLibFunction(BuiltinID)); |
| 35 | |
| 36 | // Get the name, skip over the __builtin_ prefix (if necessary). |
| 37 | StringRef Name; |
| 38 | GlobalDecl D(FD); |
| 39 | |
| 40 | // If the builtin has been declared explicitly with an assembler label, |
| 41 | // use the mangled name. This differs from the plain label on platforms |
| 42 | // that prefix labels. |
| 43 | if (FD->hasAttr<AsmLabelAttr>()) |
| 44 | Name = getMangledName(D); |
| 45 | else |
| 46 | Name = Context.BuiltinInfo.GetName(BuiltinID) + 10; |
| 47 | |
| 48 | llvm::FunctionType *Ty = |
| 49 | cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType())); |
| 50 | |
| 51 | return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false); |
| 52 | } |
| 53 | |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 54 | /// Emit the conversions required to turn the given value into an |
| 55 | /// integer of the given size. |
| 56 | static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V, |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 57 | QualType T, llvm::IntegerType *IntType) { |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 58 | V = CGF.EmitToMemory(V, T); |
Chris Lattner | a1aa9e3 | 2010-10-01 23:43:16 +0000 | [diff] [blame] | 59 | |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 60 | if (V->getType()->isPointerTy()) |
| 61 | return CGF.Builder.CreatePtrToInt(V, IntType); |
| 62 | |
| 63 | assert(V->getType() == IntType); |
| 64 | return V; |
Chandler Carruth | db4325b | 2010-07-18 07:23:17 +0000 | [diff] [blame] | 65 | } |
| 66 | |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 67 | static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 68 | QualType T, llvm::Type *ResultType) { |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 69 | V = CGF.EmitFromMemory(V, T); |
| 70 | |
| 71 | if (ResultType->isPointerTy()) |
| 72 | return CGF.Builder.CreateIntToPtr(V, ResultType); |
| 73 | |
| 74 | assert(V->getType() == ResultType); |
| 75 | return V; |
Chandler Carruth | db4325b | 2010-07-18 07:23:17 +0000 | [diff] [blame] | 76 | } |
| 77 | |
Daniel Dunbar | 0002d23 | 2009-04-07 00:55:51 +0000 | [diff] [blame] | 78 | /// Utility to insert an atomic instruction based on Instrinsic::ID |
| 79 | /// and the expression node. |
Daniel Dunbar | cb61a7b | 2010-03-20 07:04:11 +0000 | [diff] [blame] | 80 | static RValue EmitBinaryAtomic(CodeGenFunction &CGF, |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 81 | llvm::AtomicRMWInst::BinOp Kind, |
| 82 | const CallExpr *E) { |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 83 | QualType T = E->getType(); |
| 84 | assert(E->getArg(0)->getType()->isPointerType()); |
| 85 | assert(CGF.getContext().hasSameUnqualifiedType(T, |
| 86 | E->getArg(0)->getType()->getPointeeType())); |
| 87 | assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); |
| 88 | |
Chris Lattner | 4f20944 | 2010-09-21 23:40:48 +0000 | [diff] [blame] | 89 | llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); |
Micah Villmow | 956a5a1 | 2012-10-25 15:39:14 +0000 | [diff] [blame] | 90 | unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); |
John McCall | 789a159 | 2010-10-26 22:09:15 +0000 | [diff] [blame] | 91 | |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 92 | llvm::IntegerType *IntType = |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 93 | llvm::IntegerType::get(CGF.getLLVMContext(), |
| 94 | CGF.getContext().getTypeSize(T)); |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 95 | llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 96 | |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 97 | llvm::Value *Args[2]; |
| 98 | Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); |
| 99 | Args[1] = CGF.EmitScalarExpr(E->getArg(1)); |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 100 | llvm::Type *ValueType = Args[1]->getType(); |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 101 | Args[1] = EmitToInt(CGF, Args[1], T, IntType); |
| 102 | |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 103 | llvm::Value *Result = |
| 104 | CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1], |
| 105 | llvm::SequentiallyConsistent); |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 106 | Result = EmitFromInt(CGF, Result, T, ValueType); |
| 107 | return RValue::get(Result); |
Daniel Dunbar | 0002d23 | 2009-04-07 00:55:51 +0000 | [diff] [blame] | 108 | } |
| 109 | |
| 110 | /// Utility to insert an atomic instruction based Instrinsic::ID and |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 111 | /// the expression node, where the return value is the result of the |
| 112 | /// operation. |
Chris Lattner | 420b118 | 2010-05-06 05:35:16 +0000 | [diff] [blame] | 113 | static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 114 | llvm::AtomicRMWInst::BinOp Kind, |
| 115 | const CallExpr *E, |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 116 | Instruction::BinaryOps Op, |
| 117 | bool Invert = false) { |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 118 | QualType T = E->getType(); |
| 119 | assert(E->getArg(0)->getType()->isPointerType()); |
| 120 | assert(CGF.getContext().hasSameUnqualifiedType(T, |
| 121 | E->getArg(0)->getType()->getPointeeType())); |
| 122 | assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); |
| 123 | |
Chris Lattner | 4f20944 | 2010-09-21 23:40:48 +0000 | [diff] [blame] | 124 | llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0)); |
Micah Villmow | 956a5a1 | 2012-10-25 15:39:14 +0000 | [diff] [blame] | 125 | unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); |
John McCall | 789a159 | 2010-10-26 22:09:15 +0000 | [diff] [blame] | 126 | |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 127 | llvm::IntegerType *IntType = |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 128 | llvm::IntegerType::get(CGF.getLLVMContext(), |
| 129 | CGF.getContext().getTypeSize(T)); |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 130 | llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 131 | |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 132 | llvm::Value *Args[2]; |
| 133 | Args[1] = CGF.EmitScalarExpr(E->getArg(1)); |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 134 | llvm::Type *ValueType = Args[1]->getType(); |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 135 | Args[1] = EmitToInt(CGF, Args[1], T, IntType); |
| 136 | Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType); |
| 137 | |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 138 | llvm::Value *Result = |
| 139 | CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1], |
| 140 | llvm::SequentiallyConsistent); |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 141 | Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 142 | if (Invert) |
| 143 | Result = CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result, |
| 144 | llvm::ConstantInt::get(IntType, -1)); |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 145 | Result = EmitFromInt(CGF, Result, T, ValueType); |
| 146 | return RValue::get(Result); |
Mon P Wang | 1ffe281 | 2008-05-09 22:40:52 +0000 | [diff] [blame] | 147 | } |
| 148 | |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 149 | /// EmitFAbs - Emit a call to @llvm.fabs(). |
| 150 | static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) { |
| 151 | Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType()); |
| 152 | llvm::CallInst *Call = CGF.Builder.CreateCall(F, V); |
| 153 | Call->setDoesNotAccessMemory(); |
| 154 | return Call; |
Chris Lattner | 420b118 | 2010-05-06 05:35:16 +0000 | [diff] [blame] | 155 | } |
| 156 | |
John McCall | a45680b | 2011-09-13 23:05:03 +0000 | [diff] [blame] | 157 | static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, |
| 158 | const CallExpr *E, llvm::Value *calleeValue) { |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 159 | return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E, |
| 160 | ReturnValueSlot(), Fn); |
John McCall | a45680b | 2011-09-13 23:05:03 +0000 | [diff] [blame] | 161 | } |
| 162 | |
Michael Gottesman | 0cf07bc | 2013-01-13 02:22:39 +0000 | [diff] [blame] | 163 | /// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.* |
| 164 | /// depending on IntrinsicID. |
| 165 | /// |
| 166 | /// \arg CGF The current codegen function. |
| 167 | /// \arg IntrinsicID The ID for the Intrinsic we wish to generate. |
| 168 | /// \arg X The first argument to the llvm.*.with.overflow.*. |
| 169 | /// \arg Y The second argument to the llvm.*.with.overflow.*. |
| 170 | /// \arg Carry The carry returned by the llvm.*.with.overflow.*. |
| 171 | /// \returns The result (i.e. sum/product) returned by the intrinsic. |
| 172 | static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, |
| 173 | const llvm::Intrinsic::ID IntrinsicID, |
| 174 | llvm::Value *X, llvm::Value *Y, |
| 175 | llvm::Value *&Carry) { |
| 176 | // Make sure we have integers of the same width. |
| 177 | assert(X->getType() == Y->getType() && |
| 178 | "Arguments must be the same type. (Did you forget to make sure both " |
| 179 | "arguments have the same integer width?)"); |
| 180 | |
NAKAMURA Takumi | 563fb90 | 2013-01-13 11:26:44 +0000 | [diff] [blame] | 181 | llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType()); |
Michael Gottesman | 0cf07bc | 2013-01-13 02:22:39 +0000 | [diff] [blame] | 182 | llvm::Value *Tmp = CGF.Builder.CreateCall2(Callee, X, Y); |
| 183 | Carry = CGF.Builder.CreateExtractValue(Tmp, 1); |
| 184 | return CGF.Builder.CreateExtractValue(Tmp, 0); |
| 185 | } |
| 186 | |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 187 | RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, |
Daniel Dunbar | ef2abfe | 2009-02-16 22:43:43 +0000 | [diff] [blame] | 188 | unsigned BuiltinID, const CallExpr *E) { |
Chris Lattner | 564ea2a | 2008-10-06 06:56:41 +0000 | [diff] [blame] | 189 | // See if we can constant fold this builtin. If so, don't emit it at all. |
Anders Carlsson | f35d35a | 2008-12-01 02:31:41 +0000 | [diff] [blame] | 190 | Expr::EvalResult Result; |
Eli Friedman | 52a27f5 | 2012-01-06 20:03:09 +0000 | [diff] [blame] | 191 | if (E->EvaluateAsRValue(Result, CGM.getContext()) && |
Fariborz Jahanian | dd697bc | 2011-04-25 23:10:07 +0000 | [diff] [blame] | 192 | !Result.hasSideEffects()) { |
Anders Carlsson | f35d35a | 2008-12-01 02:31:41 +0000 | [diff] [blame] | 193 | if (Result.Val.isInt()) |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 194 | return RValue::get(llvm::ConstantInt::get(getLLVMContext(), |
Owen Anderson | 4a28d5d | 2009-07-24 23:12:58 +0000 | [diff] [blame] | 195 | Result.Val.getInt())); |
Chris Lattner | a1aa9e3 | 2010-10-01 23:43:16 +0000 | [diff] [blame] | 196 | if (Result.Val.isFloat()) |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 197 | return RValue::get(llvm::ConstantFP::get(getLLVMContext(), |
| 198 | Result.Val.getFloat())); |
Chris Lattner | 1f32999 | 2008-10-06 06:09:18 +0000 | [diff] [blame] | 199 | } |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 200 | |
Chris Lattner | 564ea2a | 2008-10-06 06:56:41 +0000 | [diff] [blame] | 201 | switch (BuiltinID) { |
| 202 | default: break; // Handle intrinsics and libm functions below. |
Chris Lattner | 506ff88 | 2008-10-06 07:26:43 +0000 | [diff] [blame] | 203 | case Builtin::BI__builtin___CFStringMakeConstantString: |
David Chisnall | 0d13f6f | 2010-01-23 02:40:42 +0000 | [diff] [blame] | 204 | case Builtin::BI__builtin___NSStringMakeConstantString: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 205 | return RValue::get(CGM.EmitConstantExpr(E, E->getType(), nullptr)); |
Chris Lattner | 6a705f0 | 2008-07-09 17:28:44 +0000 | [diff] [blame] | 206 | case Builtin::BI__builtin_stdarg_start: |
Anders Carlsson | 793680e | 2007-10-12 23:56:29 +0000 | [diff] [blame] | 207 | case Builtin::BI__builtin_va_start: |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 208 | case Builtin::BI__va_start: |
Anders Carlsson | 793680e | 2007-10-12 23:56:29 +0000 | [diff] [blame] | 209 | case Builtin::BI__builtin_va_end: { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 210 | Value *ArgValue = (BuiltinID == Builtin::BI__va_start) |
| 211 | ? EmitScalarExpr(E->getArg(0)) |
| 212 | : EmitVAListRef(E->getArg(0)); |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 213 | llvm::Type *DestType = Int8PtrTy; |
Anders Carlsson | 793680e | 2007-10-12 23:56:29 +0000 | [diff] [blame] | 214 | if (ArgValue->getType() != DestType) |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 215 | ArgValue = Builder.CreateBitCast(ArgValue, DestType, |
Daniel Dunbar | b27ffbe | 2009-07-26 09:28:40 +0000 | [diff] [blame] | 216 | ArgValue->getName().data()); |
Anders Carlsson | 793680e | 2007-10-12 23:56:29 +0000 | [diff] [blame] | 217 | |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 218 | Intrinsic::ID inst = (BuiltinID == Builtin::BI__builtin_va_end) ? |
Chris Lattner | 6a705f0 | 2008-07-09 17:28:44 +0000 | [diff] [blame] | 219 | Intrinsic::vaend : Intrinsic::vastart; |
Chris Lattner | 7acda7c | 2007-12-18 00:25:38 +0000 | [diff] [blame] | 220 | return RValue::get(Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue)); |
Anders Carlsson | 793680e | 2007-10-12 23:56:29 +0000 | [diff] [blame] | 221 | } |
Anders Carlsson | a28ef8b | 2008-02-09 20:26:43 +0000 | [diff] [blame] | 222 | case Builtin::BI__builtin_va_copy: { |
Eli Friedman | 4fd0aa5 | 2009-01-20 17:46:04 +0000 | [diff] [blame] | 223 | Value *DstPtr = EmitVAListRef(E->getArg(0)); |
| 224 | Value *SrcPtr = EmitVAListRef(E->getArg(1)); |
Anders Carlsson | a28ef8b | 2008-02-09 20:26:43 +0000 | [diff] [blame] | 225 | |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 226 | llvm::Type *Type = Int8PtrTy; |
Anders Carlsson | a28ef8b | 2008-02-09 20:26:43 +0000 | [diff] [blame] | 227 | |
| 228 | DstPtr = Builder.CreateBitCast(DstPtr, Type); |
| 229 | SrcPtr = Builder.CreateBitCast(SrcPtr, Type); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 230 | return RValue::get(Builder.CreateCall2(CGM.getIntrinsic(Intrinsic::vacopy), |
Chris Lattner | 3eae03e | 2008-05-06 00:56:42 +0000 | [diff] [blame] | 231 | DstPtr, SrcPtr)); |
Anders Carlsson | a28ef8b | 2008-02-09 20:26:43 +0000 | [diff] [blame] | 232 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 233 | case Builtin::BI__builtin_abs: |
Eli Friedman | f7b2d8b | 2012-01-17 22:11:30 +0000 | [diff] [blame] | 234 | case Builtin::BI__builtin_labs: |
| 235 | case Builtin::BI__builtin_llabs: { |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 236 | Value *ArgValue = EmitScalarExpr(E->getArg(0)); |
| 237 | |
Chris Lattner | 9a847f5 | 2008-07-23 06:53:34 +0000 | [diff] [blame] | 238 | Value *NegOp = Builder.CreateNeg(ArgValue, "neg"); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 239 | Value *CmpResult = |
| 240 | Builder.CreateICmpSGE(ArgValue, |
Owen Anderson | c9c88b4 | 2009-07-31 20:28:54 +0000 | [diff] [blame] | 241 | llvm::Constant::getNullValue(ArgValue->getType()), |
Chris Lattner | 9a847f5 | 2008-07-23 06:53:34 +0000 | [diff] [blame] | 242 | "abscond"); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 243 | Value *Result = |
Anders Carlsson | c2251dc | 2007-11-20 19:05:17 +0000 | [diff] [blame] | 244 | Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs"); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 245 | |
Anders Carlsson | c2251dc | 2007-11-20 19:05:17 +0000 | [diff] [blame] | 246 | return RValue::get(Result); |
| 247 | } |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 248 | case Builtin::BI__builtin_fabs: |
| 249 | case Builtin::BI__builtin_fabsf: |
| 250 | case Builtin::BI__builtin_fabsl: { |
| 251 | Value *Arg1 = EmitScalarExpr(E->getArg(0)); |
| 252 | Value *Result = EmitFAbs(*this, Arg1); |
| 253 | return RValue::get(Result); |
| 254 | } |
| 255 | case Builtin::BI__builtin_fmod: |
| 256 | case Builtin::BI__builtin_fmodf: |
| 257 | case Builtin::BI__builtin_fmodl: { |
| 258 | Value *Arg1 = EmitScalarExpr(E->getArg(0)); |
| 259 | Value *Arg2 = EmitScalarExpr(E->getArg(1)); |
| 260 | Value *Result = Builder.CreateFRem(Arg1, Arg2, "fmod"); |
| 261 | return RValue::get(Result); |
| 262 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 263 | |
Fariborz Jahanian | ce23bb7 | 2012-08-14 20:09:28 +0000 | [diff] [blame] | 264 | case Builtin::BI__builtin_conj: |
| 265 | case Builtin::BI__builtin_conjf: |
| 266 | case Builtin::BI__builtin_conjl: { |
| 267 | ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); |
| 268 | Value *Real = ComplexVal.first; |
| 269 | Value *Imag = ComplexVal.second; |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 270 | Value *Zero = |
| 271 | Imag->getType()->isFPOrFPVectorTy() |
Fariborz Jahanian | ce23bb7 | 2012-08-14 20:09:28 +0000 | [diff] [blame] | 272 | ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType()) |
| 273 | : llvm::Constant::getNullValue(Imag->getType()); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 274 | |
Fariborz Jahanian | ce23bb7 | 2012-08-14 20:09:28 +0000 | [diff] [blame] | 275 | Imag = Builder.CreateFSub(Zero, Imag, "sub"); |
| 276 | return RValue::getComplex(std::make_pair(Real, Imag)); |
| 277 | } |
| 278 | case Builtin::BI__builtin_creal: |
| 279 | case Builtin::BI__builtin_crealf: |
Meador Inge | 08cc03f | 2012-12-18 20:58:04 +0000 | [diff] [blame] | 280 | case Builtin::BI__builtin_creall: |
| 281 | case Builtin::BIcreal: |
| 282 | case Builtin::BIcrealf: |
| 283 | case Builtin::BIcreall: { |
Fariborz Jahanian | ce23bb7 | 2012-08-14 20:09:28 +0000 | [diff] [blame] | 284 | ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); |
| 285 | return RValue::get(ComplexVal.first); |
| 286 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 287 | |
Fariborz Jahanian | ce23bb7 | 2012-08-14 20:09:28 +0000 | [diff] [blame] | 288 | case Builtin::BI__builtin_cimag: |
| 289 | case Builtin::BI__builtin_cimagf: |
Meador Inge | 08cc03f | 2012-12-18 20:58:04 +0000 | [diff] [blame] | 290 | case Builtin::BI__builtin_cimagl: |
| 291 | case Builtin::BIcimag: |
| 292 | case Builtin::BIcimagf: |
| 293 | case Builtin::BIcimagl: { |
Fariborz Jahanian | ce23bb7 | 2012-08-14 20:09:28 +0000 | [diff] [blame] | 294 | ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0)); |
| 295 | return RValue::get(ComplexVal.second); |
| 296 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 297 | |
Benjamin Kramer | a49a283 | 2012-01-28 18:42:57 +0000 | [diff] [blame] | 298 | case Builtin::BI__builtin_ctzs: |
Anders Carlsson | 3a31d60 | 2008-02-06 07:19:27 +0000 | [diff] [blame] | 299 | case Builtin::BI__builtin_ctz: |
| 300 | case Builtin::BI__builtin_ctzl: |
| 301 | case Builtin::BI__builtin_ctzll: { |
| 302 | Value *ArgValue = EmitScalarExpr(E->getArg(0)); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 303 | |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 304 | llvm::Type *ArgType = ArgValue->getType(); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 305 | Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); |
Anders Carlsson | 3a31d60 | 2008-02-06 07:19:27 +0000 | [diff] [blame] | 306 | |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 307 | llvm::Type *ResultType = ConvertType(E->getType()); |
John McCall | 64aa4b3 | 2013-04-16 22:48:15 +0000 | [diff] [blame] | 308 | Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); |
Bob Wilson | 8b30a93 | 2012-01-26 22:14:27 +0000 | [diff] [blame] | 309 | Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); |
Anders Carlsson | 3a31d60 | 2008-02-06 07:19:27 +0000 | [diff] [blame] | 310 | if (Result->getType() != ResultType) |
Duncan Sands | eac73e5 | 2009-11-16 13:11:21 +0000 | [diff] [blame] | 311 | Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, |
| 312 | "cast"); |
Anders Carlsson | 3a31d60 | 2008-02-06 07:19:27 +0000 | [diff] [blame] | 313 | return RValue::get(Result); |
| 314 | } |
Benjamin Kramer | a49a283 | 2012-01-28 18:42:57 +0000 | [diff] [blame] | 315 | case Builtin::BI__builtin_clzs: |
Eli Friedman | f4e8533 | 2008-05-27 15:32:46 +0000 | [diff] [blame] | 316 | case Builtin::BI__builtin_clz: |
| 317 | case Builtin::BI__builtin_clzl: |
| 318 | case Builtin::BI__builtin_clzll: { |
| 319 | Value *ArgValue = EmitScalarExpr(E->getArg(0)); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 320 | |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 321 | llvm::Type *ArgType = ArgValue->getType(); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 322 | Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); |
Eli Friedman | f4e8533 | 2008-05-27 15:32:46 +0000 | [diff] [blame] | 323 | |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 324 | llvm::Type *ResultType = ConvertType(E->getType()); |
John McCall | 64aa4b3 | 2013-04-16 22:48:15 +0000 | [diff] [blame] | 325 | Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); |
Bob Wilson | 8b30a93 | 2012-01-26 22:14:27 +0000 | [diff] [blame] | 326 | Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); |
Eli Friedman | f4e8533 | 2008-05-27 15:32:46 +0000 | [diff] [blame] | 327 | if (Result->getType() != ResultType) |
Duncan Sands | eac73e5 | 2009-11-16 13:11:21 +0000 | [diff] [blame] | 328 | Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, |
| 329 | "cast"); |
Eli Friedman | f4e8533 | 2008-05-27 15:32:46 +0000 | [diff] [blame] | 330 | return RValue::get(Result); |
| 331 | } |
Daniel Dunbar | 04b2900 | 2008-07-21 17:19:41 +0000 | [diff] [blame] | 332 | case Builtin::BI__builtin_ffs: |
| 333 | case Builtin::BI__builtin_ffsl: |
| 334 | case Builtin::BI__builtin_ffsll: { |
| 335 | // ffs(x) -> x ? cttz(x) + 1 : 0 |
| 336 | Value *ArgValue = EmitScalarExpr(E->getArg(0)); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 337 | |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 338 | llvm::Type *ArgType = ArgValue->getType(); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 339 | Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 340 | |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 341 | llvm::Type *ResultType = ConvertType(E->getType()); |
Chandler Carruth | 50058ec | 2011-12-12 04:28:35 +0000 | [diff] [blame] | 342 | Value *Tmp = Builder.CreateAdd(Builder.CreateCall2(F, ArgValue, |
| 343 | Builder.getTrue()), |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 344 | llvm::ConstantInt::get(ArgType, 1)); |
Owen Anderson | c9c88b4 | 2009-07-31 20:28:54 +0000 | [diff] [blame] | 345 | Value *Zero = llvm::Constant::getNullValue(ArgType); |
Daniel Dunbar | 04b2900 | 2008-07-21 17:19:41 +0000 | [diff] [blame] | 346 | Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); |
| 347 | Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs"); |
| 348 | if (Result->getType() != ResultType) |
Duncan Sands | eac73e5 | 2009-11-16 13:11:21 +0000 | [diff] [blame] | 349 | Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, |
| 350 | "cast"); |
Daniel Dunbar | 04b2900 | 2008-07-21 17:19:41 +0000 | [diff] [blame] | 351 | return RValue::get(Result); |
| 352 | } |
| 353 | case Builtin::BI__builtin_parity: |
| 354 | case Builtin::BI__builtin_parityl: |
| 355 | case Builtin::BI__builtin_parityll: { |
| 356 | // parity(x) -> ctpop(x) & 1 |
| 357 | Value *ArgValue = EmitScalarExpr(E->getArg(0)); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 358 | |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 359 | llvm::Type *ArgType = ArgValue->getType(); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 360 | Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 361 | |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 362 | llvm::Type *ResultType = ConvertType(E->getType()); |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 363 | Value *Tmp = Builder.CreateCall(F, ArgValue); |
| 364 | Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1)); |
Daniel Dunbar | 04b2900 | 2008-07-21 17:19:41 +0000 | [diff] [blame] | 365 | if (Result->getType() != ResultType) |
Duncan Sands | eac73e5 | 2009-11-16 13:11:21 +0000 | [diff] [blame] | 366 | Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, |
| 367 | "cast"); |
Daniel Dunbar | 04b2900 | 2008-07-21 17:19:41 +0000 | [diff] [blame] | 368 | return RValue::get(Result); |
| 369 | } |
| 370 | case Builtin::BI__builtin_popcount: |
| 371 | case Builtin::BI__builtin_popcountl: |
| 372 | case Builtin::BI__builtin_popcountll: { |
| 373 | Value *ArgValue = EmitScalarExpr(E->getArg(0)); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 374 | |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 375 | llvm::Type *ArgType = ArgValue->getType(); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 376 | Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 377 | |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 378 | llvm::Type *ResultType = ConvertType(E->getType()); |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 379 | Value *Result = Builder.CreateCall(F, ArgValue); |
Daniel Dunbar | 04b2900 | 2008-07-21 17:19:41 +0000 | [diff] [blame] | 380 | if (Result->getType() != ResultType) |
Duncan Sands | eac73e5 | 2009-11-16 13:11:21 +0000 | [diff] [blame] | 381 | Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, |
| 382 | "cast"); |
Daniel Dunbar | 04b2900 | 2008-07-21 17:19:41 +0000 | [diff] [blame] | 383 | return RValue::get(Result); |
| 384 | } |
Fariborz Jahanian | e42b8a5 | 2010-07-26 23:11:03 +0000 | [diff] [blame] | 385 | case Builtin::BI__builtin_expect: { |
Fariborz Jahanian | dd697bc | 2011-04-25 23:10:07 +0000 | [diff] [blame] | 386 | Value *ArgValue = EmitScalarExpr(E->getArg(0)); |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 387 | llvm::Type *ArgType = ArgValue->getType(); |
Jakub Staszak | 558229f | 2011-07-08 22:45:14 +0000 | [diff] [blame] | 388 | |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 389 | Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); |
Jakub Staszak | 558229f | 2011-07-08 22:45:14 +0000 | [diff] [blame] | 390 | Value *ExpectedValue = EmitScalarExpr(E->getArg(1)); |
| 391 | |
| 392 | Value *Result = Builder.CreateCall2(FnExpect, ArgValue, ExpectedValue, |
| 393 | "expval"); |
| 394 | return RValue::get(Result); |
Fariborz Jahanian | e42b8a5 | 2010-07-26 23:11:03 +0000 | [diff] [blame] | 395 | } |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 396 | case Builtin::BI__builtin_assume_aligned: { |
| 397 | Value *PtrValue = EmitScalarExpr(E->getArg(0)); |
| 398 | Value *OffsetValue = |
| 399 | (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr; |
| 400 | |
| 401 | Value *AlignmentValue = EmitScalarExpr(E->getArg(1)); |
| 402 | ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue); |
| 403 | unsigned Alignment = (unsigned) AlignmentCI->getZExtValue(); |
| 404 | |
| 405 | EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue); |
| 406 | return RValue::get(PtrValue); |
| 407 | } |
| 408 | case Builtin::BI__assume: |
| 409 | case Builtin::BI__builtin_assume: { |
| 410 | if (E->getArg(0)->HasSideEffects(getContext())) |
| 411 | return RValue::get(nullptr); |
| 412 | |
| 413 | Value *ArgValue = EmitScalarExpr(E->getArg(0)); |
| 414 | Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume); |
| 415 | return RValue::get(Builder.CreateCall(FnAssume, ArgValue)); |
| 416 | } |
Benjamin Kramer | d190057 | 2012-10-06 14:42:22 +0000 | [diff] [blame] | 417 | case Builtin::BI__builtin_bswap16: |
Anders Carlsson | df4852a | 2007-12-02 21:58:10 +0000 | [diff] [blame] | 418 | case Builtin::BI__builtin_bswap32: |
| 419 | case Builtin::BI__builtin_bswap64: { |
Chris Lattner | 1feedd8 | 2007-12-13 07:34:23 +0000 | [diff] [blame] | 420 | Value *ArgValue = EmitScalarExpr(E->getArg(0)); |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 421 | llvm::Type *ArgType = ArgValue->getType(); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 422 | Value *F = CGM.getIntrinsic(Intrinsic::bswap, ArgType); |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 423 | return RValue::get(Builder.CreateCall(F, ArgValue)); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 424 | } |
Daniel Dunbar | d5f8a4f | 2008-09-03 21:13:56 +0000 | [diff] [blame] | 425 | case Builtin::BI__builtin_object_size: { |
Richard Smith | c679485 | 2012-05-23 04:13:20 +0000 | [diff] [blame] | 426 | // We rely on constant folding to deal with expressions with side effects. |
| 427 | assert(!E->getArg(0)->HasSideEffects(getContext()) && |
| 428 | "should have been constant folded"); |
| 429 | |
Mike Stump | b16d32f | 2009-10-26 23:39:48 +0000 | [diff] [blame] | 430 | // We pass this builtin onto the optimizer so that it can |
| 431 | // figure out the object size in more complex cases. |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 432 | llvm::Type *ResType = ConvertType(E->getType()); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 433 | |
Eric Christopher | fee667f | 2009-12-23 03:49:37 +0000 | [diff] [blame] | 434 | // LLVM only supports 0 and 2, make sure that we pass along that |
| 435 | // as a boolean. |
| 436 | Value *Ty = EmitScalarExpr(E->getArg(1)); |
| 437 | ConstantInt *CI = dyn_cast<ConstantInt>(Ty); |
| 438 | assert(CI); |
| 439 | uint64_t val = CI->getZExtValue(); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 440 | CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1); |
Matt Arsenault | 8dd4baa | 2013-10-07 19:00:18 +0000 | [diff] [blame] | 441 | // FIXME: Get right address space. |
| 442 | llvm::Type *Tys[] = { ResType, Builder.getInt8PtrTy(0) }; |
| 443 | Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); |
Nuno Lopes | 3e86a04 | 2012-05-22 15:26:48 +0000 | [diff] [blame] | 444 | return RValue::get(Builder.CreateCall2(F, EmitScalarExpr(E->getArg(0)),CI)); |
Daniel Dunbar | d5f8a4f | 2008-09-03 21:13:56 +0000 | [diff] [blame] | 445 | } |
Daniel Dunbar | 4493f79 | 2008-07-21 22:59:13 +0000 | [diff] [blame] | 446 | case Builtin::BI__builtin_prefetch: { |
| 447 | Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0)); |
| 448 | // FIXME: Technically these constants should of type 'int', yes? |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 449 | RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) : |
Chris Lattner | 77b89b8 | 2010-06-27 07:15:29 +0000 | [diff] [blame] | 450 | llvm::ConstantInt::get(Int32Ty, 0); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 451 | Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : |
Chris Lattner | 77b89b8 | 2010-06-27 07:15:29 +0000 | [diff] [blame] | 452 | llvm::ConstantInt::get(Int32Ty, 3); |
Bruno Cardoso Lopes | 2eccb67 | 2011-06-14 05:00:30 +0000 | [diff] [blame] | 453 | Value *Data = llvm::ConstantInt::get(Int32Ty, 1); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 454 | Value *F = CGM.getIntrinsic(Intrinsic::prefetch); |
Bruno Cardoso Lopes | 2eccb67 | 2011-06-14 05:00:30 +0000 | [diff] [blame] | 455 | return RValue::get(Builder.CreateCall4(F, Address, RW, Locality, Data)); |
Anders Carlsson | df4852a | 2007-12-02 21:58:10 +0000 | [diff] [blame] | 456 | } |
Hal Finkel | a841c19 | 2012-08-05 22:03:08 +0000 | [diff] [blame] | 457 | case Builtin::BI__builtin_readcyclecounter: { |
| 458 | Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); |
| 459 | return RValue::get(Builder.CreateCall(F)); |
| 460 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 461 | case Builtin::BI__builtin___clear_cache: { |
| 462 | Value *Begin = EmitScalarExpr(E->getArg(0)); |
| 463 | Value *End = EmitScalarExpr(E->getArg(1)); |
| 464 | Value *F = CGM.getIntrinsic(Intrinsic::clear_cache); |
| 465 | return RValue::get(Builder.CreateCall2(F, Begin, End)); |
| 466 | } |
Daniel Dunbar | 4493f79 | 2008-07-21 22:59:13 +0000 | [diff] [blame] | 467 | case Builtin::BI__builtin_trap: { |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 468 | Value *F = CGM.getIntrinsic(Intrinsic::trap); |
Daniel Dunbar | 4493f79 | 2008-07-21 22:59:13 +0000 | [diff] [blame] | 469 | return RValue::get(Builder.CreateCall(F)); |
| 470 | } |
Nico Weber | 385cc5f | 2012-09-26 05:40:16 +0000 | [diff] [blame] | 471 | case Builtin::BI__debugbreak: { |
| 472 | Value *F = CGM.getIntrinsic(Intrinsic::debugtrap); |
| 473 | return RValue::get(Builder.CreateCall(F)); |
| 474 | } |
Chris Lattner | 21190d5 | 2009-09-21 03:09:59 +0000 | [diff] [blame] | 475 | case Builtin::BI__builtin_unreachable: { |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 476 | if (SanOpts.has(SanitizerKind::Unreachable)) { |
| 477 | SanitizerScope SanScope(this); |
| 478 | EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()), |
| 479 | SanitizerKind::Unreachable), |
| 480 | "builtin_unreachable", EmitCheckSourceLocation(E->getExprLoc()), |
| 481 | None); |
| 482 | } else |
John McCall | cd5b22e | 2011-01-12 03:41:02 +0000 | [diff] [blame] | 483 | Builder.CreateUnreachable(); |
| 484 | |
| 485 | // We do need to preserve an insertion point. |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 486 | EmitBlock(createBasicBlock("unreachable.cont")); |
John McCall | cd5b22e | 2011-01-12 03:41:02 +0000 | [diff] [blame] | 487 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 488 | return RValue::get(nullptr); |
Chris Lattner | 21190d5 | 2009-09-21 03:09:59 +0000 | [diff] [blame] | 489 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 490 | |
Daniel Dunbar | a933c3c | 2008-07-21 18:44:41 +0000 | [diff] [blame] | 491 | case Builtin::BI__builtin_powi: |
| 492 | case Builtin::BI__builtin_powif: |
| 493 | case Builtin::BI__builtin_powil: { |
| 494 | Value *Base = EmitScalarExpr(E->getArg(0)); |
| 495 | Value *Exponent = EmitScalarExpr(E->getArg(1)); |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 496 | llvm::Type *ArgType = Base->getType(); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 497 | Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType); |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 498 | return RValue::get(Builder.CreateCall2(F, Base, Exponent)); |
Daniel Dunbar | a933c3c | 2008-07-21 18:44:41 +0000 | [diff] [blame] | 499 | } |
| 500 | |
Chris Lattner | fe23e21 | 2007-12-20 00:44:32 +0000 | [diff] [blame] | 501 | case Builtin::BI__builtin_isgreater: |
| 502 | case Builtin::BI__builtin_isgreaterequal: |
| 503 | case Builtin::BI__builtin_isless: |
| 504 | case Builtin::BI__builtin_islessequal: |
| 505 | case Builtin::BI__builtin_islessgreater: |
| 506 | case Builtin::BI__builtin_isunordered: { |
| 507 | // Ordered comparisons: we know the arguments to these are matching scalar |
| 508 | // floating point values. |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 509 | Value *LHS = EmitScalarExpr(E->getArg(0)); |
Chris Lattner | fe23e21 | 2007-12-20 00:44:32 +0000 | [diff] [blame] | 510 | Value *RHS = EmitScalarExpr(E->getArg(1)); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 511 | |
Chris Lattner | fe23e21 | 2007-12-20 00:44:32 +0000 | [diff] [blame] | 512 | switch (BuiltinID) { |
David Blaikie | b219cfc | 2011-09-23 05:06:16 +0000 | [diff] [blame] | 513 | default: llvm_unreachable("Unknown ordered comparison"); |
Chris Lattner | fe23e21 | 2007-12-20 00:44:32 +0000 | [diff] [blame] | 514 | case Builtin::BI__builtin_isgreater: |
| 515 | LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp"); |
| 516 | break; |
| 517 | case Builtin::BI__builtin_isgreaterequal: |
| 518 | LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp"); |
| 519 | break; |
| 520 | case Builtin::BI__builtin_isless: |
| 521 | LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp"); |
| 522 | break; |
| 523 | case Builtin::BI__builtin_islessequal: |
| 524 | LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp"); |
| 525 | break; |
| 526 | case Builtin::BI__builtin_islessgreater: |
| 527 | LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp"); |
| 528 | break; |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 529 | case Builtin::BI__builtin_isunordered: |
Chris Lattner | fe23e21 | 2007-12-20 00:44:32 +0000 | [diff] [blame] | 530 | LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp"); |
| 531 | break; |
| 532 | } |
| 533 | // ZExt bool to int type. |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 534 | return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType()))); |
Chris Lattner | fe23e21 | 2007-12-20 00:44:32 +0000 | [diff] [blame] | 535 | } |
Eli Friedman | d613989 | 2009-09-01 04:19:44 +0000 | [diff] [blame] | 536 | case Builtin::BI__builtin_isnan: { |
| 537 | Value *V = EmitScalarExpr(E->getArg(0)); |
| 538 | V = Builder.CreateFCmpUNO(V, V, "cmp"); |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 539 | return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); |
Eli Friedman | d613989 | 2009-09-01 04:19:44 +0000 | [diff] [blame] | 540 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 541 | |
Chris Lattner | 420b118 | 2010-05-06 05:35:16 +0000 | [diff] [blame] | 542 | case Builtin::BI__builtin_isinf: { |
| 543 | // isinf(x) --> fabs(x) == infinity |
| 544 | Value *V = EmitScalarExpr(E->getArg(0)); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 545 | V = EmitFAbs(*this, V); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 546 | |
Chris Lattner | 420b118 | 2010-05-06 05:35:16 +0000 | [diff] [blame] | 547 | V = Builder.CreateFCmpOEQ(V, ConstantFP::getInfinity(V->getType()),"isinf"); |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 548 | return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); |
Chris Lattner | 420b118 | 2010-05-06 05:35:16 +0000 | [diff] [blame] | 549 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 550 | |
Chris Lattner | 58ae5b4 | 2010-05-06 06:13:53 +0000 | [diff] [blame] | 551 | // TODO: BI__builtin_isinf_sign |
| 552 | // isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0 |
Benjamin Kramer | 6349ce9 | 2010-05-19 11:24:26 +0000 | [diff] [blame] | 553 | |
| 554 | case Builtin::BI__builtin_isnormal: { |
| 555 | // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min |
| 556 | Value *V = EmitScalarExpr(E->getArg(0)); |
| 557 | Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); |
| 558 | |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 559 | Value *Abs = EmitFAbs(*this, V); |
Benjamin Kramer | 6349ce9 | 2010-05-19 11:24:26 +0000 | [diff] [blame] | 560 | Value *IsLessThanInf = |
| 561 | Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); |
| 562 | APFloat Smallest = APFloat::getSmallestNormalized( |
| 563 | getContext().getFloatTypeSemantics(E->getArg(0)->getType())); |
| 564 | Value *IsNormal = |
| 565 | Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest), |
| 566 | "isnormal"); |
| 567 | V = Builder.CreateAnd(Eq, IsLessThanInf, "and"); |
| 568 | V = Builder.CreateAnd(V, IsNormal, "and"); |
| 569 | return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); |
| 570 | } |
| 571 | |
Chris Lattner | ed07415 | 2010-05-06 06:04:13 +0000 | [diff] [blame] | 572 | case Builtin::BI__builtin_isfinite: { |
Julien Lerouge | ef004ec | 2011-09-09 22:46:39 +0000 | [diff] [blame] | 573 | // isfinite(x) --> x == x && fabs(x) != infinity; |
Chris Lattner | ed07415 | 2010-05-06 06:04:13 +0000 | [diff] [blame] | 574 | Value *V = EmitScalarExpr(E->getArg(0)); |
| 575 | Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq"); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 576 | |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 577 | Value *Abs = EmitFAbs(*this, V); |
Chris Lattner | ed07415 | 2010-05-06 06:04:13 +0000 | [diff] [blame] | 578 | Value *IsNotInf = |
| 579 | Builder.CreateFCmpUNE(Abs, ConstantFP::getInfinity(V->getType()),"isinf"); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 580 | |
Chris Lattner | ed07415 | 2010-05-06 06:04:13 +0000 | [diff] [blame] | 581 | V = Builder.CreateAnd(Eq, IsNotInf, "and"); |
| 582 | return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); |
| 583 | } |
Benjamin Kramer | 7867f1a | 2010-06-14 10:30:41 +0000 | [diff] [blame] | 584 | |
| 585 | case Builtin::BI__builtin_fpclassify: { |
| 586 | Value *V = EmitScalarExpr(E->getArg(5)); |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 587 | llvm::Type *Ty = ConvertType(E->getArg(5)->getType()); |
Benjamin Kramer | 7867f1a | 2010-06-14 10:30:41 +0000 | [diff] [blame] | 588 | |
| 589 | // Create Result |
| 590 | BasicBlock *Begin = Builder.GetInsertBlock(); |
| 591 | BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn); |
| 592 | Builder.SetInsertPoint(End); |
| 593 | PHINode *Result = |
Jay Foad | bbf3bac | 2011-03-30 11:28:58 +0000 | [diff] [blame] | 594 | Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4, |
Benjamin Kramer | 7867f1a | 2010-06-14 10:30:41 +0000 | [diff] [blame] | 595 | "fpclassify_result"); |
| 596 | |
| 597 | // if (V==0) return FP_ZERO |
| 598 | Builder.SetInsertPoint(Begin); |
| 599 | Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty), |
| 600 | "iszero"); |
| 601 | Value *ZeroLiteral = EmitScalarExpr(E->getArg(4)); |
| 602 | BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn); |
| 603 | Builder.CreateCondBr(IsZero, End, NotZero); |
| 604 | Result->addIncoming(ZeroLiteral, Begin); |
| 605 | |
| 606 | // if (V != V) return FP_NAN |
| 607 | Builder.SetInsertPoint(NotZero); |
| 608 | Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp"); |
| 609 | Value *NanLiteral = EmitScalarExpr(E->getArg(0)); |
| 610 | BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn); |
| 611 | Builder.CreateCondBr(IsNan, End, NotNan); |
| 612 | Result->addIncoming(NanLiteral, NotZero); |
| 613 | |
| 614 | // if (fabs(V) == infinity) return FP_INFINITY |
| 615 | Builder.SetInsertPoint(NotNan); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 616 | Value *VAbs = EmitFAbs(*this, V); |
Benjamin Kramer | 7867f1a | 2010-06-14 10:30:41 +0000 | [diff] [blame] | 617 | Value *IsInf = |
| 618 | Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()), |
| 619 | "isinf"); |
| 620 | Value *InfLiteral = EmitScalarExpr(E->getArg(1)); |
| 621 | BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn); |
| 622 | Builder.CreateCondBr(IsInf, End, NotInf); |
| 623 | Result->addIncoming(InfLiteral, NotNan); |
| 624 | |
| 625 | // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL |
| 626 | Builder.SetInsertPoint(NotInf); |
| 627 | APFloat Smallest = APFloat::getSmallestNormalized( |
| 628 | getContext().getFloatTypeSemantics(E->getArg(5)->getType())); |
| 629 | Value *IsNormal = |
| 630 | Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest), |
| 631 | "isnormal"); |
| 632 | Value *NormalResult = |
| 633 | Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)), |
| 634 | EmitScalarExpr(E->getArg(3))); |
| 635 | Builder.CreateBr(End); |
| 636 | Result->addIncoming(NormalResult, NotInf); |
| 637 | |
| 638 | // return Result |
| 639 | Builder.SetInsertPoint(End); |
| 640 | return RValue::get(Result); |
| 641 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 642 | |
Eli Friedman | b52fe9c | 2009-06-02 07:10:30 +0000 | [diff] [blame] | 643 | case Builtin::BIalloca: |
Reid Kleckner | 6526de4 | 2013-11-13 22:58:53 +0000 | [diff] [blame] | 644 | case Builtin::BI_alloca: |
Chris Lattner | 9e800e3 | 2008-06-16 17:15:14 +0000 | [diff] [blame] | 645 | case Builtin::BI__builtin_alloca: { |
Chris Lattner | 9e800e3 | 2008-06-16 17:15:14 +0000 | [diff] [blame] | 646 | Value *Size = EmitScalarExpr(E->getArg(0)); |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 647 | return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size)); |
Daniel Dunbar | 1caae95 | 2008-07-22 00:26:45 +0000 | [diff] [blame] | 648 | } |
Eli Friedman | e6dddfd | 2010-01-23 19:00:10 +0000 | [diff] [blame] | 649 | case Builtin::BIbzero: |
Daniel Dunbar | 1caae95 | 2008-07-22 00:26:45 +0000 | [diff] [blame] | 650 | case Builtin::BI__builtin_bzero: { |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 651 | std::pair<llvm::Value*, unsigned> Dest = |
| 652 | EmitPointerWithAlignment(E->getArg(0)); |
Mon P Wang | 3ecd785 | 2010-04-04 03:10:52 +0000 | [diff] [blame] | 653 | Value *SizeVal = EmitScalarExpr(E->getArg(1)); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 654 | Builder.CreateMemSet(Dest.first, Builder.getInt8(0), SizeVal, |
| 655 | Dest.second, false); |
| 656 | return RValue::get(Dest.first); |
Chris Lattner | 9e800e3 | 2008-06-16 17:15:14 +0000 | [diff] [blame] | 657 | } |
Eli Friedman | e6ec205 | 2009-12-17 00:14:28 +0000 | [diff] [blame] | 658 | case Builtin::BImemcpy: |
Eli Friedman | d4b32e4 | 2008-05-19 23:27:48 +0000 | [diff] [blame] | 659 | case Builtin::BI__builtin_memcpy: { |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 660 | std::pair<llvm::Value*, unsigned> Dest = |
| 661 | EmitPointerWithAlignment(E->getArg(0)); |
| 662 | std::pair<llvm::Value*, unsigned> Src = |
| 663 | EmitPointerWithAlignment(E->getArg(1)); |
Mon P Wang | 3ecd785 | 2010-04-04 03:10:52 +0000 | [diff] [blame] | 664 | Value *SizeVal = EmitScalarExpr(E->getArg(2)); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 665 | unsigned Align = std::min(Dest.second, Src.second); |
| 666 | Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false); |
| 667 | return RValue::get(Dest.first); |
Daniel Dunbar | 1caae95 | 2008-07-22 00:26:45 +0000 | [diff] [blame] | 668 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 669 | |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 670 | case Builtin::BI__builtin___memcpy_chk: { |
Sylvestre Ledru | f3477c1 | 2012-09-27 10:16:10 +0000 | [diff] [blame] | 671 | // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2. |
Richard Smith | a6b8b2c | 2011-10-10 18:28:20 +0000 | [diff] [blame] | 672 | llvm::APSInt Size, DstSize; |
| 673 | if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || |
| 674 | !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 675 | break; |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 676 | if (Size.ugt(DstSize)) |
| 677 | break; |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 678 | std::pair<llvm::Value*, unsigned> Dest = |
| 679 | EmitPointerWithAlignment(E->getArg(0)); |
| 680 | std::pair<llvm::Value*, unsigned> Src = |
| 681 | EmitPointerWithAlignment(E->getArg(1)); |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 682 | Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 683 | unsigned Align = std::min(Dest.second, Src.second); |
| 684 | Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false); |
| 685 | return RValue::get(Dest.first); |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 686 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 687 | |
Fariborz Jahanian | 8e2eab2 | 2010-06-16 16:22:04 +0000 | [diff] [blame] | 688 | case Builtin::BI__builtin_objc_memmove_collectable: { |
Fariborz Jahanian | 55bcace | 2010-06-15 22:44:06 +0000 | [diff] [blame] | 689 | Value *Address = EmitScalarExpr(E->getArg(0)); |
| 690 | Value *SrcAddr = EmitScalarExpr(E->getArg(1)); |
| 691 | Value *SizeVal = EmitScalarExpr(E->getArg(2)); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 692 | CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, |
Fariborz Jahanian | 55bcace | 2010-06-15 22:44:06 +0000 | [diff] [blame] | 693 | Address, SrcAddr, SizeVal); |
| 694 | return RValue::get(Address); |
| 695 | } |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 696 | |
| 697 | case Builtin::BI__builtin___memmove_chk: { |
Sylvestre Ledru | f3477c1 | 2012-09-27 10:16:10 +0000 | [diff] [blame] | 698 | // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2. |
Richard Smith | a6b8b2c | 2011-10-10 18:28:20 +0000 | [diff] [blame] | 699 | llvm::APSInt Size, DstSize; |
| 700 | if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || |
| 701 | !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 702 | break; |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 703 | if (Size.ugt(DstSize)) |
| 704 | break; |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 705 | std::pair<llvm::Value*, unsigned> Dest = |
| 706 | EmitPointerWithAlignment(E->getArg(0)); |
| 707 | std::pair<llvm::Value*, unsigned> Src = |
| 708 | EmitPointerWithAlignment(E->getArg(1)); |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 709 | Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 710 | unsigned Align = std::min(Dest.second, Src.second); |
| 711 | Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false); |
| 712 | return RValue::get(Dest.first); |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 713 | } |
| 714 | |
Eli Friedman | e6ec205 | 2009-12-17 00:14:28 +0000 | [diff] [blame] | 715 | case Builtin::BImemmove: |
Daniel Dunbar | 1caae95 | 2008-07-22 00:26:45 +0000 | [diff] [blame] | 716 | case Builtin::BI__builtin_memmove: { |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 717 | std::pair<llvm::Value*, unsigned> Dest = |
| 718 | EmitPointerWithAlignment(E->getArg(0)); |
| 719 | std::pair<llvm::Value*, unsigned> Src = |
| 720 | EmitPointerWithAlignment(E->getArg(1)); |
Mon P Wang | 3ecd785 | 2010-04-04 03:10:52 +0000 | [diff] [blame] | 721 | Value *SizeVal = EmitScalarExpr(E->getArg(2)); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 722 | unsigned Align = std::min(Dest.second, Src.second); |
| 723 | Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false); |
| 724 | return RValue::get(Dest.first); |
Daniel Dunbar | 1caae95 | 2008-07-22 00:26:45 +0000 | [diff] [blame] | 725 | } |
Eli Friedman | e6ec205 | 2009-12-17 00:14:28 +0000 | [diff] [blame] | 726 | case Builtin::BImemset: |
Daniel Dunbar | 1caae95 | 2008-07-22 00:26:45 +0000 | [diff] [blame] | 727 | case Builtin::BI__builtin_memset: { |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 728 | std::pair<llvm::Value*, unsigned> Dest = |
| 729 | EmitPointerWithAlignment(E->getArg(0)); |
Benjamin Kramer | 9f0c7cc | 2010-12-30 00:13:21 +0000 | [diff] [blame] | 730 | Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), |
| 731 | Builder.getInt8Ty()); |
Mon P Wang | 3ecd785 | 2010-04-04 03:10:52 +0000 | [diff] [blame] | 732 | Value *SizeVal = EmitScalarExpr(E->getArg(2)); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 733 | Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false); |
| 734 | return RValue::get(Dest.first); |
Eli Friedman | d4b32e4 | 2008-05-19 23:27:48 +0000 | [diff] [blame] | 735 | } |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 736 | case Builtin::BI__builtin___memset_chk: { |
Sylvestre Ledru | f3477c1 | 2012-09-27 10:16:10 +0000 | [diff] [blame] | 737 | // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2. |
Richard Smith | a6b8b2c | 2011-10-10 18:28:20 +0000 | [diff] [blame] | 738 | llvm::APSInt Size, DstSize; |
| 739 | if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) || |
| 740 | !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext())) |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 741 | break; |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 742 | if (Size.ugt(DstSize)) |
| 743 | break; |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 744 | std::pair<llvm::Value*, unsigned> Dest = |
| 745 | EmitPointerWithAlignment(E->getArg(0)); |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 746 | Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), |
| 747 | Builder.getInt8Ty()); |
| 748 | Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 749 | Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false); |
| 750 | return RValue::get(Dest.first); |
Chris Lattner | a5e5e0f | 2011-04-17 00:40:24 +0000 | [diff] [blame] | 751 | } |
John McCall | fb17a56 | 2010-03-03 10:30:05 +0000 | [diff] [blame] | 752 | case Builtin::BI__builtin_dwarf_cfa: { |
| 753 | // The offset in bytes from the first argument to the CFA. |
| 754 | // |
| 755 | // Why on earth is this in the frontend? Is there any reason at |
| 756 | // all that the backend can't reasonably determine this while |
| 757 | // lowering llvm.eh.dwarf.cfa()? |
| 758 | // |
| 759 | // TODO: If there's a satisfactory reason, add a target hook for |
| 760 | // this instead of hard-coding 0, which is correct for most targets. |
| 761 | int32_t Offset = 0; |
| 762 | |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 763 | Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 764 | return RValue::get(Builder.CreateCall(F, |
Chris Lattner | 77b89b8 | 2010-06-27 07:15:29 +0000 | [diff] [blame] | 765 | llvm::ConstantInt::get(Int32Ty, Offset))); |
John McCall | fb17a56 | 2010-03-03 10:30:05 +0000 | [diff] [blame] | 766 | } |
Eli Friedman | 256f77e | 2008-05-20 08:59:34 +0000 | [diff] [blame] | 767 | case Builtin::BI__builtin_return_address: { |
Anton Korobeynikov | 83c2a98 | 2009-12-27 14:27:22 +0000 | [diff] [blame] | 768 | Value *Depth = EmitScalarExpr(E->getArg(0)); |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 769 | Depth = Builder.CreateIntCast(Depth, Int32Ty, false); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 770 | Value *F = CGM.getIntrinsic(Intrinsic::returnaddress); |
Anton Korobeynikov | 83c2a98 | 2009-12-27 14:27:22 +0000 | [diff] [blame] | 771 | return RValue::get(Builder.CreateCall(F, Depth)); |
Eli Friedman | 256f77e | 2008-05-20 08:59:34 +0000 | [diff] [blame] | 772 | } |
| 773 | case Builtin::BI__builtin_frame_address: { |
Anton Korobeynikov | 83c2a98 | 2009-12-27 14:27:22 +0000 | [diff] [blame] | 774 | Value *Depth = EmitScalarExpr(E->getArg(0)); |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 775 | Depth = Builder.CreateIntCast(Depth, Int32Ty, false); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 776 | Value *F = CGM.getIntrinsic(Intrinsic::frameaddress); |
Anton Korobeynikov | 83c2a98 | 2009-12-27 14:27:22 +0000 | [diff] [blame] | 777 | return RValue::get(Builder.CreateCall(F, Depth)); |
Eli Friedman | 256f77e | 2008-05-20 08:59:34 +0000 | [diff] [blame] | 778 | } |
Eli Friedman | 3b660ef | 2009-05-03 19:23:23 +0000 | [diff] [blame] | 779 | case Builtin::BI__builtin_extract_return_addr: { |
John McCall | 492c4f9 | 2010-03-03 04:15:11 +0000 | [diff] [blame] | 780 | Value *Address = EmitScalarExpr(E->getArg(0)); |
| 781 | Value *Result = getTargetHooks().decodeReturnAddress(*this, Address); |
| 782 | return RValue::get(Result); |
| 783 | } |
| 784 | case Builtin::BI__builtin_frob_return_addr: { |
| 785 | Value *Address = EmitScalarExpr(E->getArg(0)); |
| 786 | Value *Result = getTargetHooks().encodeReturnAddress(*this, Address); |
| 787 | return RValue::get(Result); |
Eli Friedman | 3b660ef | 2009-05-03 19:23:23 +0000 | [diff] [blame] | 788 | } |
John McCall | 6374c33 | 2010-03-06 00:35:14 +0000 | [diff] [blame] | 789 | case Builtin::BI__builtin_dwarf_sp_column: { |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 790 | llvm::IntegerType *Ty |
John McCall | 6374c33 | 2010-03-06 00:35:14 +0000 | [diff] [blame] | 791 | = cast<llvm::IntegerType>(ConvertType(E->getType())); |
| 792 | int Column = getTargetHooks().getDwarfEHStackPointer(CGM); |
| 793 | if (Column == -1) { |
| 794 | CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column"); |
| 795 | return RValue::get(llvm::UndefValue::get(Ty)); |
| 796 | } |
| 797 | return RValue::get(llvm::ConstantInt::get(Ty, Column, true)); |
| 798 | } |
| 799 | case Builtin::BI__builtin_init_dwarf_reg_size_table: { |
| 800 | Value *Address = EmitScalarExpr(E->getArg(0)); |
| 801 | if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address)) |
| 802 | CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table"); |
| 803 | return RValue::get(llvm::UndefValue::get(ConvertType(E->getType()))); |
| 804 | } |
John McCall | 7ada111 | 2010-03-03 05:38:58 +0000 | [diff] [blame] | 805 | case Builtin::BI__builtin_eh_return: { |
| 806 | Value *Int = EmitScalarExpr(E->getArg(0)); |
| 807 | Value *Ptr = EmitScalarExpr(E->getArg(1)); |
| 808 | |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 809 | llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType()); |
John McCall | 7ada111 | 2010-03-03 05:38:58 +0000 | [diff] [blame] | 810 | assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) && |
| 811 | "LLVM's __builtin_eh_return only supports 32- and 64-bit variants"); |
| 812 | Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32 |
| 813 | ? Intrinsic::eh_return_i32 |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 814 | : Intrinsic::eh_return_i64); |
John McCall | 7ada111 | 2010-03-03 05:38:58 +0000 | [diff] [blame] | 815 | Builder.CreateCall2(F, Int, Ptr); |
John McCall | cd5b22e | 2011-01-12 03:41:02 +0000 | [diff] [blame] | 816 | Builder.CreateUnreachable(); |
| 817 | |
| 818 | // We do need to preserve an insertion point. |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 819 | EmitBlock(createBasicBlock("builtin_eh_return.cont")); |
John McCall | cd5b22e | 2011-01-12 03:41:02 +0000 | [diff] [blame] | 820 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 821 | return RValue::get(nullptr); |
John McCall | 7ada111 | 2010-03-03 05:38:58 +0000 | [diff] [blame] | 822 | } |
Eli Friedman | a6d75c0 | 2009-06-02 09:37:50 +0000 | [diff] [blame] | 823 | case Builtin::BI__builtin_unwind_init: { |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 824 | Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init); |
Eli Friedman | a6d75c0 | 2009-06-02 09:37:50 +0000 | [diff] [blame] | 825 | return RValue::get(Builder.CreateCall(F)); |
| 826 | } |
John McCall | 5e11085 | 2010-03-02 02:31:24 +0000 | [diff] [blame] | 827 | case Builtin::BI__builtin_extend_pointer: { |
| 828 | // Extends a pointer to the size of an _Unwind_Word, which is |
John McCall | d0b76ca | 2010-03-02 03:50:12 +0000 | [diff] [blame] | 829 | // uint64_t on all platforms. Generally this gets poked into a |
| 830 | // register and eventually used as an address, so if the |
| 831 | // addressing registers are wider than pointers and the platform |
| 832 | // doesn't implicitly ignore high-order bits when doing |
| 833 | // addressing, we need to make sure we zext / sext based on |
| 834 | // the platform's expectations. |
John McCall | 5e11085 | 2010-03-02 02:31:24 +0000 | [diff] [blame] | 835 | // |
| 836 | // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html |
John McCall | d0b76ca | 2010-03-02 03:50:12 +0000 | [diff] [blame] | 837 | |
John McCall | d0b76ca | 2010-03-02 03:50:12 +0000 | [diff] [blame] | 838 | // Cast the pointer to intptr_t. |
John McCall | 5e11085 | 2010-03-02 02:31:24 +0000 | [diff] [blame] | 839 | Value *Ptr = EmitScalarExpr(E->getArg(0)); |
John McCall | d0b76ca | 2010-03-02 03:50:12 +0000 | [diff] [blame] | 840 | Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast"); |
| 841 | |
| 842 | // If that's 64 bits, we're done. |
| 843 | if (IntPtrTy->getBitWidth() == 64) |
| 844 | return RValue::get(Result); |
| 845 | |
| 846 | // Otherwise, ask the codegen data what to do. |
John McCall | 492c4f9 | 2010-03-03 04:15:11 +0000 | [diff] [blame] | 847 | if (getTargetHooks().extendPointerWithSExt()) |
John McCall | d0b76ca | 2010-03-02 03:50:12 +0000 | [diff] [blame] | 848 | return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext")); |
| 849 | else |
| 850 | return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext")); |
John McCall | 5e11085 | 2010-03-02 02:31:24 +0000 | [diff] [blame] | 851 | } |
Eli Friedman | a6d75c0 | 2009-06-02 09:37:50 +0000 | [diff] [blame] | 852 | case Builtin::BI__builtin_setjmp: { |
John McCall | 78673d9 | 2010-05-27 18:47:06 +0000 | [diff] [blame] | 853 | // Buffer is a void**. |
Eli Friedman | a6d75c0 | 2009-06-02 09:37:50 +0000 | [diff] [blame] | 854 | Value *Buf = EmitScalarExpr(E->getArg(0)); |
John McCall | 78673d9 | 2010-05-27 18:47:06 +0000 | [diff] [blame] | 855 | |
| 856 | // Store the frame pointer to the setjmp buffer. |
Eli Friedman | a6d75c0 | 2009-06-02 09:37:50 +0000 | [diff] [blame] | 857 | Value *FrameAddr = |
John McCall | 78673d9 | 2010-05-27 18:47:06 +0000 | [diff] [blame] | 858 | Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), |
Chris Lattner | 77b89b8 | 2010-06-27 07:15:29 +0000 | [diff] [blame] | 859 | ConstantInt::get(Int32Ty, 0)); |
Eli Friedman | a6d75c0 | 2009-06-02 09:37:50 +0000 | [diff] [blame] | 860 | Builder.CreateStore(FrameAddr, Buf); |
John McCall | 78673d9 | 2010-05-27 18:47:06 +0000 | [diff] [blame] | 861 | |
Jim Grosbach | 6d172e2 | 2010-05-27 23:54:20 +0000 | [diff] [blame] | 862 | // Store the stack pointer to the setjmp buffer. |
| 863 | Value *StackAddr = |
| 864 | Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); |
| 865 | Value *StackSaveSlot = |
Chris Lattner | 77b89b8 | 2010-06-27 07:15:29 +0000 | [diff] [blame] | 866 | Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2)); |
Jim Grosbach | 6d172e2 | 2010-05-27 23:54:20 +0000 | [diff] [blame] | 867 | Builder.CreateStore(StackAddr, StackSaveSlot); |
| 868 | |
John McCall | 78673d9 | 2010-05-27 18:47:06 +0000 | [diff] [blame] | 869 | // Call LLVM's EH setjmp, which is lightweight. |
| 870 | Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp); |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 871 | Buf = Builder.CreateBitCast(Buf, Int8PtrTy); |
Eli Friedman | a6d75c0 | 2009-06-02 09:37:50 +0000 | [diff] [blame] | 872 | return RValue::get(Builder.CreateCall(F, Buf)); |
| 873 | } |
| 874 | case Builtin::BI__builtin_longjmp: { |
Eli Friedman | a6d75c0 | 2009-06-02 09:37:50 +0000 | [diff] [blame] | 875 | Value *Buf = EmitScalarExpr(E->getArg(0)); |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 876 | Buf = Builder.CreateBitCast(Buf, Int8PtrTy); |
John McCall | 78673d9 | 2010-05-27 18:47:06 +0000 | [diff] [blame] | 877 | |
| 878 | // Call LLVM's EH longjmp, which is lightweight. |
| 879 | Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf); |
| 880 | |
John McCall | cd5b22e | 2011-01-12 03:41:02 +0000 | [diff] [blame] | 881 | // longjmp doesn't return; mark this as unreachable. |
| 882 | Builder.CreateUnreachable(); |
| 883 | |
| 884 | // We do need to preserve an insertion point. |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 885 | EmitBlock(createBasicBlock("longjmp.cont")); |
John McCall | cd5b22e | 2011-01-12 03:41:02 +0000 | [diff] [blame] | 886 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 887 | return RValue::get(nullptr); |
Eli Friedman | a6d75c0 | 2009-06-02 09:37:50 +0000 | [diff] [blame] | 888 | } |
Mon P Wang | 1ffe281 | 2008-05-09 22:40:52 +0000 | [diff] [blame] | 889 | case Builtin::BI__sync_fetch_and_add: |
Mon P Wang | 1ffe281 | 2008-05-09 22:40:52 +0000 | [diff] [blame] | 890 | case Builtin::BI__sync_fetch_and_sub: |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 891 | case Builtin::BI__sync_fetch_and_or: |
| 892 | case Builtin::BI__sync_fetch_and_and: |
| 893 | case Builtin::BI__sync_fetch_and_xor: |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 894 | case Builtin::BI__sync_fetch_and_nand: |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 895 | case Builtin::BI__sync_add_and_fetch: |
| 896 | case Builtin::BI__sync_sub_and_fetch: |
| 897 | case Builtin::BI__sync_and_and_fetch: |
| 898 | case Builtin::BI__sync_or_and_fetch: |
| 899 | case Builtin::BI__sync_xor_and_fetch: |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 900 | case Builtin::BI__sync_nand_and_fetch: |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 901 | case Builtin::BI__sync_val_compare_and_swap: |
| 902 | case Builtin::BI__sync_bool_compare_and_swap: |
| 903 | case Builtin::BI__sync_lock_test_and_set: |
| 904 | case Builtin::BI__sync_lock_release: |
Chris Lattner | 23aa9c8 | 2011-04-09 03:57:26 +0000 | [diff] [blame] | 905 | case Builtin::BI__sync_swap: |
David Blaikie | b219cfc | 2011-09-23 05:06:16 +0000 | [diff] [blame] | 906 | llvm_unreachable("Shouldn't make it through sema"); |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 907 | case Builtin::BI__sync_fetch_and_add_1: |
| 908 | case Builtin::BI__sync_fetch_and_add_2: |
| 909 | case Builtin::BI__sync_fetch_and_add_4: |
| 910 | case Builtin::BI__sync_fetch_and_add_8: |
| 911 | case Builtin::BI__sync_fetch_and_add_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 912 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E); |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 913 | case Builtin::BI__sync_fetch_and_sub_1: |
| 914 | case Builtin::BI__sync_fetch_and_sub_2: |
| 915 | case Builtin::BI__sync_fetch_and_sub_4: |
| 916 | case Builtin::BI__sync_fetch_and_sub_8: |
| 917 | case Builtin::BI__sync_fetch_and_sub_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 918 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E); |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 919 | case Builtin::BI__sync_fetch_and_or_1: |
| 920 | case Builtin::BI__sync_fetch_and_or_2: |
| 921 | case Builtin::BI__sync_fetch_and_or_4: |
| 922 | case Builtin::BI__sync_fetch_and_or_8: |
| 923 | case Builtin::BI__sync_fetch_and_or_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 924 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E); |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 925 | case Builtin::BI__sync_fetch_and_and_1: |
| 926 | case Builtin::BI__sync_fetch_and_and_2: |
| 927 | case Builtin::BI__sync_fetch_and_and_4: |
| 928 | case Builtin::BI__sync_fetch_and_and_8: |
| 929 | case Builtin::BI__sync_fetch_and_and_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 930 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E); |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 931 | case Builtin::BI__sync_fetch_and_xor_1: |
| 932 | case Builtin::BI__sync_fetch_and_xor_2: |
| 933 | case Builtin::BI__sync_fetch_and_xor_4: |
| 934 | case Builtin::BI__sync_fetch_and_xor_8: |
| 935 | case Builtin::BI__sync_fetch_and_xor_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 936 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 937 | case Builtin::BI__sync_fetch_and_nand_1: |
| 938 | case Builtin::BI__sync_fetch_and_nand_2: |
| 939 | case Builtin::BI__sync_fetch_and_nand_4: |
| 940 | case Builtin::BI__sync_fetch_and_nand_8: |
| 941 | case Builtin::BI__sync_fetch_and_nand_16: |
| 942 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 943 | |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 944 | // Clang extensions: not overloaded yet. |
Mon P Wang | 1ffe281 | 2008-05-09 22:40:52 +0000 | [diff] [blame] | 945 | case Builtin::BI__sync_fetch_and_min: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 946 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E); |
Mon P Wang | 1ffe281 | 2008-05-09 22:40:52 +0000 | [diff] [blame] | 947 | case Builtin::BI__sync_fetch_and_max: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 948 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E); |
Mon P Wang | 1ffe281 | 2008-05-09 22:40:52 +0000 | [diff] [blame] | 949 | case Builtin::BI__sync_fetch_and_umin: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 950 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E); |
Mon P Wang | 1ffe281 | 2008-05-09 22:40:52 +0000 | [diff] [blame] | 951 | case Builtin::BI__sync_fetch_and_umax: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 952 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E); |
Daniel Dunbar | 0002d23 | 2009-04-07 00:55:51 +0000 | [diff] [blame] | 953 | |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 954 | case Builtin::BI__sync_add_and_fetch_1: |
| 955 | case Builtin::BI__sync_add_and_fetch_2: |
| 956 | case Builtin::BI__sync_add_and_fetch_4: |
| 957 | case Builtin::BI__sync_add_and_fetch_8: |
| 958 | case Builtin::BI__sync_add_and_fetch_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 959 | return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E, |
Daniel Dunbar | 0002d23 | 2009-04-07 00:55:51 +0000 | [diff] [blame] | 960 | llvm::Instruction::Add); |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 961 | case Builtin::BI__sync_sub_and_fetch_1: |
| 962 | case Builtin::BI__sync_sub_and_fetch_2: |
| 963 | case Builtin::BI__sync_sub_and_fetch_4: |
| 964 | case Builtin::BI__sync_sub_and_fetch_8: |
| 965 | case Builtin::BI__sync_sub_and_fetch_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 966 | return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E, |
Daniel Dunbar | 0002d23 | 2009-04-07 00:55:51 +0000 | [diff] [blame] | 967 | llvm::Instruction::Sub); |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 968 | case Builtin::BI__sync_and_and_fetch_1: |
| 969 | case Builtin::BI__sync_and_and_fetch_2: |
| 970 | case Builtin::BI__sync_and_and_fetch_4: |
| 971 | case Builtin::BI__sync_and_and_fetch_8: |
| 972 | case Builtin::BI__sync_and_and_fetch_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 973 | return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E, |
Daniel Dunbar | 0002d23 | 2009-04-07 00:55:51 +0000 | [diff] [blame] | 974 | llvm::Instruction::And); |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 975 | case Builtin::BI__sync_or_and_fetch_1: |
| 976 | case Builtin::BI__sync_or_and_fetch_2: |
| 977 | case Builtin::BI__sync_or_and_fetch_4: |
| 978 | case Builtin::BI__sync_or_and_fetch_8: |
| 979 | case Builtin::BI__sync_or_and_fetch_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 980 | return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E, |
Daniel Dunbar | 0002d23 | 2009-04-07 00:55:51 +0000 | [diff] [blame] | 981 | llvm::Instruction::Or); |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 982 | case Builtin::BI__sync_xor_and_fetch_1: |
| 983 | case Builtin::BI__sync_xor_and_fetch_2: |
| 984 | case Builtin::BI__sync_xor_and_fetch_4: |
| 985 | case Builtin::BI__sync_xor_and_fetch_8: |
| 986 | case Builtin::BI__sync_xor_and_fetch_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 987 | return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E, |
Daniel Dunbar | 0002d23 | 2009-04-07 00:55:51 +0000 | [diff] [blame] | 988 | llvm::Instruction::Xor); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 989 | case Builtin::BI__sync_nand_and_fetch_1: |
| 990 | case Builtin::BI__sync_nand_and_fetch_2: |
| 991 | case Builtin::BI__sync_nand_and_fetch_4: |
| 992 | case Builtin::BI__sync_nand_and_fetch_8: |
| 993 | case Builtin::BI__sync_nand_and_fetch_16: |
| 994 | return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E, |
| 995 | llvm::Instruction::And, true); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 996 | |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 997 | case Builtin::BI__sync_val_compare_and_swap_1: |
| 998 | case Builtin::BI__sync_val_compare_and_swap_2: |
| 999 | case Builtin::BI__sync_val_compare_and_swap_4: |
| 1000 | case Builtin::BI__sync_val_compare_and_swap_8: |
Daniel Dunbar | cb61a7b | 2010-03-20 07:04:11 +0000 | [diff] [blame] | 1001 | case Builtin::BI__sync_val_compare_and_swap_16: { |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 1002 | QualType T = E->getType(); |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 1003 | llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0)); |
Micah Villmow | 956a5a1 | 2012-10-25 15:39:14 +0000 | [diff] [blame] | 1004 | unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 1005 | |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 1006 | llvm::IntegerType *IntType = |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 1007 | llvm::IntegerType::get(getLLVMContext(), |
| 1008 | getContext().getTypeSize(T)); |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 1009 | llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); |
Chandler Carruth | db4325b | 2010-07-18 07:23:17 +0000 | [diff] [blame] | 1010 | |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 1011 | Value *Args[3]; |
| 1012 | Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType); |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 1013 | Args[1] = EmitScalarExpr(E->getArg(1)); |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 1014 | llvm::Type *ValueType = Args[1]->getType(); |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 1015 | Args[1] = EmitToInt(*this, Args[1], T, IntType); |
| 1016 | Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType); |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 1017 | |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 1018 | Value *Result = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2], |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 1019 | llvm::SequentiallyConsistent, |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 1020 | llvm::SequentiallyConsistent); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 1021 | Result = Builder.CreateExtractValue(Result, 0); |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 1022 | Result = EmitFromInt(*this, Result, T, ValueType); |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 1023 | return RValue::get(Result); |
Anders Carlsson | 89799cf | 2007-10-29 02:59:40 +0000 | [diff] [blame] | 1024 | } |
Daniel Dunbar | 0002d23 | 2009-04-07 00:55:51 +0000 | [diff] [blame] | 1025 | |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 1026 | case Builtin::BI__sync_bool_compare_and_swap_1: |
| 1027 | case Builtin::BI__sync_bool_compare_and_swap_2: |
| 1028 | case Builtin::BI__sync_bool_compare_and_swap_4: |
| 1029 | case Builtin::BI__sync_bool_compare_and_swap_8: |
Daniel Dunbar | cb61a7b | 2010-03-20 07:04:11 +0000 | [diff] [blame] | 1030 | case Builtin::BI__sync_bool_compare_and_swap_16: { |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 1031 | QualType T = E->getArg(1)->getType(); |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 1032 | llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0)); |
Micah Villmow | 956a5a1 | 2012-10-25 15:39:14 +0000 | [diff] [blame] | 1033 | unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace(); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 1034 | |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 1035 | llvm::IntegerType *IntType = |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 1036 | llvm::IntegerType::get(getLLVMContext(), |
| 1037 | getContext().getTypeSize(T)); |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 1038 | llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace); |
Chandler Carruth | db4325b | 2010-07-18 07:23:17 +0000 | [diff] [blame] | 1039 | |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 1040 | Value *Args[3]; |
| 1041 | Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType); |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 1042 | Args[1] = EmitToInt(*this, EmitScalarExpr(E->getArg(1)), T, IntType); |
| 1043 | Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType); |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 1044 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 1045 | Value *Pair = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2], |
| 1046 | llvm::SequentiallyConsistent, |
| 1047 | llvm::SequentiallyConsistent); |
| 1048 | Value *Result = Builder.CreateExtractValue(Pair, 1); |
Daniel Dunbar | 0002d23 | 2009-04-07 00:55:51 +0000 | [diff] [blame] | 1049 | // zext bool to int. |
John McCall | 26815d9 | 2010-10-27 20:58:56 +0000 | [diff] [blame] | 1050 | Result = Builder.CreateZExt(Result, ConvertType(E->getType())); |
| 1051 | return RValue::get(Result); |
Daniel Dunbar | 0002d23 | 2009-04-07 00:55:51 +0000 | [diff] [blame] | 1052 | } |
| 1053 | |
Chris Lattner | 23aa9c8 | 2011-04-09 03:57:26 +0000 | [diff] [blame] | 1054 | case Builtin::BI__sync_swap_1: |
| 1055 | case Builtin::BI__sync_swap_2: |
| 1056 | case Builtin::BI__sync_swap_4: |
| 1057 | case Builtin::BI__sync_swap_8: |
| 1058 | case Builtin::BI__sync_swap_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 1059 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); |
Chris Lattner | 23aa9c8 | 2011-04-09 03:57:26 +0000 | [diff] [blame] | 1060 | |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 1061 | case Builtin::BI__sync_lock_test_and_set_1: |
| 1062 | case Builtin::BI__sync_lock_test_and_set_2: |
| 1063 | case Builtin::BI__sync_lock_test_and_set_4: |
| 1064 | case Builtin::BI__sync_lock_test_and_set_8: |
| 1065 | case Builtin::BI__sync_lock_test_and_set_16: |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 1066 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); |
Daniel Dunbar | cb61a7b | 2010-03-20 07:04:11 +0000 | [diff] [blame] | 1067 | |
Chris Lattner | 5caa370 | 2009-05-08 06:58:22 +0000 | [diff] [blame] | 1068 | case Builtin::BI__sync_lock_release_1: |
| 1069 | case Builtin::BI__sync_lock_release_2: |
| 1070 | case Builtin::BI__sync_lock_release_4: |
| 1071 | case Builtin::BI__sync_lock_release_8: |
Chris Lattner | f58cd9b | 2009-05-13 04:46:13 +0000 | [diff] [blame] | 1072 | case Builtin::BI__sync_lock_release_16: { |
| 1073 | Value *Ptr = EmitScalarExpr(E->getArg(0)); |
Eli Friedman | eb43f4a | 2011-09-13 22:21:56 +0000 | [diff] [blame] | 1074 | QualType ElTy = E->getArg(0)->getType()->getPointeeType(); |
| 1075 | CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy); |
Eli Friedman | ff99320 | 2012-03-16 01:48:04 +0000 | [diff] [blame] | 1076 | llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(), |
| 1077 | StoreSize.getQuantity() * 8); |
| 1078 | Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo()); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 1079 | llvm::StoreInst *Store = |
Eli Friedman | ff99320 | 2012-03-16 01:48:04 +0000 | [diff] [blame] | 1080 | Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr); |
Eli Friedman | eb43f4a | 2011-09-13 22:21:56 +0000 | [diff] [blame] | 1081 | Store->setAlignment(StoreSize.getQuantity()); |
| 1082 | Store->setAtomic(llvm::Release); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 1083 | return RValue::get(nullptr); |
Chris Lattner | f58cd9b | 2009-05-13 04:46:13 +0000 | [diff] [blame] | 1084 | } |
Daniel Dunbar | ef2abfe | 2009-02-16 22:43:43 +0000 | [diff] [blame] | 1085 | |
Chris Lattner | f58cd9b | 2009-05-13 04:46:13 +0000 | [diff] [blame] | 1086 | case Builtin::BI__sync_synchronize: { |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 1087 | // We assume this is supposed to correspond to a C++0x-style |
| 1088 | // sequentially-consistent fence (i.e. this is only usable for |
| 1089 | // synchonization, not device I/O or anything like that). This intrinsic |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 1090 | // is really badly designed in the sense that in theory, there isn't |
Eli Friedman | c83b975 | 2011-09-07 01:41:24 +0000 | [diff] [blame] | 1091 | // any way to safely use it... but in practice, it mostly works |
| 1092 | // to use it with non-atomic loads and stores to get acquire/release |
| 1093 | // semantics. |
| 1094 | Builder.CreateFence(llvm::SequentiallyConsistent); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 1095 | return RValue::get(nullptr); |
Chris Lattner | f58cd9b | 2009-05-13 04:46:13 +0000 | [diff] [blame] | 1096 | } |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1097 | |
Richard Smith | 2c39d71 | 2012-04-13 00:45:38 +0000 | [diff] [blame] | 1098 | case Builtin::BI__c11_atomic_is_lock_free: |
| 1099 | case Builtin::BI__atomic_is_lock_free: { |
| 1100 | // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the |
| 1101 | // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since |
| 1102 | // _Atomic(T) is always properly-aligned. |
| 1103 | const char *LibCallName = "__atomic_is_lock_free"; |
| 1104 | CallArgList Args; |
| 1105 | Args.add(RValue::get(EmitScalarExpr(E->getArg(0))), |
| 1106 | getContext().getSizeType()); |
| 1107 | if (BuiltinID == Builtin::BI__atomic_is_lock_free) |
| 1108 | Args.add(RValue::get(EmitScalarExpr(E->getArg(1))), |
| 1109 | getContext().VoidPtrTy); |
| 1110 | else |
| 1111 | Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)), |
| 1112 | getContext().VoidPtrTy); |
| 1113 | const CGFunctionInfo &FuncInfo = |
John McCall | 0f3d097 | 2012-07-07 06:41:13 +0000 | [diff] [blame] | 1114 | CGM.getTypes().arrangeFreeFunctionCall(E->getType(), Args, |
| 1115 | FunctionType::ExtInfo(), |
| 1116 | RequiredArgs::All); |
Richard Smith | 2c39d71 | 2012-04-13 00:45:38 +0000 | [diff] [blame] | 1117 | llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); |
| 1118 | llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); |
| 1119 | return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); |
| 1120 | } |
| 1121 | |
| 1122 | case Builtin::BI__atomic_test_and_set: { |
| 1123 | // Look at the argument type to determine whether this is a volatile |
| 1124 | // operation. The parameter type is always volatile. |
| 1125 | QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); |
| 1126 | bool Volatile = |
| 1127 | PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); |
| 1128 | |
| 1129 | Value *Ptr = EmitScalarExpr(E->getArg(0)); |
Micah Villmow | 956a5a1 | 2012-10-25 15:39:14 +0000 | [diff] [blame] | 1130 | unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); |
Richard Smith | 2c39d71 | 2012-04-13 00:45:38 +0000 | [diff] [blame] | 1131 | Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); |
| 1132 | Value *NewVal = Builder.getInt8(1); |
| 1133 | Value *Order = EmitScalarExpr(E->getArg(1)); |
| 1134 | if (isa<llvm::ConstantInt>(Order)) { |
| 1135 | int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 1136 | AtomicRMWInst *Result = nullptr; |
Richard Smith | 2c39d71 | 2012-04-13 00:45:38 +0000 | [diff] [blame] | 1137 | switch (ord) { |
| 1138 | case 0: // memory_order_relaxed |
| 1139 | default: // invalid order |
| 1140 | Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, |
| 1141 | Ptr, NewVal, |
| 1142 | llvm::Monotonic); |
| 1143 | break; |
| 1144 | case 1: // memory_order_consume |
| 1145 | case 2: // memory_order_acquire |
| 1146 | Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, |
| 1147 | Ptr, NewVal, |
| 1148 | llvm::Acquire); |
| 1149 | break; |
| 1150 | case 3: // memory_order_release |
| 1151 | Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, |
| 1152 | Ptr, NewVal, |
| 1153 | llvm::Release); |
| 1154 | break; |
| 1155 | case 4: // memory_order_acq_rel |
| 1156 | Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, |
| 1157 | Ptr, NewVal, |
| 1158 | llvm::AcquireRelease); |
| 1159 | break; |
| 1160 | case 5: // memory_order_seq_cst |
| 1161 | Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, |
| 1162 | Ptr, NewVal, |
| 1163 | llvm::SequentiallyConsistent); |
| 1164 | break; |
| 1165 | } |
| 1166 | Result->setVolatile(Volatile); |
| 1167 | return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); |
| 1168 | } |
| 1169 | |
| 1170 | llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); |
| 1171 | |
| 1172 | llvm::BasicBlock *BBs[5] = { |
| 1173 | createBasicBlock("monotonic", CurFn), |
| 1174 | createBasicBlock("acquire", CurFn), |
| 1175 | createBasicBlock("release", CurFn), |
| 1176 | createBasicBlock("acqrel", CurFn), |
| 1177 | createBasicBlock("seqcst", CurFn) |
| 1178 | }; |
| 1179 | llvm::AtomicOrdering Orders[5] = { |
| 1180 | llvm::Monotonic, llvm::Acquire, llvm::Release, |
| 1181 | llvm::AcquireRelease, llvm::SequentiallyConsistent |
| 1182 | }; |
| 1183 | |
| 1184 | Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); |
| 1185 | llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); |
| 1186 | |
| 1187 | Builder.SetInsertPoint(ContBB); |
| 1188 | PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set"); |
| 1189 | |
| 1190 | for (unsigned i = 0; i < 5; ++i) { |
| 1191 | Builder.SetInsertPoint(BBs[i]); |
| 1192 | AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, |
| 1193 | Ptr, NewVal, Orders[i]); |
| 1194 | RMW->setVolatile(Volatile); |
| 1195 | Result->addIncoming(RMW, BBs[i]); |
| 1196 | Builder.CreateBr(ContBB); |
| 1197 | } |
| 1198 | |
| 1199 | SI->addCase(Builder.getInt32(0), BBs[0]); |
| 1200 | SI->addCase(Builder.getInt32(1), BBs[1]); |
| 1201 | SI->addCase(Builder.getInt32(2), BBs[1]); |
| 1202 | SI->addCase(Builder.getInt32(3), BBs[2]); |
| 1203 | SI->addCase(Builder.getInt32(4), BBs[3]); |
| 1204 | SI->addCase(Builder.getInt32(5), BBs[4]); |
| 1205 | |
| 1206 | Builder.SetInsertPoint(ContBB); |
| 1207 | return RValue::get(Builder.CreateIsNotNull(Result, "tobool")); |
| 1208 | } |
| 1209 | |
| 1210 | case Builtin::BI__atomic_clear: { |
| 1211 | QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType(); |
| 1212 | bool Volatile = |
| 1213 | PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified(); |
| 1214 | |
| 1215 | Value *Ptr = EmitScalarExpr(E->getArg(0)); |
Micah Villmow | 956a5a1 | 2012-10-25 15:39:14 +0000 | [diff] [blame] | 1216 | unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace(); |
Richard Smith | 2c39d71 | 2012-04-13 00:45:38 +0000 | [diff] [blame] | 1217 | Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace)); |
| 1218 | Value *NewVal = Builder.getInt8(0); |
| 1219 | Value *Order = EmitScalarExpr(E->getArg(1)); |
| 1220 | if (isa<llvm::ConstantInt>(Order)) { |
| 1221 | int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); |
| 1222 | StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); |
| 1223 | Store->setAlignment(1); |
| 1224 | switch (ord) { |
| 1225 | case 0: // memory_order_relaxed |
| 1226 | default: // invalid order |
| 1227 | Store->setOrdering(llvm::Monotonic); |
| 1228 | break; |
| 1229 | case 3: // memory_order_release |
| 1230 | Store->setOrdering(llvm::Release); |
| 1231 | break; |
| 1232 | case 5: // memory_order_seq_cst |
| 1233 | Store->setOrdering(llvm::SequentiallyConsistent); |
| 1234 | break; |
| 1235 | } |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 1236 | return RValue::get(nullptr); |
Richard Smith | 2c39d71 | 2012-04-13 00:45:38 +0000 | [diff] [blame] | 1237 | } |
| 1238 | |
| 1239 | llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); |
| 1240 | |
| 1241 | llvm::BasicBlock *BBs[3] = { |
| 1242 | createBasicBlock("monotonic", CurFn), |
| 1243 | createBasicBlock("release", CurFn), |
| 1244 | createBasicBlock("seqcst", CurFn) |
| 1245 | }; |
| 1246 | llvm::AtomicOrdering Orders[3] = { |
| 1247 | llvm::Monotonic, llvm::Release, llvm::SequentiallyConsistent |
| 1248 | }; |
| 1249 | |
| 1250 | Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); |
| 1251 | llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]); |
| 1252 | |
| 1253 | for (unsigned i = 0; i < 3; ++i) { |
| 1254 | Builder.SetInsertPoint(BBs[i]); |
| 1255 | StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile); |
| 1256 | Store->setAlignment(1); |
| 1257 | Store->setOrdering(Orders[i]); |
| 1258 | Builder.CreateBr(ContBB); |
| 1259 | } |
| 1260 | |
| 1261 | SI->addCase(Builder.getInt32(0), BBs[0]); |
| 1262 | SI->addCase(Builder.getInt32(3), BBs[1]); |
| 1263 | SI->addCase(Builder.getInt32(5), BBs[2]); |
| 1264 | |
| 1265 | Builder.SetInsertPoint(ContBB); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 1266 | return RValue::get(nullptr); |
Richard Smith | 2c39d71 | 2012-04-13 00:45:38 +0000 | [diff] [blame] | 1267 | } |
| 1268 | |
Eli Friedman | 276b061 | 2011-10-11 02:20:01 +0000 | [diff] [blame] | 1269 | case Builtin::BI__atomic_thread_fence: |
Richard Smith | fafbf06 | 2012-04-11 17:55:32 +0000 | [diff] [blame] | 1270 | case Builtin::BI__atomic_signal_fence: |
| 1271 | case Builtin::BI__c11_atomic_thread_fence: |
| 1272 | case Builtin::BI__c11_atomic_signal_fence: { |
Eli Friedman | 276b061 | 2011-10-11 02:20:01 +0000 | [diff] [blame] | 1273 | llvm::SynchronizationScope Scope; |
Richard Smith | fafbf06 | 2012-04-11 17:55:32 +0000 | [diff] [blame] | 1274 | if (BuiltinID == Builtin::BI__atomic_signal_fence || |
| 1275 | BuiltinID == Builtin::BI__c11_atomic_signal_fence) |
Eli Friedman | 276b061 | 2011-10-11 02:20:01 +0000 | [diff] [blame] | 1276 | Scope = llvm::SingleThread; |
| 1277 | else |
| 1278 | Scope = llvm::CrossThread; |
| 1279 | Value *Order = EmitScalarExpr(E->getArg(0)); |
| 1280 | if (isa<llvm::ConstantInt>(Order)) { |
| 1281 | int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); |
| 1282 | switch (ord) { |
| 1283 | case 0: // memory_order_relaxed |
| 1284 | default: // invalid order |
| 1285 | break; |
| 1286 | case 1: // memory_order_consume |
| 1287 | case 2: // memory_order_acquire |
| 1288 | Builder.CreateFence(llvm::Acquire, Scope); |
| 1289 | break; |
| 1290 | case 3: // memory_order_release |
| 1291 | Builder.CreateFence(llvm::Release, Scope); |
| 1292 | break; |
| 1293 | case 4: // memory_order_acq_rel |
| 1294 | Builder.CreateFence(llvm::AcquireRelease, Scope); |
| 1295 | break; |
| 1296 | case 5: // memory_order_seq_cst |
| 1297 | Builder.CreateFence(llvm::SequentiallyConsistent, Scope); |
| 1298 | break; |
| 1299 | } |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 1300 | return RValue::get(nullptr); |
Eli Friedman | 276b061 | 2011-10-11 02:20:01 +0000 | [diff] [blame] | 1301 | } |
| 1302 | |
| 1303 | llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB; |
| 1304 | AcquireBB = createBasicBlock("acquire", CurFn); |
| 1305 | ReleaseBB = createBasicBlock("release", CurFn); |
| 1306 | AcqRelBB = createBasicBlock("acqrel", CurFn); |
| 1307 | SeqCstBB = createBasicBlock("seqcst", CurFn); |
| 1308 | llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); |
| 1309 | |
| 1310 | Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); |
| 1311 | llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB); |
| 1312 | |
| 1313 | Builder.SetInsertPoint(AcquireBB); |
| 1314 | Builder.CreateFence(llvm::Acquire, Scope); |
| 1315 | Builder.CreateBr(ContBB); |
| 1316 | SI->addCase(Builder.getInt32(1), AcquireBB); |
| 1317 | SI->addCase(Builder.getInt32(2), AcquireBB); |
| 1318 | |
| 1319 | Builder.SetInsertPoint(ReleaseBB); |
| 1320 | Builder.CreateFence(llvm::Release, Scope); |
| 1321 | Builder.CreateBr(ContBB); |
| 1322 | SI->addCase(Builder.getInt32(3), ReleaseBB); |
| 1323 | |
| 1324 | Builder.SetInsertPoint(AcqRelBB); |
| 1325 | Builder.CreateFence(llvm::AcquireRelease, Scope); |
| 1326 | Builder.CreateBr(ContBB); |
| 1327 | SI->addCase(Builder.getInt32(4), AcqRelBB); |
| 1328 | |
| 1329 | Builder.SetInsertPoint(SeqCstBB); |
| 1330 | Builder.CreateFence(llvm::SequentiallyConsistent, Scope); |
| 1331 | Builder.CreateBr(ContBB); |
| 1332 | SI->addCase(Builder.getInt32(5), SeqCstBB); |
| 1333 | |
| 1334 | Builder.SetInsertPoint(ContBB); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 1335 | return RValue::get(nullptr); |
Eli Friedman | 276b061 | 2011-10-11 02:20:01 +0000 | [diff] [blame] | 1336 | } |
| 1337 | |
Daniel Dunbar | ef2abfe | 2009-02-16 22:43:43 +0000 | [diff] [blame] | 1338 | // Library functions with special handling. |
Daniel Dunbar | ef2abfe | 2009-02-16 22:43:43 +0000 | [diff] [blame] | 1339 | case Builtin::BIsqrt: |
| 1340 | case Builtin::BIsqrtf: |
| 1341 | case Builtin::BIsqrtl: { |
Hal Finkel | ce4ad40 | 2013-09-12 23:57:55 +0000 | [diff] [blame] | 1342 | // Transform a call to sqrt* into a @llvm.sqrt.* intrinsic call, but only |
| 1343 | // in finite- or unsafe-math mode (the intrinsic has different semantics |
| 1344 | // for handling negative numbers compared to the library function, so |
| 1345 | // -fmath-errno=0 is not enough). |
| 1346 | if (!FD->hasAttr<ConstAttr>()) |
| 1347 | break; |
| 1348 | if (!(CGM.getCodeGenOpts().UnsafeFPMath || |
| 1349 | CGM.getCodeGenOpts().NoNaNsFPMath)) |
| 1350 | break; |
| 1351 | Value *Arg0 = EmitScalarExpr(E->getArg(0)); |
| 1352 | llvm::Type *ArgType = Arg0->getType(); |
| 1353 | Value *F = CGM.getIntrinsic(Intrinsic::sqrt, ArgType); |
| 1354 | return RValue::get(Builder.CreateCall(F, Arg0)); |
Daniel Dunbar | ef2abfe | 2009-02-16 22:43:43 +0000 | [diff] [blame] | 1355 | } |
| 1356 | |
| 1357 | case Builtin::BIpow: |
| 1358 | case Builtin::BIpowf: |
| 1359 | case Builtin::BIpowl: { |
Eli Bendersky | 0323a78 | 2013-07-24 21:22:01 +0000 | [diff] [blame] | 1360 | // Transform a call to pow* into a @llvm.pow.* intrinsic call. |
| 1361 | if (!FD->hasAttr<ConstAttr>()) |
| 1362 | break; |
| 1363 | Value *Base = EmitScalarExpr(E->getArg(0)); |
| 1364 | Value *Exponent = EmitScalarExpr(E->getArg(1)); |
| 1365 | llvm::Type *ArgType = Base->getType(); |
| 1366 | Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType); |
| 1367 | return RValue::get(Builder.CreateCall2(F, Base, Exponent)); |
Daniel Dunbar | ef2abfe | 2009-02-16 22:43:43 +0000 | [diff] [blame] | 1368 | } |
Eli Friedman | ba68b08 | 2010-03-06 02:17:52 +0000 | [diff] [blame] | 1369 | |
Cameron Zwarich | 094240a | 2011-07-08 21:39:34 +0000 | [diff] [blame] | 1370 | case Builtin::BIfma: |
| 1371 | case Builtin::BIfmaf: |
| 1372 | case Builtin::BIfmal: |
| 1373 | case Builtin::BI__builtin_fma: |
| 1374 | case Builtin::BI__builtin_fmaf: |
| 1375 | case Builtin::BI__builtin_fmal: { |
| 1376 | // Rewrite fma to intrinsic. |
| 1377 | Value *FirstArg = EmitScalarExpr(E->getArg(0)); |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 1378 | llvm::Type *ArgType = FirstArg->getType(); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 1379 | Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType); |
Cameron Zwarich | 094240a | 2011-07-08 21:39:34 +0000 | [diff] [blame] | 1380 | return RValue::get(Builder.CreateCall3(F, FirstArg, |
| 1381 | EmitScalarExpr(E->getArg(1)), |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 1382 | EmitScalarExpr(E->getArg(2)))); |
Cameron Zwarich | 094240a | 2011-07-08 21:39:34 +0000 | [diff] [blame] | 1383 | } |
| 1384 | |
Eli Friedman | ba68b08 | 2010-03-06 02:17:52 +0000 | [diff] [blame] | 1385 | case Builtin::BI__builtin_signbit: |
| 1386 | case Builtin::BI__builtin_signbitf: |
| 1387 | case Builtin::BI__builtin_signbitl: { |
| 1388 | LLVMContext &C = CGM.getLLVMContext(); |
| 1389 | |
| 1390 | Value *Arg = EmitScalarExpr(E->getArg(0)); |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 1391 | llvm::Type *ArgTy = Arg->getType(); |
Eli Friedman | ba68b08 | 2010-03-06 02:17:52 +0000 | [diff] [blame] | 1392 | int ArgWidth = ArgTy->getPrimitiveSizeInBits(); |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 1393 | llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth); |
Eli Friedman | ba68b08 | 2010-03-06 02:17:52 +0000 | [diff] [blame] | 1394 | Value *BCArg = Builder.CreateBitCast(Arg, ArgIntTy); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 1395 | if (ArgTy->isPPC_FP128Ty()) { |
| 1396 | // The higher-order double comes first, and so we need to truncate the |
| 1397 | // pair to extract the overall sign. The order of the pair is the same |
| 1398 | // in both little- and big-Endian modes. |
| 1399 | ArgWidth >>= 1; |
| 1400 | ArgIntTy = llvm::IntegerType::get(C, ArgWidth); |
| 1401 | BCArg = Builder.CreateTrunc(BCArg, ArgIntTy); |
| 1402 | } |
Eli Friedman | ba68b08 | 2010-03-06 02:17:52 +0000 | [diff] [blame] | 1403 | Value *ZeroCmp = llvm::Constant::getNullValue(ArgIntTy); |
| 1404 | Value *Result = Builder.CreateICmpSLT(BCArg, ZeroCmp); |
| 1405 | return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType()))); |
| 1406 | } |
Julien Lerouge | 77f68bb | 2011-09-09 22:41:49 +0000 | [diff] [blame] | 1407 | case Builtin::BI__builtin_annotation: { |
| 1408 | llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0)); |
| 1409 | llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation, |
| 1410 | AnnVal->getType()); |
| 1411 | |
| 1412 | // Get the annotation string, go through casts. Sema requires this to be a |
| 1413 | // non-wide string literal, potentially casted, so the cast<> is safe. |
| 1414 | const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts(); |
Dmitri Gribenko | cfa88f8 | 2013-01-12 19:30:44 +0000 | [diff] [blame] | 1415 | StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString(); |
Julien Lerouge | 77f68bb | 2011-09-09 22:41:49 +0000 | [diff] [blame] | 1416 | return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc())); |
| 1417 | } |
Michael Gottesman | ee76e72 | 2013-06-18 20:40:40 +0000 | [diff] [blame] | 1418 | case Builtin::BI__builtin_addcb: |
Michael Gottesman | 0cf07bc | 2013-01-13 02:22:39 +0000 | [diff] [blame] | 1419 | case Builtin::BI__builtin_addcs: |
| 1420 | case Builtin::BI__builtin_addc: |
| 1421 | case Builtin::BI__builtin_addcl: |
Michael Gottesman | 7c9dbb7 | 2013-01-14 21:44:30 +0000 | [diff] [blame] | 1422 | case Builtin::BI__builtin_addcll: |
Michael Gottesman | ee76e72 | 2013-06-18 20:40:40 +0000 | [diff] [blame] | 1423 | case Builtin::BI__builtin_subcb: |
Michael Gottesman | 7c9dbb7 | 2013-01-14 21:44:30 +0000 | [diff] [blame] | 1424 | case Builtin::BI__builtin_subcs: |
| 1425 | case Builtin::BI__builtin_subc: |
| 1426 | case Builtin::BI__builtin_subcl: |
| 1427 | case Builtin::BI__builtin_subcll: { |
Michael Gottesman | 0cf07bc | 2013-01-13 02:22:39 +0000 | [diff] [blame] | 1428 | |
| 1429 | // We translate all of these builtins from expressions of the form: |
| 1430 | // int x = ..., y = ..., carryin = ..., carryout, result; |
| 1431 | // result = __builtin_addc(x, y, carryin, &carryout); |
| 1432 | // |
| 1433 | // to LLVM IR of the form: |
| 1434 | // |
| 1435 | // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y) |
| 1436 | // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0 |
| 1437 | // %carry1 = extractvalue {i32, i1} %tmp1, 1 |
| 1438 | // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1, |
| 1439 | // i32 %carryin) |
| 1440 | // %result = extractvalue {i32, i1} %tmp2, 0 |
| 1441 | // %carry2 = extractvalue {i32, i1} %tmp2, 1 |
| 1442 | // %tmp3 = or i1 %carry1, %carry2 |
| 1443 | // %tmp4 = zext i1 %tmp3 to i32 |
| 1444 | // store i32 %tmp4, i32* %carryout |
| 1445 | |
| 1446 | // Scalarize our inputs. |
| 1447 | llvm::Value *X = EmitScalarExpr(E->getArg(0)); |
| 1448 | llvm::Value *Y = EmitScalarExpr(E->getArg(1)); |
| 1449 | llvm::Value *Carryin = EmitScalarExpr(E->getArg(2)); |
| 1450 | std::pair<llvm::Value*, unsigned> CarryOutPtr = |
| 1451 | EmitPointerWithAlignment(E->getArg(3)); |
| 1452 | |
Michael Gottesman | 7c9dbb7 | 2013-01-14 21:44:30 +0000 | [diff] [blame] | 1453 | // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow. |
| 1454 | llvm::Intrinsic::ID IntrinsicId; |
| 1455 | switch (BuiltinID) { |
| 1456 | default: llvm_unreachable("Unknown multiprecision builtin id."); |
Michael Gottesman | ee76e72 | 2013-06-18 20:40:40 +0000 | [diff] [blame] | 1457 | case Builtin::BI__builtin_addcb: |
Michael Gottesman | 7c9dbb7 | 2013-01-14 21:44:30 +0000 | [diff] [blame] | 1458 | case Builtin::BI__builtin_addcs: |
| 1459 | case Builtin::BI__builtin_addc: |
| 1460 | case Builtin::BI__builtin_addcl: |
| 1461 | case Builtin::BI__builtin_addcll: |
| 1462 | IntrinsicId = llvm::Intrinsic::uadd_with_overflow; |
| 1463 | break; |
Michael Gottesman | ee76e72 | 2013-06-18 20:40:40 +0000 | [diff] [blame] | 1464 | case Builtin::BI__builtin_subcb: |
Michael Gottesman | 7c9dbb7 | 2013-01-14 21:44:30 +0000 | [diff] [blame] | 1465 | case Builtin::BI__builtin_subcs: |
| 1466 | case Builtin::BI__builtin_subc: |
| 1467 | case Builtin::BI__builtin_subcl: |
| 1468 | case Builtin::BI__builtin_subcll: |
| 1469 | IntrinsicId = llvm::Intrinsic::usub_with_overflow; |
| 1470 | break; |
| 1471 | } |
Michael Gottesman | 0cf07bc | 2013-01-13 02:22:39 +0000 | [diff] [blame] | 1472 | |
| 1473 | // Construct our resulting LLVM IR expression. |
| 1474 | llvm::Value *Carry1; |
| 1475 | llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId, |
| 1476 | X, Y, Carry1); |
| 1477 | llvm::Value *Carry2; |
| 1478 | llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId, |
| 1479 | Sum1, Carryin, Carry2); |
| 1480 | llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2), |
| 1481 | X->getType()); |
| 1482 | llvm::StoreInst *CarryOutStore = Builder.CreateStore(CarryOut, |
| 1483 | CarryOutPtr.first); |
| 1484 | CarryOutStore->setAlignment(CarryOutPtr.second); |
| 1485 | return RValue::get(Sum2); |
| 1486 | } |
Michael Gottesman | 98d1ec1 | 2013-06-20 23:28:10 +0000 | [diff] [blame] | 1487 | case Builtin::BI__builtin_uadd_overflow: |
| 1488 | case Builtin::BI__builtin_uaddl_overflow: |
| 1489 | case Builtin::BI__builtin_uaddll_overflow: |
| 1490 | case Builtin::BI__builtin_usub_overflow: |
| 1491 | case Builtin::BI__builtin_usubl_overflow: |
| 1492 | case Builtin::BI__builtin_usubll_overflow: |
| 1493 | case Builtin::BI__builtin_umul_overflow: |
| 1494 | case Builtin::BI__builtin_umull_overflow: |
| 1495 | case Builtin::BI__builtin_umulll_overflow: |
| 1496 | case Builtin::BI__builtin_sadd_overflow: |
| 1497 | case Builtin::BI__builtin_saddl_overflow: |
| 1498 | case Builtin::BI__builtin_saddll_overflow: |
| 1499 | case Builtin::BI__builtin_ssub_overflow: |
| 1500 | case Builtin::BI__builtin_ssubl_overflow: |
| 1501 | case Builtin::BI__builtin_ssubll_overflow: |
| 1502 | case Builtin::BI__builtin_smul_overflow: |
| 1503 | case Builtin::BI__builtin_smull_overflow: |
| 1504 | case Builtin::BI__builtin_smulll_overflow: { |
| 1505 | |
| 1506 | // We translate all of these builtins directly to the relevant llvm IR node. |
| 1507 | |
| 1508 | // Scalarize our inputs. |
| 1509 | llvm::Value *X = EmitScalarExpr(E->getArg(0)); |
| 1510 | llvm::Value *Y = EmitScalarExpr(E->getArg(1)); |
| 1511 | std::pair<llvm::Value *, unsigned> SumOutPtr = |
| 1512 | EmitPointerWithAlignment(E->getArg(2)); |
| 1513 | |
| 1514 | // Decide which of the overflow intrinsics we are lowering to: |
| 1515 | llvm::Intrinsic::ID IntrinsicId; |
| 1516 | switch (BuiltinID) { |
| 1517 | default: llvm_unreachable("Unknown security overflow builtin id."); |
| 1518 | case Builtin::BI__builtin_uadd_overflow: |
| 1519 | case Builtin::BI__builtin_uaddl_overflow: |
| 1520 | case Builtin::BI__builtin_uaddll_overflow: |
| 1521 | IntrinsicId = llvm::Intrinsic::uadd_with_overflow; |
| 1522 | break; |
| 1523 | case Builtin::BI__builtin_usub_overflow: |
| 1524 | case Builtin::BI__builtin_usubl_overflow: |
| 1525 | case Builtin::BI__builtin_usubll_overflow: |
| 1526 | IntrinsicId = llvm::Intrinsic::usub_with_overflow; |
| 1527 | break; |
| 1528 | case Builtin::BI__builtin_umul_overflow: |
| 1529 | case Builtin::BI__builtin_umull_overflow: |
| 1530 | case Builtin::BI__builtin_umulll_overflow: |
| 1531 | IntrinsicId = llvm::Intrinsic::umul_with_overflow; |
| 1532 | break; |
| 1533 | case Builtin::BI__builtin_sadd_overflow: |
| 1534 | case Builtin::BI__builtin_saddl_overflow: |
| 1535 | case Builtin::BI__builtin_saddll_overflow: |
| 1536 | IntrinsicId = llvm::Intrinsic::sadd_with_overflow; |
| 1537 | break; |
| 1538 | case Builtin::BI__builtin_ssub_overflow: |
| 1539 | case Builtin::BI__builtin_ssubl_overflow: |
| 1540 | case Builtin::BI__builtin_ssubll_overflow: |
| 1541 | IntrinsicId = llvm::Intrinsic::ssub_with_overflow; |
| 1542 | break; |
| 1543 | case Builtin::BI__builtin_smul_overflow: |
| 1544 | case Builtin::BI__builtin_smull_overflow: |
| 1545 | case Builtin::BI__builtin_smulll_overflow: |
| 1546 | IntrinsicId = llvm::Intrinsic::smul_with_overflow; |
| 1547 | break; |
| 1548 | } |
| 1549 | |
| 1550 | |
| 1551 | llvm::Value *Carry; |
| 1552 | llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry); |
| 1553 | llvm::StoreInst *SumOutStore = Builder.CreateStore(Sum, SumOutPtr.first); |
| 1554 | SumOutStore->setAlignment(SumOutPtr.second); |
| 1555 | |
| 1556 | return RValue::get(Carry); |
| 1557 | } |
Richard Smith | 5154dce | 2013-07-11 02:27:57 +0000 | [diff] [blame] | 1558 | case Builtin::BI__builtin_addressof: |
| 1559 | return RValue::get(EmitLValue(E->getArg(0)).getAddress()); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 1560 | case Builtin::BI__builtin_operator_new: |
| 1561 | return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), |
| 1562 | E->getArg(0), false); |
| 1563 | case Builtin::BI__builtin_operator_delete: |
| 1564 | return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(), |
| 1565 | E->getArg(0), true); |
Nico Weber | 2784453 | 2012-10-13 22:30:41 +0000 | [diff] [blame] | 1566 | case Builtin::BI__noop: |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 1567 | // __noop always evaluates to an integer literal zero. |
| 1568 | return RValue::get(ConstantInt::get(IntTy, 0)); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 1569 | case Builtin::BI_InterlockedExchange: |
| 1570 | case Builtin::BI_InterlockedExchangePointer: |
| 1571 | return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E); |
| 1572 | case Builtin::BI_InterlockedCompareExchangePointer: { |
| 1573 | llvm::Type *RTy; |
| 1574 | llvm::IntegerType *IntType = |
| 1575 | IntegerType::get(getLLVMContext(), |
| 1576 | getContext().getTypeSize(E->getType())); |
| 1577 | llvm::Type *IntPtrType = IntType->getPointerTo(); |
| 1578 | |
| 1579 | llvm::Value *Destination = |
| 1580 | Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType); |
| 1581 | |
| 1582 | llvm::Value *Exchange = EmitScalarExpr(E->getArg(1)); |
| 1583 | RTy = Exchange->getType(); |
| 1584 | Exchange = Builder.CreatePtrToInt(Exchange, IntType); |
| 1585 | |
| 1586 | llvm::Value *Comparand = |
| 1587 | Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType); |
| 1588 | |
| 1589 | auto Result = Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange, |
| 1590 | SequentiallyConsistent, |
| 1591 | SequentiallyConsistent); |
| 1592 | Result->setVolatile(true); |
| 1593 | |
| 1594 | return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result, |
| 1595 | 0), |
| 1596 | RTy)); |
| 1597 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 1598 | case Builtin::BI_InterlockedCompareExchange: { |
| 1599 | AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg( |
| 1600 | EmitScalarExpr(E->getArg(0)), |
| 1601 | EmitScalarExpr(E->getArg(2)), |
| 1602 | EmitScalarExpr(E->getArg(1)), |
| 1603 | SequentiallyConsistent, |
| 1604 | SequentiallyConsistent); |
| 1605 | CXI->setVolatile(true); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 1606 | return RValue::get(Builder.CreateExtractValue(CXI, 0)); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 1607 | } |
| 1608 | case Builtin::BI_InterlockedIncrement: { |
| 1609 | AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( |
| 1610 | AtomicRMWInst::Add, |
| 1611 | EmitScalarExpr(E->getArg(0)), |
| 1612 | ConstantInt::get(Int32Ty, 1), |
| 1613 | llvm::SequentiallyConsistent); |
| 1614 | RMWI->setVolatile(true); |
| 1615 | return RValue::get(Builder.CreateAdd(RMWI, ConstantInt::get(Int32Ty, 1))); |
| 1616 | } |
| 1617 | case Builtin::BI_InterlockedDecrement: { |
| 1618 | AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( |
| 1619 | AtomicRMWInst::Sub, |
| 1620 | EmitScalarExpr(E->getArg(0)), |
| 1621 | ConstantInt::get(Int32Ty, 1), |
| 1622 | llvm::SequentiallyConsistent); |
| 1623 | RMWI->setVolatile(true); |
| 1624 | return RValue::get(Builder.CreateSub(RMWI, ConstantInt::get(Int32Ty, 1))); |
| 1625 | } |
| 1626 | case Builtin::BI_InterlockedExchangeAdd: { |
| 1627 | AtomicRMWInst *RMWI = Builder.CreateAtomicRMW( |
| 1628 | AtomicRMWInst::Add, |
| 1629 | EmitScalarExpr(E->getArg(0)), |
| 1630 | EmitScalarExpr(E->getArg(1)), |
| 1631 | llvm::SequentiallyConsistent); |
| 1632 | RMWI->setVolatile(true); |
| 1633 | return RValue::get(RMWI); |
| 1634 | } |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 1635 | case Builtin::BI__readfsdword: { |
| 1636 | Value *IntToPtr = |
| 1637 | Builder.CreateIntToPtr(EmitScalarExpr(E->getArg(0)), |
| 1638 | llvm::PointerType::get(CGM.Int32Ty, 257)); |
| 1639 | LoadInst *Load = |
| 1640 | Builder.CreateAlignedLoad(IntToPtr, /*Align=*/4, /*isVolatile=*/true); |
| 1641 | return RValue::get(Load); |
| 1642 | } |
Nate Begeman | 7ea2e3f | 2008-05-15 07:38:03 +0000 | [diff] [blame] | 1643 | } |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1644 | |
John McCall | a45680b | 2011-09-13 23:05:03 +0000 | [diff] [blame] | 1645 | // If this is an alias for a lib function (e.g. __builtin_sin), emit |
| 1646 | // the call using the normal call path, but using the unmangled |
| 1647 | // version of the function name. |
| 1648 | if (getContext().BuiltinInfo.isLibFunction(BuiltinID)) |
| 1649 | return emitLibraryCall(*this, FD, E, |
| 1650 | CGM.getBuiltinLibFunction(FD, BuiltinID)); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 1651 | |
John McCall | a45680b | 2011-09-13 23:05:03 +0000 | [diff] [blame] | 1652 | // If this is a predefined lib function (e.g. malloc), emit the call |
| 1653 | // using exactly the normal call path. |
| 1654 | if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID)) |
| 1655 | return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee())); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1656 | |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1657 | // See if we have a target specific intrinsic. |
Dale Johannesen | a6f80ef | 2009-02-05 01:50:47 +0000 | [diff] [blame] | 1658 | const char *Name = getContext().BuiltinInfo.GetName(BuiltinID); |
Daniel Dunbar | 55cc2ed | 2009-08-24 09:54:37 +0000 | [diff] [blame] | 1659 | Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic; |
| 1660 | if (const char *Prefix = |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 1661 | llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) { |
Daniel Dunbar | 55cc2ed | 2009-08-24 09:54:37 +0000 | [diff] [blame] | 1662 | IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 1663 | // NOTE we dont need to perform a compatibility flag check here since the |
| 1664 | // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the |
| 1665 | // MS builtins via ALL_MS_LANGUAGES and are filtered earlier. |
| 1666 | if (IntrinsicID == Intrinsic::not_intrinsic) |
| 1667 | IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix, Name); |
| 1668 | } |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1669 | |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1670 | if (IntrinsicID != Intrinsic::not_intrinsic) { |
| 1671 | SmallVector<Value*, 16> Args; |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1672 | |
Chris Lattner | 46c5591 | 2010-10-02 00:09:12 +0000 | [diff] [blame] | 1673 | // Find out if any arguments are required to be integer constant |
| 1674 | // expressions. |
| 1675 | unsigned ICEArguments = 0; |
| 1676 | ASTContext::GetBuiltinTypeError Error; |
| 1677 | getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); |
| 1678 | assert(Error == ASTContext::GE_None && "Should not codegen an error"); |
| 1679 | |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1680 | Function *F = CGM.getIntrinsic(IntrinsicID); |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 1681 | llvm::FunctionType *FTy = F->getFunctionType(); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1682 | |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1683 | for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) { |
Chris Lattner | 46c5591 | 2010-10-02 00:09:12 +0000 | [diff] [blame] | 1684 | Value *ArgValue; |
| 1685 | // If this is a normal argument, just emit it as a scalar. |
| 1686 | if ((ICEArguments & (1 << i)) == 0) { |
| 1687 | ArgValue = EmitScalarExpr(E->getArg(i)); |
| 1688 | } else { |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 1689 | // If this is required to be a constant, constant fold it so that we |
Chris Lattner | 46c5591 | 2010-10-02 00:09:12 +0000 | [diff] [blame] | 1690 | // know that the generated intrinsic gets a ConstantInt. |
| 1691 | llvm::APSInt Result; |
| 1692 | bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext()); |
| 1693 | assert(IsConst && "Constant arg isn't actually constant?"); |
| 1694 | (void)IsConst; |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 1695 | ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result); |
Chris Lattner | 46c5591 | 2010-10-02 00:09:12 +0000 | [diff] [blame] | 1696 | } |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1697 | |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1698 | // If the intrinsic arg type is different from the builtin arg type |
| 1699 | // we need to do a bit cast. |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 1700 | llvm::Type *PTy = FTy->getParamType(i); |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1701 | if (PTy != ArgValue->getType()) { |
| 1702 | assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) && |
| 1703 | "Must be able to losslessly bit cast to param"); |
| 1704 | ArgValue = Builder.CreateBitCast(ArgValue, PTy); |
| 1705 | } |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1706 | |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1707 | Args.push_back(ArgValue); |
| 1708 | } |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1709 | |
Jay Foad | 4c7d9f1 | 2011-07-15 08:37:34 +0000 | [diff] [blame] | 1710 | Value *V = Builder.CreateCall(F, Args); |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1711 | QualType BuiltinRetType = E->getType(); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1712 | |
Chris Lattner | 8b41868 | 2012-02-07 00:39:47 +0000 | [diff] [blame] | 1713 | llvm::Type *RetTy = VoidTy; |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 1714 | if (!BuiltinRetType->isVoidType()) |
Chris Lattner | 8b41868 | 2012-02-07 00:39:47 +0000 | [diff] [blame] | 1715 | RetTy = ConvertType(BuiltinRetType); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1716 | |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1717 | if (RetTy != V->getType()) { |
| 1718 | assert(V->getType()->canLosslesslyBitCastTo(RetTy) && |
| 1719 | "Must be able to losslessly bit cast result type"); |
| 1720 | V = Builder.CreateBitCast(V, RetTy); |
| 1721 | } |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1722 | |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1723 | return RValue::get(V); |
| 1724 | } |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1725 | |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1726 | // See if we have a target specific builtin that needs to be lowered. |
Daniel Dunbar | f02e9dd | 2008-10-10 00:24:54 +0000 | [diff] [blame] | 1727 | if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E)) |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1728 | return RValue::get(V); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1729 | |
Daniel Dunbar | 488e993 | 2008-08-16 00:56:44 +0000 | [diff] [blame] | 1730 | ErrorUnsupported(E, "builtin function"); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1731 | |
Chris Lattner | b7cfe88 | 2008-06-30 18:32:54 +0000 | [diff] [blame] | 1732 | // Unknown builtin, for now just dump it out and return undef. |
John McCall | 9d232c8 | 2013-03-07 21:37:08 +0000 | [diff] [blame] | 1733 | return GetUndefRValue(E->getType()); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1734 | } |
Anders Carlsson | 564f1de | 2007-12-09 23:17:02 +0000 | [diff] [blame] | 1735 | |
Daniel Dunbar | f02e9dd | 2008-10-10 00:24:54 +0000 | [diff] [blame] | 1736 | Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, |
| 1737 | const CallExpr *E) { |
John McCall | 64aa4b3 | 2013-04-16 22:48:15 +0000 | [diff] [blame] | 1738 | switch (getTarget().getTriple().getArch()) { |
Chris Lattner | 2752c01 | 2010-03-03 19:03:45 +0000 | [diff] [blame] | 1739 | case llvm::Triple::arm: |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 1740 | case llvm::Triple::armeb: |
Chris Lattner | 2752c01 | 2010-03-03 19:03:45 +0000 | [diff] [blame] | 1741 | case llvm::Triple::thumb: |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 1742 | case llvm::Triple::thumbeb: |
Chris Lattner | 2752c01 | 2010-03-03 19:03:45 +0000 | [diff] [blame] | 1743 | return EmitARMBuiltinExpr(BuiltinID, E); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 1744 | case llvm::Triple::aarch64: |
| 1745 | case llvm::Triple::aarch64_be: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 1746 | return EmitAArch64BuiltinExpr(BuiltinID, E); |
Daniel Dunbar | 55cc2ed | 2009-08-24 09:54:37 +0000 | [diff] [blame] | 1747 | case llvm::Triple::x86: |
| 1748 | case llvm::Triple::x86_64: |
Daniel Dunbar | f02e9dd | 2008-10-10 00:24:54 +0000 | [diff] [blame] | 1749 | return EmitX86BuiltinExpr(BuiltinID, E); |
Daniel Dunbar | 55cc2ed | 2009-08-24 09:54:37 +0000 | [diff] [blame] | 1750 | case llvm::Triple::ppc: |
| 1751 | case llvm::Triple::ppc64: |
Bill Schmidt | ea7fb0c | 2013-07-26 01:36:11 +0000 | [diff] [blame] | 1752 | case llvm::Triple::ppc64le: |
Daniel Dunbar | f02e9dd | 2008-10-10 00:24:54 +0000 | [diff] [blame] | 1753 | return EmitPPCBuiltinExpr(BuiltinID, E); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 1754 | case llvm::Triple::r600: |
| 1755 | return EmitR600BuiltinExpr(BuiltinID, E); |
Daniel Dunbar | 55cc2ed | 2009-08-24 09:54:37 +0000 | [diff] [blame] | 1756 | default: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 1757 | return nullptr; |
Daniel Dunbar | 55cc2ed | 2009-08-24 09:54:37 +0000 | [diff] [blame] | 1758 | } |
Daniel Dunbar | f02e9dd | 2008-10-10 00:24:54 +0000 | [diff] [blame] | 1759 | } |
| 1760 | |
Chris Lattner | 8b41868 | 2012-02-07 00:39:47 +0000 | [diff] [blame] | 1761 | static llvm::VectorType *GetNeonType(CodeGenFunction *CGF, |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 1762 | NeonTypeFlags TypeFlags, |
| 1763 | bool V1Ty=false) { |
NAKAMURA Takumi | 83084c8 | 2011-11-08 03:27:04 +0000 | [diff] [blame] | 1764 | int IsQuad = TypeFlags.isQuad(); |
| 1765 | switch (TypeFlags.getEltType()) { |
Bob Wilson | da95f73 | 2011-11-08 01:16:11 +0000 | [diff] [blame] | 1766 | case NeonTypeFlags::Int8: |
| 1767 | case NeonTypeFlags::Poly8: |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 1768 | return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad)); |
Bob Wilson | da95f73 | 2011-11-08 01:16:11 +0000 | [diff] [blame] | 1769 | case NeonTypeFlags::Int16: |
| 1770 | case NeonTypeFlags::Poly16: |
| 1771 | case NeonTypeFlags::Float16: |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 1772 | return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad)); |
Bob Wilson | da95f73 | 2011-11-08 01:16:11 +0000 | [diff] [blame] | 1773 | case NeonTypeFlags::Int32: |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 1774 | return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad)); |
Bob Wilson | da95f73 | 2011-11-08 01:16:11 +0000 | [diff] [blame] | 1775 | case NeonTypeFlags::Int64: |
Kevin Qin | 624bb5e | 2013-11-14 03:29:16 +0000 | [diff] [blame] | 1776 | case NeonTypeFlags::Poly64: |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 1777 | return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad)); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 1778 | case NeonTypeFlags::Poly128: |
| 1779 | // FIXME: i128 and f128 doesn't get fully support in Clang and llvm. |
| 1780 | // There is a lot of i128 and f128 API missing. |
| 1781 | // so we use v16i8 to represent poly128 and get pattern matched. |
| 1782 | return llvm::VectorType::get(CGF->Int8Ty, 16); |
Bob Wilson | da95f73 | 2011-11-08 01:16:11 +0000 | [diff] [blame] | 1783 | case NeonTypeFlags::Float32: |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 1784 | return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad)); |
Tim Northover | b793f0d | 2013-08-01 09:23:19 +0000 | [diff] [blame] | 1785 | case NeonTypeFlags::Float64: |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 1786 | return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad)); |
David Blaikie | 561d3ab | 2012-01-17 02:30:50 +0000 | [diff] [blame] | 1787 | } |
Benjamin Kramer | 0cd6bd6 | 2013-09-26 16:36:08 +0000 | [diff] [blame] | 1788 | llvm_unreachable("Unknown vector element type!"); |
Nate Begeman | 998622c | 2010-06-07 16:01:56 +0000 | [diff] [blame] | 1789 | } |
| 1790 | |
Bob Wilson | cf55652 | 2010-12-07 22:40:02 +0000 | [diff] [blame] | 1791 | Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) { |
Nate Begeman | d075c01 | 2010-06-10 00:17:56 +0000 | [diff] [blame] | 1792 | unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements(); |
Chris Lattner | 2ce8842 | 2012-01-25 05:34:41 +0000 | [diff] [blame] | 1793 | Value* SV = llvm::ConstantVector::getSplat(nElts, C); |
Nate Begeman | d075c01 | 2010-06-10 00:17:56 +0000 | [diff] [blame] | 1794 | return Builder.CreateShuffleVector(V, V, SV, "lane"); |
| 1795 | } |
| 1796 | |
Nate Begeman | 30d9171 | 2010-06-08 06:03:01 +0000 | [diff] [blame] | 1797 | Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, |
Bob Wilson | db3d4d0 | 2010-12-08 22:37:56 +0000 | [diff] [blame] | 1798 | const char *name, |
Nate Begeman | 61eecf5 | 2010-06-14 05:21:25 +0000 | [diff] [blame] | 1799 | unsigned shift, bool rightshift) { |
Nate Begeman | 30d9171 | 2010-06-08 06:03:01 +0000 | [diff] [blame] | 1800 | unsigned j = 0; |
| 1801 | for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); |
| 1802 | ai != ae; ++ai, ++j) |
Nate Begeman | 61eecf5 | 2010-06-14 05:21:25 +0000 | [diff] [blame] | 1803 | if (shift > 0 && shift == j) |
| 1804 | Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); |
| 1805 | else |
| 1806 | Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); |
Nate Begeman | 30d9171 | 2010-06-08 06:03:01 +0000 | [diff] [blame] | 1807 | |
Jay Foad | 4c7d9f1 | 2011-07-15 08:37:34 +0000 | [diff] [blame] | 1808 | return Builder.CreateCall(F, Ops, name); |
Nate Begeman | 30d9171 | 2010-06-08 06:03:01 +0000 | [diff] [blame] | 1809 | } |
| 1810 | |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 1811 | Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty, |
Nate Begeman | 464ccb6 | 2010-06-11 22:57:12 +0000 | [diff] [blame] | 1812 | bool neg) { |
Chris Lattner | 2ce8842 | 2012-01-25 05:34:41 +0000 | [diff] [blame] | 1813 | int SV = cast<ConstantInt>(V)->getSExtValue(); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 1814 | |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 1815 | llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); |
Nate Begeman | 464ccb6 | 2010-06-11 22:57:12 +0000 | [diff] [blame] | 1816 | llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV); |
Chris Lattner | 2ce8842 | 2012-01-25 05:34:41 +0000 | [diff] [blame] | 1817 | return llvm::ConstantVector::getSplat(VTy->getNumElements(), C); |
Nate Begeman | 464ccb6 | 2010-06-11 22:57:12 +0000 | [diff] [blame] | 1818 | } |
| 1819 | |
Amaury de la Vieuville | 7f0ff70 | 2013-10-04 13:13:15 +0000 | [diff] [blame] | 1820 | // \brief Right-shift a vector by a constant. |
| 1821 | Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift, |
| 1822 | llvm::Type *Ty, bool usgn, |
| 1823 | const char *name) { |
| 1824 | llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); |
| 1825 | |
| 1826 | int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue(); |
| 1827 | int EltSize = VTy->getScalarSizeInBits(); |
| 1828 | |
| 1829 | Vec = Builder.CreateBitCast(Vec, Ty); |
| 1830 | |
| 1831 | // lshr/ashr are undefined when the shift amount is equal to the vector |
| 1832 | // element size. |
| 1833 | if (ShiftAmt == EltSize) { |
| 1834 | if (usgn) { |
| 1835 | // Right-shifting an unsigned value by its size yields 0. |
| 1836 | llvm::Constant *Zero = ConstantInt::get(VTy->getElementType(), 0); |
| 1837 | return llvm::ConstantVector::getSplat(VTy->getNumElements(), Zero); |
| 1838 | } else { |
| 1839 | // Right-shifting a signed value by its size is equivalent |
| 1840 | // to a shift of size-1. |
| 1841 | --ShiftAmt; |
| 1842 | Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt); |
| 1843 | } |
| 1844 | } |
| 1845 | |
| 1846 | Shift = EmitNeonShiftVector(Shift, Ty, false); |
| 1847 | if (usgn) |
| 1848 | return Builder.CreateLShr(Vec, Shift, name); |
| 1849 | else |
| 1850 | return Builder.CreateAShr(Vec, Shift, name); |
| 1851 | } |
| 1852 | |
Bob Wilson | 06b6c58 | 2010-08-27 17:14:29 +0000 | [diff] [blame] | 1853 | /// GetPointeeAlignment - Given an expression with a pointer type, find the |
| 1854 | /// alignment of the type referenced by the pointer. Skip over implicit |
| 1855 | /// casts. |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 1856 | std::pair<llvm::Value*, unsigned> |
| 1857 | CodeGenFunction::EmitPointerWithAlignment(const Expr *Addr) { |
| 1858 | assert(Addr->getType()->isPointerType()); |
| 1859 | Addr = Addr->IgnoreParens(); |
| 1860 | if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Addr)) { |
Eli Friedman | a484fc7 | 2012-08-29 21:21:11 +0000 | [diff] [blame] | 1861 | if ((ICE->getCastKind() == CK_BitCast || ICE->getCastKind() == CK_NoOp) && |
| 1862 | ICE->getSubExpr()->getType()->isPointerType()) { |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 1863 | std::pair<llvm::Value*, unsigned> Ptr = |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 1864 | EmitPointerWithAlignment(ICE->getSubExpr()); |
| 1865 | Ptr.first = Builder.CreateBitCast(Ptr.first, |
| 1866 | ConvertType(Addr->getType())); |
| 1867 | return Ptr; |
| 1868 | } else if (ICE->getCastKind() == CK_ArrayToPointerDecay) { |
| 1869 | LValue LV = EmitLValue(ICE->getSubExpr()); |
Eli Friedman | 8e4c189 | 2012-08-23 11:27:56 +0000 | [diff] [blame] | 1870 | unsigned Align = LV.getAlignment().getQuantity(); |
| 1871 | if (!Align) { |
| 1872 | // FIXME: Once LValues are fixed to always set alignment, |
| 1873 | // zap this code. |
| 1874 | QualType PtTy = ICE->getSubExpr()->getType(); |
| 1875 | if (!PtTy->isIncompleteType()) |
| 1876 | Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); |
| 1877 | else |
| 1878 | Align = 1; |
| 1879 | } |
| 1880 | return std::make_pair(LV.getAddress(), Align); |
Chris Lattner | d6e7356 | 2012-03-04 00:52:12 +0000 | [diff] [blame] | 1881 | } |
Bob Wilson | 06b6c58 | 2010-08-27 17:14:29 +0000 | [diff] [blame] | 1882 | } |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 1883 | if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Addr)) { |
| 1884 | if (UO->getOpcode() == UO_AddrOf) { |
| 1885 | LValue LV = EmitLValue(UO->getSubExpr()); |
Eli Friedman | 8e4c189 | 2012-08-23 11:27:56 +0000 | [diff] [blame] | 1886 | unsigned Align = LV.getAlignment().getQuantity(); |
| 1887 | if (!Align) { |
| 1888 | // FIXME: Once LValues are fixed to always set alignment, |
| 1889 | // zap this code. |
| 1890 | QualType PtTy = UO->getSubExpr()->getType(); |
| 1891 | if (!PtTy->isIncompleteType()) |
| 1892 | Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); |
| 1893 | else |
| 1894 | Align = 1; |
| 1895 | } |
| 1896 | return std::make_pair(LV.getAddress(), Align); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 1897 | } |
| 1898 | } |
Jay Foad | f4c3db1 | 2012-03-02 18:34:30 +0000 | [diff] [blame] | 1899 | |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 1900 | unsigned Align = 1; |
| 1901 | QualType PtTy = Addr->getType()->getPointeeType(); |
| 1902 | if (!PtTy->isIncompleteType()) |
| 1903 | Align = getContext().getTypeAlignInChars(PtTy).getQuantity(); |
| 1904 | |
| 1905 | return std::make_pair(EmitScalarExpr(Addr), Align); |
Bob Wilson | 06b6c58 | 2010-08-27 17:14:29 +0000 | [diff] [blame] | 1906 | } |
| 1907 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 1908 | enum { |
| 1909 | AddRetType = (1 << 0), |
| 1910 | Add1ArgType = (1 << 1), |
| 1911 | Add2ArgTypes = (1 << 2), |
| 1912 | |
| 1913 | VectorizeRetType = (1 << 3), |
| 1914 | VectorizeArgTypes = (1 << 4), |
| 1915 | |
| 1916 | InventFloatType = (1 << 5), |
| 1917 | UnsignedAlts = (1 << 6), |
| 1918 | |
| 1919 | Use64BitVectors = (1 << 7), |
| 1920 | Use128BitVectors = (1 << 8), |
| 1921 | |
| 1922 | Vectorize1ArgType = Add1ArgType | VectorizeArgTypes, |
| 1923 | VectorRet = AddRetType | VectorizeRetType, |
| 1924 | VectorRetGetArgs01 = |
| 1925 | AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes, |
| 1926 | FpCmpzModifiers = |
| 1927 | AddRetType | VectorizeRetType | Add1ArgType | InventFloatType |
| 1928 | }; |
| 1929 | |
| 1930 | struct NeonIntrinsicInfo { |
| 1931 | unsigned BuiltinID; |
| 1932 | unsigned LLVMIntrinsic; |
| 1933 | unsigned AltLLVMIntrinsic; |
| 1934 | const char *NameHint; |
| 1935 | unsigned TypeModifier; |
| 1936 | |
| 1937 | bool operator<(unsigned RHSBuiltinID) const { |
| 1938 | return BuiltinID < RHSBuiltinID; |
| 1939 | } |
| 1940 | }; |
| 1941 | |
| 1942 | #define NEONMAP0(NameBase) \ |
| 1943 | { NEON::BI__builtin_neon_ ## NameBase, 0, 0, #NameBase, 0 } |
| 1944 | |
| 1945 | #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ |
| 1946 | { NEON:: BI__builtin_neon_ ## NameBase, \ |
| 1947 | Intrinsic::LLVMIntrinsic, 0, #NameBase, TypeModifier } |
| 1948 | |
| 1949 | #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \ |
| 1950 | { NEON:: BI__builtin_neon_ ## NameBase, \ |
| 1951 | Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ |
| 1952 | #NameBase, TypeModifier } |
| 1953 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 1954 | static NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { |
| 1955 | NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), |
| 1956 | NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), |
| 1957 | NEONMAP1(vabs_v, arm_neon_vabs, 0), |
| 1958 | NEONMAP1(vabsq_v, arm_neon_vabs, 0), |
| 1959 | NEONMAP0(vaddhn_v), |
| 1960 | NEONMAP1(vaesdq_v, arm_neon_aesd, 0), |
| 1961 | NEONMAP1(vaeseq_v, arm_neon_aese, 0), |
| 1962 | NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0), |
| 1963 | NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0), |
| 1964 | NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType), |
| 1965 | NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType), |
| 1966 | NEONMAP1(vcage_v, arm_neon_vacge, 0), |
| 1967 | NEONMAP1(vcageq_v, arm_neon_vacge, 0), |
| 1968 | NEONMAP1(vcagt_v, arm_neon_vacgt, 0), |
| 1969 | NEONMAP1(vcagtq_v, arm_neon_vacgt, 0), |
| 1970 | NEONMAP1(vcale_v, arm_neon_vacge, 0), |
| 1971 | NEONMAP1(vcaleq_v, arm_neon_vacge, 0), |
| 1972 | NEONMAP1(vcalt_v, arm_neon_vacgt, 0), |
| 1973 | NEONMAP1(vcaltq_v, arm_neon_vacgt, 0), |
| 1974 | NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType), |
| 1975 | NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType), |
| 1976 | NEONMAP1(vclz_v, ctlz, Add1ArgType), |
| 1977 | NEONMAP1(vclzq_v, ctlz, Add1ArgType), |
| 1978 | NEONMAP1(vcnt_v, ctpop, Add1ArgType), |
| 1979 | NEONMAP1(vcntq_v, ctpop, Add1ArgType), |
| 1980 | NEONMAP1(vcvt_f16_v, arm_neon_vcvtfp2hf, 0), |
| 1981 | NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0), |
| 1982 | NEONMAP0(vcvt_f32_v), |
| 1983 | NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), |
| 1984 | NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0), |
| 1985 | NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0), |
| 1986 | NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0), |
| 1987 | NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0), |
| 1988 | NEONMAP0(vcvt_s32_v), |
| 1989 | NEONMAP0(vcvt_s64_v), |
| 1990 | NEONMAP0(vcvt_u32_v), |
| 1991 | NEONMAP0(vcvt_u64_v), |
| 1992 | NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0), |
| 1993 | NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0), |
| 1994 | NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0), |
| 1995 | NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0), |
| 1996 | NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0), |
| 1997 | NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0), |
| 1998 | NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0), |
| 1999 | NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0), |
| 2000 | NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0), |
| 2001 | NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0), |
| 2002 | NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0), |
| 2003 | NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0), |
| 2004 | NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0), |
| 2005 | NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0), |
| 2006 | NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0), |
| 2007 | NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0), |
| 2008 | NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0), |
| 2009 | NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0), |
| 2010 | NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0), |
| 2011 | NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0), |
| 2012 | NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0), |
| 2013 | NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0), |
| 2014 | NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0), |
| 2015 | NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0), |
| 2016 | NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0), |
| 2017 | NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0), |
| 2018 | NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0), |
| 2019 | NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0), |
| 2020 | NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0), |
| 2021 | NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0), |
| 2022 | NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0), |
| 2023 | NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0), |
| 2024 | NEONMAP0(vcvtq_f32_v), |
| 2025 | NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0), |
| 2026 | NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0), |
| 2027 | NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0), |
| 2028 | NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0), |
| 2029 | NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0), |
| 2030 | NEONMAP0(vcvtq_s32_v), |
| 2031 | NEONMAP0(vcvtq_s64_v), |
| 2032 | NEONMAP0(vcvtq_u32_v), |
| 2033 | NEONMAP0(vcvtq_u64_v), |
| 2034 | NEONMAP0(vext_v), |
| 2035 | NEONMAP0(vextq_v), |
| 2036 | NEONMAP0(vfma_v), |
| 2037 | NEONMAP0(vfmaq_v), |
| 2038 | NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), |
| 2039 | NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts), |
| 2040 | NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), |
| 2041 | NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts), |
| 2042 | NEONMAP0(vld1_dup_v), |
| 2043 | NEONMAP1(vld1_v, arm_neon_vld1, 0), |
| 2044 | NEONMAP0(vld1q_dup_v), |
| 2045 | NEONMAP1(vld1q_v, arm_neon_vld1, 0), |
| 2046 | NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0), |
| 2047 | NEONMAP1(vld2_v, arm_neon_vld2, 0), |
| 2048 | NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0), |
| 2049 | NEONMAP1(vld2q_v, arm_neon_vld2, 0), |
| 2050 | NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0), |
| 2051 | NEONMAP1(vld3_v, arm_neon_vld3, 0), |
| 2052 | NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0), |
| 2053 | NEONMAP1(vld3q_v, arm_neon_vld3, 0), |
| 2054 | NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0), |
| 2055 | NEONMAP1(vld4_v, arm_neon_vld4, 0), |
| 2056 | NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0), |
| 2057 | NEONMAP1(vld4q_v, arm_neon_vld4, 0), |
| 2058 | NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 2059 | NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType), |
| 2060 | NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2061 | NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts), |
| 2062 | NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 2063 | NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType), |
| 2064 | NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2065 | NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts), |
| 2066 | NEONMAP0(vmovl_v), |
| 2067 | NEONMAP0(vmovn_v), |
| 2068 | NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType), |
| 2069 | NEONMAP0(vmull_v), |
| 2070 | NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType), |
| 2071 | NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), |
| 2072 | NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts), |
| 2073 | NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType), |
| 2074 | NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), |
| 2075 | NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts), |
| 2076 | NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType), |
| 2077 | NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts), |
| 2078 | NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts), |
| 2079 | NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType), |
| 2080 | NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType), |
| 2081 | NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), |
| 2082 | NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts), |
| 2083 | NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0), |
| 2084 | NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0), |
| 2085 | NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType), |
| 2086 | NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType), |
| 2087 | NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType), |
| 2088 | NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts), |
| 2089 | NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType), |
| 2090 | NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType), |
| 2091 | NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType), |
| 2092 | NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType), |
| 2093 | NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType), |
| 2094 | NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), |
| 2095 | NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts), |
| 2096 | NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), |
| 2097 | NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), |
| 2098 | NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts), |
| 2099 | NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts), |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 2100 | NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0), |
| 2101 | NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2102 | NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), |
| 2103 | NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts), |
| 2104 | NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType), |
| 2105 | NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), |
| 2106 | NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0), |
| 2107 | NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType), |
| 2108 | NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType), |
| 2109 | NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), |
| 2110 | NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts), |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 2111 | NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType), |
| 2112 | NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType), |
| 2113 | NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType), |
| 2114 | NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType), |
| 2115 | NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType), |
| 2116 | NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType), |
| 2117 | NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType), |
| 2118 | NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType), |
| 2119 | NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType), |
| 2120 | NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType), |
| 2121 | NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType), |
| 2122 | NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2123 | NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), |
| 2124 | NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts), |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 2125 | NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), |
| 2126 | NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2127 | NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), |
| 2128 | NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0), |
| 2129 | NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType), |
| 2130 | NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType), |
| 2131 | NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType), |
| 2132 | NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0), |
| 2133 | NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0), |
| 2134 | NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0), |
| 2135 | NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0), |
| 2136 | NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0), |
| 2137 | NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0), |
| 2138 | NEONMAP0(vshl_n_v), |
| 2139 | NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), |
| 2140 | NEONMAP0(vshll_n_v), |
| 2141 | NEONMAP0(vshlq_n_v), |
| 2142 | NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts), |
| 2143 | NEONMAP0(vshr_n_v), |
| 2144 | NEONMAP0(vshrn_n_v), |
| 2145 | NEONMAP0(vshrq_n_v), |
| 2146 | NEONMAP1(vst1_v, arm_neon_vst1, 0), |
| 2147 | NEONMAP1(vst1q_v, arm_neon_vst1, 0), |
| 2148 | NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0), |
| 2149 | NEONMAP1(vst2_v, arm_neon_vst2, 0), |
| 2150 | NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0), |
| 2151 | NEONMAP1(vst2q_v, arm_neon_vst2, 0), |
| 2152 | NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0), |
| 2153 | NEONMAP1(vst3_v, arm_neon_vst3, 0), |
| 2154 | NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0), |
| 2155 | NEONMAP1(vst3q_v, arm_neon_vst3, 0), |
| 2156 | NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0), |
| 2157 | NEONMAP1(vst4_v, arm_neon_vst4, 0), |
| 2158 | NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0), |
| 2159 | NEONMAP1(vst4q_v, arm_neon_vst4, 0), |
| 2160 | NEONMAP0(vsubhn_v), |
| 2161 | NEONMAP0(vtrn_v), |
| 2162 | NEONMAP0(vtrnq_v), |
| 2163 | NEONMAP0(vtst_v), |
| 2164 | NEONMAP0(vtstq_v), |
| 2165 | NEONMAP0(vuzp_v), |
| 2166 | NEONMAP0(vuzpq_v), |
| 2167 | NEONMAP0(vzip_v), |
| 2168 | NEONMAP0(vzipq_v) |
| 2169 | }; |
| 2170 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2171 | static NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { |
| 2172 | NEONMAP1(vabs_v, aarch64_neon_abs, 0), |
| 2173 | NEONMAP1(vabsq_v, aarch64_neon_abs, 0), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2174 | NEONMAP0(vaddhn_v), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2175 | NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0), |
| 2176 | NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0), |
| 2177 | NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0), |
| 2178 | NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0), |
| 2179 | NEONMAP1(vcage_v, aarch64_neon_facge, 0), |
| 2180 | NEONMAP1(vcageq_v, aarch64_neon_facge, 0), |
| 2181 | NEONMAP1(vcagt_v, aarch64_neon_facgt, 0), |
| 2182 | NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0), |
| 2183 | NEONMAP1(vcale_v, aarch64_neon_facge, 0), |
| 2184 | NEONMAP1(vcaleq_v, aarch64_neon_facge, 0), |
| 2185 | NEONMAP1(vcalt_v, aarch64_neon_facgt, 0), |
| 2186 | NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0), |
| 2187 | NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType), |
| 2188 | NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2189 | NEONMAP1(vclz_v, ctlz, Add1ArgType), |
| 2190 | NEONMAP1(vclzq_v, ctlz, Add1ArgType), |
| 2191 | NEONMAP1(vcnt_v, ctpop, Add1ArgType), |
| 2192 | NEONMAP1(vcntq_v, ctpop, Add1ArgType), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2193 | NEONMAP1(vcvt_f16_v, aarch64_neon_vcvtfp2hf, 0), |
| 2194 | NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2195 | NEONMAP0(vcvt_f32_v), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2196 | NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), |
| 2197 | NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), |
| 2198 | NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), |
| 2199 | NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), |
| 2200 | NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), |
| 2201 | NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2202 | NEONMAP0(vcvtq_f32_v), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2203 | NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), |
| 2204 | NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0), |
| 2205 | NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0), |
| 2206 | NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0), |
| 2207 | NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0), |
| 2208 | NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0), |
| 2209 | NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2210 | NEONMAP0(vext_v), |
| 2211 | NEONMAP0(vextq_v), |
| 2212 | NEONMAP0(vfma_v), |
| 2213 | NEONMAP0(vfmaq_v), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2214 | NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), |
| 2215 | NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts), |
| 2216 | NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), |
| 2217 | NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2218 | NEONMAP0(vmovl_v), |
| 2219 | NEONMAP0(vmovn_v), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2220 | NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType), |
| 2221 | NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType), |
| 2222 | NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType), |
| 2223 | NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), |
| 2224 | NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts), |
| 2225 | NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType), |
| 2226 | NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType), |
| 2227 | NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType), |
| 2228 | NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), |
| 2229 | NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts), |
| 2230 | NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0), |
| 2231 | NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0), |
| 2232 | NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType), |
| 2233 | NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType), |
| 2234 | NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType), |
| 2235 | NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts), |
| 2236 | NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType), |
| 2237 | NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType), |
| 2238 | NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType), |
| 2239 | NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType), |
| 2240 | NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType), |
| 2241 | NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), |
| 2242 | NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts), |
| 2243 | NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts), |
| 2244 | NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), |
| 2245 | NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts), |
| 2246 | NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts), |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 2247 | NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0), |
| 2248 | NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2249 | NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), |
| 2250 | NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts), |
| 2251 | NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType), |
| 2252 | NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), |
| 2253 | NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0), |
| 2254 | NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType), |
| 2255 | NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType), |
| 2256 | NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), |
| 2257 | NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts), |
| 2258 | NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), |
| 2259 | NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts), |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 2260 | NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), |
| 2261 | NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2262 | NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), |
| 2263 | NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0), |
| 2264 | NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType), |
| 2265 | NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType), |
| 2266 | NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType), |
| 2267 | NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0), |
| 2268 | NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0), |
| 2269 | NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0), |
| 2270 | NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0), |
| 2271 | NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0), |
| 2272 | NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2273 | NEONMAP0(vshl_n_v), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2274 | NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2275 | NEONMAP0(vshll_n_v), |
| 2276 | NEONMAP0(vshlq_n_v), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2277 | NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2278 | NEONMAP0(vshr_n_v), |
| 2279 | NEONMAP0(vshrn_n_v), |
| 2280 | NEONMAP0(vshrq_n_v), |
| 2281 | NEONMAP0(vsubhn_v), |
| 2282 | NEONMAP0(vtst_v), |
| 2283 | NEONMAP0(vtstq_v), |
| 2284 | }; |
| 2285 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2286 | static NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = { |
| 2287 | NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType), |
| 2288 | NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType), |
| 2289 | NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType), |
| 2290 | NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), |
| 2291 | NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), |
| 2292 | NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType), |
| 2293 | NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType), |
| 2294 | NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), |
| 2295 | NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), |
| 2296 | NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), |
| 2297 | NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType), |
| 2298 | NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType), |
| 2299 | NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType), |
| 2300 | NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType), |
| 2301 | NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType), |
| 2302 | NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), |
| 2303 | NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType), |
| 2304 | NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType), |
| 2305 | NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), |
| 2306 | NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), |
| 2307 | NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType), |
| 2308 | NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType), |
| 2309 | NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType), |
| 2310 | NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType), |
| 2311 | NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType), |
| 2312 | NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType), |
| 2313 | NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType), |
| 2314 | NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType), |
| 2315 | NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), |
| 2316 | NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), |
| 2317 | NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), |
| 2318 | NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), |
| 2319 | NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType), |
| 2320 | NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), |
| 2321 | NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType), |
| 2322 | NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType), |
| 2323 | NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType), |
| 2324 | NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), |
| 2325 | NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType), |
| 2326 | NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType), |
| 2327 | NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType), |
| 2328 | NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), |
| 2329 | NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType), |
| 2330 | NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType), |
| 2331 | NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType), |
| 2332 | NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType), |
| 2333 | NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType), |
| 2334 | NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType), |
| 2335 | NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0), |
| 2336 | NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), |
| 2337 | NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), |
| 2338 | NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), |
| 2339 | NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), |
| 2340 | NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), |
| 2341 | NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), |
| 2342 | NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), |
| 2343 | NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), |
| 2344 | NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType), |
| 2345 | NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType), |
| 2346 | NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), |
| 2347 | NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), |
| 2348 | NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), |
| 2349 | NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), |
| 2350 | NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), |
| 2351 | NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), |
| 2352 | NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), |
| 2353 | NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), |
| 2354 | NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType), |
| 2355 | NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType), |
| 2356 | NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0), |
| 2357 | NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType), |
| 2358 | NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType), |
| 2359 | NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType), |
| 2360 | NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType), |
| 2361 | NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), |
| 2362 | NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType), |
| 2363 | NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType), |
| 2364 | NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType), |
| 2365 | NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType), |
| 2366 | NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType), |
| 2367 | NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType), |
| 2368 | NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType), |
| 2369 | NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), |
| 2370 | NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType), |
| 2371 | NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors), |
| 2372 | NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType), |
| 2373 | NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), |
| 2374 | NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), |
| 2375 | NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType), |
| 2376 | NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType), |
| 2377 | NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors), |
| 2378 | NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors), |
| 2379 | NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType), |
| 2380 | NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType), |
| 2381 | NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors), |
| 2382 | NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType), |
| 2383 | NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors), |
| 2384 | NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0), |
| 2385 | NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType), |
| 2386 | NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType), |
| 2387 | NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), |
| 2388 | NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), |
| 2389 | NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors), |
| 2390 | NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors), |
| 2391 | NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType), |
| 2392 | NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), |
| 2393 | NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors), |
| 2394 | NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), |
| 2395 | NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType), |
| 2396 | NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors), |
| 2397 | NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType), |
| 2398 | NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors), |
| 2399 | NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType), |
| 2400 | NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), |
| 2401 | NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), |
| 2402 | NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType), |
| 2403 | NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType), |
| 2404 | NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors), |
| 2405 | NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors), |
| 2406 | NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType), |
| 2407 | NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType), |
| 2408 | NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType), |
| 2409 | NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType), |
| 2410 | NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), |
| 2411 | NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), |
| 2412 | NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors), |
| 2413 | NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors), |
| 2414 | NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType), |
| 2415 | NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), |
| 2416 | NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors), |
| 2417 | NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), |
| 2418 | NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), |
| 2419 | NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), |
| 2420 | NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), |
| 2421 | NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType), |
| 2422 | NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType), |
| 2423 | NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), |
| 2424 | NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), |
| 2425 | NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors), |
| 2426 | NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors), |
| 2427 | NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType), |
| 2428 | NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType), |
| 2429 | NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType), |
| 2430 | NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType), |
| 2431 | NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), |
| 2432 | NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors), |
| 2433 | NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType), |
| 2434 | NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType), |
| 2435 | NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType), |
| 2436 | NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), |
| 2437 | NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), |
| 2438 | NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors), |
| 2439 | NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors), |
| 2440 | NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType), |
| 2441 | NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), |
| 2442 | NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors), |
| 2443 | NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), |
| 2444 | NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), |
| 2445 | NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType), |
| 2446 | NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType), |
| 2447 | NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors), |
| 2448 | NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors), |
| 2449 | NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType), |
| 2450 | NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType), |
| 2451 | NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType), |
| 2452 | NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType), |
| 2453 | NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType), |
| 2454 | NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType), |
| 2455 | NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType), |
| 2456 | NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType), |
| 2457 | NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType), |
| 2458 | NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType), |
| 2459 | NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType), |
| 2460 | NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType), |
| 2461 | NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0), |
| 2462 | NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0), |
| 2463 | NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0), |
| 2464 | NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0), |
| 2465 | NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType), |
| 2466 | NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType), |
| 2467 | NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType), |
| 2468 | NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType), |
| 2469 | NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), |
| 2470 | NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType), |
| 2471 | NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors), |
| 2472 | NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType), |
| 2473 | NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType), |
| 2474 | NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType), |
| 2475 | NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), |
| 2476 | NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType), |
| 2477 | NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors), |
| 2478 | NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2479 | }; |
| 2480 | |
| 2481 | #undef NEONMAP0 |
| 2482 | #undef NEONMAP1 |
| 2483 | #undef NEONMAP2 |
| 2484 | |
| 2485 | static bool NEONSIMDIntrinsicsProvenSorted = false; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2486 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2487 | static bool AArch64SIMDIntrinsicsProvenSorted = false; |
| 2488 | static bool AArch64SISDIntrinsicsProvenSorted = false; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2489 | |
| 2490 | |
| 2491 | static const NeonIntrinsicInfo * |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 2492 | findNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap, |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2493 | unsigned BuiltinID, bool &MapProvenSorted) { |
| 2494 | |
| 2495 | #ifndef NDEBUG |
| 2496 | if (!MapProvenSorted) { |
| 2497 | // FIXME: use std::is_sorted once C++11 is allowed |
| 2498 | for (unsigned i = 0; i < IntrinsicMap.size() - 1; ++i) |
| 2499 | assert(IntrinsicMap[i].BuiltinID <= IntrinsicMap[i + 1].BuiltinID); |
| 2500 | MapProvenSorted = true; |
| 2501 | } |
| 2502 | #endif |
| 2503 | |
| 2504 | const NeonIntrinsicInfo *Builtin = |
| 2505 | std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID); |
| 2506 | |
| 2507 | if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID) |
| 2508 | return Builtin; |
| 2509 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2510 | return nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2511 | } |
| 2512 | |
| 2513 | Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID, |
| 2514 | unsigned Modifier, |
| 2515 | llvm::Type *ArgType, |
| 2516 | const CallExpr *E) { |
| 2517 | int VectorSize = 0; |
| 2518 | if (Modifier & Use64BitVectors) |
| 2519 | VectorSize = 64; |
| 2520 | else if (Modifier & Use128BitVectors) |
| 2521 | VectorSize = 128; |
| 2522 | |
| 2523 | // Return type. |
| 2524 | SmallVector<llvm::Type *, 3> Tys; |
| 2525 | if (Modifier & AddRetType) { |
| 2526 | llvm::Type *Ty = ConvertType(E->getCallReturnType()); |
| 2527 | if (Modifier & VectorizeRetType) |
| 2528 | Ty = llvm::VectorType::get( |
| 2529 | Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1); |
| 2530 | |
| 2531 | Tys.push_back(Ty); |
| 2532 | } |
| 2533 | |
| 2534 | // Arguments. |
| 2535 | if (Modifier & VectorizeArgTypes) { |
| 2536 | int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1; |
| 2537 | ArgType = llvm::VectorType::get(ArgType, Elts); |
| 2538 | } |
| 2539 | |
| 2540 | if (Modifier & (Add1ArgType | Add2ArgTypes)) |
| 2541 | Tys.push_back(ArgType); |
| 2542 | |
| 2543 | if (Modifier & Add2ArgTypes) |
| 2544 | Tys.push_back(ArgType); |
| 2545 | |
| 2546 | if (Modifier & InventFloatType) |
| 2547 | Tys.push_back(FloatTy); |
| 2548 | |
| 2549 | return CGM.getIntrinsic(IntrinsicID, Tys); |
| 2550 | } |
| 2551 | |
| 2552 | static Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, |
| 2553 | const NeonIntrinsicInfo &SISDInfo, |
| 2554 | SmallVectorImpl<Value *> &Ops, |
| 2555 | const CallExpr *E) { |
| 2556 | unsigned BuiltinID = SISDInfo.BuiltinID; |
| 2557 | unsigned int Int = SISDInfo.LLVMIntrinsic; |
| 2558 | unsigned Modifier = SISDInfo.TypeModifier; |
| 2559 | const char *s = SISDInfo.NameHint; |
| 2560 | |
| 2561 | switch (BuiltinID) { |
| 2562 | case NEON::BI__builtin_neon_vcled_s64: |
| 2563 | case NEON::BI__builtin_neon_vcled_u64: |
| 2564 | case NEON::BI__builtin_neon_vcles_f32: |
| 2565 | case NEON::BI__builtin_neon_vcled_f64: |
| 2566 | case NEON::BI__builtin_neon_vcltd_s64: |
| 2567 | case NEON::BI__builtin_neon_vcltd_u64: |
| 2568 | case NEON::BI__builtin_neon_vclts_f32: |
| 2569 | case NEON::BI__builtin_neon_vcltd_f64: |
| 2570 | case NEON::BI__builtin_neon_vcales_f32: |
| 2571 | case NEON::BI__builtin_neon_vcaled_f64: |
| 2572 | case NEON::BI__builtin_neon_vcalts_f32: |
| 2573 | case NEON::BI__builtin_neon_vcaltd_f64: |
| 2574 | // Only one direction of comparisons actually exist, cmle is actually a cmge |
| 2575 | // with swapped operands. The table gives us the right intrinsic but we |
| 2576 | // still need to do the swap. |
| 2577 | std::swap(Ops[0], Ops[1]); |
| 2578 | break; |
| 2579 | } |
| 2580 | |
| 2581 | assert(Int && "Generic code assumes a valid intrinsic"); |
| 2582 | |
| 2583 | // Determine the type(s) of this overloaded AArch64 intrinsic. |
| 2584 | const Expr *Arg = E->getArg(0); |
| 2585 | llvm::Type *ArgTy = CGF.ConvertType(Arg->getType()); |
| 2586 | Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E); |
| 2587 | |
| 2588 | int j = 0; |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 2589 | ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2590 | for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end(); |
| 2591 | ai != ae; ++ai, ++j) { |
| 2592 | llvm::Type *ArgTy = ai->getType(); |
| 2593 | if (Ops[j]->getType()->getPrimitiveSizeInBits() == |
| 2594 | ArgTy->getPrimitiveSizeInBits()) |
| 2595 | continue; |
| 2596 | |
| 2597 | assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); |
| 2598 | // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate |
| 2599 | // it before inserting. |
| 2600 | Ops[j] = |
| 2601 | CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType()); |
| 2602 | Ops[j] = |
| 2603 | CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); |
| 2604 | } |
| 2605 | |
| 2606 | Value *Result = CGF.EmitNeonCall(F, Ops, s); |
| 2607 | llvm::Type *ResultType = CGF.ConvertType(E->getType()); |
| 2608 | if (ResultType->getPrimitiveSizeInBits() < |
| 2609 | Result->getType()->getPrimitiveSizeInBits()) |
| 2610 | return CGF.Builder.CreateExtractElement(Result, C0); |
| 2611 | |
| 2612 | return CGF.Builder.CreateBitCast(Result, ResultType, s); |
| 2613 | } |
| 2614 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2615 | Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( |
| 2616 | unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, |
| 2617 | const char *NameHint, unsigned Modifier, const CallExpr *E, |
| 2618 | SmallVectorImpl<llvm::Value *> &Ops, llvm::Value *Align) { |
| 2619 | // Get the last argument, which specifies the vector type. |
| 2620 | llvm::APSInt NeonTypeConst; |
| 2621 | const Expr *Arg = E->getArg(E->getNumArgs() - 1); |
| 2622 | if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext())) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2623 | return nullptr; |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 2624 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2625 | // Determine the type of this overloaded NEON intrinsic. |
| 2626 | NeonTypeFlags Type(NeonTypeConst.getZExtValue()); |
| 2627 | bool Usgn = Type.isUnsigned(); |
| 2628 | bool Quad = Type.isQuad(); |
| 2629 | |
| 2630 | llvm::VectorType *VTy = GetNeonType(this, Type); |
| 2631 | llvm::Type *Ty = VTy; |
| 2632 | if (!Ty) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2633 | return nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2634 | |
| 2635 | unsigned Int = LLVMIntrinsic; |
| 2636 | if ((Modifier & UnsignedAlts) && !Usgn) |
| 2637 | Int = AltLLVMIntrinsic; |
| 2638 | |
| 2639 | switch (BuiltinID) { |
| 2640 | default: break; |
| 2641 | case NEON::BI__builtin_neon_vabs_v: |
| 2642 | case NEON::BI__builtin_neon_vabsq_v: |
| 2643 | if (VTy->getElementType()->isFloatingPointTy()) |
| 2644 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); |
| 2645 | return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); |
| 2646 | case NEON::BI__builtin_neon_vaddhn_v: { |
| 2647 | llvm::VectorType *SrcTy = |
| 2648 | llvm::VectorType::getExtendedElementVectorType(VTy); |
| 2649 | |
| 2650 | // %sum = add <4 x i32> %lhs, %rhs |
| 2651 | Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); |
| 2652 | Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); |
| 2653 | Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); |
| 2654 | |
| 2655 | // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> |
| 2656 | Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(), |
| 2657 | SrcTy->getScalarSizeInBits() / 2); |
| 2658 | ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt); |
| 2659 | Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); |
| 2660 | |
| 2661 | // %res = trunc <4 x i32> %high to <4 x i16> |
| 2662 | return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); |
| 2663 | } |
| 2664 | case NEON::BI__builtin_neon_vcale_v: |
| 2665 | case NEON::BI__builtin_neon_vcaleq_v: |
| 2666 | case NEON::BI__builtin_neon_vcalt_v: |
| 2667 | case NEON::BI__builtin_neon_vcaltq_v: |
| 2668 | std::swap(Ops[0], Ops[1]); |
| 2669 | case NEON::BI__builtin_neon_vcage_v: |
| 2670 | case NEON::BI__builtin_neon_vcageq_v: |
| 2671 | case NEON::BI__builtin_neon_vcagt_v: |
| 2672 | case NEON::BI__builtin_neon_vcagtq_v: { |
| 2673 | llvm::Type *VecFlt = llvm::VectorType::get( |
| 2674 | VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy, |
| 2675 | VTy->getNumElements()); |
| 2676 | llvm::Type *Tys[] = { VTy, VecFlt }; |
| 2677 | Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); |
| 2678 | return EmitNeonCall(F, Ops, NameHint); |
| 2679 | } |
| 2680 | case NEON::BI__builtin_neon_vclz_v: |
| 2681 | case NEON::BI__builtin_neon_vclzq_v: |
| 2682 | // We generate target-independent intrinsic, which needs a second argument |
| 2683 | // for whether or not clz of zero is undefined; on ARM it isn't. |
| 2684 | Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); |
| 2685 | break; |
| 2686 | case NEON::BI__builtin_neon_vcvt_f32_v: |
| 2687 | case NEON::BI__builtin_neon_vcvtq_f32_v: |
| 2688 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 2689 | Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad)); |
| 2690 | return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") |
| 2691 | : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); |
| 2692 | case NEON::BI__builtin_neon_vcvt_n_f32_v: |
| 2693 | case NEON::BI__builtin_neon_vcvt_n_f64_v: |
| 2694 | case NEON::BI__builtin_neon_vcvtq_n_f32_v: |
| 2695 | case NEON::BI__builtin_neon_vcvtq_n_f64_v: { |
| 2696 | bool Double = |
| 2697 | (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); |
| 2698 | llvm::Type *FloatTy = |
| 2699 | GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 |
| 2700 | : NeonTypeFlags::Float32, |
| 2701 | false, Quad)); |
| 2702 | llvm::Type *Tys[2] = { FloatTy, Ty }; |
| 2703 | Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic; |
| 2704 | Function *F = CGM.getIntrinsic(Int, Tys); |
| 2705 | return EmitNeonCall(F, Ops, "vcvt_n"); |
| 2706 | } |
| 2707 | case NEON::BI__builtin_neon_vcvt_n_s32_v: |
| 2708 | case NEON::BI__builtin_neon_vcvt_n_u32_v: |
| 2709 | case NEON::BI__builtin_neon_vcvt_n_s64_v: |
| 2710 | case NEON::BI__builtin_neon_vcvt_n_u64_v: |
| 2711 | case NEON::BI__builtin_neon_vcvtq_n_s32_v: |
| 2712 | case NEON::BI__builtin_neon_vcvtq_n_u32_v: |
| 2713 | case NEON::BI__builtin_neon_vcvtq_n_s64_v: |
| 2714 | case NEON::BI__builtin_neon_vcvtq_n_u64_v: { |
| 2715 | bool Double = |
| 2716 | (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); |
| 2717 | llvm::Type *FloatTy = |
| 2718 | GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 |
| 2719 | : NeonTypeFlags::Float32, |
| 2720 | false, Quad)); |
| 2721 | llvm::Type *Tys[2] = { Ty, FloatTy }; |
| 2722 | Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys); |
| 2723 | return EmitNeonCall(F, Ops, "vcvt_n"); |
| 2724 | } |
| 2725 | case NEON::BI__builtin_neon_vcvt_s32_v: |
| 2726 | case NEON::BI__builtin_neon_vcvt_u32_v: |
| 2727 | case NEON::BI__builtin_neon_vcvt_s64_v: |
| 2728 | case NEON::BI__builtin_neon_vcvt_u64_v: |
| 2729 | case NEON::BI__builtin_neon_vcvtq_s32_v: |
| 2730 | case NEON::BI__builtin_neon_vcvtq_u32_v: |
| 2731 | case NEON::BI__builtin_neon_vcvtq_s64_v: |
| 2732 | case NEON::BI__builtin_neon_vcvtq_u64_v: { |
| 2733 | bool Double = |
| 2734 | (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); |
| 2735 | llvm::Type *FloatTy = |
| 2736 | GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64 |
| 2737 | : NeonTypeFlags::Float32, |
| 2738 | false, Quad)); |
| 2739 | Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); |
| 2740 | return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") |
| 2741 | : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); |
| 2742 | } |
| 2743 | case NEON::BI__builtin_neon_vcvta_s32_v: |
| 2744 | case NEON::BI__builtin_neon_vcvta_s64_v: |
| 2745 | case NEON::BI__builtin_neon_vcvta_u32_v: |
| 2746 | case NEON::BI__builtin_neon_vcvta_u64_v: |
| 2747 | case NEON::BI__builtin_neon_vcvtaq_s32_v: |
| 2748 | case NEON::BI__builtin_neon_vcvtaq_s64_v: |
| 2749 | case NEON::BI__builtin_neon_vcvtaq_u32_v: |
| 2750 | case NEON::BI__builtin_neon_vcvtaq_u64_v: |
| 2751 | case NEON::BI__builtin_neon_vcvtn_s32_v: |
| 2752 | case NEON::BI__builtin_neon_vcvtn_s64_v: |
| 2753 | case NEON::BI__builtin_neon_vcvtn_u32_v: |
| 2754 | case NEON::BI__builtin_neon_vcvtn_u64_v: |
| 2755 | case NEON::BI__builtin_neon_vcvtnq_s32_v: |
| 2756 | case NEON::BI__builtin_neon_vcvtnq_s64_v: |
| 2757 | case NEON::BI__builtin_neon_vcvtnq_u32_v: |
| 2758 | case NEON::BI__builtin_neon_vcvtnq_u64_v: |
| 2759 | case NEON::BI__builtin_neon_vcvtp_s32_v: |
| 2760 | case NEON::BI__builtin_neon_vcvtp_s64_v: |
| 2761 | case NEON::BI__builtin_neon_vcvtp_u32_v: |
| 2762 | case NEON::BI__builtin_neon_vcvtp_u64_v: |
| 2763 | case NEON::BI__builtin_neon_vcvtpq_s32_v: |
| 2764 | case NEON::BI__builtin_neon_vcvtpq_s64_v: |
| 2765 | case NEON::BI__builtin_neon_vcvtpq_u32_v: |
| 2766 | case NEON::BI__builtin_neon_vcvtpq_u64_v: |
| 2767 | case NEON::BI__builtin_neon_vcvtm_s32_v: |
| 2768 | case NEON::BI__builtin_neon_vcvtm_s64_v: |
| 2769 | case NEON::BI__builtin_neon_vcvtm_u32_v: |
| 2770 | case NEON::BI__builtin_neon_vcvtm_u64_v: |
| 2771 | case NEON::BI__builtin_neon_vcvtmq_s32_v: |
| 2772 | case NEON::BI__builtin_neon_vcvtmq_s64_v: |
| 2773 | case NEON::BI__builtin_neon_vcvtmq_u32_v: |
| 2774 | case NEON::BI__builtin_neon_vcvtmq_u64_v: { |
| 2775 | bool Double = |
| 2776 | (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); |
| 2777 | llvm::Type *InTy = |
| 2778 | GetNeonType(this, |
| 2779 | NeonTypeFlags(Double ? NeonTypeFlags::Float64 |
| 2780 | : NeonTypeFlags::Float32, false, Quad)); |
| 2781 | llvm::Type *Tys[2] = { Ty, InTy }; |
| 2782 | return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); |
| 2783 | } |
| 2784 | case NEON::BI__builtin_neon_vext_v: |
| 2785 | case NEON::BI__builtin_neon_vextq_v: { |
| 2786 | int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); |
| 2787 | SmallVector<Constant*, 16> Indices; |
| 2788 | for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) |
| 2789 | Indices.push_back(ConstantInt::get(Int32Ty, i+CV)); |
| 2790 | |
| 2791 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 2792 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 2793 | Value *SV = llvm::ConstantVector::get(Indices); |
| 2794 | return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext"); |
| 2795 | } |
| 2796 | case NEON::BI__builtin_neon_vfma_v: |
| 2797 | case NEON::BI__builtin_neon_vfmaq_v: { |
| 2798 | Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); |
| 2799 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 2800 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 2801 | Ops[2] = Builder.CreateBitCast(Ops[2], Ty); |
| 2802 | |
| 2803 | // NEON intrinsic puts accumulator first, unlike the LLVM fma. |
| 2804 | return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); |
| 2805 | } |
| 2806 | case NEON::BI__builtin_neon_vld1_v: |
| 2807 | case NEON::BI__builtin_neon_vld1q_v: |
| 2808 | Ops.push_back(Align); |
| 2809 | return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vld1"); |
| 2810 | case NEON::BI__builtin_neon_vld2_v: |
| 2811 | case NEON::BI__builtin_neon_vld2q_v: |
| 2812 | case NEON::BI__builtin_neon_vld3_v: |
| 2813 | case NEON::BI__builtin_neon_vld3q_v: |
| 2814 | case NEON::BI__builtin_neon_vld4_v: |
| 2815 | case NEON::BI__builtin_neon_vld4q_v: { |
| 2816 | Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); |
| 2817 | Ops[1] = Builder.CreateCall2(F, Ops[1], Align, NameHint); |
| 2818 | Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); |
| 2819 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 2820 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 2821 | } |
| 2822 | case NEON::BI__builtin_neon_vld1_dup_v: |
| 2823 | case NEON::BI__builtin_neon_vld1q_dup_v: { |
| 2824 | Value *V = UndefValue::get(Ty); |
| 2825 | Ty = llvm::PointerType::getUnqual(VTy->getElementType()); |
| 2826 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 2827 | LoadInst *Ld = Builder.CreateLoad(Ops[0]); |
| 2828 | Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 2829 | llvm::Constant *CI = ConstantInt::get(SizeTy, 0); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2830 | Ops[0] = Builder.CreateInsertElement(V, Ld, CI); |
| 2831 | return EmitNeonSplat(Ops[0], CI); |
| 2832 | } |
| 2833 | case NEON::BI__builtin_neon_vld2_lane_v: |
| 2834 | case NEON::BI__builtin_neon_vld2q_lane_v: |
| 2835 | case NEON::BI__builtin_neon_vld3_lane_v: |
| 2836 | case NEON::BI__builtin_neon_vld3q_lane_v: |
| 2837 | case NEON::BI__builtin_neon_vld4_lane_v: |
| 2838 | case NEON::BI__builtin_neon_vld4q_lane_v: { |
| 2839 | Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); |
| 2840 | for (unsigned I = 2; I < Ops.size() - 1; ++I) |
| 2841 | Ops[I] = Builder.CreateBitCast(Ops[I], Ty); |
| 2842 | Ops.push_back(Align); |
| 2843 | Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); |
| 2844 | Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); |
| 2845 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 2846 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 2847 | } |
| 2848 | case NEON::BI__builtin_neon_vmovl_v: { |
| 2849 | llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy); |
| 2850 | Ops[0] = Builder.CreateBitCast(Ops[0], DTy); |
| 2851 | if (Usgn) |
| 2852 | return Builder.CreateZExt(Ops[0], Ty, "vmovl"); |
| 2853 | return Builder.CreateSExt(Ops[0], Ty, "vmovl"); |
| 2854 | } |
| 2855 | case NEON::BI__builtin_neon_vmovn_v: { |
| 2856 | llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy); |
| 2857 | Ops[0] = Builder.CreateBitCast(Ops[0], QTy); |
| 2858 | return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); |
| 2859 | } |
| 2860 | case NEON::BI__builtin_neon_vmull_v: |
| 2861 | // FIXME: the integer vmull operations could be emitted in terms of pure |
| 2862 | // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of |
| 2863 | // hoisting the exts outside loops. Until global ISel comes along that can |
| 2864 | // see through such movement this leads to bad CodeGen. So we need an |
| 2865 | // intrinsic for now. |
| 2866 | Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls; |
| 2867 | Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int; |
| 2868 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); |
| 2869 | case NEON::BI__builtin_neon_vpadal_v: |
| 2870 | case NEON::BI__builtin_neon_vpadalq_v: { |
| 2871 | // The source operand type has twice as many elements of half the size. |
| 2872 | unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); |
| 2873 | llvm::Type *EltTy = |
| 2874 | llvm::IntegerType::get(getLLVMContext(), EltBits / 2); |
| 2875 | llvm::Type *NarrowTy = |
| 2876 | llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); |
| 2877 | llvm::Type *Tys[2] = { Ty, NarrowTy }; |
| 2878 | return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); |
| 2879 | } |
| 2880 | case NEON::BI__builtin_neon_vpaddl_v: |
| 2881 | case NEON::BI__builtin_neon_vpaddlq_v: { |
| 2882 | // The source operand type has twice as many elements of half the size. |
| 2883 | unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); |
| 2884 | llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2); |
| 2885 | llvm::Type *NarrowTy = |
| 2886 | llvm::VectorType::get(EltTy, VTy->getNumElements() * 2); |
| 2887 | llvm::Type *Tys[2] = { Ty, NarrowTy }; |
| 2888 | return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); |
| 2889 | } |
| 2890 | case NEON::BI__builtin_neon_vqdmlal_v: |
| 2891 | case NEON::BI__builtin_neon_vqdmlsl_v: { |
| 2892 | SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); |
| 2893 | Value *Mul = EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), |
| 2894 | MulOps, "vqdmlal"); |
| 2895 | |
| 2896 | SmallVector<Value *, 2> AccumOps; |
| 2897 | AccumOps.push_back(Ops[0]); |
| 2898 | AccumOps.push_back(Mul); |
| 2899 | return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), |
| 2900 | AccumOps, NameHint); |
| 2901 | } |
| 2902 | case NEON::BI__builtin_neon_vqshl_n_v: |
| 2903 | case NEON::BI__builtin_neon_vqshlq_n_v: |
| 2904 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", |
| 2905 | 1, false); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 2906 | case NEON::BI__builtin_neon_vqshlu_n_v: |
| 2907 | case NEON::BI__builtin_neon_vqshluq_n_v: |
| 2908 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", |
| 2909 | 1, false); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2910 | case NEON::BI__builtin_neon_vrecpe_v: |
| 2911 | case NEON::BI__builtin_neon_vrecpeq_v: |
| 2912 | case NEON::BI__builtin_neon_vrsqrte_v: |
| 2913 | case NEON::BI__builtin_neon_vrsqrteq_v: |
| 2914 | Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic; |
| 2915 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); |
| 2916 | |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 2917 | case NEON::BI__builtin_neon_vrshr_n_v: |
| 2918 | case NEON::BI__builtin_neon_vrshrq_n_v: |
| 2919 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", |
| 2920 | 1, true); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2921 | case NEON::BI__builtin_neon_vshl_n_v: |
| 2922 | case NEON::BI__builtin_neon_vshlq_n_v: |
| 2923 | Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); |
| 2924 | return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], |
| 2925 | "vshl_n"); |
| 2926 | case NEON::BI__builtin_neon_vshll_n_v: { |
| 2927 | llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy); |
| 2928 | Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); |
| 2929 | if (Usgn) |
| 2930 | Ops[0] = Builder.CreateZExt(Ops[0], VTy); |
| 2931 | else |
| 2932 | Ops[0] = Builder.CreateSExt(Ops[0], VTy); |
| 2933 | Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); |
| 2934 | return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); |
| 2935 | } |
| 2936 | case NEON::BI__builtin_neon_vshrn_n_v: { |
| 2937 | llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy); |
| 2938 | Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); |
| 2939 | Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); |
| 2940 | if (Usgn) |
| 2941 | Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); |
| 2942 | else |
| 2943 | Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); |
| 2944 | return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); |
| 2945 | } |
| 2946 | case NEON::BI__builtin_neon_vshr_n_v: |
| 2947 | case NEON::BI__builtin_neon_vshrq_n_v: |
| 2948 | return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); |
| 2949 | case NEON::BI__builtin_neon_vst1_v: |
| 2950 | case NEON::BI__builtin_neon_vst1q_v: |
| 2951 | case NEON::BI__builtin_neon_vst2_v: |
| 2952 | case NEON::BI__builtin_neon_vst2q_v: |
| 2953 | case NEON::BI__builtin_neon_vst3_v: |
| 2954 | case NEON::BI__builtin_neon_vst3q_v: |
| 2955 | case NEON::BI__builtin_neon_vst4_v: |
| 2956 | case NEON::BI__builtin_neon_vst4q_v: |
| 2957 | case NEON::BI__builtin_neon_vst2_lane_v: |
| 2958 | case NEON::BI__builtin_neon_vst2q_lane_v: |
| 2959 | case NEON::BI__builtin_neon_vst3_lane_v: |
| 2960 | case NEON::BI__builtin_neon_vst3q_lane_v: |
| 2961 | case NEON::BI__builtin_neon_vst4_lane_v: |
| 2962 | case NEON::BI__builtin_neon_vst4q_lane_v: |
| 2963 | Ops.push_back(Align); |
| 2964 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, ""); |
| 2965 | case NEON::BI__builtin_neon_vsubhn_v: { |
| 2966 | llvm::VectorType *SrcTy = |
| 2967 | llvm::VectorType::getExtendedElementVectorType(VTy); |
| 2968 | |
| 2969 | // %sum = add <4 x i32> %lhs, %rhs |
| 2970 | Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); |
| 2971 | Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); |
| 2972 | Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); |
| 2973 | |
| 2974 | // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16> |
| 2975 | Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(), |
| 2976 | SrcTy->getScalarSizeInBits() / 2); |
| 2977 | ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt); |
| 2978 | Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); |
| 2979 | |
| 2980 | // %res = trunc <4 x i32> %high to <4 x i16> |
| 2981 | return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); |
| 2982 | } |
| 2983 | case NEON::BI__builtin_neon_vtrn_v: |
| 2984 | case NEON::BI__builtin_neon_vtrnq_v: { |
| 2985 | Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); |
| 2986 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 2987 | Ops[2] = Builder.CreateBitCast(Ops[2], Ty); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 2988 | Value *SV = nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 2989 | |
| 2990 | for (unsigned vi = 0; vi != 2; ++vi) { |
| 2991 | SmallVector<Constant*, 16> Indices; |
| 2992 | for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { |
| 2993 | Indices.push_back(Builder.getInt32(i+vi)); |
| 2994 | Indices.push_back(Builder.getInt32(i+e+vi)); |
| 2995 | } |
| 2996 | Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); |
| 2997 | SV = llvm::ConstantVector::get(Indices); |
| 2998 | SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); |
| 2999 | SV = Builder.CreateStore(SV, Addr); |
Jiangning Liu | 0391691 | 2013-10-05 08:22:55 +0000 | [diff] [blame] | 3000 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3001 | return SV; |
| 3002 | } |
| 3003 | case NEON::BI__builtin_neon_vtst_v: |
| 3004 | case NEON::BI__builtin_neon_vtstq_v: { |
| 3005 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 3006 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 3007 | Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); |
| 3008 | Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], |
| 3009 | ConstantAggregateZero::get(Ty)); |
| 3010 | return Builder.CreateSExt(Ops[0], Ty, "vtst"); |
| 3011 | } |
| 3012 | case NEON::BI__builtin_neon_vuzp_v: |
| 3013 | case NEON::BI__builtin_neon_vuzpq_v: { |
| 3014 | Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); |
| 3015 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 3016 | Ops[2] = Builder.CreateBitCast(Ops[2], Ty); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3017 | Value *SV = nullptr; |
Jiangning Liu | 0391691 | 2013-10-05 08:22:55 +0000 | [diff] [blame] | 3018 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3019 | for (unsigned vi = 0; vi != 2; ++vi) { |
| 3020 | SmallVector<Constant*, 16> Indices; |
| 3021 | for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) |
| 3022 | Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); |
Jiangning Liu | 0391691 | 2013-10-05 08:22:55 +0000 | [diff] [blame] | 3023 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3024 | Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); |
| 3025 | SV = llvm::ConstantVector::get(Indices); |
| 3026 | SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); |
| 3027 | SV = Builder.CreateStore(SV, Addr); |
| 3028 | } |
| 3029 | return SV; |
| 3030 | } |
| 3031 | case NEON::BI__builtin_neon_vzip_v: |
| 3032 | case NEON::BI__builtin_neon_vzipq_v: { |
| 3033 | Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); |
| 3034 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 3035 | Ops[2] = Builder.CreateBitCast(Ops[2], Ty); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3036 | Value *SV = nullptr; |
Chad Rosier | 5640209 | 2013-10-30 15:20:07 +0000 | [diff] [blame] | 3037 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3038 | for (unsigned vi = 0; vi != 2; ++vi) { |
| 3039 | SmallVector<Constant*, 16> Indices; |
| 3040 | for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { |
| 3041 | Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); |
| 3042 | Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); |
| 3043 | } |
| 3044 | Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); |
| 3045 | SV = llvm::ConstantVector::get(Indices); |
| 3046 | SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); |
| 3047 | SV = Builder.CreateStore(SV, Addr); |
| 3048 | } |
| 3049 | return SV; |
| 3050 | } |
| 3051 | } |
Bill Wendling | c02b18c | 2013-12-01 03:04:49 +0000 | [diff] [blame] | 3052 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3053 | assert(Int && "Expected valid intrinsic number"); |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 3054 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3055 | // Determine the type(s) of this overloaded AArch64 intrinsic. |
| 3056 | Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E); |
| 3057 | |
| 3058 | Value *Result = EmitNeonCall(F, Ops, NameHint); |
| 3059 | llvm::Type *ResultType = ConvertType(E->getType()); |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 3060 | // AArch64 intrinsic one-element vector type cast to |
| 3061 | // scalar type expected by the builtin |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3062 | return Builder.CreateBitCast(Result, ResultType, NameHint); |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 3063 | } |
| 3064 | |
Kevin Qin | 8137a60 | 2013-11-14 02:45:18 +0000 | [diff] [blame] | 3065 | Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr( |
| 3066 | Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp, |
| 3067 | const CmpInst::Predicate Ip, const Twine &Name) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3068 | llvm::Type *OTy = Op->getType(); |
| 3069 | |
| 3070 | // FIXME: this is utterly horrific. We should not be looking at previous |
| 3071 | // codegen context to find out what needs doing. Unfortunately TableGen |
| 3072 | // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32 |
| 3073 | // (etc). |
| 3074 | if (BitCastInst *BI = dyn_cast<BitCastInst>(Op)) |
| 3075 | OTy = BI->getOperand(0)->getType(); |
| 3076 | |
Kevin Qin | 8137a60 | 2013-11-14 02:45:18 +0000 | [diff] [blame] | 3077 | Op = Builder.CreateBitCast(Op, OTy); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3078 | if (OTy->getScalarType()->isFloatingPointTy()) { |
| 3079 | Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy)); |
Kevin Qin | 8137a60 | 2013-11-14 02:45:18 +0000 | [diff] [blame] | 3080 | } else { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3081 | Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy)); |
Kevin Qin | 8137a60 | 2013-11-14 02:45:18 +0000 | [diff] [blame] | 3082 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3083 | return Builder.CreateSExt(Op, Ty, Name); |
Kevin Qin | 8137a60 | 2013-11-14 02:45:18 +0000 | [diff] [blame] | 3084 | } |
| 3085 | |
Jiangning Liu | 2e22f29 | 2013-11-14 01:57:55 +0000 | [diff] [blame] | 3086 | static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, |
| 3087 | Value *ExtOp, Value *IndexOp, |
| 3088 | llvm::Type *ResTy, unsigned IntID, |
| 3089 | const char *Name) { |
| 3090 | SmallVector<Value *, 2> TblOps; |
| 3091 | if (ExtOp) |
| 3092 | TblOps.push_back(ExtOp); |
Jiangning Liu | aee8e16 | 2013-09-24 02:48:06 +0000 | [diff] [blame] | 3093 | |
Jiangning Liu | 2e22f29 | 2013-11-14 01:57:55 +0000 | [diff] [blame] | 3094 | // Build a vector containing sequential number like (0, 1, 2, ..., 15) |
| 3095 | SmallVector<Constant*, 16> Indices; |
| 3096 | llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType()); |
| 3097 | for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) { |
| 3098 | Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i)); |
| 3099 | Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i+1)); |
| 3100 | } |
| 3101 | Value *SV = llvm::ConstantVector::get(Indices); |
| 3102 | |
| 3103 | int PairPos = 0, End = Ops.size() - 1; |
| 3104 | while (PairPos < End) { |
| 3105 | TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], |
| 3106 | Ops[PairPos+1], SV, Name)); |
| 3107 | PairPos += 2; |
| 3108 | } |
| 3109 | |
| 3110 | // If there's an odd number of 64-bit lookup table, fill the high 64-bit |
| 3111 | // of the 128-bit lookup table with zero. |
| 3112 | if (PairPos == End) { |
| 3113 | Value *ZeroTbl = ConstantAggregateZero::get(TblTy); |
| 3114 | TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], |
| 3115 | ZeroTbl, SV, Name)); |
| 3116 | } |
| 3117 | |
Jiangning Liu | 2e22f29 | 2013-11-14 01:57:55 +0000 | [diff] [blame] | 3118 | Function *TblF; |
| 3119 | TblOps.push_back(IndexOp); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3120 | TblF = CGF.CGM.getIntrinsic(IntID, ResTy); |
Jiangning Liu | 2e22f29 | 2013-11-14 01:57:55 +0000 | [diff] [blame] | 3121 | |
| 3122 | return CGF.EmitNeonCall(TblF, TblOps, Name); |
| 3123 | } |
| 3124 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3125 | Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, |
| 3126 | const CallExpr *E) { |
| 3127 | unsigned HintID = static_cast<unsigned>(-1); |
Jiangning Liu | 2e22f29 | 2013-11-14 01:57:55 +0000 | [diff] [blame] | 3128 | switch (BuiltinID) { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3129 | default: break; |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 3130 | case ARM::BI__builtin_arm_nop: |
| 3131 | HintID = 0; |
| 3132 | break; |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3133 | case ARM::BI__builtin_arm_yield: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3134 | case ARM::BI__yield: |
| 3135 | HintID = 1; |
| 3136 | break; |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3137 | case ARM::BI__builtin_arm_wfe: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3138 | case ARM::BI__wfe: |
| 3139 | HintID = 2; |
| 3140 | break; |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3141 | case ARM::BI__builtin_arm_wfi: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3142 | case ARM::BI__wfi: |
| 3143 | HintID = 3; |
| 3144 | break; |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3145 | case ARM::BI__builtin_arm_sev: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3146 | case ARM::BI__sev: |
| 3147 | HintID = 4; |
| 3148 | break; |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3149 | case ARM::BI__builtin_arm_sevl: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3150 | case ARM::BI__sevl: |
| 3151 | HintID = 5; |
Jiangning Liu | 2e22f29 | 2013-11-14 01:57:55 +0000 | [diff] [blame] | 3152 | break; |
| 3153 | } |
| 3154 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3155 | if (HintID != static_cast<unsigned>(-1)) { |
| 3156 | Function *F = CGM.getIntrinsic(Intrinsic::arm_hint); |
| 3157 | return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); |
Jiangning Liu | 2e22f29 | 2013-11-14 01:57:55 +0000 | [diff] [blame] | 3158 | } |
| 3159 | |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 3160 | if (BuiltinID == ARM::BI__builtin_arm_dbg) { |
| 3161 | Value *Option = EmitScalarExpr(E->getArg(0)); |
| 3162 | return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option); |
| 3163 | } |
| 3164 | |
| 3165 | if (BuiltinID == ARM::BI__builtin_arm_prefetch) { |
| 3166 | Value *Address = EmitScalarExpr(E->getArg(0)); |
| 3167 | Value *RW = EmitScalarExpr(E->getArg(1)); |
| 3168 | Value *IsData = EmitScalarExpr(E->getArg(2)); |
| 3169 | |
| 3170 | // Locality is not supported on ARM target |
| 3171 | Value *Locality = llvm::ConstantInt::get(Int32Ty, 3); |
| 3172 | |
| 3173 | Value *F = CGM.getIntrinsic(Intrinsic::prefetch); |
| 3174 | return Builder.CreateCall4(F, Address, RW, Locality, IsData); |
| 3175 | } |
| 3176 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3177 | if (BuiltinID == ARM::BI__builtin_arm_rbit) { |
| 3178 | return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_rbit), |
| 3179 | EmitScalarExpr(E->getArg(0)), |
| 3180 | "rbit"); |
| 3181 | } |
| 3182 | |
Rafael Espindola | e140af3 | 2010-06-09 03:48:40 +0000 | [diff] [blame] | 3183 | if (BuiltinID == ARM::BI__clear_cache) { |
Rafael Espindola | 4537d6e | 2013-05-14 12:45:47 +0000 | [diff] [blame] | 3184 | assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); |
Rafael Espindola | 79ba509 | 2010-06-07 17:26:50 +0000 | [diff] [blame] | 3185 | const FunctionDecl *FD = E->getDirectCallee(); |
Chris Lattner | 5f9e272 | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 3186 | SmallVector<Value*, 2> Ops; |
Rafael Espindola | 4537d6e | 2013-05-14 12:45:47 +0000 | [diff] [blame] | 3187 | for (unsigned i = 0; i < 2; i++) |
Eric Christopher | 8a37c79 | 2011-03-14 20:30:34 +0000 | [diff] [blame] | 3188 | Ops.push_back(EmitScalarExpr(E->getArg(i))); |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 3189 | llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); |
| 3190 | llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); |
Chris Lattner | 5f9e272 | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 3191 | StringRef Name = FD->getName(); |
John McCall | bd7370a | 2013-02-28 19:01:20 +0000 | [diff] [blame] | 3192 | return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); |
Chris Lattner | 2752c01 | 2010-03-03 19:03:45 +0000 | [diff] [blame] | 3193 | } |
Rafael Espindola | e140af3 | 2010-06-09 03:48:40 +0000 | [diff] [blame] | 3194 | |
Tim Northover | 09df2b0 | 2013-07-16 09:47:53 +0000 | [diff] [blame] | 3195 | if (BuiltinID == ARM::BI__builtin_arm_ldrexd || |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3196 | ((BuiltinID == ARM::BI__builtin_arm_ldrex || |
| 3197 | BuiltinID == ARM::BI__builtin_arm_ldaex) && |
| 3198 | getContext().getTypeSize(E->getType()) == 64) || |
| 3199 | BuiltinID == ARM::BI__ldrexd) { |
| 3200 | Function *F; |
| 3201 | |
| 3202 | switch (BuiltinID) { |
| 3203 | default: llvm_unreachable("unexpected builtin"); |
| 3204 | case ARM::BI__builtin_arm_ldaex: |
| 3205 | F = CGM.getIntrinsic(Intrinsic::arm_ldaexd); |
| 3206 | break; |
| 3207 | case ARM::BI__builtin_arm_ldrexd: |
| 3208 | case ARM::BI__builtin_arm_ldrex: |
| 3209 | case ARM::BI__ldrexd: |
| 3210 | F = CGM.getIntrinsic(Intrinsic::arm_ldrexd); |
| 3211 | break; |
| 3212 | } |
Bruno Cardoso Lopes | 26c1b8d | 2011-05-28 04:11:33 +0000 | [diff] [blame] | 3213 | |
| 3214 | Value *LdPtr = EmitScalarExpr(E->getArg(0)); |
Tim Northover | 09df2b0 | 2013-07-16 09:47:53 +0000 | [diff] [blame] | 3215 | Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), |
| 3216 | "ldrexd"); |
Bruno Cardoso Lopes | 26c1b8d | 2011-05-28 04:11:33 +0000 | [diff] [blame] | 3217 | |
| 3218 | Value *Val0 = Builder.CreateExtractValue(Val, 1); |
| 3219 | Value *Val1 = Builder.CreateExtractValue(Val, 0); |
| 3220 | Val0 = Builder.CreateZExt(Val0, Int64Ty); |
| 3221 | Val1 = Builder.CreateZExt(Val1, Int64Ty); |
| 3222 | |
| 3223 | Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32); |
| 3224 | Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); |
Tim Northover | 09df2b0 | 2013-07-16 09:47:53 +0000 | [diff] [blame] | 3225 | Val = Builder.CreateOr(Val, Val1); |
| 3226 | return Builder.CreateBitCast(Val, ConvertType(E->getType())); |
Bruno Cardoso Lopes | 26c1b8d | 2011-05-28 04:11:33 +0000 | [diff] [blame] | 3227 | } |
| 3228 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3229 | if (BuiltinID == ARM::BI__builtin_arm_ldrex || |
| 3230 | BuiltinID == ARM::BI__builtin_arm_ldaex) { |
Tim Northover | 09df2b0 | 2013-07-16 09:47:53 +0000 | [diff] [blame] | 3231 | Value *LoadAddr = EmitScalarExpr(E->getArg(0)); |
| 3232 | |
| 3233 | QualType Ty = E->getType(); |
| 3234 | llvm::Type *RealResTy = ConvertType(Ty); |
| 3235 | llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), |
| 3236 | getContext().getTypeSize(Ty)); |
| 3237 | LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); |
| 3238 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3239 | Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex |
| 3240 | ? Intrinsic::arm_ldaex |
| 3241 | : Intrinsic::arm_ldrex, |
| 3242 | LoadAddr->getType()); |
Tim Northover | 09df2b0 | 2013-07-16 09:47:53 +0000 | [diff] [blame] | 3243 | Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex"); |
| 3244 | |
| 3245 | if (RealResTy->isPointerTy()) |
| 3246 | return Builder.CreateIntToPtr(Val, RealResTy); |
| 3247 | else { |
| 3248 | Val = Builder.CreateTruncOrBitCast(Val, IntResTy); |
| 3249 | return Builder.CreateBitCast(Val, RealResTy); |
| 3250 | } |
| 3251 | } |
| 3252 | |
| 3253 | if (BuiltinID == ARM::BI__builtin_arm_strexd || |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3254 | ((BuiltinID == ARM::BI__builtin_arm_stlex || |
| 3255 | BuiltinID == ARM::BI__builtin_arm_strex) && |
Tim Northover | 09df2b0 | 2013-07-16 09:47:53 +0000 | [diff] [blame] | 3256 | getContext().getTypeSize(E->getArg(0)->getType()) == 64)) { |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3257 | Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex |
| 3258 | ? Intrinsic::arm_stlexd |
| 3259 | : Intrinsic::arm_strexd); |
Chris Lattner | 7650d95 | 2011-06-18 22:49:11 +0000 | [diff] [blame] | 3260 | llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, NULL); |
Bruno Cardoso Lopes | 26c1b8d | 2011-05-28 04:11:33 +0000 | [diff] [blame] | 3261 | |
Juergen Ributzka | 3fc7791 | 2013-08-19 22:20:37 +0000 | [diff] [blame] | 3262 | Value *Tmp = CreateMemTemp(E->getArg(0)->getType()); |
Bruno Cardoso Lopes | 26c1b8d | 2011-05-28 04:11:33 +0000 | [diff] [blame] | 3263 | Value *Val = EmitScalarExpr(E->getArg(0)); |
| 3264 | Builder.CreateStore(Val, Tmp); |
| 3265 | |
| 3266 | Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); |
| 3267 | Val = Builder.CreateLoad(LdPtr); |
| 3268 | |
| 3269 | Value *Arg0 = Builder.CreateExtractValue(Val, 0); |
| 3270 | Value *Arg1 = Builder.CreateExtractValue(Val, 1); |
Tim Northover | 09df2b0 | 2013-07-16 09:47:53 +0000 | [diff] [blame] | 3271 | Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); |
Bruno Cardoso Lopes | 26c1b8d | 2011-05-28 04:11:33 +0000 | [diff] [blame] | 3272 | return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "strexd"); |
| 3273 | } |
| 3274 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3275 | if (BuiltinID == ARM::BI__builtin_arm_strex || |
| 3276 | BuiltinID == ARM::BI__builtin_arm_stlex) { |
Tim Northover | 09df2b0 | 2013-07-16 09:47:53 +0000 | [diff] [blame] | 3277 | Value *StoreVal = EmitScalarExpr(E->getArg(0)); |
| 3278 | Value *StoreAddr = EmitScalarExpr(E->getArg(1)); |
| 3279 | |
| 3280 | QualType Ty = E->getArg(0)->getType(); |
| 3281 | llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), |
| 3282 | getContext().getTypeSize(Ty)); |
| 3283 | StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); |
| 3284 | |
| 3285 | if (StoreVal->getType()->isPointerTy()) |
| 3286 | StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty); |
| 3287 | else { |
| 3288 | StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); |
| 3289 | StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty); |
| 3290 | } |
| 3291 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3292 | Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex |
| 3293 | ? Intrinsic::arm_stlex |
| 3294 | : Intrinsic::arm_strex, |
| 3295 | StoreAddr->getType()); |
Tim Northover | 09df2b0 | 2013-07-16 09:47:53 +0000 | [diff] [blame] | 3296 | return Builder.CreateCall2(F, StoreVal, StoreAddr, "strex"); |
| 3297 | } |
| 3298 | |
| 3299 | if (BuiltinID == ARM::BI__builtin_arm_clrex) { |
| 3300 | Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex); |
| 3301 | return Builder.CreateCall(F); |
| 3302 | } |
| 3303 | |
Joey Gouly | 520ec1e | 2013-09-18 10:07:09 +0000 | [diff] [blame] | 3304 | // CRC32 |
| 3305 | Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; |
| 3306 | switch (BuiltinID) { |
| 3307 | case ARM::BI__builtin_arm_crc32b: |
| 3308 | CRCIntrinsicID = Intrinsic::arm_crc32b; break; |
| 3309 | case ARM::BI__builtin_arm_crc32cb: |
| 3310 | CRCIntrinsicID = Intrinsic::arm_crc32cb; break; |
| 3311 | case ARM::BI__builtin_arm_crc32h: |
| 3312 | CRCIntrinsicID = Intrinsic::arm_crc32h; break; |
| 3313 | case ARM::BI__builtin_arm_crc32ch: |
| 3314 | CRCIntrinsicID = Intrinsic::arm_crc32ch; break; |
| 3315 | case ARM::BI__builtin_arm_crc32w: |
| 3316 | case ARM::BI__builtin_arm_crc32d: |
| 3317 | CRCIntrinsicID = Intrinsic::arm_crc32w; break; |
| 3318 | case ARM::BI__builtin_arm_crc32cw: |
| 3319 | case ARM::BI__builtin_arm_crc32cd: |
| 3320 | CRCIntrinsicID = Intrinsic::arm_crc32cw; break; |
| 3321 | } |
| 3322 | |
| 3323 | if (CRCIntrinsicID != Intrinsic::not_intrinsic) { |
| 3324 | Value *Arg0 = EmitScalarExpr(E->getArg(0)); |
| 3325 | Value *Arg1 = EmitScalarExpr(E->getArg(1)); |
| 3326 | |
| 3327 | // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w |
| 3328 | // intrinsics, hence we need different codegen for these cases. |
| 3329 | if (BuiltinID == ARM::BI__builtin_arm_crc32d || |
| 3330 | BuiltinID == ARM::BI__builtin_arm_crc32cd) { |
| 3331 | Value *C1 = llvm::ConstantInt::get(Int64Ty, 32); |
| 3332 | Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty); |
| 3333 | Value *Arg1b = Builder.CreateLShr(Arg1, C1); |
| 3334 | Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty); |
| 3335 | |
| 3336 | Function *F = CGM.getIntrinsic(CRCIntrinsicID); |
| 3337 | Value *Res = Builder.CreateCall2(F, Arg0, Arg1a); |
| 3338 | return Builder.CreateCall2(F, Res, Arg1b); |
| 3339 | } else { |
| 3340 | Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty); |
| 3341 | |
| 3342 | Function *F = CGM.getIntrinsic(CRCIntrinsicID); |
| 3343 | return Builder.CreateCall2(F, Arg0, Arg1); |
| 3344 | } |
| 3345 | } |
| 3346 | |
Chris Lattner | 5f9e272 | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 3347 | SmallVector<Value*, 4> Ops; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3348 | llvm::Value *Align = nullptr; |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 3349 | for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) { |
| 3350 | if (i == 0) { |
| 3351 | switch (BuiltinID) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3352 | case NEON::BI__builtin_neon_vld1_v: |
| 3353 | case NEON::BI__builtin_neon_vld1q_v: |
| 3354 | case NEON::BI__builtin_neon_vld1q_lane_v: |
| 3355 | case NEON::BI__builtin_neon_vld1_lane_v: |
| 3356 | case NEON::BI__builtin_neon_vld1_dup_v: |
| 3357 | case NEON::BI__builtin_neon_vld1q_dup_v: |
| 3358 | case NEON::BI__builtin_neon_vst1_v: |
| 3359 | case NEON::BI__builtin_neon_vst1q_v: |
| 3360 | case NEON::BI__builtin_neon_vst1q_lane_v: |
| 3361 | case NEON::BI__builtin_neon_vst1_lane_v: |
| 3362 | case NEON::BI__builtin_neon_vst2_v: |
| 3363 | case NEON::BI__builtin_neon_vst2q_v: |
| 3364 | case NEON::BI__builtin_neon_vst2_lane_v: |
| 3365 | case NEON::BI__builtin_neon_vst2q_lane_v: |
| 3366 | case NEON::BI__builtin_neon_vst3_v: |
| 3367 | case NEON::BI__builtin_neon_vst3q_v: |
| 3368 | case NEON::BI__builtin_neon_vst3_lane_v: |
| 3369 | case NEON::BI__builtin_neon_vst3q_lane_v: |
| 3370 | case NEON::BI__builtin_neon_vst4_v: |
| 3371 | case NEON::BI__builtin_neon_vst4q_v: |
| 3372 | case NEON::BI__builtin_neon_vst4_lane_v: |
| 3373 | case NEON::BI__builtin_neon_vst4q_lane_v: |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 3374 | // Get the alignment for the argument in addition to the value; |
| 3375 | // we'll use it later. |
| 3376 | std::pair<llvm::Value*, unsigned> Src = |
| 3377 | EmitPointerWithAlignment(E->getArg(0)); |
| 3378 | Ops.push_back(Src.first); |
| 3379 | Align = Builder.getInt32(Src.second); |
| 3380 | continue; |
| 3381 | } |
| 3382 | } |
| 3383 | if (i == 1) { |
| 3384 | switch (BuiltinID) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3385 | case NEON::BI__builtin_neon_vld2_v: |
| 3386 | case NEON::BI__builtin_neon_vld2q_v: |
| 3387 | case NEON::BI__builtin_neon_vld3_v: |
| 3388 | case NEON::BI__builtin_neon_vld3q_v: |
| 3389 | case NEON::BI__builtin_neon_vld4_v: |
| 3390 | case NEON::BI__builtin_neon_vld4q_v: |
| 3391 | case NEON::BI__builtin_neon_vld2_lane_v: |
| 3392 | case NEON::BI__builtin_neon_vld2q_lane_v: |
| 3393 | case NEON::BI__builtin_neon_vld3_lane_v: |
| 3394 | case NEON::BI__builtin_neon_vld3q_lane_v: |
| 3395 | case NEON::BI__builtin_neon_vld4_lane_v: |
| 3396 | case NEON::BI__builtin_neon_vld4q_lane_v: |
| 3397 | case NEON::BI__builtin_neon_vld2_dup_v: |
| 3398 | case NEON::BI__builtin_neon_vld3_dup_v: |
| 3399 | case NEON::BI__builtin_neon_vld4_dup_v: |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 3400 | // Get the alignment for the argument in addition to the value; |
| 3401 | // we'll use it later. |
| 3402 | std::pair<llvm::Value*, unsigned> Src = |
| 3403 | EmitPointerWithAlignment(E->getArg(1)); |
| 3404 | Ops.push_back(Src.first); |
| 3405 | Align = Builder.getInt32(Src.second); |
| 3406 | continue; |
| 3407 | } |
| 3408 | } |
Rafael Espindola | e140af3 | 2010-06-09 03:48:40 +0000 | [diff] [blame] | 3409 | Ops.push_back(EmitScalarExpr(E->getArg(i))); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 3410 | } |
Rafael Espindola | e140af3 | 2010-06-09 03:48:40 +0000 | [diff] [blame] | 3411 | |
Bob Wilson | 83bbba1 | 2011-08-13 05:03:46 +0000 | [diff] [blame] | 3412 | switch (BuiltinID) { |
| 3413 | default: break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3414 | // vget_lane and vset_lane are not overloaded and do not have an extra |
| 3415 | // argument that specifies the vector type. |
| 3416 | case NEON::BI__builtin_neon_vget_lane_i8: |
| 3417 | case NEON::BI__builtin_neon_vget_lane_i16: |
| 3418 | case NEON::BI__builtin_neon_vget_lane_i32: |
| 3419 | case NEON::BI__builtin_neon_vget_lane_i64: |
| 3420 | case NEON::BI__builtin_neon_vget_lane_f32: |
| 3421 | case NEON::BI__builtin_neon_vgetq_lane_i8: |
| 3422 | case NEON::BI__builtin_neon_vgetq_lane_i16: |
| 3423 | case NEON::BI__builtin_neon_vgetq_lane_i32: |
| 3424 | case NEON::BI__builtin_neon_vgetq_lane_i64: |
| 3425 | case NEON::BI__builtin_neon_vgetq_lane_f32: |
Bob Wilson | 83bbba1 | 2011-08-13 05:03:46 +0000 | [diff] [blame] | 3426 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 3427 | "vget_lane"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3428 | case NEON::BI__builtin_neon_vset_lane_i8: |
| 3429 | case NEON::BI__builtin_neon_vset_lane_i16: |
| 3430 | case NEON::BI__builtin_neon_vset_lane_i32: |
| 3431 | case NEON::BI__builtin_neon_vset_lane_i64: |
| 3432 | case NEON::BI__builtin_neon_vset_lane_f32: |
| 3433 | case NEON::BI__builtin_neon_vsetq_lane_i8: |
| 3434 | case NEON::BI__builtin_neon_vsetq_lane_i16: |
| 3435 | case NEON::BI__builtin_neon_vsetq_lane_i32: |
| 3436 | case NEON::BI__builtin_neon_vsetq_lane_i64: |
| 3437 | case NEON::BI__builtin_neon_vsetq_lane_f32: |
Bob Wilson | 83bbba1 | 2011-08-13 05:03:46 +0000 | [diff] [blame] | 3438 | Ops.push_back(EmitScalarExpr(E->getArg(2))); |
| 3439 | return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3440 | |
| 3441 | // Non-polymorphic crypto instructions also not overloaded |
| 3442 | case NEON::BI__builtin_neon_vsha1h_u32: |
| 3443 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 3444 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, |
| 3445 | "vsha1h"); |
| 3446 | case NEON::BI__builtin_neon_vsha1cq_u32: |
| 3447 | Ops.push_back(EmitScalarExpr(E->getArg(2))); |
| 3448 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, |
| 3449 | "vsha1h"); |
| 3450 | case NEON::BI__builtin_neon_vsha1pq_u32: |
| 3451 | Ops.push_back(EmitScalarExpr(E->getArg(2))); |
| 3452 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, |
| 3453 | "vsha1h"); |
| 3454 | case NEON::BI__builtin_neon_vsha1mq_u32: |
| 3455 | Ops.push_back(EmitScalarExpr(E->getArg(2))); |
| 3456 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, |
| 3457 | "vsha1h"); |
Bob Wilson | 83bbba1 | 2011-08-13 05:03:46 +0000 | [diff] [blame] | 3458 | } |
| 3459 | |
| 3460 | // Get the last argument, which specifies the vector type. |
Rafael Espindola | e140af3 | 2010-06-09 03:48:40 +0000 | [diff] [blame] | 3461 | llvm::APSInt Result; |
| 3462 | const Expr *Arg = E->getArg(E->getNumArgs()-1); |
| 3463 | if (!Arg->isIntegerConstantExpr(Result, getContext())) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3464 | return nullptr; |
Rafael Espindola | e140af3 | 2010-06-09 03:48:40 +0000 | [diff] [blame] | 3465 | |
Nate Begeman | 99c40bb | 2010-08-03 21:32:34 +0000 | [diff] [blame] | 3466 | if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f || |
| 3467 | BuiltinID == ARM::BI__builtin_arm_vcvtr_d) { |
| 3468 | // Determine the overloaded type of this builtin. |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 3469 | llvm::Type *Ty; |
Nate Begeman | 99c40bb | 2010-08-03 21:32:34 +0000 | [diff] [blame] | 3470 | if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f) |
Chris Lattner | 8b41868 | 2012-02-07 00:39:47 +0000 | [diff] [blame] | 3471 | Ty = FloatTy; |
Nate Begeman | 99c40bb | 2010-08-03 21:32:34 +0000 | [diff] [blame] | 3472 | else |
Chris Lattner | 8b41868 | 2012-02-07 00:39:47 +0000 | [diff] [blame] | 3473 | Ty = DoubleTy; |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3474 | |
Nate Begeman | 99c40bb | 2010-08-03 21:32:34 +0000 | [diff] [blame] | 3475 | // Determine whether this is an unsigned conversion or not. |
| 3476 | bool usgn = Result.getZExtValue() == 1; |
| 3477 | unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr; |
| 3478 | |
| 3479 | // Call the appropriate intrinsic. |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 3480 | Function *F = CGM.getIntrinsic(Int, Ty); |
Jay Foad | 4c7d9f1 | 2011-07-15 08:37:34 +0000 | [diff] [blame] | 3481 | return Builder.CreateCall(F, Ops, "vcvtr"); |
Nate Begeman | 99c40bb | 2010-08-03 21:32:34 +0000 | [diff] [blame] | 3482 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3483 | |
Nate Begeman | 99c40bb | 2010-08-03 21:32:34 +0000 | [diff] [blame] | 3484 | // Determine the type of this overloaded NEON intrinsic. |
Bob Wilson | da95f73 | 2011-11-08 01:16:11 +0000 | [diff] [blame] | 3485 | NeonTypeFlags Type(Result.getZExtValue()); |
| 3486 | bool usgn = Type.isUnsigned(); |
Bob Wilson | 7965396 | 2010-12-03 17:10:22 +0000 | [diff] [blame] | 3487 | bool rightShift = false; |
Rafael Espindola | e140af3 | 2010-06-09 03:48:40 +0000 | [diff] [blame] | 3488 | |
Chris Lattner | 8b41868 | 2012-02-07 00:39:47 +0000 | [diff] [blame] | 3489 | llvm::VectorType *VTy = GetNeonType(this, Type); |
Chris Lattner | 9cbe4f0 | 2011-07-09 17:41:47 +0000 | [diff] [blame] | 3490 | llvm::Type *Ty = VTy; |
Rafael Espindola | e140af3 | 2010-06-09 03:48:40 +0000 | [diff] [blame] | 3491 | if (!Ty) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3492 | return nullptr; |
Rafael Espindola | e140af3 | 2010-06-09 03:48:40 +0000 | [diff] [blame] | 3493 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3494 | // Many NEON builtins have identical semantics and uses in ARM and |
| 3495 | // AArch64. Emit these in a single function. |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 3496 | auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3497 | const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( |
| 3498 | IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted); |
| 3499 | if (Builtin) |
| 3500 | return EmitCommonNeonBuiltinExpr( |
| 3501 | Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, |
| 3502 | Builtin->NameHint, Builtin->TypeModifier, E, Ops, Align); |
| 3503 | |
Rafael Espindola | e140af3 | 2010-06-09 03:48:40 +0000 | [diff] [blame] | 3504 | unsigned Int; |
| 3505 | switch (BuiltinID) { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3506 | default: return nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3507 | case NEON::BI__builtin_neon_vld1q_lane_v: |
Bob Wilson | 550a9d8 | 2012-08-14 17:27:04 +0000 | [diff] [blame] | 3508 | // Handle 64-bit integer elements as a special case. Use shuffles of |
| 3509 | // one-element vectors to avoid poor code for i64 in the backend. |
| 3510 | if (VTy->getElementType()->isIntegerTy(64)) { |
| 3511 | // Extract the other lane. |
| 3512 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 3513 | int Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); |
| 3514 | Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane)); |
| 3515 | Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); |
| 3516 | // Load the value as a one-element vector. |
| 3517 | Ty = llvm::VectorType::get(VTy->getElementType(), 1); |
| 3518 | Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 3519 | Value *Ld = Builder.CreateCall2(F, Ops[0], Align); |
Bob Wilson | 550a9d8 | 2012-08-14 17:27:04 +0000 | [diff] [blame] | 3520 | // Combine them. |
| 3521 | SmallVector<Constant*, 2> Indices; |
| 3522 | Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane)); |
| 3523 | Indices.push_back(ConstantInt::get(Int32Ty, Lane)); |
| 3524 | SV = llvm::ConstantVector::get(Indices); |
| 3525 | return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); |
| 3526 | } |
| 3527 | // fall through |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3528 | case NEON::BI__builtin_neon_vld1_lane_v: { |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 3529 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 3530 | Ty = llvm::PointerType::getUnqual(VTy->getElementType()); |
| 3531 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
Bob Wilson | eac1f67 | 2012-02-04 23:58:08 +0000 | [diff] [blame] | 3532 | LoadInst *Ld = Builder.CreateLoad(Ops[0]); |
Bob Wilson | eac1f67 | 2012-02-04 23:58:08 +0000 | [diff] [blame] | 3533 | Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); |
| 3534 | return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); |
| 3535 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3536 | case NEON::BI__builtin_neon_vld2_dup_v: |
| 3537 | case NEON::BI__builtin_neon_vld3_dup_v: |
| 3538 | case NEON::BI__builtin_neon_vld4_dup_v: { |
Bob Wilson | a0eb23b | 2010-12-10 22:54:58 +0000 | [diff] [blame] | 3539 | // Handle 64-bit elements as a special-case. There is no "dup" needed. |
| 3540 | if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) { |
| 3541 | switch (BuiltinID) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3542 | case NEON::BI__builtin_neon_vld2_dup_v: |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3543 | Int = Intrinsic::arm_neon_vld2; |
Bob Wilson | a0eb23b | 2010-12-10 22:54:58 +0000 | [diff] [blame] | 3544 | break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3545 | case NEON::BI__builtin_neon_vld3_dup_v: |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3546 | Int = Intrinsic::arm_neon_vld3; |
Bob Wilson | a0eb23b | 2010-12-10 22:54:58 +0000 | [diff] [blame] | 3547 | break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3548 | case NEON::BI__builtin_neon_vld4_dup_v: |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3549 | Int = Intrinsic::arm_neon_vld4; |
Bob Wilson | a0eb23b | 2010-12-10 22:54:58 +0000 | [diff] [blame] | 3550 | break; |
David Blaikie | b219cfc | 2011-09-23 05:06:16 +0000 | [diff] [blame] | 3551 | default: llvm_unreachable("unknown vld_dup intrinsic?"); |
Bob Wilson | a0eb23b | 2010-12-10 22:54:58 +0000 | [diff] [blame] | 3552 | } |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 3553 | Function *F = CGM.getIntrinsic(Int, Ty); |
Bob Wilson | a0eb23b | 2010-12-10 22:54:58 +0000 | [diff] [blame] | 3554 | Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup"); |
| 3555 | Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); |
| 3556 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 3557 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 3558 | } |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 3559 | switch (BuiltinID) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3560 | case NEON::BI__builtin_neon_vld2_dup_v: |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3561 | Int = Intrinsic::arm_neon_vld2lane; |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 3562 | break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3563 | case NEON::BI__builtin_neon_vld3_dup_v: |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3564 | Int = Intrinsic::arm_neon_vld3lane; |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 3565 | break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3566 | case NEON::BI__builtin_neon_vld4_dup_v: |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3567 | Int = Intrinsic::arm_neon_vld4lane; |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 3568 | break; |
David Blaikie | b219cfc | 2011-09-23 05:06:16 +0000 | [diff] [blame] | 3569 | default: llvm_unreachable("unknown vld_dup intrinsic?"); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 3570 | } |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 3571 | Function *F = CGM.getIntrinsic(Int, Ty); |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 3572 | llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType()); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3573 | |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 3574 | SmallVector<Value*, 6> Args; |
| 3575 | Args.push_back(Ops[1]); |
| 3576 | Args.append(STy->getNumElements(), UndefValue::get(Ty)); |
| 3577 | |
Chris Lattner | 77b89b8 | 2010-06-27 07:15:29 +0000 | [diff] [blame] | 3578 | llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 3579 | Args.push_back(CI); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 3580 | Args.push_back(Align); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3581 | |
Jay Foad | 4c7d9f1 | 2011-07-15 08:37:34 +0000 | [diff] [blame] | 3582 | Ops[1] = Builder.CreateCall(F, Args, "vld_dup"); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 3583 | // splat lane 0 to all elts in each vector of the result. |
| 3584 | for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { |
| 3585 | Value *Val = Builder.CreateExtractValue(Ops[1], i); |
| 3586 | Value *Elt = Builder.CreateBitCast(Val, Ty); |
| 3587 | Elt = EmitNeonSplat(Elt, CI); |
| 3588 | Elt = Builder.CreateBitCast(Elt, Val->getType()); |
| 3589 | Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i); |
| 3590 | } |
| 3591 | Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); |
| 3592 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 3593 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 3594 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3595 | case NEON::BI__builtin_neon_vqrshrn_n_v: |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3596 | Int = |
| 3597 | usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns; |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 3598 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", |
Nate Begeman | 61eecf5 | 2010-06-14 05:21:25 +0000 | [diff] [blame] | 3599 | 1, true); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3600 | case NEON::BI__builtin_neon_vqrshrun_n_v: |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 3601 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty), |
Bob Wilson | db3d4d0 | 2010-12-08 22:37:56 +0000 | [diff] [blame] | 3602 | Ops, "vqrshrun_n", 1, true); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3603 | case NEON::BI__builtin_neon_vqshrn_n_v: |
Nate Begeman | 61eecf5 | 2010-06-14 05:21:25 +0000 | [diff] [blame] | 3604 | Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns; |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 3605 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", |
Nate Begeman | 61eecf5 | 2010-06-14 05:21:25 +0000 | [diff] [blame] | 3606 | 1, true); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3607 | case NEON::BI__builtin_neon_vqshrun_n_v: |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 3608 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty), |
Bob Wilson | db3d4d0 | 2010-12-08 22:37:56 +0000 | [diff] [blame] | 3609 | Ops, "vqshrun_n", 1, true); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3610 | case NEON::BI__builtin_neon_vrecpe_v: |
| 3611 | case NEON::BI__builtin_neon_vrecpeq_v: |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 3612 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty), |
Nate Begeman | 464ccb6 | 2010-06-11 22:57:12 +0000 | [diff] [blame] | 3613 | Ops, "vrecpe"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3614 | case NEON::BI__builtin_neon_vrshrn_n_v: |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 3615 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty), |
Bob Wilson | db3d4d0 | 2010-12-08 22:37:56 +0000 | [diff] [blame] | 3616 | Ops, "vrshrn_n", 1, true); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3617 | case NEON::BI__builtin_neon_vrsra_n_v: |
| 3618 | case NEON::BI__builtin_neon_vrsraq_n_v: |
Nate Begeman | 5af93ef | 2010-06-12 06:06:07 +0000 | [diff] [blame] | 3619 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 3620 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 3621 | Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); |
| 3622 | Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts; |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 3623 | Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]); |
Nate Begeman | 5af93ef | 2010-06-12 06:06:07 +0000 | [diff] [blame] | 3624 | return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3625 | case NEON::BI__builtin_neon_vsri_n_v: |
| 3626 | case NEON::BI__builtin_neon_vsriq_n_v: |
Bob Wilson | 7965396 | 2010-12-03 17:10:22 +0000 | [diff] [blame] | 3627 | rightShift = true; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3628 | case NEON::BI__builtin_neon_vsli_n_v: |
| 3629 | case NEON::BI__builtin_neon_vsliq_n_v: |
Bob Wilson | 7965396 | 2010-12-03 17:10:22 +0000 | [diff] [blame] | 3630 | Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); |
Benjamin Kramer | 8dd55a3 | 2011-07-14 17:45:50 +0000 | [diff] [blame] | 3631 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty), |
Nate Begeman | 464ccb6 | 2010-06-11 22:57:12 +0000 | [diff] [blame] | 3632 | Ops, "vsli_n"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3633 | case NEON::BI__builtin_neon_vsra_n_v: |
| 3634 | case NEON::BI__builtin_neon_vsraq_n_v: |
Nate Begeman | 464ccb6 | 2010-06-11 22:57:12 +0000 | [diff] [blame] | 3635 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
Amaury de la Vieuville | 7f0ff70 | 2013-10-04 13:13:15 +0000 | [diff] [blame] | 3636 | Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); |
Nate Begeman | 464ccb6 | 2010-06-11 22:57:12 +0000 | [diff] [blame] | 3637 | return Builder.CreateAdd(Ops[0], Ops[1]); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3638 | case NEON::BI__builtin_neon_vst1q_lane_v: |
Bob Wilson | 550a9d8 | 2012-08-14 17:27:04 +0000 | [diff] [blame] | 3639 | // Handle 64-bit integer elements as a special case. Use a shuffle to get |
| 3640 | // a one-element vector and avoid poor code for i64 in the backend. |
| 3641 | if (VTy->getElementType()->isIntegerTy(64)) { |
| 3642 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 3643 | Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); |
| 3644 | Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); |
Eli Friedman | ea93e40 | 2012-08-23 03:10:17 +0000 | [diff] [blame] | 3645 | Ops[2] = Align; |
Bob Wilson | 550a9d8 | 2012-08-14 17:27:04 +0000 | [diff] [blame] | 3646 | return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, |
| 3647 | Ops[1]->getType()), Ops); |
| 3648 | } |
| 3649 | // fall through |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3650 | case NEON::BI__builtin_neon_vst1_lane_v: { |
Nate Begeman | 464ccb6 | 2010-06-11 22:57:12 +0000 | [diff] [blame] | 3651 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 3652 | Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); |
| 3653 | Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); |
Bob Wilson | eac1f67 | 2012-02-04 23:58:08 +0000 | [diff] [blame] | 3654 | StoreInst *St = Builder.CreateStore(Ops[1], |
| 3655 | Builder.CreateBitCast(Ops[0], Ty)); |
Bob Wilson | eac1f67 | 2012-02-04 23:58:08 +0000 | [diff] [blame] | 3656 | St->setAlignment(cast<ConstantInt>(Align)->getZExtValue()); |
| 3657 | return St; |
| 3658 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3659 | case NEON::BI__builtin_neon_vtbl1_v: |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 3660 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1), |
| 3661 | Ops, "vtbl1"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3662 | case NEON::BI__builtin_neon_vtbl2_v: |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 3663 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2), |
| 3664 | Ops, "vtbl2"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3665 | case NEON::BI__builtin_neon_vtbl3_v: |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 3666 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3), |
| 3667 | Ops, "vtbl3"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3668 | case NEON::BI__builtin_neon_vtbl4_v: |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 3669 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4), |
| 3670 | Ops, "vtbl4"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3671 | case NEON::BI__builtin_neon_vtbx1_v: |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 3672 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1), |
| 3673 | Ops, "vtbx1"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3674 | case NEON::BI__builtin_neon_vtbx2_v: |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 3675 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2), |
| 3676 | Ops, "vtbx2"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3677 | case NEON::BI__builtin_neon_vtbx3_v: |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 3678 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3), |
| 3679 | Ops, "vtbx3"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3680 | case NEON::BI__builtin_neon_vtbx4_v: |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 3681 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4), |
| 3682 | Ops, "vtbx4"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3683 | } |
| 3684 | } |
| 3685 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3686 | static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3687 | const CallExpr *E, |
| 3688 | SmallVectorImpl<Value *> &Ops) { |
| 3689 | unsigned int Int = 0; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3690 | const char *s = nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3691 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3692 | switch (BuiltinID) { |
| 3693 | default: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3694 | return nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3695 | case NEON::BI__builtin_neon_vtbl1_v: |
| 3696 | case NEON::BI__builtin_neon_vqtbl1_v: |
| 3697 | case NEON::BI__builtin_neon_vqtbl1q_v: |
| 3698 | case NEON::BI__builtin_neon_vtbl2_v: |
| 3699 | case NEON::BI__builtin_neon_vqtbl2_v: |
| 3700 | case NEON::BI__builtin_neon_vqtbl2q_v: |
| 3701 | case NEON::BI__builtin_neon_vtbl3_v: |
| 3702 | case NEON::BI__builtin_neon_vqtbl3_v: |
| 3703 | case NEON::BI__builtin_neon_vqtbl3q_v: |
| 3704 | case NEON::BI__builtin_neon_vtbl4_v: |
| 3705 | case NEON::BI__builtin_neon_vqtbl4_v: |
| 3706 | case NEON::BI__builtin_neon_vqtbl4q_v: |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3707 | break; |
| 3708 | case NEON::BI__builtin_neon_vtbx1_v: |
| 3709 | case NEON::BI__builtin_neon_vqtbx1_v: |
| 3710 | case NEON::BI__builtin_neon_vqtbx1q_v: |
| 3711 | case NEON::BI__builtin_neon_vtbx2_v: |
| 3712 | case NEON::BI__builtin_neon_vqtbx2_v: |
| 3713 | case NEON::BI__builtin_neon_vqtbx2q_v: |
| 3714 | case NEON::BI__builtin_neon_vtbx3_v: |
| 3715 | case NEON::BI__builtin_neon_vqtbx3_v: |
| 3716 | case NEON::BI__builtin_neon_vqtbx3q_v: |
| 3717 | case NEON::BI__builtin_neon_vtbx4_v: |
| 3718 | case NEON::BI__builtin_neon_vqtbx4_v: |
| 3719 | case NEON::BI__builtin_neon_vqtbx4q_v: |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3720 | break; |
| 3721 | } |
| 3722 | |
| 3723 | assert(E->getNumArgs() >= 3); |
| 3724 | |
| 3725 | // Get the last argument, which specifies the vector type. |
| 3726 | llvm::APSInt Result; |
| 3727 | const Expr *Arg = E->getArg(E->getNumArgs() - 1); |
| 3728 | if (!Arg->isIntegerConstantExpr(Result, CGF.getContext())) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3729 | return nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3730 | |
| 3731 | // Determine the type of this overloaded NEON intrinsic. |
| 3732 | NeonTypeFlags Type(Result.getZExtValue()); |
| 3733 | llvm::VectorType *VTy = GetNeonType(&CGF, Type); |
| 3734 | llvm::Type *Ty = VTy; |
| 3735 | if (!Ty) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3736 | return nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3737 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3738 | unsigned nElts = VTy->getNumElements(); |
| 3739 | |
| 3740 | CodeGen::CGBuilderTy &Builder = CGF.Builder; |
| 3741 | |
| 3742 | // AArch64 scalar builtins are not overloaded, they do not have an extra |
| 3743 | // argument that specifies the vector type, need to handle each case. |
| 3744 | SmallVector<Value *, 2> TblOps; |
| 3745 | switch (BuiltinID) { |
| 3746 | case NEON::BI__builtin_neon_vtbl1_v: { |
| 3747 | TblOps.push_back(Ops[0]); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3748 | return packTBLDVectorList(CGF, TblOps, nullptr, Ops[1], Ty, |
| 3749 | Intrinsic::aarch64_neon_tbl1, "vtbl1"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3750 | } |
| 3751 | case NEON::BI__builtin_neon_vtbl2_v: { |
| 3752 | TblOps.push_back(Ops[0]); |
| 3753 | TblOps.push_back(Ops[1]); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3754 | return packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty, |
| 3755 | Intrinsic::aarch64_neon_tbl1, "vtbl1"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3756 | } |
| 3757 | case NEON::BI__builtin_neon_vtbl3_v: { |
| 3758 | TblOps.push_back(Ops[0]); |
| 3759 | TblOps.push_back(Ops[1]); |
| 3760 | TblOps.push_back(Ops[2]); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3761 | return packTBLDVectorList(CGF, TblOps, nullptr, Ops[3], Ty, |
| 3762 | Intrinsic::aarch64_neon_tbl2, "vtbl2"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3763 | } |
| 3764 | case NEON::BI__builtin_neon_vtbl4_v: { |
| 3765 | TblOps.push_back(Ops[0]); |
| 3766 | TblOps.push_back(Ops[1]); |
| 3767 | TblOps.push_back(Ops[2]); |
| 3768 | TblOps.push_back(Ops[3]); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3769 | return packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty, |
| 3770 | Intrinsic::aarch64_neon_tbl2, "vtbl2"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3771 | } |
| 3772 | case NEON::BI__builtin_neon_vtbx1_v: { |
| 3773 | TblOps.push_back(Ops[1]); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3774 | Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty, |
| 3775 | Intrinsic::aarch64_neon_tbl1, "vtbl1"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3776 | |
| 3777 | llvm::Constant *Eight = ConstantInt::get(VTy->getElementType(), 8); |
| 3778 | Value* EightV = llvm::ConstantVector::getSplat(nElts, Eight); |
| 3779 | Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); |
| 3780 | CmpRes = Builder.CreateSExt(CmpRes, Ty); |
| 3781 | |
| 3782 | Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); |
| 3783 | Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); |
| 3784 | return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); |
| 3785 | } |
| 3786 | case NEON::BI__builtin_neon_vtbx2_v: { |
| 3787 | TblOps.push_back(Ops[1]); |
| 3788 | TblOps.push_back(Ops[2]); |
| 3789 | return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[3], Ty, |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3790 | Intrinsic::aarch64_neon_tbx1, "vtbx1"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3791 | } |
| 3792 | case NEON::BI__builtin_neon_vtbx3_v: { |
| 3793 | TblOps.push_back(Ops[1]); |
| 3794 | TblOps.push_back(Ops[2]); |
| 3795 | TblOps.push_back(Ops[3]); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3796 | Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty, |
| 3797 | Intrinsic::aarch64_neon_tbl2, "vtbl2"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3798 | |
| 3799 | llvm::Constant *TwentyFour = ConstantInt::get(VTy->getElementType(), 24); |
| 3800 | Value* TwentyFourV = llvm::ConstantVector::getSplat(nElts, TwentyFour); |
| 3801 | Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], |
| 3802 | TwentyFourV); |
| 3803 | CmpRes = Builder.CreateSExt(CmpRes, Ty); |
| 3804 | |
| 3805 | Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); |
| 3806 | Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes); |
| 3807 | return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx"); |
| 3808 | } |
| 3809 | case NEON::BI__builtin_neon_vtbx4_v: { |
| 3810 | TblOps.push_back(Ops[1]); |
| 3811 | TblOps.push_back(Ops[2]); |
| 3812 | TblOps.push_back(Ops[3]); |
| 3813 | TblOps.push_back(Ops[4]); |
| 3814 | return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[5], Ty, |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3815 | Intrinsic::aarch64_neon_tbx2, "vtbx2"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3816 | } |
| 3817 | case NEON::BI__builtin_neon_vqtbl1_v: |
| 3818 | case NEON::BI__builtin_neon_vqtbl1q_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3819 | Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3820 | case NEON::BI__builtin_neon_vqtbl2_v: |
| 3821 | case NEON::BI__builtin_neon_vqtbl2q_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3822 | Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3823 | case NEON::BI__builtin_neon_vqtbl3_v: |
| 3824 | case NEON::BI__builtin_neon_vqtbl3q_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3825 | Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3826 | case NEON::BI__builtin_neon_vqtbl4_v: |
| 3827 | case NEON::BI__builtin_neon_vqtbl4q_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3828 | Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3829 | case NEON::BI__builtin_neon_vqtbx1_v: |
| 3830 | case NEON::BI__builtin_neon_vqtbx1q_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3831 | Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3832 | case NEON::BI__builtin_neon_vqtbx2_v: |
| 3833 | case NEON::BI__builtin_neon_vqtbx2q_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3834 | Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3835 | case NEON::BI__builtin_neon_vqtbx3_v: |
| 3836 | case NEON::BI__builtin_neon_vqtbx3q_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3837 | Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3838 | case NEON::BI__builtin_neon_vqtbx4_v: |
| 3839 | case NEON::BI__builtin_neon_vqtbx4q_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3840 | Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3841 | } |
| 3842 | } |
| 3843 | |
| 3844 | if (!Int) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3845 | return nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3846 | |
| 3847 | Function *F = CGF.CGM.getIntrinsic(Int, Ty); |
| 3848 | return CGF.EmitNeonCall(F, Ops, s); |
| 3849 | } |
| 3850 | |
| 3851 | Value *CodeGenFunction::vectorWrapScalar16(Value *Op) { |
| 3852 | llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); |
| 3853 | Op = Builder.CreateBitCast(Op, Int16Ty); |
| 3854 | Value *V = UndefValue::get(VTy); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3855 | llvm::Constant *CI = ConstantInt::get(SizeTy, 0); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3856 | Op = Builder.CreateInsertElement(V, Op, CI); |
| 3857 | return Op; |
| 3858 | } |
| 3859 | |
| 3860 | Value *CodeGenFunction::vectorWrapScalar8(Value *Op) { |
| 3861 | llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8); |
| 3862 | Op = Builder.CreateBitCast(Op, Int8Ty); |
| 3863 | Value *V = UndefValue::get(VTy); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3864 | llvm::Constant *CI = ConstantInt::get(SizeTy, 0); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3865 | Op = Builder.CreateInsertElement(V, Op, CI); |
| 3866 | return Op; |
| 3867 | } |
| 3868 | |
| 3869 | Value *CodeGenFunction:: |
| 3870 | emitVectorWrappedScalar8Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops, |
| 3871 | const char *Name) { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3872 | // i8 is not a legal types for AArch64, so we can't just use |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3873 | // a normal overloaded intrinsic call for these scalar types. Instead |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3874 | // we'll build 64-bit vectors w/ lane zero being our input values and |
| 3875 | // perform the operation on that. The back end can pattern match directly |
| 3876 | // to the scalar instruction. |
| 3877 | Ops[0] = vectorWrapScalar8(Ops[0]); |
| 3878 | Ops[1] = vectorWrapScalar8(Ops[1]); |
| 3879 | llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8); |
| 3880 | Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3881 | Constant *CI = ConstantInt::get(SizeTy, 0); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3882 | return Builder.CreateExtractElement(V, CI, "lane0"); |
| 3883 | } |
| 3884 | |
| 3885 | Value *CodeGenFunction:: |
| 3886 | emitVectorWrappedScalar16Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops, |
| 3887 | const char *Name) { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3888 | // i16 is not a legal types for AArch64, so we can't just use |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3889 | // a normal overloaded intrinsic call for these scalar types. Instead |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3890 | // we'll build 64-bit vectors w/ lane zero being our input values and |
| 3891 | // perform the operation on that. The back end can pattern match directly |
| 3892 | // to the scalar instruction. |
| 3893 | Ops[0] = vectorWrapScalar16(Ops[0]); |
| 3894 | Ops[1] = vectorWrapScalar16(Ops[1]); |
| 3895 | llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4); |
| 3896 | Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3897 | Constant *CI = ConstantInt::get(SizeTy, 0); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3898 | return Builder.CreateExtractElement(V, CI, "lane0"); |
| 3899 | } |
| 3900 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3901 | Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, |
| 3902 | const CallExpr *E) { |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 3903 | unsigned HintID = static_cast<unsigned>(-1); |
| 3904 | switch (BuiltinID) { |
| 3905 | default: break; |
| 3906 | case AArch64::BI__builtin_arm_nop: |
| 3907 | HintID = 0; |
| 3908 | break; |
| 3909 | case AArch64::BI__builtin_arm_yield: |
| 3910 | HintID = 1; |
| 3911 | break; |
| 3912 | case AArch64::BI__builtin_arm_wfe: |
| 3913 | HintID = 2; |
| 3914 | break; |
| 3915 | case AArch64::BI__builtin_arm_wfi: |
| 3916 | HintID = 3; |
| 3917 | break; |
| 3918 | case AArch64::BI__builtin_arm_sev: |
| 3919 | HintID = 4; |
| 3920 | break; |
| 3921 | case AArch64::BI__builtin_arm_sevl: |
| 3922 | HintID = 5; |
| 3923 | break; |
| 3924 | } |
| 3925 | |
| 3926 | if (HintID != static_cast<unsigned>(-1)) { |
| 3927 | Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint); |
| 3928 | return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); |
| 3929 | } |
| 3930 | |
| 3931 | if (BuiltinID == AArch64::BI__builtin_arm_prefetch) { |
| 3932 | Value *Address = EmitScalarExpr(E->getArg(0)); |
| 3933 | Value *RW = EmitScalarExpr(E->getArg(1)); |
| 3934 | Value *CacheLevel = EmitScalarExpr(E->getArg(2)); |
| 3935 | Value *RetentionPolicy = EmitScalarExpr(E->getArg(3)); |
| 3936 | Value *IsData = EmitScalarExpr(E->getArg(4)); |
| 3937 | |
| 3938 | Value *Locality = nullptr; |
| 3939 | if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) { |
| 3940 | // Temporal fetch, needs to convert cache level to locality. |
| 3941 | Locality = llvm::ConstantInt::get(Int32Ty, |
| 3942 | -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3); |
| 3943 | } else { |
| 3944 | // Streaming fetch. |
| 3945 | Locality = llvm::ConstantInt::get(Int32Ty, 0); |
| 3946 | } |
| 3947 | |
| 3948 | // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify |
| 3949 | // PLDL3STRM or PLDL2STRM. |
| 3950 | Value *F = CGM.getIntrinsic(Intrinsic::prefetch); |
| 3951 | return Builder.CreateCall4(F, Address, RW, Locality, IsData); |
| 3952 | } |
| 3953 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3954 | if (BuiltinID == AArch64::BI__builtin_arm_rbit) { |
| 3955 | assert((getContext().getTypeSize(E->getType()) == 32) && |
| 3956 | "rbit of unusual size!"); |
| 3957 | llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); |
| 3958 | return Builder.CreateCall( |
| 3959 | CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); |
| 3960 | } |
| 3961 | if (BuiltinID == AArch64::BI__builtin_arm_rbit64) { |
| 3962 | assert((getContext().getTypeSize(E->getType()) == 64) && |
| 3963 | "rbit of unusual size!"); |
| 3964 | llvm::Value *Arg = EmitScalarExpr(E->getArg(0)); |
| 3965 | return Builder.CreateCall( |
| 3966 | CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit"); |
| 3967 | } |
| 3968 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 3969 | if (BuiltinID == AArch64::BI__clear_cache) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3970 | assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments"); |
| 3971 | const FunctionDecl *FD = E->getDirectCallee(); |
| 3972 | SmallVector<Value*, 2> Ops; |
| 3973 | for (unsigned i = 0; i < 2; i++) |
| 3974 | Ops.push_back(EmitScalarExpr(E->getArg(i))); |
| 3975 | llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType()); |
| 3976 | llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty); |
| 3977 | StringRef Name = FD->getName(); |
| 3978 | return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); |
| 3979 | } |
| 3980 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3981 | if ((BuiltinID == AArch64::BI__builtin_arm_ldrex || |
| 3982 | BuiltinID == AArch64::BI__builtin_arm_ldaex) && |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3983 | getContext().getTypeSize(E->getType()) == 128) { |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 3984 | Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex |
| 3985 | ? Intrinsic::aarch64_ldaxp |
| 3986 | : Intrinsic::aarch64_ldxp); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 3987 | |
| 3988 | Value *LdPtr = EmitScalarExpr(E->getArg(0)); |
| 3989 | Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy), |
| 3990 | "ldxp"); |
| 3991 | |
| 3992 | Value *Val0 = Builder.CreateExtractValue(Val, 1); |
| 3993 | Value *Val1 = Builder.CreateExtractValue(Val, 0); |
| 3994 | llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128); |
| 3995 | Val0 = Builder.CreateZExt(Val0, Int128Ty); |
| 3996 | Val1 = Builder.CreateZExt(Val1, Int128Ty); |
| 3997 | |
| 3998 | Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64); |
| 3999 | Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */); |
| 4000 | Val = Builder.CreateOr(Val, Val1); |
| 4001 | return Builder.CreateBitCast(Val, ConvertType(E->getType())); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 4002 | } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex || |
| 4003 | BuiltinID == AArch64::BI__builtin_arm_ldaex) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4004 | Value *LoadAddr = EmitScalarExpr(E->getArg(0)); |
| 4005 | |
| 4006 | QualType Ty = E->getType(); |
| 4007 | llvm::Type *RealResTy = ConvertType(Ty); |
| 4008 | llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(), |
| 4009 | getContext().getTypeSize(Ty)); |
| 4010 | LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo()); |
| 4011 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 4012 | Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex |
| 4013 | ? Intrinsic::aarch64_ldaxr |
| 4014 | : Intrinsic::aarch64_ldxr, |
| 4015 | LoadAddr->getType()); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4016 | Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr"); |
| 4017 | |
| 4018 | if (RealResTy->isPointerTy()) |
| 4019 | return Builder.CreateIntToPtr(Val, RealResTy); |
| 4020 | |
| 4021 | Val = Builder.CreateTruncOrBitCast(Val, IntResTy); |
| 4022 | return Builder.CreateBitCast(Val, RealResTy); |
| 4023 | } |
| 4024 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 4025 | if ((BuiltinID == AArch64::BI__builtin_arm_strex || |
| 4026 | BuiltinID == AArch64::BI__builtin_arm_stlex) && |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4027 | getContext().getTypeSize(E->getArg(0)->getType()) == 128) { |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 4028 | Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex |
| 4029 | ? Intrinsic::aarch64_stlxp |
| 4030 | : Intrinsic::aarch64_stxp); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4031 | llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, NULL); |
| 4032 | |
| 4033 | Value *One = llvm::ConstantInt::get(Int32Ty, 1); |
| 4034 | Value *Tmp = Builder.CreateAlloca(ConvertType(E->getArg(0)->getType()), |
| 4035 | One); |
| 4036 | Value *Val = EmitScalarExpr(E->getArg(0)); |
| 4037 | Builder.CreateStore(Val, Tmp); |
| 4038 | |
| 4039 | Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy)); |
| 4040 | Val = Builder.CreateLoad(LdPtr); |
| 4041 | |
| 4042 | Value *Arg0 = Builder.CreateExtractValue(Val, 0); |
| 4043 | Value *Arg1 = Builder.CreateExtractValue(Val, 1); |
| 4044 | Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), |
| 4045 | Int8PtrTy); |
| 4046 | return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "stxp"); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 4047 | } else if (BuiltinID == AArch64::BI__builtin_arm_strex || |
| 4048 | BuiltinID == AArch64::BI__builtin_arm_stlex) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4049 | Value *StoreVal = EmitScalarExpr(E->getArg(0)); |
| 4050 | Value *StoreAddr = EmitScalarExpr(E->getArg(1)); |
| 4051 | |
| 4052 | QualType Ty = E->getArg(0)->getType(); |
| 4053 | llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(), |
| 4054 | getContext().getTypeSize(Ty)); |
| 4055 | StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo()); |
| 4056 | |
| 4057 | if (StoreVal->getType()->isPointerTy()) |
| 4058 | StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty); |
| 4059 | else { |
| 4060 | StoreVal = Builder.CreateBitCast(StoreVal, StoreTy); |
| 4061 | StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty); |
| 4062 | } |
| 4063 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 4064 | Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex |
| 4065 | ? Intrinsic::aarch64_stlxr |
| 4066 | : Intrinsic::aarch64_stxr, |
| 4067 | StoreAddr->getType()); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4068 | return Builder.CreateCall2(F, StoreVal, StoreAddr, "stxr"); |
| 4069 | } |
| 4070 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4071 | if (BuiltinID == AArch64::BI__builtin_arm_clrex) { |
| 4072 | Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4073 | return Builder.CreateCall(F); |
| 4074 | } |
| 4075 | |
| 4076 | // CRC32 |
| 4077 | Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic; |
| 4078 | switch (BuiltinID) { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4079 | case AArch64::BI__builtin_arm_crc32b: |
| 4080 | CRCIntrinsicID = Intrinsic::aarch64_crc32b; break; |
| 4081 | case AArch64::BI__builtin_arm_crc32cb: |
| 4082 | CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break; |
| 4083 | case AArch64::BI__builtin_arm_crc32h: |
| 4084 | CRCIntrinsicID = Intrinsic::aarch64_crc32h; break; |
| 4085 | case AArch64::BI__builtin_arm_crc32ch: |
| 4086 | CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break; |
| 4087 | case AArch64::BI__builtin_arm_crc32w: |
| 4088 | CRCIntrinsicID = Intrinsic::aarch64_crc32w; break; |
| 4089 | case AArch64::BI__builtin_arm_crc32cw: |
| 4090 | CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break; |
| 4091 | case AArch64::BI__builtin_arm_crc32d: |
| 4092 | CRCIntrinsicID = Intrinsic::aarch64_crc32x; break; |
| 4093 | case AArch64::BI__builtin_arm_crc32cd: |
| 4094 | CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4095 | } |
| 4096 | |
| 4097 | if (CRCIntrinsicID != Intrinsic::not_intrinsic) { |
| 4098 | Value *Arg0 = EmitScalarExpr(E->getArg(0)); |
| 4099 | Value *Arg1 = EmitScalarExpr(E->getArg(1)); |
| 4100 | Function *F = CGM.getIntrinsic(CRCIntrinsicID); |
| 4101 | |
| 4102 | llvm::Type *DataTy = F->getFunctionType()->getParamType(1); |
| 4103 | Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy); |
| 4104 | |
| 4105 | return Builder.CreateCall2(F, Arg0, Arg1); |
| 4106 | } |
| 4107 | |
| 4108 | llvm::SmallVector<Value*, 4> Ops; |
| 4109 | for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) |
| 4110 | Ops.push_back(EmitScalarExpr(E->getArg(i))); |
| 4111 | |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 4112 | auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4113 | const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4114 | SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4115 | |
| 4116 | if (Builtin) { |
| 4117 | Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); |
| 4118 | Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); |
| 4119 | assert(Result && "SISD intrinsic should have been handled"); |
| 4120 | return Result; |
| 4121 | } |
| 4122 | |
| 4123 | llvm::APSInt Result; |
| 4124 | const Expr *Arg = E->getArg(E->getNumArgs()-1); |
| 4125 | NeonTypeFlags Type(0); |
| 4126 | if (Arg->isIntegerConstantExpr(Result, getContext())) |
| 4127 | // Determine the type of this overloaded NEON intrinsic. |
| 4128 | Type = NeonTypeFlags(Result.getZExtValue()); |
| 4129 | |
| 4130 | bool usgn = Type.isUnsigned(); |
| 4131 | bool quad = Type.isQuad(); |
| 4132 | |
| 4133 | // Handle non-overloaded intrinsics first. |
| 4134 | switch (BuiltinID) { |
| 4135 | default: break; |
| 4136 | case NEON::BI__builtin_neon_vldrq_p128: { |
| 4137 | llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); |
| 4138 | Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy); |
| 4139 | return Builder.CreateLoad(Ptr); |
| 4140 | } |
| 4141 | case NEON::BI__builtin_neon_vstrq_p128: { |
| 4142 | llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128); |
| 4143 | Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); |
| 4144 | return Builder.CreateStore(EmitScalarExpr(E->getArg(1)), Ptr); |
| 4145 | } |
| 4146 | case NEON::BI__builtin_neon_vcvts_u32_f32: |
| 4147 | case NEON::BI__builtin_neon_vcvtd_u64_f64: |
| 4148 | usgn = true; |
| 4149 | // FALL THROUGH |
| 4150 | case NEON::BI__builtin_neon_vcvts_s32_f32: |
| 4151 | case NEON::BI__builtin_neon_vcvtd_s64_f64: { |
| 4152 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 4153 | bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; |
| 4154 | llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; |
| 4155 | llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; |
| 4156 | Ops[0] = Builder.CreateBitCast(Ops[0], FTy); |
| 4157 | if (usgn) |
| 4158 | return Builder.CreateFPToUI(Ops[0], InTy); |
| 4159 | return Builder.CreateFPToSI(Ops[0], InTy); |
| 4160 | } |
| 4161 | case NEON::BI__builtin_neon_vcvts_f32_u32: |
| 4162 | case NEON::BI__builtin_neon_vcvtd_f64_u64: |
| 4163 | usgn = true; |
| 4164 | // FALL THROUGH |
| 4165 | case NEON::BI__builtin_neon_vcvts_f32_s32: |
| 4166 | case NEON::BI__builtin_neon_vcvtd_f64_s64: { |
| 4167 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 4168 | bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; |
| 4169 | llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty; |
| 4170 | llvm::Type *FTy = Is64 ? DoubleTy : FloatTy; |
| 4171 | Ops[0] = Builder.CreateBitCast(Ops[0], InTy); |
| 4172 | if (usgn) |
| 4173 | return Builder.CreateUIToFP(Ops[0], FTy); |
| 4174 | return Builder.CreateSIToFP(Ops[0], FTy); |
| 4175 | } |
| 4176 | case NEON::BI__builtin_neon_vpaddd_s64: { |
| 4177 | llvm::Type *Ty = |
| 4178 | llvm::VectorType::get(llvm::Type::getInt64Ty(getLLVMContext()), 2); |
| 4179 | Value *Vec = EmitScalarExpr(E->getArg(0)); |
| 4180 | // The vector is v2f64, so make sure it's bitcast to that. |
| 4181 | Vec = Builder.CreateBitCast(Vec, Ty, "v2i64"); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 4182 | llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); |
| 4183 | llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4184 | Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); |
| 4185 | Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); |
| 4186 | // Pairwise addition of a v2f64 into a scalar f64. |
| 4187 | return Builder.CreateAdd(Op0, Op1, "vpaddd"); |
| 4188 | } |
| 4189 | case NEON::BI__builtin_neon_vpaddd_f64: { |
| 4190 | llvm::Type *Ty = |
| 4191 | llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2); |
| 4192 | Value *Vec = EmitScalarExpr(E->getArg(0)); |
| 4193 | // The vector is v2f64, so make sure it's bitcast to that. |
| 4194 | Vec = Builder.CreateBitCast(Vec, Ty, "v2f64"); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 4195 | llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); |
| 4196 | llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4197 | Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); |
| 4198 | Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); |
| 4199 | // Pairwise addition of a v2f64 into a scalar f64. |
| 4200 | return Builder.CreateFAdd(Op0, Op1, "vpaddd"); |
| 4201 | } |
| 4202 | case NEON::BI__builtin_neon_vpadds_f32: { |
| 4203 | llvm::Type *Ty = |
| 4204 | llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2); |
| 4205 | Value *Vec = EmitScalarExpr(E->getArg(0)); |
| 4206 | // The vector is v2f32, so make sure it's bitcast to that. |
| 4207 | Vec = Builder.CreateBitCast(Vec, Ty, "v2f32"); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 4208 | llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0); |
| 4209 | llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4210 | Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0"); |
| 4211 | Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1"); |
| 4212 | // Pairwise addition of a v2f32 into a scalar f32. |
| 4213 | return Builder.CreateFAdd(Op0, Op1, "vpaddd"); |
| 4214 | } |
| 4215 | case NEON::BI__builtin_neon_vceqzd_s64: |
| 4216 | case NEON::BI__builtin_neon_vceqzd_f64: |
| 4217 | case NEON::BI__builtin_neon_vceqzs_f32: |
| 4218 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 4219 | return EmitAArch64CompareBuiltinExpr( |
| 4220 | Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OEQ, |
| 4221 | ICmpInst::ICMP_EQ, "vceqz"); |
| 4222 | case NEON::BI__builtin_neon_vcgezd_s64: |
| 4223 | case NEON::BI__builtin_neon_vcgezd_f64: |
| 4224 | case NEON::BI__builtin_neon_vcgezs_f32: |
| 4225 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 4226 | return EmitAArch64CompareBuiltinExpr( |
| 4227 | Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGE, |
| 4228 | ICmpInst::ICMP_SGE, "vcgez"); |
| 4229 | case NEON::BI__builtin_neon_vclezd_s64: |
| 4230 | case NEON::BI__builtin_neon_vclezd_f64: |
| 4231 | case NEON::BI__builtin_neon_vclezs_f32: |
| 4232 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 4233 | return EmitAArch64CompareBuiltinExpr( |
| 4234 | Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLE, |
| 4235 | ICmpInst::ICMP_SLE, "vclez"); |
| 4236 | case NEON::BI__builtin_neon_vcgtzd_s64: |
| 4237 | case NEON::BI__builtin_neon_vcgtzd_f64: |
| 4238 | case NEON::BI__builtin_neon_vcgtzs_f32: |
| 4239 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 4240 | return EmitAArch64CompareBuiltinExpr( |
| 4241 | Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGT, |
| 4242 | ICmpInst::ICMP_SGT, "vcgtz"); |
| 4243 | case NEON::BI__builtin_neon_vcltzd_s64: |
| 4244 | case NEON::BI__builtin_neon_vcltzd_f64: |
| 4245 | case NEON::BI__builtin_neon_vcltzs_f32: |
| 4246 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 4247 | return EmitAArch64CompareBuiltinExpr( |
| 4248 | Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLT, |
| 4249 | ICmpInst::ICMP_SLT, "vcltz"); |
| 4250 | |
| 4251 | case NEON::BI__builtin_neon_vceqzd_u64: { |
| 4252 | llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); |
| 4253 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 4254 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 4255 | Ops[0] = Builder.CreateICmp(llvm::ICmpInst::ICMP_EQ, Ops[0], |
| 4256 | llvm::Constant::getNullValue(Ty)); |
| 4257 | return Builder.CreateSExt(Ops[0], Ty, "vceqzd"); |
| 4258 | } |
| 4259 | case NEON::BI__builtin_neon_vceqd_f64: |
| 4260 | case NEON::BI__builtin_neon_vcled_f64: |
| 4261 | case NEON::BI__builtin_neon_vcltd_f64: |
| 4262 | case NEON::BI__builtin_neon_vcged_f64: |
| 4263 | case NEON::BI__builtin_neon_vcgtd_f64: { |
| 4264 | llvm::CmpInst::Predicate P; |
| 4265 | switch (BuiltinID) { |
| 4266 | default: llvm_unreachable("missing builtin ID in switch!"); |
| 4267 | case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break; |
| 4268 | case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break; |
| 4269 | case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break; |
| 4270 | case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break; |
| 4271 | case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break; |
| 4272 | } |
| 4273 | Ops.push_back(EmitScalarExpr(E->getArg(1))); |
| 4274 | Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); |
| 4275 | Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); |
| 4276 | Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); |
| 4277 | return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); |
| 4278 | } |
| 4279 | case NEON::BI__builtin_neon_vceqs_f32: |
| 4280 | case NEON::BI__builtin_neon_vcles_f32: |
| 4281 | case NEON::BI__builtin_neon_vclts_f32: |
| 4282 | case NEON::BI__builtin_neon_vcges_f32: |
| 4283 | case NEON::BI__builtin_neon_vcgts_f32: { |
| 4284 | llvm::CmpInst::Predicate P; |
| 4285 | switch (BuiltinID) { |
| 4286 | default: llvm_unreachable("missing builtin ID in switch!"); |
| 4287 | case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break; |
| 4288 | case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break; |
| 4289 | case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break; |
| 4290 | case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break; |
| 4291 | case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break; |
| 4292 | } |
| 4293 | Ops.push_back(EmitScalarExpr(E->getArg(1))); |
| 4294 | Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); |
| 4295 | Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); |
| 4296 | Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); |
| 4297 | return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); |
| 4298 | } |
| 4299 | case NEON::BI__builtin_neon_vceqd_s64: |
| 4300 | case NEON::BI__builtin_neon_vceqd_u64: |
| 4301 | case NEON::BI__builtin_neon_vcgtd_s64: |
| 4302 | case NEON::BI__builtin_neon_vcgtd_u64: |
| 4303 | case NEON::BI__builtin_neon_vcltd_s64: |
| 4304 | case NEON::BI__builtin_neon_vcltd_u64: |
| 4305 | case NEON::BI__builtin_neon_vcged_u64: |
| 4306 | case NEON::BI__builtin_neon_vcged_s64: |
| 4307 | case NEON::BI__builtin_neon_vcled_u64: |
| 4308 | case NEON::BI__builtin_neon_vcled_s64: { |
| 4309 | llvm::CmpInst::Predicate P; |
| 4310 | switch (BuiltinID) { |
| 4311 | default: llvm_unreachable("missing builtin ID in switch!"); |
| 4312 | case NEON::BI__builtin_neon_vceqd_s64: |
| 4313 | case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break; |
| 4314 | case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break; |
| 4315 | case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break; |
| 4316 | case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break; |
| 4317 | case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break; |
| 4318 | case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break; |
| 4319 | case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break; |
| 4320 | case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break; |
| 4321 | case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break; |
| 4322 | } |
| 4323 | Ops.push_back(EmitScalarExpr(E->getArg(1))); |
| 4324 | Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); |
| 4325 | Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); |
| 4326 | Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); |
| 4327 | return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); |
| 4328 | } |
| 4329 | case NEON::BI__builtin_neon_vtstd_s64: |
| 4330 | case NEON::BI__builtin_neon_vtstd_u64: { |
| 4331 | llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); |
| 4332 | Ops.push_back(EmitScalarExpr(E->getArg(1))); |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 4333 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 4334 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 4335 | Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 4336 | Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4337 | llvm::Constant::getNullValue(Ty)); |
| 4338 | return Builder.CreateSExt(Ops[0], Ty, "vtstd"); |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 4339 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4340 | case NEON::BI__builtin_neon_vset_lane_i8: |
| 4341 | case NEON::BI__builtin_neon_vset_lane_i16: |
| 4342 | case NEON::BI__builtin_neon_vset_lane_i32: |
| 4343 | case NEON::BI__builtin_neon_vset_lane_i64: |
| 4344 | case NEON::BI__builtin_neon_vset_lane_f32: |
| 4345 | case NEON::BI__builtin_neon_vsetq_lane_i8: |
| 4346 | case NEON::BI__builtin_neon_vsetq_lane_i16: |
| 4347 | case NEON::BI__builtin_neon_vsetq_lane_i32: |
| 4348 | case NEON::BI__builtin_neon_vsetq_lane_i64: |
| 4349 | case NEON::BI__builtin_neon_vsetq_lane_f32: |
| 4350 | Ops.push_back(EmitScalarExpr(E->getArg(2))); |
| 4351 | return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); |
| 4352 | case NEON::BI__builtin_neon_vset_lane_f64: |
| 4353 | // The vector type needs a cast for the v1f64 variant. |
| 4354 | Ops[1] = Builder.CreateBitCast(Ops[1], |
| 4355 | llvm::VectorType::get(DoubleTy, 1)); |
| 4356 | Ops.push_back(EmitScalarExpr(E->getArg(2))); |
| 4357 | return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); |
| 4358 | case NEON::BI__builtin_neon_vsetq_lane_f64: |
| 4359 | // The vector type needs a cast for the v2f64 variant. |
| 4360 | Ops[1] = Builder.CreateBitCast(Ops[1], |
| 4361 | llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2)); |
| 4362 | Ops.push_back(EmitScalarExpr(E->getArg(2))); |
| 4363 | return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); |
| 4364 | |
| 4365 | case NEON::BI__builtin_neon_vget_lane_i8: |
| 4366 | case NEON::BI__builtin_neon_vdupb_lane_i8: |
| 4367 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4368 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8)); |
| 4369 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4370 | "vget_lane"); |
| 4371 | case NEON::BI__builtin_neon_vgetq_lane_i8: |
| 4372 | case NEON::BI__builtin_neon_vdupb_laneq_i8: |
| 4373 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4374 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16)); |
| 4375 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4376 | "vgetq_lane"); |
| 4377 | case NEON::BI__builtin_neon_vget_lane_i16: |
| 4378 | case NEON::BI__builtin_neon_vduph_lane_i16: |
| 4379 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4380 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4)); |
| 4381 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4382 | "vget_lane"); |
| 4383 | case NEON::BI__builtin_neon_vgetq_lane_i16: |
| 4384 | case NEON::BI__builtin_neon_vduph_laneq_i16: |
| 4385 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4386 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8)); |
| 4387 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4388 | "vgetq_lane"); |
| 4389 | case NEON::BI__builtin_neon_vget_lane_i32: |
| 4390 | case NEON::BI__builtin_neon_vdups_lane_i32: |
| 4391 | Ops[0] = Builder.CreateBitCast( |
| 4392 | Ops[0], |
| 4393 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 2)); |
| 4394 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4395 | "vget_lane"); |
| 4396 | case NEON::BI__builtin_neon_vdups_lane_f32: |
| 4397 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4398 | llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2)); |
| 4399 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4400 | "vdups_lane"); |
| 4401 | case NEON::BI__builtin_neon_vgetq_lane_i32: |
| 4402 | case NEON::BI__builtin_neon_vdups_laneq_i32: |
| 4403 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4404 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 4)); |
| 4405 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4406 | "vgetq_lane"); |
| 4407 | case NEON::BI__builtin_neon_vget_lane_i64: |
| 4408 | case NEON::BI__builtin_neon_vdupd_lane_i64: |
| 4409 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4410 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 1)); |
| 4411 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4412 | "vget_lane"); |
| 4413 | case NEON::BI__builtin_neon_vdupd_lane_f64: |
| 4414 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4415 | llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1)); |
| 4416 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4417 | "vdupd_lane"); |
| 4418 | case NEON::BI__builtin_neon_vgetq_lane_i64: |
| 4419 | case NEON::BI__builtin_neon_vdupd_laneq_i64: |
| 4420 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4421 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 2)); |
| 4422 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4423 | "vgetq_lane"); |
| 4424 | case NEON::BI__builtin_neon_vget_lane_f32: |
| 4425 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4426 | llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2)); |
| 4427 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4428 | "vget_lane"); |
| 4429 | case NEON::BI__builtin_neon_vget_lane_f64: |
| 4430 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4431 | llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1)); |
| 4432 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4433 | "vget_lane"); |
| 4434 | case NEON::BI__builtin_neon_vgetq_lane_f32: |
| 4435 | case NEON::BI__builtin_neon_vdups_laneq_f32: |
| 4436 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4437 | llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 4)); |
| 4438 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4439 | "vgetq_lane"); |
| 4440 | case NEON::BI__builtin_neon_vgetq_lane_f64: |
| 4441 | case NEON::BI__builtin_neon_vdupd_laneq_f64: |
| 4442 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 4443 | llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2)); |
| 4444 | return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), |
| 4445 | "vgetq_lane"); |
| 4446 | case NEON::BI__builtin_neon_vaddd_s64: |
| 4447 | case NEON::BI__builtin_neon_vaddd_u64: |
| 4448 | return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); |
| 4449 | case NEON::BI__builtin_neon_vsubd_s64: |
| 4450 | case NEON::BI__builtin_neon_vsubd_u64: |
| 4451 | return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); |
| 4452 | case NEON::BI__builtin_neon_vqdmlalh_s16: |
| 4453 | case NEON::BI__builtin_neon_vqdmlslh_s16: { |
| 4454 | SmallVector<Value *, 2> ProductOps; |
| 4455 | ProductOps.push_back(vectorWrapScalar16(Ops[1])); |
| 4456 | ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2)))); |
| 4457 | llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4458 | Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4459 | ProductOps, "vqdmlXl"); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 4460 | Constant *CI = ConstantInt::get(SizeTy, 0); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4461 | Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); |
| 4462 | |
| 4463 | unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16 |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4464 | ? Intrinsic::aarch64_neon_sqadd |
| 4465 | : Intrinsic::aarch64_neon_sqsub; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4466 | return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); |
| 4467 | } |
| 4468 | case NEON::BI__builtin_neon_vqshlud_n_s64: { |
| 4469 | Ops.push_back(EmitScalarExpr(E->getArg(1))); |
| 4470 | Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4471 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty), |
| 4472 | Ops, "vqshlu_n"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4473 | } |
| 4474 | case NEON::BI__builtin_neon_vqshld_n_u64: |
| 4475 | case NEON::BI__builtin_neon_vqshld_n_s64: { |
| 4476 | unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64 |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4477 | ? Intrinsic::aarch64_neon_uqshl |
| 4478 | : Intrinsic::aarch64_neon_sqshl; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4479 | Ops.push_back(EmitScalarExpr(E->getArg(1))); |
| 4480 | Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4481 | return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4482 | } |
| 4483 | case NEON::BI__builtin_neon_vrshrd_n_u64: |
| 4484 | case NEON::BI__builtin_neon_vrshrd_n_s64: { |
| 4485 | unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64 |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4486 | ? Intrinsic::aarch64_neon_urshl |
| 4487 | : Intrinsic::aarch64_neon_srshl; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4488 | Ops.push_back(EmitScalarExpr(E->getArg(1))); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4489 | int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); |
| 4490 | Ops[1] = ConstantInt::get(Int64Ty, -SV); |
| 4491 | return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4492 | } |
| 4493 | case NEON::BI__builtin_neon_vrsrad_n_u64: |
| 4494 | case NEON::BI__builtin_neon_vrsrad_n_s64: { |
| 4495 | unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64 |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4496 | ? Intrinsic::aarch64_neon_urshl |
| 4497 | : Intrinsic::aarch64_neon_srshl; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4498 | Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); |
| 4499 | Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); |
| 4500 | Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Int64Ty), Ops[1], |
| 4501 | Builder.CreateSExt(Ops[2], Int64Ty)); |
| 4502 | return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); |
| 4503 | } |
| 4504 | case NEON::BI__builtin_neon_vshld_n_s64: |
| 4505 | case NEON::BI__builtin_neon_vshld_n_u64: { |
| 4506 | llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); |
| 4507 | return Builder.CreateShl( |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4508 | Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4509 | } |
| 4510 | case NEON::BI__builtin_neon_vshrd_n_s64: { |
| 4511 | llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); |
| 4512 | return Builder.CreateAShr( |
| 4513 | Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), |
| 4514 | Amt->getZExtValue())), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4515 | "shrd_n"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4516 | } |
| 4517 | case NEON::BI__builtin_neon_vshrd_n_u64: { |
| 4518 | llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1))); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4519 | uint64_t ShiftAmt = Amt->getZExtValue(); |
| 4520 | // Right-shifting an unsigned value by its size yields 0. |
| 4521 | if (ShiftAmt == 64) |
| 4522 | return ConstantInt::get(Int64Ty, 0); |
| 4523 | return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), |
| 4524 | "shrd_n"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4525 | } |
| 4526 | case NEON::BI__builtin_neon_vsrad_n_s64: { |
| 4527 | llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); |
| 4528 | Ops[1] = Builder.CreateAShr( |
| 4529 | Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), |
| 4530 | Amt->getZExtValue())), |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4531 | "shrd_n"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4532 | return Builder.CreateAdd(Ops[0], Ops[1]); |
| 4533 | } |
| 4534 | case NEON::BI__builtin_neon_vsrad_n_u64: { |
| 4535 | llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2))); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4536 | uint64_t ShiftAmt = Amt->getZExtValue(); |
| 4537 | // Right-shifting an unsigned value by its size yields 0. |
| 4538 | // As Op + 0 = Op, return Ops[0] directly. |
| 4539 | if (ShiftAmt == 64) |
| 4540 | return Ops[0]; |
| 4541 | Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), |
| 4542 | "shrd_n"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4543 | return Builder.CreateAdd(Ops[0], Ops[1]); |
| 4544 | } |
| 4545 | case NEON::BI__builtin_neon_vqdmlalh_lane_s16: |
| 4546 | case NEON::BI__builtin_neon_vqdmlalh_laneq_s16: |
| 4547 | case NEON::BI__builtin_neon_vqdmlslh_lane_s16: |
| 4548 | case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: { |
| 4549 | Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), |
| 4550 | "lane"); |
| 4551 | SmallVector<Value *, 2> ProductOps; |
| 4552 | ProductOps.push_back(vectorWrapScalar16(Ops[1])); |
| 4553 | ProductOps.push_back(vectorWrapScalar16(Ops[2])); |
| 4554 | llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4555 | Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4556 | ProductOps, "vqdmlXl"); |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 4557 | Constant *CI = ConstantInt::get(SizeTy, 0); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4558 | Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); |
| 4559 | Ops.pop_back(); |
| 4560 | |
| 4561 | unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 || |
| 4562 | BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4563 | ? Intrinsic::aarch64_neon_sqadd |
| 4564 | : Intrinsic::aarch64_neon_sqsub; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4565 | return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); |
| 4566 | } |
| 4567 | case NEON::BI__builtin_neon_vqdmlals_s32: |
| 4568 | case NEON::BI__builtin_neon_vqdmlsls_s32: { |
| 4569 | SmallVector<Value *, 2> ProductOps; |
| 4570 | ProductOps.push_back(Ops[1]); |
| 4571 | ProductOps.push_back(EmitScalarExpr(E->getArg(2))); |
| 4572 | Ops[1] = |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4573 | EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4574 | ProductOps, "vqdmlXl"); |
| 4575 | |
| 4576 | unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32 |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4577 | ? Intrinsic::aarch64_neon_sqadd |
| 4578 | : Intrinsic::aarch64_neon_sqsub; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4579 | return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); |
| 4580 | } |
| 4581 | case NEON::BI__builtin_neon_vqdmlals_lane_s32: |
| 4582 | case NEON::BI__builtin_neon_vqdmlals_laneq_s32: |
| 4583 | case NEON::BI__builtin_neon_vqdmlsls_lane_s32: |
| 4584 | case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: { |
| 4585 | Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), |
| 4586 | "lane"); |
| 4587 | SmallVector<Value *, 2> ProductOps; |
| 4588 | ProductOps.push_back(Ops[1]); |
| 4589 | ProductOps.push_back(Ops[2]); |
| 4590 | Ops[1] = |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4591 | EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4592 | ProductOps, "vqdmlXl"); |
| 4593 | Ops.pop_back(); |
| 4594 | |
| 4595 | unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 || |
| 4596 | BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4597 | ? Intrinsic::aarch64_neon_sqadd |
| 4598 | : Intrinsic::aarch64_neon_sqsub; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4599 | return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); |
| 4600 | } |
| 4601 | } |
| 4602 | |
| 4603 | llvm::VectorType *VTy = GetNeonType(this, Type); |
| 4604 | llvm::Type *Ty = VTy; |
| 4605 | if (!Ty) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4606 | return nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4607 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4608 | // Not all intrinsics handled by the common case work for AArch64 yet, so only |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4609 | // defer to common code if it's been added to our special map. |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4610 | Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID, |
| 4611 | AArch64SIMDIntrinsicsProvenSorted); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4612 | |
| 4613 | if (Builtin) |
| 4614 | return EmitCommonNeonBuiltinExpr( |
| 4615 | Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic, |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4616 | Builtin->NameHint, Builtin->TypeModifier, E, Ops, nullptr); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4617 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4618 | if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops)) |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4619 | return V; |
| 4620 | |
| 4621 | unsigned Int; |
| 4622 | switch (BuiltinID) { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4623 | default: return nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4624 | case NEON::BI__builtin_neon_vbsl_v: |
| 4625 | case NEON::BI__builtin_neon_vbslq_v: { |
| 4626 | llvm::Type *BitTy = llvm::VectorType::getInteger(VTy); |
| 4627 | Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); |
| 4628 | Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); |
| 4629 | Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); |
| 4630 | |
| 4631 | Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); |
| 4632 | Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); |
| 4633 | Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); |
| 4634 | return Builder.CreateBitCast(Ops[0], Ty); |
| 4635 | } |
| 4636 | case NEON::BI__builtin_neon_vfma_lane_v: |
| 4637 | case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types |
| 4638 | // The ARM builtins (and instructions) have the addend as the first |
| 4639 | // operand, but the 'fma' intrinsics have it last. Swap it around here. |
| 4640 | Value *Addend = Ops[0]; |
| 4641 | Value *Multiplicand = Ops[1]; |
| 4642 | Value *LaneSource = Ops[2]; |
| 4643 | Ops[0] = Multiplicand; |
| 4644 | Ops[1] = LaneSource; |
| 4645 | Ops[2] = Addend; |
| 4646 | |
| 4647 | // Now adjust things to handle the lane access. |
| 4648 | llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ? |
| 4649 | llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) : |
| 4650 | VTy; |
| 4651 | llvm::Constant *cst = cast<Constant>(Ops[3]); |
| 4652 | Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst); |
| 4653 | Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); |
| 4654 | Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); |
| 4655 | |
| 4656 | Ops.pop_back(); |
| 4657 | Int = Intrinsic::fma; |
| 4658 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); |
| 4659 | } |
| 4660 | case NEON::BI__builtin_neon_vfma_laneq_v: { |
| 4661 | llvm::VectorType *VTy = cast<llvm::VectorType>(Ty); |
| 4662 | // v1f64 fma should be mapped to Neon scalar f64 fma |
| 4663 | if (VTy && VTy->getElementType() == DoubleTy) { |
| 4664 | Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); |
| 4665 | Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); |
| 4666 | llvm::Type *VTy = GetNeonType(this, |
| 4667 | NeonTypeFlags(NeonTypeFlags::Float64, false, true)); |
| 4668 | Ops[2] = Builder.CreateBitCast(Ops[2], VTy); |
| 4669 | Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); |
| 4670 | Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy); |
| 4671 | Value *Result = Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); |
| 4672 | return Builder.CreateBitCast(Result, Ty); |
| 4673 | } |
| 4674 | Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); |
| 4675 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 4676 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 4677 | |
| 4678 | llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(), |
| 4679 | VTy->getNumElements() * 2); |
| 4680 | Ops[2] = Builder.CreateBitCast(Ops[2], STy); |
| 4681 | Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), |
| 4682 | cast<ConstantInt>(Ops[3])); |
| 4683 | Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); |
| 4684 | |
| 4685 | return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); |
| 4686 | } |
| 4687 | case NEON::BI__builtin_neon_vfmaq_laneq_v: { |
| 4688 | Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); |
| 4689 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 4690 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 4691 | |
| 4692 | Ops[2] = Builder.CreateBitCast(Ops[2], Ty); |
| 4693 | Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); |
| 4694 | return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); |
| 4695 | } |
| 4696 | case NEON::BI__builtin_neon_vfmas_lane_f32: |
| 4697 | case NEON::BI__builtin_neon_vfmas_laneq_f32: |
| 4698 | case NEON::BI__builtin_neon_vfmad_lane_f64: |
| 4699 | case NEON::BI__builtin_neon_vfmad_laneq_f64: { |
| 4700 | Ops.push_back(EmitScalarExpr(E->getArg(3))); |
| 4701 | llvm::Type *Ty = ConvertType(E->getCallReturnType()); |
| 4702 | Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); |
| 4703 | Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); |
| 4704 | return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); |
| 4705 | } |
| 4706 | case NEON::BI__builtin_neon_vfms_v: |
| 4707 | case NEON::BI__builtin_neon_vfmsq_v: { // Only used for FP types |
| 4708 | // FIXME: probably remove when we no longer support aarch64_simd.h |
| 4709 | // (arm_neon.h delegates to vfma). |
| 4710 | |
| 4711 | // The ARM builtins (and instructions) have the addend as the first |
| 4712 | // operand, but the 'fma' intrinsics have it last. Swap it around here. |
| 4713 | Value *Subtrahend = Ops[0]; |
| 4714 | Value *Multiplicand = Ops[2]; |
| 4715 | Ops[0] = Multiplicand; |
| 4716 | Ops[2] = Subtrahend; |
| 4717 | Ops[1] = Builder.CreateBitCast(Ops[1], VTy); |
| 4718 | Ops[1] = Builder.CreateFNeg(Ops[1]); |
| 4719 | Int = Intrinsic::fma; |
| 4720 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmls"); |
| 4721 | } |
| 4722 | case NEON::BI__builtin_neon_vmull_v: |
| 4723 | // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4724 | Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull; |
| 4725 | if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4726 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); |
| 4727 | case NEON::BI__builtin_neon_vmax_v: |
| 4728 | case NEON::BI__builtin_neon_vmaxq_v: |
| 4729 | // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4730 | Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax; |
| 4731 | if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4732 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); |
| 4733 | case NEON::BI__builtin_neon_vmin_v: |
| 4734 | case NEON::BI__builtin_neon_vminq_v: |
| 4735 | // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4736 | Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin; |
| 4737 | if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4738 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); |
| 4739 | case NEON::BI__builtin_neon_vabd_v: |
| 4740 | case NEON::BI__builtin_neon_vabdq_v: |
| 4741 | // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4742 | Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd; |
| 4743 | if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4744 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); |
| 4745 | case NEON::BI__builtin_neon_vpadal_v: |
| 4746 | case NEON::BI__builtin_neon_vpadalq_v: { |
| 4747 | unsigned ArgElts = VTy->getNumElements(); |
| 4748 | llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType()); |
| 4749 | unsigned BitWidth = EltTy->getBitWidth(); |
| 4750 | llvm::Type *ArgTy = llvm::VectorType::get( |
| 4751 | llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts); |
| 4752 | llvm::Type* Tys[2] = { VTy, ArgTy }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4753 | Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4754 | SmallVector<llvm::Value*, 1> TmpOps; |
| 4755 | TmpOps.push_back(Ops[1]); |
| 4756 | Function *F = CGM.getIntrinsic(Int, Tys); |
| 4757 | llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal"); |
| 4758 | llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); |
| 4759 | return Builder.CreateAdd(tmp, addend); |
| 4760 | } |
| 4761 | case NEON::BI__builtin_neon_vpmin_v: |
| 4762 | case NEON::BI__builtin_neon_vpminq_v: |
| 4763 | // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4764 | Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp; |
| 4765 | if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4766 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); |
| 4767 | case NEON::BI__builtin_neon_vpmax_v: |
| 4768 | case NEON::BI__builtin_neon_vpmaxq_v: |
| 4769 | // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics. |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4770 | Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp; |
| 4771 | if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4772 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); |
| 4773 | case NEON::BI__builtin_neon_vminnm_v: |
| 4774 | case NEON::BI__builtin_neon_vminnmq_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4775 | Int = Intrinsic::aarch64_neon_fminnm; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4776 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); |
| 4777 | case NEON::BI__builtin_neon_vmaxnm_v: |
| 4778 | case NEON::BI__builtin_neon_vmaxnmq_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4779 | Int = Intrinsic::aarch64_neon_fmaxnm; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4780 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); |
| 4781 | case NEON::BI__builtin_neon_vrecpss_f32: { |
| 4782 | llvm::Type *f32Type = llvm::Type::getFloatTy(getLLVMContext()); |
| 4783 | Ops.push_back(EmitScalarExpr(E->getArg(1))); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4784 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f32Type), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4785 | Ops, "vrecps"); |
| 4786 | } |
| 4787 | case NEON::BI__builtin_neon_vrecpsd_f64: { |
| 4788 | llvm::Type *f64Type = llvm::Type::getDoubleTy(getLLVMContext()); |
| 4789 | Ops.push_back(EmitScalarExpr(E->getArg(1))); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4790 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f64Type), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4791 | Ops, "vrecps"); |
| 4792 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4793 | case NEON::BI__builtin_neon_vqshrun_n_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4794 | Int = Intrinsic::aarch64_neon_sqshrun; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4795 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); |
| 4796 | case NEON::BI__builtin_neon_vqrshrun_n_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4797 | Int = Intrinsic::aarch64_neon_sqrshrun; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4798 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); |
| 4799 | case NEON::BI__builtin_neon_vqshrn_n_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4800 | Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4801 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); |
| 4802 | case NEON::BI__builtin_neon_vrshrn_n_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4803 | Int = Intrinsic::aarch64_neon_rshrn; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4804 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); |
| 4805 | case NEON::BI__builtin_neon_vqrshrn_n_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4806 | Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4807 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); |
| 4808 | case NEON::BI__builtin_neon_vrnda_v: |
| 4809 | case NEON::BI__builtin_neon_vrndaq_v: { |
| 4810 | Int = Intrinsic::round; |
| 4811 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); |
| 4812 | } |
| 4813 | case NEON::BI__builtin_neon_vrndi_v: |
| 4814 | case NEON::BI__builtin_neon_vrndiq_v: { |
| 4815 | Int = Intrinsic::nearbyint; |
| 4816 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi"); |
| 4817 | } |
| 4818 | case NEON::BI__builtin_neon_vrndm_v: |
| 4819 | case NEON::BI__builtin_neon_vrndmq_v: { |
| 4820 | Int = Intrinsic::floor; |
| 4821 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); |
| 4822 | } |
| 4823 | case NEON::BI__builtin_neon_vrndn_v: |
| 4824 | case NEON::BI__builtin_neon_vrndnq_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4825 | Int = Intrinsic::aarch64_neon_frintn; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4826 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); |
| 4827 | } |
| 4828 | case NEON::BI__builtin_neon_vrndp_v: |
| 4829 | case NEON::BI__builtin_neon_vrndpq_v: { |
| 4830 | Int = Intrinsic::ceil; |
| 4831 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); |
| 4832 | } |
| 4833 | case NEON::BI__builtin_neon_vrndx_v: |
| 4834 | case NEON::BI__builtin_neon_vrndxq_v: { |
| 4835 | Int = Intrinsic::rint; |
| 4836 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); |
| 4837 | } |
| 4838 | case NEON::BI__builtin_neon_vrnd_v: |
| 4839 | case NEON::BI__builtin_neon_vrndq_v: { |
| 4840 | Int = Intrinsic::trunc; |
| 4841 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); |
| 4842 | } |
| 4843 | case NEON::BI__builtin_neon_vceqz_v: |
| 4844 | case NEON::BI__builtin_neon_vceqzq_v: |
| 4845 | return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, |
| 4846 | ICmpInst::ICMP_EQ, "vceqz"); |
| 4847 | case NEON::BI__builtin_neon_vcgez_v: |
| 4848 | case NEON::BI__builtin_neon_vcgezq_v: |
| 4849 | return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, |
| 4850 | ICmpInst::ICMP_SGE, "vcgez"); |
| 4851 | case NEON::BI__builtin_neon_vclez_v: |
| 4852 | case NEON::BI__builtin_neon_vclezq_v: |
| 4853 | return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, |
| 4854 | ICmpInst::ICMP_SLE, "vclez"); |
| 4855 | case NEON::BI__builtin_neon_vcgtz_v: |
| 4856 | case NEON::BI__builtin_neon_vcgtzq_v: |
| 4857 | return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, |
| 4858 | ICmpInst::ICMP_SGT, "vcgtz"); |
| 4859 | case NEON::BI__builtin_neon_vcltz_v: |
| 4860 | case NEON::BI__builtin_neon_vcltzq_v: |
| 4861 | return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, |
| 4862 | ICmpInst::ICMP_SLT, "vcltz"); |
| 4863 | case NEON::BI__builtin_neon_vcvt_f64_v: |
| 4864 | case NEON::BI__builtin_neon_vcvtq_f64_v: |
| 4865 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 4866 | Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad)); |
| 4867 | return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") |
| 4868 | : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); |
| 4869 | case NEON::BI__builtin_neon_vcvt_f64_f32: { |
| 4870 | assert(Type.getEltType() == NeonTypeFlags::Float64 && quad && |
| 4871 | "unexpected vcvt_f64_f32 builtin"); |
| 4872 | NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false); |
| 4873 | Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); |
| 4874 | |
| 4875 | return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); |
| 4876 | } |
| 4877 | case NEON::BI__builtin_neon_vcvt_f32_f64: { |
| 4878 | assert(Type.getEltType() == NeonTypeFlags::Float32 && |
| 4879 | "unexpected vcvt_f32_f64 builtin"); |
| 4880 | NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true); |
| 4881 | Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); |
| 4882 | |
| 4883 | return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); |
| 4884 | } |
| 4885 | case NEON::BI__builtin_neon_vcvt_s32_v: |
| 4886 | case NEON::BI__builtin_neon_vcvt_u32_v: |
| 4887 | case NEON::BI__builtin_neon_vcvt_s64_v: |
| 4888 | case NEON::BI__builtin_neon_vcvt_u64_v: |
| 4889 | case NEON::BI__builtin_neon_vcvtq_s32_v: |
| 4890 | case NEON::BI__builtin_neon_vcvtq_u32_v: |
| 4891 | case NEON::BI__builtin_neon_vcvtq_s64_v: |
| 4892 | case NEON::BI__builtin_neon_vcvtq_u64_v: { |
| 4893 | bool Double = |
| 4894 | (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); |
| 4895 | llvm::Type *InTy = |
| 4896 | GetNeonType(this, |
| 4897 | NeonTypeFlags(Double ? NeonTypeFlags::Float64 |
| 4898 | : NeonTypeFlags::Float32, false, quad)); |
| 4899 | Ops[0] = Builder.CreateBitCast(Ops[0], InTy); |
| 4900 | if (usgn) |
| 4901 | return Builder.CreateFPToUI(Ops[0], Ty); |
| 4902 | return Builder.CreateFPToSI(Ops[0], Ty); |
| 4903 | } |
| 4904 | case NEON::BI__builtin_neon_vcvta_s32_v: |
| 4905 | case NEON::BI__builtin_neon_vcvtaq_s32_v: |
| 4906 | case NEON::BI__builtin_neon_vcvta_u32_v: |
| 4907 | case NEON::BI__builtin_neon_vcvtaq_u32_v: |
| 4908 | case NEON::BI__builtin_neon_vcvta_s64_v: |
| 4909 | case NEON::BI__builtin_neon_vcvtaq_s64_v: |
| 4910 | case NEON::BI__builtin_neon_vcvta_u64_v: |
| 4911 | case NEON::BI__builtin_neon_vcvtaq_u64_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4912 | Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4913 | bool Double = |
| 4914 | (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); |
| 4915 | llvm::Type *InTy = |
| 4916 | GetNeonType(this, |
| 4917 | NeonTypeFlags(Double ? NeonTypeFlags::Float64 |
| 4918 | : NeonTypeFlags::Float32, false, quad)); |
| 4919 | llvm::Type *Tys[2] = { Ty, InTy }; |
| 4920 | return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); |
| 4921 | } |
| 4922 | case NEON::BI__builtin_neon_vcvtm_s32_v: |
| 4923 | case NEON::BI__builtin_neon_vcvtmq_s32_v: |
| 4924 | case NEON::BI__builtin_neon_vcvtm_u32_v: |
| 4925 | case NEON::BI__builtin_neon_vcvtmq_u32_v: |
| 4926 | case NEON::BI__builtin_neon_vcvtm_s64_v: |
| 4927 | case NEON::BI__builtin_neon_vcvtmq_s64_v: |
| 4928 | case NEON::BI__builtin_neon_vcvtm_u64_v: |
| 4929 | case NEON::BI__builtin_neon_vcvtmq_u64_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4930 | Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4931 | bool Double = |
| 4932 | (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); |
| 4933 | llvm::Type *InTy = |
| 4934 | GetNeonType(this, |
| 4935 | NeonTypeFlags(Double ? NeonTypeFlags::Float64 |
| 4936 | : NeonTypeFlags::Float32, false, quad)); |
| 4937 | llvm::Type *Tys[2] = { Ty, InTy }; |
| 4938 | return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); |
| 4939 | } |
| 4940 | case NEON::BI__builtin_neon_vcvtn_s32_v: |
| 4941 | case NEON::BI__builtin_neon_vcvtnq_s32_v: |
| 4942 | case NEON::BI__builtin_neon_vcvtn_u32_v: |
| 4943 | case NEON::BI__builtin_neon_vcvtnq_u32_v: |
| 4944 | case NEON::BI__builtin_neon_vcvtn_s64_v: |
| 4945 | case NEON::BI__builtin_neon_vcvtnq_s64_v: |
| 4946 | case NEON::BI__builtin_neon_vcvtn_u64_v: |
| 4947 | case NEON::BI__builtin_neon_vcvtnq_u64_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4948 | Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4949 | bool Double = |
| 4950 | (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); |
| 4951 | llvm::Type *InTy = |
| 4952 | GetNeonType(this, |
| 4953 | NeonTypeFlags(Double ? NeonTypeFlags::Float64 |
| 4954 | : NeonTypeFlags::Float32, false, quad)); |
| 4955 | llvm::Type *Tys[2] = { Ty, InTy }; |
| 4956 | return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); |
| 4957 | } |
| 4958 | case NEON::BI__builtin_neon_vcvtp_s32_v: |
| 4959 | case NEON::BI__builtin_neon_vcvtpq_s32_v: |
| 4960 | case NEON::BI__builtin_neon_vcvtp_u32_v: |
| 4961 | case NEON::BI__builtin_neon_vcvtpq_u32_v: |
| 4962 | case NEON::BI__builtin_neon_vcvtp_s64_v: |
| 4963 | case NEON::BI__builtin_neon_vcvtpq_s64_v: |
| 4964 | case NEON::BI__builtin_neon_vcvtp_u64_v: |
| 4965 | case NEON::BI__builtin_neon_vcvtpq_u64_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4966 | Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4967 | bool Double = |
| 4968 | (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64); |
| 4969 | llvm::Type *InTy = |
| 4970 | GetNeonType(this, |
| 4971 | NeonTypeFlags(Double ? NeonTypeFlags::Float64 |
| 4972 | : NeonTypeFlags::Float32, false, quad)); |
| 4973 | llvm::Type *Tys[2] = { Ty, InTy }; |
| 4974 | return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); |
| 4975 | } |
| 4976 | case NEON::BI__builtin_neon_vmulx_v: |
| 4977 | case NEON::BI__builtin_neon_vmulxq_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4978 | Int = Intrinsic::aarch64_neon_fmulx; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 4979 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); |
| 4980 | } |
| 4981 | case NEON::BI__builtin_neon_vmul_lane_v: |
| 4982 | case NEON::BI__builtin_neon_vmul_laneq_v: { |
| 4983 | // v1f64 vmul_lane should be mapped to Neon scalar mul lane |
| 4984 | bool Quad = false; |
| 4985 | if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v) |
| 4986 | Quad = true; |
| 4987 | Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); |
| 4988 | llvm::Type *VTy = GetNeonType(this, |
| 4989 | NeonTypeFlags(NeonTypeFlags::Float64, false, Quad)); |
| 4990 | Ops[1] = Builder.CreateBitCast(Ops[1], VTy); |
| 4991 | Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); |
| 4992 | Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); |
| 4993 | return Builder.CreateBitCast(Result, Ty); |
| 4994 | } |
| 4995 | case NEON::BI__builtin_neon_vnegd_s64: |
| 4996 | return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd"); |
| 4997 | case NEON::BI__builtin_neon_vpmaxnm_v: |
| 4998 | case NEON::BI__builtin_neon_vpmaxnmq_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 4999 | Int = Intrinsic::aarch64_neon_fmaxnmp; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5000 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); |
| 5001 | } |
| 5002 | case NEON::BI__builtin_neon_vpminnm_v: |
| 5003 | case NEON::BI__builtin_neon_vpminnmq_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5004 | Int = Intrinsic::aarch64_neon_fminnmp; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5005 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); |
| 5006 | } |
| 5007 | case NEON::BI__builtin_neon_vsqrt_v: |
| 5008 | case NEON::BI__builtin_neon_vsqrtq_v: { |
| 5009 | Int = Intrinsic::sqrt; |
| 5010 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 5011 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); |
| 5012 | } |
| 5013 | case NEON::BI__builtin_neon_vrbit_v: |
| 5014 | case NEON::BI__builtin_neon_vrbitq_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5015 | Int = Intrinsic::aarch64_neon_rbit; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5016 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); |
| 5017 | } |
| 5018 | case NEON::BI__builtin_neon_vaddv_u8: |
| 5019 | // FIXME: These are handled by the AArch64 scalar code. |
| 5020 | usgn = true; |
| 5021 | // FALLTHROUGH |
| 5022 | case NEON::BI__builtin_neon_vaddv_s8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5023 | Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5024 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5025 | VTy = |
| 5026 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); |
| 5027 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5028 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5029 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); |
| 5030 | return Builder.CreateTrunc(Ops[0], |
| 5031 | llvm::IntegerType::get(getLLVMContext(), 8)); |
| 5032 | } |
| 5033 | case NEON::BI__builtin_neon_vaddv_u16: |
| 5034 | usgn = true; |
| 5035 | // FALLTHROUGH |
| 5036 | case NEON::BI__builtin_neon_vaddv_s16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5037 | Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5038 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5039 | VTy = |
| 5040 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); |
| 5041 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5042 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5043 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); |
| 5044 | return Builder.CreateTrunc(Ops[0], |
| 5045 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5046 | } |
| 5047 | case NEON::BI__builtin_neon_vaddvq_u8: |
| 5048 | usgn = true; |
| 5049 | // FALLTHROUGH |
| 5050 | case NEON::BI__builtin_neon_vaddvq_s8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5051 | Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5052 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5053 | VTy = |
| 5054 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); |
| 5055 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5056 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5057 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); |
| 5058 | return Builder.CreateTrunc(Ops[0], |
| 5059 | llvm::IntegerType::get(getLLVMContext(), 8)); |
| 5060 | } |
| 5061 | case NEON::BI__builtin_neon_vaddvq_u16: |
| 5062 | usgn = true; |
| 5063 | // FALLTHROUGH |
| 5064 | case NEON::BI__builtin_neon_vaddvq_s16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5065 | Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5066 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5067 | VTy = |
| 5068 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); |
| 5069 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5070 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5071 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); |
| 5072 | return Builder.CreateTrunc(Ops[0], |
| 5073 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5074 | } |
| 5075 | case NEON::BI__builtin_neon_vmaxv_u8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5076 | Int = Intrinsic::aarch64_neon_umaxv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5077 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5078 | VTy = |
| 5079 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); |
| 5080 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5081 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5082 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); |
| 5083 | return Builder.CreateTrunc(Ops[0], |
| 5084 | llvm::IntegerType::get(getLLVMContext(), 8)); |
| 5085 | } |
| 5086 | case NEON::BI__builtin_neon_vmaxv_u16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5087 | Int = Intrinsic::aarch64_neon_umaxv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5088 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5089 | VTy = |
| 5090 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); |
| 5091 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5092 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5093 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); |
| 5094 | return Builder.CreateTrunc(Ops[0], |
| 5095 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5096 | } |
| 5097 | case NEON::BI__builtin_neon_vmaxvq_u8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5098 | Int = Intrinsic::aarch64_neon_umaxv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5099 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5100 | VTy = |
| 5101 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); |
| 5102 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5103 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5104 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); |
| 5105 | return Builder.CreateTrunc(Ops[0], |
| 5106 | llvm::IntegerType::get(getLLVMContext(), 8)); |
| 5107 | } |
| 5108 | case NEON::BI__builtin_neon_vmaxvq_u16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5109 | Int = Intrinsic::aarch64_neon_umaxv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5110 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5111 | VTy = |
| 5112 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); |
| 5113 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5114 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5115 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); |
| 5116 | return Builder.CreateTrunc(Ops[0], |
| 5117 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5118 | } |
| 5119 | case NEON::BI__builtin_neon_vmaxv_s8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5120 | Int = Intrinsic::aarch64_neon_smaxv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5121 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5122 | VTy = |
| 5123 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); |
| 5124 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5125 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5126 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); |
| 5127 | return Builder.CreateTrunc(Ops[0], |
| 5128 | llvm::IntegerType::get(getLLVMContext(), 8)); |
| 5129 | } |
| 5130 | case NEON::BI__builtin_neon_vmaxv_s16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5131 | Int = Intrinsic::aarch64_neon_smaxv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5132 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5133 | VTy = |
| 5134 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); |
| 5135 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5136 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5137 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); |
| 5138 | return Builder.CreateTrunc(Ops[0], |
| 5139 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5140 | } |
| 5141 | case NEON::BI__builtin_neon_vmaxvq_s8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5142 | Int = Intrinsic::aarch64_neon_smaxv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5143 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5144 | VTy = |
| 5145 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); |
| 5146 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5147 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5148 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); |
| 5149 | return Builder.CreateTrunc(Ops[0], |
| 5150 | llvm::IntegerType::get(getLLVMContext(), 8)); |
| 5151 | } |
| 5152 | case NEON::BI__builtin_neon_vmaxvq_s16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5153 | Int = Intrinsic::aarch64_neon_smaxv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5154 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5155 | VTy = |
| 5156 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); |
| 5157 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5158 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5159 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); |
| 5160 | return Builder.CreateTrunc(Ops[0], |
| 5161 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5162 | } |
| 5163 | case NEON::BI__builtin_neon_vminv_u8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5164 | Int = Intrinsic::aarch64_neon_uminv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5165 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5166 | VTy = |
| 5167 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); |
| 5168 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5169 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5170 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); |
| 5171 | return Builder.CreateTrunc(Ops[0], |
| 5172 | llvm::IntegerType::get(getLLVMContext(), 8)); |
| 5173 | } |
| 5174 | case NEON::BI__builtin_neon_vminv_u16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5175 | Int = Intrinsic::aarch64_neon_uminv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5176 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5177 | VTy = |
| 5178 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); |
| 5179 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5180 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5181 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); |
| 5182 | return Builder.CreateTrunc(Ops[0], |
| 5183 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5184 | } |
| 5185 | case NEON::BI__builtin_neon_vminvq_u8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5186 | Int = Intrinsic::aarch64_neon_uminv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5187 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5188 | VTy = |
| 5189 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); |
| 5190 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5191 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5192 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); |
| 5193 | return Builder.CreateTrunc(Ops[0], |
| 5194 | llvm::IntegerType::get(getLLVMContext(), 8)); |
| 5195 | } |
| 5196 | case NEON::BI__builtin_neon_vminvq_u16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5197 | Int = Intrinsic::aarch64_neon_uminv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5198 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5199 | VTy = |
| 5200 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); |
| 5201 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5202 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5203 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); |
| 5204 | return Builder.CreateTrunc(Ops[0], |
| 5205 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5206 | } |
| 5207 | case NEON::BI__builtin_neon_vminv_s8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5208 | Int = Intrinsic::aarch64_neon_sminv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5209 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5210 | VTy = |
| 5211 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); |
| 5212 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5213 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5214 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); |
| 5215 | return Builder.CreateTrunc(Ops[0], |
| 5216 | llvm::IntegerType::get(getLLVMContext(), 8)); |
| 5217 | } |
| 5218 | case NEON::BI__builtin_neon_vminv_s16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5219 | Int = Intrinsic::aarch64_neon_sminv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5220 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5221 | VTy = |
| 5222 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); |
| 5223 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5224 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5225 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); |
| 5226 | return Builder.CreateTrunc(Ops[0], |
| 5227 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5228 | } |
| 5229 | case NEON::BI__builtin_neon_vminvq_s8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5230 | Int = Intrinsic::aarch64_neon_sminv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5231 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5232 | VTy = |
| 5233 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); |
| 5234 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5235 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5236 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); |
| 5237 | return Builder.CreateTrunc(Ops[0], |
| 5238 | llvm::IntegerType::get(getLLVMContext(), 8)); |
| 5239 | } |
| 5240 | case NEON::BI__builtin_neon_vminvq_s16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5241 | Int = Intrinsic::aarch64_neon_sminv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5242 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5243 | VTy = |
| 5244 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); |
| 5245 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5246 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5247 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); |
| 5248 | return Builder.CreateTrunc(Ops[0], |
| 5249 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5250 | } |
| 5251 | case NEON::BI__builtin_neon_vmul_n_f64: { |
| 5252 | Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); |
| 5253 | Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy); |
| 5254 | return Builder.CreateFMul(Ops[0], RHS); |
| 5255 | } |
| 5256 | case NEON::BI__builtin_neon_vaddlv_u8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5257 | Int = Intrinsic::aarch64_neon_uaddlv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5258 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5259 | VTy = |
| 5260 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); |
| 5261 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5262 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5263 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); |
| 5264 | return Builder.CreateTrunc(Ops[0], |
| 5265 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5266 | } |
| 5267 | case NEON::BI__builtin_neon_vaddlv_u16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5268 | Int = Intrinsic::aarch64_neon_uaddlv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5269 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5270 | VTy = |
| 5271 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); |
| 5272 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5273 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5274 | return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); |
| 5275 | } |
| 5276 | case NEON::BI__builtin_neon_vaddlvq_u8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5277 | Int = Intrinsic::aarch64_neon_uaddlv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5278 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5279 | VTy = |
| 5280 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); |
| 5281 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5282 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5283 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); |
| 5284 | return Builder.CreateTrunc(Ops[0], |
| 5285 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5286 | } |
| 5287 | case NEON::BI__builtin_neon_vaddlvq_u16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5288 | Int = Intrinsic::aarch64_neon_uaddlv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5289 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5290 | VTy = |
| 5291 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); |
| 5292 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5293 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5294 | return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); |
| 5295 | } |
| 5296 | case NEON::BI__builtin_neon_vaddlv_s8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5297 | Int = Intrinsic::aarch64_neon_saddlv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5298 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5299 | VTy = |
| 5300 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8); |
| 5301 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5302 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5303 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); |
| 5304 | return Builder.CreateTrunc(Ops[0], |
| 5305 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5306 | } |
| 5307 | case NEON::BI__builtin_neon_vaddlv_s16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5308 | Int = Intrinsic::aarch64_neon_saddlv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5309 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5310 | VTy = |
| 5311 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4); |
| 5312 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5313 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5314 | return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); |
| 5315 | } |
| 5316 | case NEON::BI__builtin_neon_vaddlvq_s8: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5317 | Int = Intrinsic::aarch64_neon_saddlv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5318 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5319 | VTy = |
| 5320 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16); |
| 5321 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5322 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5323 | Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); |
| 5324 | return Builder.CreateTrunc(Ops[0], |
| 5325 | llvm::IntegerType::get(getLLVMContext(), 16)); |
| 5326 | } |
| 5327 | case NEON::BI__builtin_neon_vaddlvq_s16: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5328 | Int = Intrinsic::aarch64_neon_saddlv; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5329 | Ty = llvm::IntegerType::get(getLLVMContext(), 32); |
| 5330 | VTy = |
| 5331 | llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8); |
| 5332 | llvm::Type *Tys[2] = { Ty, VTy }; |
| 5333 | Ops.push_back(EmitScalarExpr(E->getArg(0))); |
| 5334 | return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); |
| 5335 | } |
| 5336 | case NEON::BI__builtin_neon_vsri_n_v: |
| 5337 | case NEON::BI__builtin_neon_vsriq_n_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5338 | Int = Intrinsic::aarch64_neon_vsri; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5339 | llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); |
| 5340 | return EmitNeonCall(Intrin, Ops, "vsri_n"); |
| 5341 | } |
| 5342 | case NEON::BI__builtin_neon_vsli_n_v: |
| 5343 | case NEON::BI__builtin_neon_vsliq_n_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5344 | Int = Intrinsic::aarch64_neon_vsli; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5345 | llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty); |
| 5346 | return EmitNeonCall(Intrin, Ops, "vsli_n"); |
| 5347 | } |
| 5348 | case NEON::BI__builtin_neon_vsra_n_v: |
| 5349 | case NEON::BI__builtin_neon_vsraq_n_v: |
| 5350 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 5351 | Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); |
| 5352 | return Builder.CreateAdd(Ops[0], Ops[1]); |
| 5353 | case NEON::BI__builtin_neon_vrsra_n_v: |
| 5354 | case NEON::BI__builtin_neon_vrsraq_n_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5355 | Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5356 | SmallVector<llvm::Value*,2> TmpOps; |
| 5357 | TmpOps.push_back(Ops[1]); |
| 5358 | TmpOps.push_back(Ops[2]); |
| 5359 | Function* F = CGM.getIntrinsic(Int, Ty); |
| 5360 | llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true); |
| 5361 | Ops[0] = Builder.CreateBitCast(Ops[0], VTy); |
| 5362 | return Builder.CreateAdd(Ops[0], tmp); |
| 5363 | } |
| 5364 | // FIXME: Sharing loads & stores with 32-bit is complicated by the absence |
| 5365 | // of an Align parameter here. |
| 5366 | case NEON::BI__builtin_neon_vld1_x2_v: |
| 5367 | case NEON::BI__builtin_neon_vld1q_x2_v: |
| 5368 | case NEON::BI__builtin_neon_vld1_x3_v: |
| 5369 | case NEON::BI__builtin_neon_vld1q_x3_v: |
| 5370 | case NEON::BI__builtin_neon_vld1_x4_v: |
| 5371 | case NEON::BI__builtin_neon_vld1q_x4_v: { |
| 5372 | llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); |
| 5373 | Ops[1] = Builder.CreateBitCast(Ops[1], PTy); |
| 5374 | llvm::Type *Tys[2] = { VTy, PTy }; |
| 5375 | unsigned Int; |
| 5376 | switch (BuiltinID) { |
| 5377 | case NEON::BI__builtin_neon_vld1_x2_v: |
| 5378 | case NEON::BI__builtin_neon_vld1q_x2_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5379 | Int = Intrinsic::aarch64_neon_ld1x2; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5380 | break; |
| 5381 | case NEON::BI__builtin_neon_vld1_x3_v: |
| 5382 | case NEON::BI__builtin_neon_vld1q_x3_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5383 | Int = Intrinsic::aarch64_neon_ld1x3; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5384 | break; |
| 5385 | case NEON::BI__builtin_neon_vld1_x4_v: |
| 5386 | case NEON::BI__builtin_neon_vld1q_x4_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5387 | Int = Intrinsic::aarch64_neon_ld1x4; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5388 | break; |
| 5389 | } |
| 5390 | Function *F = CGM.getIntrinsic(Int, Tys); |
| 5391 | Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); |
| 5392 | Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); |
| 5393 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 5394 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5395 | } |
| 5396 | case NEON::BI__builtin_neon_vst1_x2_v: |
| 5397 | case NEON::BI__builtin_neon_vst1q_x2_v: |
| 5398 | case NEON::BI__builtin_neon_vst1_x3_v: |
| 5399 | case NEON::BI__builtin_neon_vst1q_x3_v: |
| 5400 | case NEON::BI__builtin_neon_vst1_x4_v: |
| 5401 | case NEON::BI__builtin_neon_vst1q_x4_v: { |
| 5402 | llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType()); |
| 5403 | llvm::Type *Tys[2] = { VTy, PTy }; |
| 5404 | unsigned Int; |
| 5405 | switch (BuiltinID) { |
| 5406 | case NEON::BI__builtin_neon_vst1_x2_v: |
| 5407 | case NEON::BI__builtin_neon_vst1q_x2_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5408 | Int = Intrinsic::aarch64_neon_st1x2; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5409 | break; |
| 5410 | case NEON::BI__builtin_neon_vst1_x3_v: |
| 5411 | case NEON::BI__builtin_neon_vst1q_x3_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5412 | Int = Intrinsic::aarch64_neon_st1x3; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5413 | break; |
| 5414 | case NEON::BI__builtin_neon_vst1_x4_v: |
| 5415 | case NEON::BI__builtin_neon_vst1q_x4_v: |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5416 | Int = Intrinsic::aarch64_neon_st1x4; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5417 | break; |
| 5418 | } |
| 5419 | SmallVector<Value *, 4> IntOps(Ops.begin()+1, Ops.end()); |
| 5420 | IntOps.push_back(Ops[0]); |
| 5421 | return EmitNeonCall(CGM.getIntrinsic(Int, Tys), IntOps, ""); |
| 5422 | } |
| 5423 | case NEON::BI__builtin_neon_vld1_v: |
| 5424 | case NEON::BI__builtin_neon_vld1q_v: |
| 5425 | Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); |
| 5426 | return Builder.CreateLoad(Ops[0]); |
| 5427 | case NEON::BI__builtin_neon_vst1_v: |
| 5428 | case NEON::BI__builtin_neon_vst1q_v: |
| 5429 | Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); |
| 5430 | Ops[1] = Builder.CreateBitCast(Ops[1], VTy); |
| 5431 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5432 | case NEON::BI__builtin_neon_vld1_lane_v: |
| 5433 | case NEON::BI__builtin_neon_vld1q_lane_v: |
| 5434 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 5435 | Ty = llvm::PointerType::getUnqual(VTy->getElementType()); |
| 5436 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 5437 | Ops[0] = Builder.CreateLoad(Ops[0]); |
| 5438 | return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); |
| 5439 | case NEON::BI__builtin_neon_vld1_dup_v: |
| 5440 | case NEON::BI__builtin_neon_vld1q_dup_v: { |
| 5441 | Value *V = UndefValue::get(Ty); |
| 5442 | Ty = llvm::PointerType::getUnqual(VTy->getElementType()); |
| 5443 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 5444 | Ops[0] = Builder.CreateLoad(Ops[0]); |
| 5445 | llvm::Constant *CI = ConstantInt::get(Int32Ty, 0); |
| 5446 | Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); |
| 5447 | return EmitNeonSplat(Ops[0], CI); |
| 5448 | } |
| 5449 | case NEON::BI__builtin_neon_vst1_lane_v: |
| 5450 | case NEON::BI__builtin_neon_vst1q_lane_v: |
| 5451 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 5452 | Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); |
| 5453 | Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); |
| 5454 | return Builder.CreateStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty)); |
| 5455 | case NEON::BI__builtin_neon_vld2_v: |
| 5456 | case NEON::BI__builtin_neon_vld2q_v: { |
| 5457 | llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); |
| 5458 | Ops[1] = Builder.CreateBitCast(Ops[1], PTy); |
| 5459 | llvm::Type *Tys[2] = { VTy, PTy }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5460 | Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5461 | Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); |
| 5462 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 5463 | llvm::PointerType::getUnqual(Ops[1]->getType())); |
| 5464 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5465 | } |
| 5466 | case NEON::BI__builtin_neon_vld3_v: |
| 5467 | case NEON::BI__builtin_neon_vld3q_v: { |
| 5468 | llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); |
| 5469 | Ops[1] = Builder.CreateBitCast(Ops[1], PTy); |
| 5470 | llvm::Type *Tys[2] = { VTy, PTy }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5471 | Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5472 | Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); |
| 5473 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 5474 | llvm::PointerType::getUnqual(Ops[1]->getType())); |
| 5475 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5476 | } |
| 5477 | case NEON::BI__builtin_neon_vld4_v: |
| 5478 | case NEON::BI__builtin_neon_vld4q_v: { |
| 5479 | llvm::Type *PTy = llvm::PointerType::getUnqual(VTy); |
| 5480 | Ops[1] = Builder.CreateBitCast(Ops[1], PTy); |
| 5481 | llvm::Type *Tys[2] = { VTy, PTy }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5482 | Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5483 | Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); |
| 5484 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 5485 | llvm::PointerType::getUnqual(Ops[1]->getType())); |
| 5486 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5487 | } |
| 5488 | case NEON::BI__builtin_neon_vld2_dup_v: |
| 5489 | case NEON::BI__builtin_neon_vld2q_dup_v: { |
| 5490 | llvm::Type *PTy = |
| 5491 | llvm::PointerType::getUnqual(VTy->getElementType()); |
| 5492 | Ops[1] = Builder.CreateBitCast(Ops[1], PTy); |
| 5493 | llvm::Type *Tys[2] = { VTy, PTy }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5494 | Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5495 | Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); |
| 5496 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 5497 | llvm::PointerType::getUnqual(Ops[1]->getType())); |
| 5498 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5499 | } |
| 5500 | case NEON::BI__builtin_neon_vld3_dup_v: |
| 5501 | case NEON::BI__builtin_neon_vld3q_dup_v: { |
| 5502 | llvm::Type *PTy = |
| 5503 | llvm::PointerType::getUnqual(VTy->getElementType()); |
| 5504 | Ops[1] = Builder.CreateBitCast(Ops[1], PTy); |
| 5505 | llvm::Type *Tys[2] = { VTy, PTy }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5506 | Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5507 | Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); |
| 5508 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 5509 | llvm::PointerType::getUnqual(Ops[1]->getType())); |
| 5510 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5511 | } |
| 5512 | case NEON::BI__builtin_neon_vld4_dup_v: |
| 5513 | case NEON::BI__builtin_neon_vld4q_dup_v: { |
| 5514 | llvm::Type *PTy = |
| 5515 | llvm::PointerType::getUnqual(VTy->getElementType()); |
| 5516 | Ops[1] = Builder.CreateBitCast(Ops[1], PTy); |
| 5517 | llvm::Type *Tys[2] = { VTy, PTy }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5518 | Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5519 | Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); |
| 5520 | Ops[0] = Builder.CreateBitCast(Ops[0], |
| 5521 | llvm::PointerType::getUnqual(Ops[1]->getType())); |
| 5522 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5523 | } |
| 5524 | case NEON::BI__builtin_neon_vld2_lane_v: |
| 5525 | case NEON::BI__builtin_neon_vld2q_lane_v: { |
| 5526 | llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5527 | Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5528 | Ops.push_back(Ops[1]); |
| 5529 | Ops.erase(Ops.begin()+1); |
| 5530 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 5531 | Ops[2] = Builder.CreateBitCast(Ops[2], Ty); |
| 5532 | Ops[3] = Builder.CreateZExt(Ops[3], |
| 5533 | llvm::IntegerType::get(getLLVMContext(), 64)); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 5534 | Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5535 | Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); |
| 5536 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 5537 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5538 | } |
| 5539 | case NEON::BI__builtin_neon_vld3_lane_v: |
| 5540 | case NEON::BI__builtin_neon_vld3q_lane_v: { |
| 5541 | llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5542 | Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5543 | Ops.push_back(Ops[1]); |
| 5544 | Ops.erase(Ops.begin()+1); |
| 5545 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 5546 | Ops[2] = Builder.CreateBitCast(Ops[2], Ty); |
| 5547 | Ops[3] = Builder.CreateBitCast(Ops[3], Ty); |
| 5548 | Ops[4] = Builder.CreateZExt(Ops[4], |
| 5549 | llvm::IntegerType::get(getLLVMContext(), 64)); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 5550 | Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5551 | Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); |
| 5552 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 5553 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5554 | } |
| 5555 | case NEON::BI__builtin_neon_vld4_lane_v: |
| 5556 | case NEON::BI__builtin_neon_vld4q_lane_v: { |
| 5557 | llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5558 | Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5559 | Ops.push_back(Ops[1]); |
| 5560 | Ops.erase(Ops.begin()+1); |
| 5561 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
| 5562 | Ops[2] = Builder.CreateBitCast(Ops[2], Ty); |
| 5563 | Ops[3] = Builder.CreateBitCast(Ops[3], Ty); |
| 5564 | Ops[4] = Builder.CreateBitCast(Ops[4], Ty); |
| 5565 | Ops[5] = Builder.CreateZExt(Ops[5], |
| 5566 | llvm::IntegerType::get(getLLVMContext(), 64)); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 5567 | Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5568 | Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); |
| 5569 | Ops[0] = Builder.CreateBitCast(Ops[0], Ty); |
| 5570 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5571 | } |
| 5572 | case NEON::BI__builtin_neon_vst2_v: |
| 5573 | case NEON::BI__builtin_neon_vst2q_v: { |
| 5574 | Ops.push_back(Ops[0]); |
| 5575 | Ops.erase(Ops.begin()); |
| 5576 | llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5577 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5578 | Ops, ""); |
| 5579 | } |
| 5580 | case NEON::BI__builtin_neon_vst2_lane_v: |
| 5581 | case NEON::BI__builtin_neon_vst2q_lane_v: { |
| 5582 | Ops.push_back(Ops[0]); |
| 5583 | Ops.erase(Ops.begin()); |
| 5584 | Ops[2] = Builder.CreateZExt(Ops[2], |
| 5585 | llvm::IntegerType::get(getLLVMContext(), 64)); |
| 5586 | llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5587 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5588 | Ops, ""); |
| 5589 | } |
| 5590 | case NEON::BI__builtin_neon_vst3_v: |
| 5591 | case NEON::BI__builtin_neon_vst3q_v: { |
| 5592 | Ops.push_back(Ops[0]); |
| 5593 | Ops.erase(Ops.begin()); |
| 5594 | llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5595 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5596 | Ops, ""); |
| 5597 | } |
| 5598 | case NEON::BI__builtin_neon_vst3_lane_v: |
| 5599 | case NEON::BI__builtin_neon_vst3q_lane_v: { |
| 5600 | Ops.push_back(Ops[0]); |
| 5601 | Ops.erase(Ops.begin()); |
| 5602 | Ops[3] = Builder.CreateZExt(Ops[3], |
| 5603 | llvm::IntegerType::get(getLLVMContext(), 64)); |
| 5604 | llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5605 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5606 | Ops, ""); |
| 5607 | } |
| 5608 | case NEON::BI__builtin_neon_vst4_v: |
| 5609 | case NEON::BI__builtin_neon_vst4q_v: { |
| 5610 | Ops.push_back(Ops[0]); |
| 5611 | Ops.erase(Ops.begin()); |
| 5612 | llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5613 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5614 | Ops, ""); |
| 5615 | } |
| 5616 | case NEON::BI__builtin_neon_vst4_lane_v: |
| 5617 | case NEON::BI__builtin_neon_vst4q_lane_v: { |
| 5618 | Ops.push_back(Ops[0]); |
| 5619 | Ops.erase(Ops.begin()); |
| 5620 | Ops[4] = Builder.CreateZExt(Ops[4], |
| 5621 | llvm::IntegerType::get(getLLVMContext(), 64)); |
| 5622 | llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5623 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5624 | Ops, ""); |
| 5625 | } |
| 5626 | case NEON::BI__builtin_neon_vtrn_v: |
| 5627 | case NEON::BI__builtin_neon_vtrnq_v: { |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5628 | Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 5629 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5630 | Ops[2] = Builder.CreateBitCast(Ops[2], Ty); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5631 | Value *SV = nullptr; |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5632 | |
| 5633 | for (unsigned vi = 0; vi != 2; ++vi) { |
| 5634 | SmallVector<Constant*, 16> Indices; |
| 5635 | for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5636 | Indices.push_back(ConstantInt::get(Int32Ty, i+vi)); |
| 5637 | Indices.push_back(ConstantInt::get(Int32Ty, i+e+vi)); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5638 | } |
| 5639 | Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); |
Chris Lattner | fb018d1 | 2011-02-15 00:14:06 +0000 | [diff] [blame] | 5640 | SV = llvm::ConstantVector::get(Indices); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5641 | SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); |
| 5642 | SV = Builder.CreateStore(SV, Addr); |
| 5643 | } |
| 5644 | return SV; |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 5645 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5646 | case NEON::BI__builtin_neon_vuzp_v: |
| 5647 | case NEON::BI__builtin_neon_vuzpq_v: { |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5648 | Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 5649 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5650 | Ops[2] = Builder.CreateBitCast(Ops[2], Ty); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5651 | Value *SV = nullptr; |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5652 | |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5653 | for (unsigned vi = 0; vi != 2; ++vi) { |
| 5654 | SmallVector<Constant*, 16> Indices; |
| 5655 | for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) |
Chris Lattner | 77b89b8 | 2010-06-27 07:15:29 +0000 | [diff] [blame] | 5656 | Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5657 | |
| 5658 | Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); |
Chris Lattner | fb018d1 | 2011-02-15 00:14:06 +0000 | [diff] [blame] | 5659 | SV = llvm::ConstantVector::get(Indices); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5660 | SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); |
| 5661 | SV = Builder.CreateStore(SV, Addr); |
| 5662 | } |
| 5663 | return SV; |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 5664 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5665 | case NEON::BI__builtin_neon_vzip_v: |
| 5666 | case NEON::BI__builtin_neon_vzipq_v: { |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5667 | Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); |
Nate Begeman | 1c2a88c | 2010-06-09 01:10:23 +0000 | [diff] [blame] | 5668 | Ops[1] = Builder.CreateBitCast(Ops[1], Ty); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5669 | Ops[2] = Builder.CreateBitCast(Ops[2], Ty); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5670 | Value *SV = nullptr; |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5671 | |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5672 | for (unsigned vi = 0; vi != 2; ++vi) { |
| 5673 | SmallVector<Constant*, 16> Indices; |
| 5674 | for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) { |
Daniel Dunbar | e361cc3 | 2010-08-26 00:55:57 +0000 | [diff] [blame] | 5675 | Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); |
| 5676 | Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5677 | } |
| 5678 | Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); |
Chris Lattner | fb018d1 | 2011-02-15 00:14:06 +0000 | [diff] [blame] | 5679 | SV = llvm::ConstantVector::get(Indices); |
Nate Begeman | 4be5430 | 2010-06-20 23:05:28 +0000 | [diff] [blame] | 5680 | SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); |
| 5681 | SV = Builder.CreateStore(SV, Addr); |
| 5682 | } |
| 5683 | return SV; |
Nate Begeman | 9eb65a5 | 2010-06-08 00:17:19 +0000 | [diff] [blame] | 5684 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5685 | case NEON::BI__builtin_neon_vqtbl1q_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5686 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5687 | Ops, "vtbl1"); |
| 5688 | } |
| 5689 | case NEON::BI__builtin_neon_vqtbl2q_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5690 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5691 | Ops, "vtbl2"); |
| 5692 | } |
| 5693 | case NEON::BI__builtin_neon_vqtbl3q_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5694 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5695 | Ops, "vtbl3"); |
| 5696 | } |
| 5697 | case NEON::BI__builtin_neon_vqtbl4q_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5698 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5699 | Ops, "vtbl4"); |
| 5700 | } |
| 5701 | case NEON::BI__builtin_neon_vqtbx1q_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5702 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5703 | Ops, "vtbx1"); |
| 5704 | } |
| 5705 | case NEON::BI__builtin_neon_vqtbx2q_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5706 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5707 | Ops, "vtbx2"); |
| 5708 | } |
| 5709 | case NEON::BI__builtin_neon_vqtbx3q_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5710 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5711 | Ops, "vtbx3"); |
| 5712 | } |
| 5713 | case NEON::BI__builtin_neon_vqtbx4q_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5714 | return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty), |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5715 | Ops, "vtbx4"); |
| 5716 | } |
| 5717 | case NEON::BI__builtin_neon_vsqadd_v: |
| 5718 | case NEON::BI__builtin_neon_vsqaddq_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5719 | Int = Intrinsic::aarch64_neon_usqadd; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5720 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); |
| 5721 | } |
| 5722 | case NEON::BI__builtin_neon_vuqadd_v: |
| 5723 | case NEON::BI__builtin_neon_vuqaddq_v: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5724 | Int = Intrinsic::aarch64_neon_suqadd; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5725 | return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); |
| 5726 | } |
Chris Lattner | 2752c01 | 2010-03-03 19:03:45 +0000 | [diff] [blame] | 5727 | } |
| 5728 | } |
| 5729 | |
Bill Wendling | aa51e51 | 2010-10-09 08:47:25 +0000 | [diff] [blame] | 5730 | llvm::Value *CodeGenFunction:: |
Bill Wendling | 795b100 | 2012-02-22 09:30:11 +0000 | [diff] [blame] | 5731 | BuildVector(ArrayRef<llvm::Value*> Ops) { |
Bill Wendling | aa51e51 | 2010-10-09 08:47:25 +0000 | [diff] [blame] | 5732 | assert((Ops.size() & (Ops.size() - 1)) == 0 && |
| 5733 | "Not a power-of-two sized vector!"); |
| 5734 | bool AllConstants = true; |
| 5735 | for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) |
| 5736 | AllConstants &= isa<Constant>(Ops[i]); |
| 5737 | |
| 5738 | // If this is a constant vector, create a ConstantVector. |
| 5739 | if (AllConstants) { |
Chris Lattner | 2ce8842 | 2012-01-25 05:34:41 +0000 | [diff] [blame] | 5740 | SmallVector<llvm::Constant*, 16> CstOps; |
Bill Wendling | aa51e51 | 2010-10-09 08:47:25 +0000 | [diff] [blame] | 5741 | for (unsigned i = 0, e = Ops.size(); i != e; ++i) |
| 5742 | CstOps.push_back(cast<Constant>(Ops[i])); |
| 5743 | return llvm::ConstantVector::get(CstOps); |
| 5744 | } |
| 5745 | |
| 5746 | // Otherwise, insertelement the values to build the vector. |
| 5747 | Value *Result = |
| 5748 | llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size())); |
| 5749 | |
| 5750 | for (unsigned i = 0, e = Ops.size(); i != e; ++i) |
Chris Lattner | 2ce8842 | 2012-01-25 05:34:41 +0000 | [diff] [blame] | 5751 | Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); |
Bill Wendling | aa51e51 | 2010-10-09 08:47:25 +0000 | [diff] [blame] | 5752 | |
| 5753 | return Result; |
| 5754 | } |
| 5755 | |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 5756 | Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, |
Chris Lattner | 1feedd8 | 2007-12-13 07:34:23 +0000 | [diff] [blame] | 5757 | const CallExpr *E) { |
Chris Lattner | 5f9e272 | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 5758 | SmallVector<Value*, 4> Ops; |
Anders Carlsson | 2929cfa | 2007-12-14 17:48:24 +0000 | [diff] [blame] | 5759 | |
Chris Lattner | 46c5591 | 2010-10-02 00:09:12 +0000 | [diff] [blame] | 5760 | // Find out if any arguments are required to be integer constant expressions. |
| 5761 | unsigned ICEArguments = 0; |
| 5762 | ASTContext::GetBuiltinTypeError Error; |
| 5763 | getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); |
| 5764 | assert(Error == ASTContext::GE_None && "Should not codegen an error"); |
| 5765 | |
| 5766 | for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) { |
| 5767 | // If this is a normal argument, just emit it as a scalar. |
| 5768 | if ((ICEArguments & (1 << i)) == 0) { |
| 5769 | Ops.push_back(EmitScalarExpr(E->getArg(i))); |
| 5770 | continue; |
| 5771 | } |
| 5772 | |
| 5773 | // If this is required to be a constant, constant fold it so that we know |
| 5774 | // that the generated intrinsic gets a ConstantInt. |
| 5775 | llvm::APSInt Result; |
| 5776 | bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext()); |
| 5777 | assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst; |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 5778 | Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); |
Chris Lattner | 46c5591 | 2010-10-02 00:09:12 +0000 | [diff] [blame] | 5779 | } |
Anders Carlsson | 2929cfa | 2007-12-14 17:48:24 +0000 | [diff] [blame] | 5780 | |
Anders Carlsson | 564f1de | 2007-12-09 23:17:02 +0000 | [diff] [blame] | 5781 | switch (BuiltinID) { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5782 | default: return nullptr; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 5783 | case X86::BI_mm_prefetch: { |
| 5784 | Value *Address = EmitScalarExpr(E->getArg(0)); |
| 5785 | Value *RW = ConstantInt::get(Int32Ty, 0); |
| 5786 | Value *Locality = EmitScalarExpr(E->getArg(1)); |
| 5787 | Value *Data = ConstantInt::get(Int32Ty, 1); |
| 5788 | Value *F = CGM.getIntrinsic(Intrinsic::prefetch); |
| 5789 | return Builder.CreateCall4(F, Address, RW, Locality, Data); |
| 5790 | } |
Bill Wendling | aa51e51 | 2010-10-09 08:47:25 +0000 | [diff] [blame] | 5791 | case X86::BI__builtin_ia32_vec_init_v8qi: |
| 5792 | case X86::BI__builtin_ia32_vec_init_v4hi: |
| 5793 | case X86::BI__builtin_ia32_vec_init_v2si: |
| 5794 | return Builder.CreateBitCast(BuildVector(Ops), |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 5795 | llvm::Type::getX86_MMXTy(getLLVMContext())); |
Argyrios Kyrtzidis | 1944ec1 | 2010-10-10 03:19:11 +0000 | [diff] [blame] | 5796 | case X86::BI__builtin_ia32_vec_ext_v2si: |
| 5797 | return Builder.CreateExtractElement(Ops[0], |
| 5798 | llvm::ConstantInt::get(Ops[1]->getType(), 0)); |
Nate Begeman | e772210 | 2008-04-14 04:49:57 +0000 | [diff] [blame] | 5799 | case X86::BI__builtin_ia32_ldmxcsr: { |
Juergen Ributzka | 3fc7791 | 2013-08-19 22:20:37 +0000 | [diff] [blame] | 5800 | Value *Tmp = CreateMemTemp(E->getArg(0)->getType()); |
Nate Begeman | e772210 | 2008-04-14 04:49:57 +0000 | [diff] [blame] | 5801 | Builder.CreateStore(Ops[0], Tmp); |
| 5802 | return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr), |
Juergen Ributzka | 3fc7791 | 2013-08-19 22:20:37 +0000 | [diff] [blame] | 5803 | Builder.CreateBitCast(Tmp, Int8PtrTy)); |
Nate Begeman | e772210 | 2008-04-14 04:49:57 +0000 | [diff] [blame] | 5804 | } |
| 5805 | case X86::BI__builtin_ia32_stmxcsr: { |
Juergen Ributzka | fe0af45 | 2013-08-19 23:08:53 +0000 | [diff] [blame] | 5806 | Value *Tmp = CreateMemTemp(E->getType()); |
Ted Kremenek | 012614e | 2011-08-17 21:04:19 +0000 | [diff] [blame] | 5807 | Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr), |
Juergen Ributzka | 3fc7791 | 2013-08-19 22:20:37 +0000 | [diff] [blame] | 5808 | Builder.CreateBitCast(Tmp, Int8PtrTy)); |
Nate Begeman | e772210 | 2008-04-14 04:49:57 +0000 | [diff] [blame] | 5809 | return Builder.CreateLoad(Tmp, "stmxcsr"); |
| 5810 | } |
Nate Begeman | e772210 | 2008-04-14 04:49:57 +0000 | [diff] [blame] | 5811 | case X86::BI__builtin_ia32_storehps: |
| 5812 | case X86::BI__builtin_ia32_storelps: { |
Chris Lattner | 77b89b8 | 2010-06-27 07:15:29 +0000 | [diff] [blame] | 5813 | llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty); |
| 5814 | llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 5815 | |
Nate Begeman | e772210 | 2008-04-14 04:49:57 +0000 | [diff] [blame] | 5816 | // cast val v2i64 |
| 5817 | Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast"); |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 5818 | |
Nate Begeman | e772210 | 2008-04-14 04:49:57 +0000 | [diff] [blame] | 5819 | // extract (0, 1) |
| 5820 | unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1; |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 5821 | llvm::Value *Idx = llvm::ConstantInt::get(SizeTy, Index); |
Nate Begeman | e772210 | 2008-04-14 04:49:57 +0000 | [diff] [blame] | 5822 | Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract"); |
| 5823 | |
| 5824 | // cast pointer to i64 & store |
| 5825 | Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); |
| 5826 | return Builder.CreateStore(Ops[1], Ops[0]); |
| 5827 | } |
Bill Wendling | 28cab38 | 2010-09-28 01:28:56 +0000 | [diff] [blame] | 5828 | case X86::BI__builtin_ia32_palignr: { |
| 5829 | unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5830 | |
Bill Wendling | 28cab38 | 2010-09-28 01:28:56 +0000 | [diff] [blame] | 5831 | // If palignr is shifting the pair of input vectors less than 9 bytes, |
| 5832 | // emit a shuffle instruction. |
| 5833 | if (shiftVal <= 8) { |
Chris Lattner | 5f9e272 | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 5834 | SmallVector<llvm::Constant*, 8> Indices; |
Bill Wendling | 28cab38 | 2010-09-28 01:28:56 +0000 | [diff] [blame] | 5835 | for (unsigned i = 0; i != 8; ++i) |
| 5836 | Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5837 | |
Chris Lattner | fb018d1 | 2011-02-15 00:14:06 +0000 | [diff] [blame] | 5838 | Value* SV = llvm::ConstantVector::get(Indices); |
Bill Wendling | 28cab38 | 2010-09-28 01:28:56 +0000 | [diff] [blame] | 5839 | return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); |
| 5840 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5841 | |
Bill Wendling | 28cab38 | 2010-09-28 01:28:56 +0000 | [diff] [blame] | 5842 | // If palignr is shifting the pair of input vectors more than 8 but less |
| 5843 | // than 16 bytes, emit a logical right shift of the destination. |
| 5844 | if (shiftVal < 16) { |
| 5845 | // MMX has these as 1 x i64 vectors for some odd optimization reasons. |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 5846 | llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5847 | |
Bill Wendling | 28cab38 | 2010-09-28 01:28:56 +0000 | [diff] [blame] | 5848 | Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); |
| 5849 | Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5850 | |
Bill Wendling | 28cab38 | 2010-09-28 01:28:56 +0000 | [diff] [blame] | 5851 | // create i32 constant |
| 5852 | llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 5853 | return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); |
Bill Wendling | 28cab38 | 2010-09-28 01:28:56 +0000 | [diff] [blame] | 5854 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5855 | |
Eli Friedman | 5c22ad2 | 2011-09-14 00:52:45 +0000 | [diff] [blame] | 5856 | // If palignr is shifting the pair of vectors more than 16 bytes, emit zero. |
Bill Wendling | 28cab38 | 2010-09-28 01:28:56 +0000 | [diff] [blame] | 5857 | return llvm::Constant::getNullValue(ConvertType(E->getType())); |
| 5858 | } |
Nate Begeman | c3420ff | 2009-12-14 05:15:02 +0000 | [diff] [blame] | 5859 | case X86::BI__builtin_ia32_palignr128: { |
Nate Begeman | ce5818a | 2009-12-14 04:57:03 +0000 | [diff] [blame] | 5860 | unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5861 | |
Nate Begeman | ce5818a | 2009-12-14 04:57:03 +0000 | [diff] [blame] | 5862 | // If palignr is shifting the pair of input vectors less than 17 bytes, |
| 5863 | // emit a shuffle instruction. |
| 5864 | if (shiftVal <= 16) { |
Chris Lattner | 5f9e272 | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 5865 | SmallVector<llvm::Constant*, 16> Indices; |
Nate Begeman | ce5818a | 2009-12-14 04:57:03 +0000 | [diff] [blame] | 5866 | for (unsigned i = 0; i != 16; ++i) |
Chris Lattner | 77b89b8 | 2010-06-27 07:15:29 +0000 | [diff] [blame] | 5867 | Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5868 | |
Chris Lattner | fb018d1 | 2011-02-15 00:14:06 +0000 | [diff] [blame] | 5869 | Value* SV = llvm::ConstantVector::get(Indices); |
Nate Begeman | ce5818a | 2009-12-14 04:57:03 +0000 | [diff] [blame] | 5870 | return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); |
| 5871 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5872 | |
Nate Begeman | ce5818a | 2009-12-14 04:57:03 +0000 | [diff] [blame] | 5873 | // If palignr is shifting the pair of input vectors more than 16 but less |
| 5874 | // than 32 bytes, emit a logical right shift of the destination. |
| 5875 | if (shiftVal < 32) { |
Chris Lattner | 2acc6e3 | 2011-07-18 04:24:23 +0000 | [diff] [blame] | 5876 | llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5877 | |
Nate Begeman | ce5818a | 2009-12-14 04:57:03 +0000 | [diff] [blame] | 5878 | Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); |
Chris Lattner | 77b89b8 | 2010-06-27 07:15:29 +0000 | [diff] [blame] | 5879 | Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5880 | |
Nate Begeman | ce5818a | 2009-12-14 04:57:03 +0000 | [diff] [blame] | 5881 | // create i32 constant |
| 5882 | llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 5883 | return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); |
Nate Begeman | ce5818a | 2009-12-14 04:57:03 +0000 | [diff] [blame] | 5884 | } |
Jim Grosbach | 258f930 | 2012-09-21 00:18:27 +0000 | [diff] [blame] | 5885 | |
Nate Begeman | ce5818a | 2009-12-14 04:57:03 +0000 | [diff] [blame] | 5886 | // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. |
| 5887 | return llvm::Constant::getNullValue(ConvertType(E->getType())); |
Eric Christopher | 91b5927 | 2009-12-01 05:00:51 +0000 | [diff] [blame] | 5888 | } |
Craig Topper | 9c2ffd8 | 2011-12-19 07:03:25 +0000 | [diff] [blame] | 5889 | case X86::BI__builtin_ia32_palignr256: { |
| 5890 | unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); |
| 5891 | |
| 5892 | // If palignr is shifting the pair of input vectors less than 17 bytes, |
| 5893 | // emit a shuffle instruction. |
| 5894 | if (shiftVal <= 16) { |
| 5895 | SmallVector<llvm::Constant*, 32> Indices; |
| 5896 | // 256-bit palignr operates on 128-bit lanes so we need to handle that |
| 5897 | for (unsigned l = 0; l != 2; ++l) { |
| 5898 | unsigned LaneStart = l * 16; |
| 5899 | unsigned LaneEnd = (l+1) * 16; |
| 5900 | for (unsigned i = 0; i != 16; ++i) { |
| 5901 | unsigned Idx = shiftVal + i + LaneStart; |
| 5902 | if (Idx >= LaneEnd) Idx += 16; // end of lane, switch operand |
| 5903 | Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx)); |
| 5904 | } |
| 5905 | } |
| 5906 | |
| 5907 | Value* SV = llvm::ConstantVector::get(Indices); |
| 5908 | return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); |
| 5909 | } |
| 5910 | |
| 5911 | // If palignr is shifting the pair of input vectors more than 16 but less |
| 5912 | // than 32 bytes, emit a logical right shift of the destination. |
| 5913 | if (shiftVal < 32) { |
| 5914 | llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 4); |
| 5915 | |
| 5916 | Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); |
| 5917 | Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); |
| 5918 | |
| 5919 | // create i32 constant |
| 5920 | llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_avx2_psrl_dq); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 5921 | return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); |
Craig Topper | 9c2ffd8 | 2011-12-19 07:03:25 +0000 | [diff] [blame] | 5922 | } |
| 5923 | |
| 5924 | // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. |
| 5925 | return llvm::Constant::getNullValue(ConvertType(E->getType())); |
| 5926 | } |
Bill Wendling | b51bdda | 2011-05-04 02:40:38 +0000 | [diff] [blame] | 5927 | case X86::BI__builtin_ia32_movntps: |
Craig Topper | 4a7376d | 2012-05-07 06:25:45 +0000 | [diff] [blame] | 5928 | case X86::BI__builtin_ia32_movntps256: |
Bill Wendling | b51bdda | 2011-05-04 02:40:38 +0000 | [diff] [blame] | 5929 | case X86::BI__builtin_ia32_movntpd: |
Craig Topper | 4a7376d | 2012-05-07 06:25:45 +0000 | [diff] [blame] | 5930 | case X86::BI__builtin_ia32_movntpd256: |
Bill Wendling | b51bdda | 2011-05-04 02:40:38 +0000 | [diff] [blame] | 5931 | case X86::BI__builtin_ia32_movntdq: |
Craig Topper | 4a7376d | 2012-05-07 06:25:45 +0000 | [diff] [blame] | 5932 | case X86::BI__builtin_ia32_movntdq256: |
Eli Friedman | 440a5f4 | 2013-09-23 23:38:39 +0000 | [diff] [blame] | 5933 | case X86::BI__builtin_ia32_movnti: |
| 5934 | case X86::BI__builtin_ia32_movnti64: { |
Bill Wendling | b107dd0 | 2011-05-04 20:28:12 +0000 | [diff] [blame] | 5935 | llvm::MDNode *Node = llvm::MDNode::get(getLLVMContext(), |
| 5936 | Builder.getInt32(1)); |
Bill Wendling | b51bdda | 2011-05-04 02:40:38 +0000 | [diff] [blame] | 5937 | |
| 5938 | // Convert the type of the pointer to a pointer to the stored type. |
| 5939 | Value *BC = Builder.CreateBitCast(Ops[0], |
| 5940 | llvm::PointerType::getUnqual(Ops[1]->getType()), |
| 5941 | "cast"); |
| 5942 | StoreInst *SI = Builder.CreateStore(Ops[1], BC); |
| 5943 | SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); |
Eli Friedman | 440a5f4 | 2013-09-23 23:38:39 +0000 | [diff] [blame] | 5944 | |
| 5945 | // If the operand is an integer, we can't assume alignment. Otherwise, |
| 5946 | // assume natural alignment. |
| 5947 | QualType ArgTy = E->getArg(1)->getType(); |
| 5948 | unsigned Align; |
| 5949 | if (ArgTy->isIntegerType()) |
| 5950 | Align = 1; |
| 5951 | else |
| 5952 | Align = getContext().getTypeSizeInChars(ArgTy).getQuantity(); |
| 5953 | SI->setAlignment(Align); |
Bill Wendling | b51bdda | 2011-05-04 02:40:38 +0000 | [diff] [blame] | 5954 | return SI; |
| 5955 | } |
Michael J. Spencer | 8b36a9e | 2011-04-15 15:07:13 +0000 | [diff] [blame] | 5956 | // 3DNow! |
Michael J. Spencer | 8b36a9e | 2011-04-15 15:07:13 +0000 | [diff] [blame] | 5957 | case X86::BI__builtin_ia32_pswapdsf: |
| 5958 | case X86::BI__builtin_ia32_pswapdsi: { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 5959 | const char *name = nullptr; |
Michael J. Spencer | 8b36a9e | 2011-04-15 15:07:13 +0000 | [diff] [blame] | 5960 | Intrinsic::ID ID = Intrinsic::not_intrinsic; |
| 5961 | switch(BuiltinID) { |
Craig Topper | f8495d6 | 2012-01-30 08:18:19 +0000 | [diff] [blame] | 5962 | default: llvm_unreachable("Unsupported intrinsic!"); |
Michael J. Spencer | 8b36a9e | 2011-04-15 15:07:13 +0000 | [diff] [blame] | 5963 | case X86::BI__builtin_ia32_pswapdsf: |
| 5964 | case X86::BI__builtin_ia32_pswapdsi: |
| 5965 | name = "pswapd"; |
| 5966 | ID = Intrinsic::x86_3dnowa_pswapd; |
| 5967 | break; |
| 5968 | } |
Chandler Carruth | 345032a | 2012-02-20 07:35:45 +0000 | [diff] [blame] | 5969 | llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext()); |
| 5970 | Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); |
Michael J. Spencer | 8b36a9e | 2011-04-15 15:07:13 +0000 | [diff] [blame] | 5971 | llvm::Function *F = CGM.getIntrinsic(ID); |
Jay Foad | 4c7d9f1 | 2011-07-15 08:37:34 +0000 | [diff] [blame] | 5972 | return Builder.CreateCall(F, Ops, name); |
Michael J. Spencer | 8b36a9e | 2011-04-15 15:07:13 +0000 | [diff] [blame] | 5973 | } |
Benjamin Kramer | 9a50249 | 2012-07-12 09:33:03 +0000 | [diff] [blame] | 5974 | case X86::BI__builtin_ia32_rdrand16_step: |
| 5975 | case X86::BI__builtin_ia32_rdrand32_step: |
Michael Liao | 1bfc28c | 2013-03-29 05:17:55 +0000 | [diff] [blame] | 5976 | case X86::BI__builtin_ia32_rdrand64_step: |
| 5977 | case X86::BI__builtin_ia32_rdseed16_step: |
| 5978 | case X86::BI__builtin_ia32_rdseed32_step: |
| 5979 | case X86::BI__builtin_ia32_rdseed64_step: { |
Benjamin Kramer | 9a50249 | 2012-07-12 09:33:03 +0000 | [diff] [blame] | 5980 | Intrinsic::ID ID; |
| 5981 | switch (BuiltinID) { |
| 5982 | default: llvm_unreachable("Unsupported intrinsic!"); |
| 5983 | case X86::BI__builtin_ia32_rdrand16_step: |
| 5984 | ID = Intrinsic::x86_rdrand_16; |
| 5985 | break; |
| 5986 | case X86::BI__builtin_ia32_rdrand32_step: |
| 5987 | ID = Intrinsic::x86_rdrand_32; |
| 5988 | break; |
| 5989 | case X86::BI__builtin_ia32_rdrand64_step: |
| 5990 | ID = Intrinsic::x86_rdrand_64; |
| 5991 | break; |
Michael Liao | 1bfc28c | 2013-03-29 05:17:55 +0000 | [diff] [blame] | 5992 | case X86::BI__builtin_ia32_rdseed16_step: |
| 5993 | ID = Intrinsic::x86_rdseed_16; |
| 5994 | break; |
| 5995 | case X86::BI__builtin_ia32_rdseed32_step: |
| 5996 | ID = Intrinsic::x86_rdseed_32; |
| 5997 | break; |
| 5998 | case X86::BI__builtin_ia32_rdseed64_step: |
| 5999 | ID = Intrinsic::x86_rdseed_64; |
| 6000 | break; |
Benjamin Kramer | 9a50249 | 2012-07-12 09:33:03 +0000 | [diff] [blame] | 6001 | } |
| 6002 | |
| 6003 | Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID)); |
| 6004 | Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]); |
| 6005 | return Builder.CreateExtractValue(Call, 1); |
| 6006 | } |
Juergen Ributzka | 2766deb | 2013-08-17 16:40:09 +0000 | [diff] [blame] | 6007 | // AVX2 broadcast |
| 6008 | case X86::BI__builtin_ia32_vbroadcastsi256: { |
Juergen Ributzka | 3fc7791 | 2013-08-19 22:20:37 +0000 | [diff] [blame] | 6009 | Value *VecTmp = CreateMemTemp(E->getArg(0)->getType()); |
| 6010 | Builder.CreateStore(Ops[0], VecTmp); |
| 6011 | Value *F = CGM.getIntrinsic(Intrinsic::x86_avx2_vbroadcasti128); |
| 6012 | return Builder.CreateCall(F, Builder.CreateBitCast(VecTmp, Int8PtrTy)); |
Juergen Ributzka | 2766deb | 2013-08-17 16:40:09 +0000 | [diff] [blame] | 6013 | } |
Anders Carlsson | 564f1de | 2007-12-09 23:17:02 +0000 | [diff] [blame] | 6014 | } |
| 6015 | } |
| 6016 | |
Tony Linthicum | 9631939 | 2011-12-12 21:14:55 +0000 | [diff] [blame] | 6017 | |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 6018 | Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, |
Chris Lattner | 1feedd8 | 2007-12-13 07:34:23 +0000 | [diff] [blame] | 6019 | const CallExpr *E) { |
Chris Lattner | 5f9e272 | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 6020 | SmallVector<Value*, 4> Ops; |
Chris Lattner | dd17394 | 2010-04-14 03:54:58 +0000 | [diff] [blame] | 6021 | |
| 6022 | for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) |
| 6023 | Ops.push_back(EmitScalarExpr(E->getArg(i))); |
| 6024 | |
| 6025 | Intrinsic::ID ID = Intrinsic::not_intrinsic; |
| 6026 | |
| 6027 | switch (BuiltinID) { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 6028 | default: return nullptr; |
Chris Lattner | dd17394 | 2010-04-14 03:54:58 +0000 | [diff] [blame] | 6029 | |
Anton Korobeynikov | 4d3a7b0 | 2010-06-19 09:47:18 +0000 | [diff] [blame] | 6030 | // vec_ld, vec_lvsl, vec_lvsr |
| 6031 | case PPC::BI__builtin_altivec_lvx: |
| 6032 | case PPC::BI__builtin_altivec_lvxl: |
| 6033 | case PPC::BI__builtin_altivec_lvebx: |
| 6034 | case PPC::BI__builtin_altivec_lvehx: |
| 6035 | case PPC::BI__builtin_altivec_lvewx: |
| 6036 | case PPC::BI__builtin_altivec_lvsl: |
| 6037 | case PPC::BI__builtin_altivec_lvsr: |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 6038 | case PPC::BI__builtin_vsx_lxvd2x: |
| 6039 | case PPC::BI__builtin_vsx_lxvw4x: |
Anton Korobeynikov | 4d3a7b0 | 2010-06-19 09:47:18 +0000 | [diff] [blame] | 6040 | { |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 6041 | Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); |
Anton Korobeynikov | 4d3a7b0 | 2010-06-19 09:47:18 +0000 | [diff] [blame] | 6042 | |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 6043 | Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); |
Anton Korobeynikov | 4d3a7b0 | 2010-06-19 09:47:18 +0000 | [diff] [blame] | 6044 | Ops.pop_back(); |
| 6045 | |
| 6046 | switch (BuiltinID) { |
David Blaikie | b219cfc | 2011-09-23 05:06:16 +0000 | [diff] [blame] | 6047 | default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!"); |
Anton Korobeynikov | 4d3a7b0 | 2010-06-19 09:47:18 +0000 | [diff] [blame] | 6048 | case PPC::BI__builtin_altivec_lvx: |
| 6049 | ID = Intrinsic::ppc_altivec_lvx; |
| 6050 | break; |
| 6051 | case PPC::BI__builtin_altivec_lvxl: |
| 6052 | ID = Intrinsic::ppc_altivec_lvxl; |
| 6053 | break; |
| 6054 | case PPC::BI__builtin_altivec_lvebx: |
| 6055 | ID = Intrinsic::ppc_altivec_lvebx; |
| 6056 | break; |
| 6057 | case PPC::BI__builtin_altivec_lvehx: |
| 6058 | ID = Intrinsic::ppc_altivec_lvehx; |
| 6059 | break; |
| 6060 | case PPC::BI__builtin_altivec_lvewx: |
| 6061 | ID = Intrinsic::ppc_altivec_lvewx; |
| 6062 | break; |
| 6063 | case PPC::BI__builtin_altivec_lvsl: |
| 6064 | ID = Intrinsic::ppc_altivec_lvsl; |
| 6065 | break; |
| 6066 | case PPC::BI__builtin_altivec_lvsr: |
| 6067 | ID = Intrinsic::ppc_altivec_lvsr; |
| 6068 | break; |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 6069 | case PPC::BI__builtin_vsx_lxvd2x: |
| 6070 | ID = Intrinsic::ppc_vsx_lxvd2x; |
| 6071 | break; |
| 6072 | case PPC::BI__builtin_vsx_lxvw4x: |
| 6073 | ID = Intrinsic::ppc_vsx_lxvw4x; |
| 6074 | break; |
Anton Korobeynikov | 4d3a7b0 | 2010-06-19 09:47:18 +0000 | [diff] [blame] | 6075 | } |
| 6076 | llvm::Function *F = CGM.getIntrinsic(ID); |
Jay Foad | 4c7d9f1 | 2011-07-15 08:37:34 +0000 | [diff] [blame] | 6077 | return Builder.CreateCall(F, Ops, ""); |
Anton Korobeynikov | 4d3a7b0 | 2010-06-19 09:47:18 +0000 | [diff] [blame] | 6078 | } |
| 6079 | |
Chris Lattner | dd17394 | 2010-04-14 03:54:58 +0000 | [diff] [blame] | 6080 | // vec_st |
| 6081 | case PPC::BI__builtin_altivec_stvx: |
| 6082 | case PPC::BI__builtin_altivec_stvxl: |
| 6083 | case PPC::BI__builtin_altivec_stvebx: |
| 6084 | case PPC::BI__builtin_altivec_stvehx: |
| 6085 | case PPC::BI__builtin_altivec_stvewx: |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 6086 | case PPC::BI__builtin_vsx_stxvd2x: |
| 6087 | case PPC::BI__builtin_vsx_stxvw4x: |
Chris Lattner | dd17394 | 2010-04-14 03:54:58 +0000 | [diff] [blame] | 6088 | { |
John McCall | d16c2cf | 2011-02-08 08:22:06 +0000 | [diff] [blame] | 6089 | Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); |
Benjamin Kramer | 578faa8 | 2011-09-27 21:06:10 +0000 | [diff] [blame] | 6090 | Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); |
Chris Lattner | dd17394 | 2010-04-14 03:54:58 +0000 | [diff] [blame] | 6091 | Ops.pop_back(); |
| 6092 | |
| 6093 | switch (BuiltinID) { |
David Blaikie | b219cfc | 2011-09-23 05:06:16 +0000 | [diff] [blame] | 6094 | default: llvm_unreachable("Unsupported st intrinsic!"); |
Chris Lattner | dd17394 | 2010-04-14 03:54:58 +0000 | [diff] [blame] | 6095 | case PPC::BI__builtin_altivec_stvx: |
| 6096 | ID = Intrinsic::ppc_altivec_stvx; |
| 6097 | break; |
| 6098 | case PPC::BI__builtin_altivec_stvxl: |
| 6099 | ID = Intrinsic::ppc_altivec_stvxl; |
| 6100 | break; |
| 6101 | case PPC::BI__builtin_altivec_stvebx: |
| 6102 | ID = Intrinsic::ppc_altivec_stvebx; |
| 6103 | break; |
| 6104 | case PPC::BI__builtin_altivec_stvehx: |
| 6105 | ID = Intrinsic::ppc_altivec_stvehx; |
| 6106 | break; |
| 6107 | case PPC::BI__builtin_altivec_stvewx: |
| 6108 | ID = Intrinsic::ppc_altivec_stvewx; |
| 6109 | break; |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 6110 | case PPC::BI__builtin_vsx_stxvd2x: |
| 6111 | ID = Intrinsic::ppc_vsx_stxvd2x; |
| 6112 | break; |
| 6113 | case PPC::BI__builtin_vsx_stxvw4x: |
| 6114 | ID = Intrinsic::ppc_vsx_stxvw4x; |
| 6115 | break; |
Chris Lattner | dd17394 | 2010-04-14 03:54:58 +0000 | [diff] [blame] | 6116 | } |
| 6117 | llvm::Function *F = CGM.getIntrinsic(ID); |
Jay Foad | 4c7d9f1 | 2011-07-15 08:37:34 +0000 | [diff] [blame] | 6118 | return Builder.CreateCall(F, Ops, ""); |
Chris Lattner | dd17394 | 2010-04-14 03:54:58 +0000 | [diff] [blame] | 6119 | } |
| 6120 | } |
Mike Stump | 1eb4433 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 6121 | } |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 6122 | |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 6123 | // Emit an intrinsic that has 1 float or double. |
| 6124 | static Value *emitUnaryFPBuiltin(CodeGenFunction &CGF, |
| 6125 | const CallExpr *E, |
| 6126 | unsigned IntrinsicID) { |
| 6127 | llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); |
| 6128 | |
| 6129 | Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); |
| 6130 | return CGF.Builder.CreateCall(F, Src0); |
| 6131 | } |
| 6132 | |
| 6133 | // Emit an intrinsic that has 3 float or double operands. |
| 6134 | static Value *emitTernaryFPBuiltin(CodeGenFunction &CGF, |
| 6135 | const CallExpr *E, |
| 6136 | unsigned IntrinsicID) { |
| 6137 | llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); |
| 6138 | llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); |
| 6139 | llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2)); |
| 6140 | |
| 6141 | Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); |
| 6142 | return CGF.Builder.CreateCall3(F, Src0, Src1, Src2); |
| 6143 | } |
| 6144 | |
| 6145 | // Emit an intrinsic that has 1 float or double operand, and 1 integer. |
| 6146 | static Value *emitFPIntBuiltin(CodeGenFunction &CGF, |
| 6147 | const CallExpr *E, |
| 6148 | unsigned IntrinsicID) { |
| 6149 | llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0)); |
| 6150 | llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); |
| 6151 | |
| 6152 | Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); |
| 6153 | return CGF.Builder.CreateCall2(F, Src0, Src1); |
| 6154 | } |
| 6155 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 6156 | Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, |
| 6157 | const CallExpr *E) { |
| 6158 | switch (BuiltinID) { |
| 6159 | case R600::BI__builtin_amdgpu_div_scale: |
| 6160 | case R600::BI__builtin_amdgpu_div_scalef: { |
| 6161 | // Translate from the intrinsics's struct return to the builtin's out |
| 6162 | // argument. |
| 6163 | |
| 6164 | std::pair<llvm::Value *, unsigned> FlagOutPtr |
| 6165 | = EmitPointerWithAlignment(E->getArg(3)); |
| 6166 | |
| 6167 | llvm::Value *X = EmitScalarExpr(E->getArg(0)); |
| 6168 | llvm::Value *Y = EmitScalarExpr(E->getArg(1)); |
| 6169 | llvm::Value *Z = EmitScalarExpr(E->getArg(2)); |
| 6170 | |
| 6171 | llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::AMDGPU_div_scale, |
| 6172 | X->getType()); |
| 6173 | |
| 6174 | llvm::Value *Tmp = Builder.CreateCall3(Callee, X, Y, Z); |
| 6175 | |
| 6176 | llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0); |
| 6177 | llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1); |
| 6178 | |
| 6179 | llvm::Type *RealFlagType |
| 6180 | = FlagOutPtr.first->getType()->getPointerElementType(); |
| 6181 | |
| 6182 | llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType); |
| 6183 | llvm::StoreInst *FlagStore = Builder.CreateStore(FlagExt, FlagOutPtr.first); |
| 6184 | FlagStore->setAlignment(FlagOutPtr.second); |
| 6185 | return Result; |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 6186 | } |
| 6187 | case R600::BI__builtin_amdgpu_div_fmas: |
| 6188 | case R600::BI__builtin_amdgpu_div_fmasf: { |
| 6189 | llvm::Value *Src0 = EmitScalarExpr(E->getArg(0)); |
| 6190 | llvm::Value *Src1 = EmitScalarExpr(E->getArg(1)); |
| 6191 | llvm::Value *Src2 = EmitScalarExpr(E->getArg(2)); |
| 6192 | llvm::Value *Src3 = EmitScalarExpr(E->getArg(3)); |
| 6193 | |
| 6194 | llvm::Value *F = CGM.getIntrinsic(Intrinsic::AMDGPU_div_fmas, |
| 6195 | Src0->getType()); |
| 6196 | llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3); |
| 6197 | return Builder.CreateCall4(F, Src0, Src1, Src2, Src3ToBool); |
| 6198 | } |
| 6199 | case R600::BI__builtin_amdgpu_div_fixup: |
| 6200 | case R600::BI__builtin_amdgpu_div_fixupf: |
| 6201 | return emitTernaryFPBuiltin(*this, E, Intrinsic::AMDGPU_div_fixup); |
| 6202 | case R600::BI__builtin_amdgpu_trig_preop: |
| 6203 | case R600::BI__builtin_amdgpu_trig_preopf: |
| 6204 | return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_trig_preop); |
| 6205 | case R600::BI__builtin_amdgpu_rcp: |
| 6206 | case R600::BI__builtin_amdgpu_rcpf: |
| 6207 | return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rcp); |
| 6208 | case R600::BI__builtin_amdgpu_rsq: |
| 6209 | case R600::BI__builtin_amdgpu_rsqf: |
| 6210 | return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq); |
| 6211 | case R600::BI__builtin_amdgpu_rsq_clamped: |
| 6212 | case R600::BI__builtin_amdgpu_rsq_clampedf: |
| 6213 | return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq_clamped); |
| 6214 | case R600::BI__builtin_amdgpu_ldexp: |
| 6215 | case R600::BI__builtin_amdgpu_ldexpf: |
| 6216 | return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_ldexp); |
| 6217 | default: |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 6218 | return nullptr; |
| 6219 | } |
| 6220 | } |