blob: 10b29d15a36f0909384d695b1b2b6917b728f605 [file] [log] [blame]
Nate Begeman0b71e002005-10-18 00:28:58 +00001//===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===//
Chris Lattnerf22556d2005-08-16 17:14:42 +00002//
3// The LLVM Compiler Infrastructure
4//
Chris Lattnerf3ebc3f2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Chris Lattnerf22556d2005-08-16 17:14:42 +00007//
8//===----------------------------------------------------------------------===//
9//
Nate Begeman6cca84e2005-10-16 05:39:50 +000010// This file implements the PPCISelLowering class.
Chris Lattnerf22556d2005-08-16 17:14:42 +000011//
12//===----------------------------------------------------------------------===//
13
Chris Lattner6f3b9542005-10-14 23:59:06 +000014#include "PPCISelLowering.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000015#include "MCTargetDesc/PPCPredicates.h"
Hal Finkel934361a2015-01-14 01:07:51 +000016#include "PPCCallingConv.h"
Jim Laskey48850c12006-11-16 22:43:37 +000017#include "PPCMachineFunctionInfo.h"
Bill Wendlingdd3fe942010-03-12 02:00:43 +000018#include "PPCPerfectShuffle.h"
Chris Lattner6f3b9542005-10-14 23:59:06 +000019#include "PPCTargetMachine.h"
Bill Schmidt22d40dc2013-05-13 19:34:37 +000020#include "PPCTargetObjectFile.h"
Owen Andersone2f23a32007-09-07 04:06:50 +000021#include "llvm/ADT/STLExtras.h"
Hal Finkel0d8db462014-05-11 19:29:11 +000022#include "llvm/ADT/StringSwitch.h"
Eric Christopher89958332014-05-31 00:07:32 +000023#include "llvm/ADT/Triple.h"
Chris Lattner4f2e4e02007-03-06 00:59:59 +000024#include "llvm/CodeGen/CallingConvLower.h"
Chris Lattnerf22556d2005-08-16 17:14:42 +000025#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
Chris Lattner9b577f12005-08-26 21:23:58 +000027#include "llvm/CodeGen/MachineInstrBuilder.h"
Hal Finkel57725662015-01-03 17:58:24 +000028#include "llvm/CodeGen/MachineLoopInfo.h"
Chris Lattnera10fff52007-12-31 04:13:23 +000029#include "llvm/CodeGen/MachineRegisterInfo.h"
Chris Lattnerf22556d2005-08-16 17:14:42 +000030#include "llvm/CodeGen/SelectionDAG.h"
Anton Korobeynikovab663a02010-02-15 22:37:53 +000031#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000032#include "llvm/IR/CallingConv.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DerivedTypes.h"
35#include "llvm/IR/Function.h"
36#include "llvm/IR/Intrinsics.h"
Chris Lattnerce645542006-11-10 02:08:47 +000037#include "llvm/Support/CommandLine.h"
Torok Edwinfb8d6d52009-07-08 20:53:28 +000038#include "llvm/Support/ErrorHandling.h"
Craig Topperb25fda92012-03-17 18:46:09 +000039#include "llvm/Support/MathExtras.h"
Torok Edwinfb8d6d52009-07-08 20:53:28 +000040#include "llvm/Support/raw_ostream.h"
Craig Topperb25fda92012-03-17 18:46:09 +000041#include "llvm/Target/TargetOptions.h"
Chris Lattnerf22556d2005-08-16 17:14:42 +000042using namespace llvm;
43
Joerg Sonnenbergereb8655a2014-08-08 16:46:10 +000044// FIXME: Remove this once soft-float is supported.
45static cl::opt<bool> DisablePPCFloatInVariadic("disable-ppc-float-in-variadic",
46cl::desc("disable saving float registers for va_start on PPC"), cl::Hidden);
47
Hal Finkel595817e2012-06-04 02:21:00 +000048static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc",
49cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden);
Chris Lattnerce645542006-11-10 02:08:47 +000050
Hal Finkel4e9f1a82012-06-10 19:32:29 +000051static cl::opt<bool> DisableILPPref("disable-ppc-ilp-pref",
52cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden);
53
Hal Finkel8d7fbc92013-03-15 15:27:13 +000054static cl::opt<bool> DisablePPCUnaligned("disable-ppc-unaligned",
55cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden);
56
Hal Finkel940ab932014-02-28 00:27:01 +000057// FIXME: Remove this once the bug has been fixed!
58extern cl::opt<bool> ANDIGlueBug;
59
Eric Christophercccae792015-01-30 22:02:31 +000060PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
61 const PPCSubtarget &STI)
62 : TargetLowering(TM), Subtarget(STI) {
Chris Lattnera028e7a2005-09-27 22:18:25 +000063 // Use _setjmp/_longjmp instead of setjmp/longjmp.
Anton Korobeynikov3b7c2572006-12-10 23:12:42 +000064 setUseUnderscoreSetJmp(true);
65 setUseUnderscoreLongJmp(true);
Scott Michelcf0da6c2009-02-17 22:15:04 +000066
Chris Lattnerd10babf2010-10-10 18:34:00 +000067 // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all
68 // arguments are at least 4/8 bytes aligned.
Eric Christopherb1aaebe2014-06-12 22:38:18 +000069 bool isPPC64 = Subtarget.isPPC64();
Evan Cheng39e90022012-07-02 22:39:56 +000070 setMinStackArgumentAlignment(isPPC64 ? 8:4);
Wesley Peck527da1b2010-11-23 03:31:01 +000071
Chris Lattnerf22556d2005-08-16 17:14:42 +000072 // Set up the register classes.
Craig Topperabadc662012-04-20 06:31:50 +000073 addRegisterClass(MVT::i32, &PPC::GPRCRegClass);
74 addRegisterClass(MVT::f32, &PPC::F4RCRegClass);
75 addRegisterClass(MVT::f64, &PPC::F8RCRegClass);
Scott Michelcf0da6c2009-02-17 22:15:04 +000076
Evan Cheng5d9fd972006-10-04 00:56:09 +000077 // PowerPC has an i16 but no i8 (or i1) SEXTLOAD
Ahmed Bougacha2b6917b2015-01-08 00:51:32 +000078 for (MVT VT : MVT::integer_valuetypes()) {
79 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
80 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Expand);
81 }
Duncan Sands95d46ef2008-01-23 20:39:46 +000082
Owen Anderson9f944592009-08-11 20:47:22 +000083 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
Scott Michelcf0da6c2009-02-17 22:15:04 +000084
Chris Lattnerc9fa36d2006-11-10 23:58:45 +000085 // PowerPC has pre-inc load and store's.
Owen Anderson9f944592009-08-11 20:47:22 +000086 setIndexedLoadAction(ISD::PRE_INC, MVT::i1, Legal);
87 setIndexedLoadAction(ISD::PRE_INC, MVT::i8, Legal);
88 setIndexedLoadAction(ISD::PRE_INC, MVT::i16, Legal);
89 setIndexedLoadAction(ISD::PRE_INC, MVT::i32, Legal);
90 setIndexedLoadAction(ISD::PRE_INC, MVT::i64, Legal);
Hal Finkel65d1cbf2015-02-05 18:42:53 +000091 setIndexedLoadAction(ISD::PRE_INC, MVT::f32, Legal);
92 setIndexedLoadAction(ISD::PRE_INC, MVT::f64, Legal);
Owen Anderson9f944592009-08-11 20:47:22 +000093 setIndexedStoreAction(ISD::PRE_INC, MVT::i1, Legal);
94 setIndexedStoreAction(ISD::PRE_INC, MVT::i8, Legal);
95 setIndexedStoreAction(ISD::PRE_INC, MVT::i16, Legal);
96 setIndexedStoreAction(ISD::PRE_INC, MVT::i32, Legal);
97 setIndexedStoreAction(ISD::PRE_INC, MVT::i64, Legal);
Hal Finkel65d1cbf2015-02-05 18:42:53 +000098 setIndexedStoreAction(ISD::PRE_INC, MVT::f32, Legal);
99 setIndexedStoreAction(ISD::PRE_INC, MVT::f64, Legal);
Evan Cheng36a8fbf2006-11-09 19:11:50 +0000100
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000101 if (Subtarget.useCRBits()) {
Hal Finkel940ab932014-02-28 00:27:01 +0000102 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
103
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000104 if (isPPC64 || Subtarget.hasFPCVT()) {
Hal Finkel6a56b212014-03-05 22:14:00 +0000105 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote);
106 AddPromotedToType (ISD::SINT_TO_FP, MVT::i1,
107 isPPC64 ? MVT::i64 : MVT::i32);
108 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote);
109 AddPromotedToType (ISD::UINT_TO_FP, MVT::i1,
110 isPPC64 ? MVT::i64 : MVT::i32);
111 } else {
112 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Custom);
113 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Custom);
114 }
Hal Finkel940ab932014-02-28 00:27:01 +0000115
116 // PowerPC does not support direct load / store of condition registers
117 setOperationAction(ISD::LOAD, MVT::i1, Custom);
118 setOperationAction(ISD::STORE, MVT::i1, Custom);
119
120 // FIXME: Remove this once the ANDI glue bug is fixed:
121 if (ANDIGlueBug)
122 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom);
123
Ahmed Bougacha2b6917b2015-01-08 00:51:32 +0000124 for (MVT VT : MVT::integer_valuetypes()) {
125 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
126 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
127 setTruncStoreAction(VT, MVT::i1, Expand);
128 }
Hal Finkel940ab932014-02-28 00:27:01 +0000129
130 addRegisterClass(MVT::i1, &PPC::CRBITRCRegClass);
131 }
132
Dale Johannesen666323e2007-10-10 01:01:31 +0000133 // This is used in the ppcf128->int sequence. Note it has different semantics
134 // from FP_ROUND: that rounds to nearest, this rounds to zero.
Owen Anderson9f944592009-08-11 20:47:22 +0000135 setOperationAction(ISD::FP_ROUND_INREG, MVT::ppcf128, Custom);
Dale Johannesenf864ac92007-10-06 01:24:11 +0000136
Roman Divacky1faf5b02012-08-16 18:19:29 +0000137 // We do not currently implement these libm ops for PowerPC.
Owen Anderson0b9b9da2011-12-08 19:32:14 +0000138 setOperationAction(ISD::FFLOOR, MVT::ppcf128, Expand);
139 setOperationAction(ISD::FCEIL, MVT::ppcf128, Expand);
140 setOperationAction(ISD::FTRUNC, MVT::ppcf128, Expand);
141 setOperationAction(ISD::FRINT, MVT::ppcf128, Expand);
142 setOperationAction(ISD::FNEARBYINT, MVT::ppcf128, Expand);
Bill Schmidt92e26642013-04-03 13:05:44 +0000143 setOperationAction(ISD::FREM, MVT::ppcf128, Expand);
Owen Anderson0b9b9da2011-12-08 19:32:14 +0000144
Chris Lattnerf22556d2005-08-16 17:14:42 +0000145 // PowerPC has no SREM/UREM instructions
Owen Anderson9f944592009-08-11 20:47:22 +0000146 setOperationAction(ISD::SREM, MVT::i32, Expand);
147 setOperationAction(ISD::UREM, MVT::i32, Expand);
148 setOperationAction(ISD::SREM, MVT::i64, Expand);
149 setOperationAction(ISD::UREM, MVT::i64, Expand);
Dan Gohman71f0d7d2007-10-08 17:28:24 +0000150
151 // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM.
Owen Anderson9f944592009-08-11 20:47:22 +0000152 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
153 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
154 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
155 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
156 setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
157 setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
158 setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
159 setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000160
Dan Gohman482732a2007-10-11 23:21:31 +0000161 // We don't support sin/cos/sqrt/fmod/pow
Owen Anderson9f944592009-08-11 20:47:22 +0000162 setOperationAction(ISD::FSIN , MVT::f64, Expand);
163 setOperationAction(ISD::FCOS , MVT::f64, Expand);
Evan Cheng0e88c7d2013-01-29 02:32:37 +0000164 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
Owen Anderson9f944592009-08-11 20:47:22 +0000165 setOperationAction(ISD::FREM , MVT::f64, Expand);
166 setOperationAction(ISD::FPOW , MVT::f64, Expand);
Hal Finkel0a479ae2012-06-22 00:49:52 +0000167 setOperationAction(ISD::FMA , MVT::f64, Legal);
Owen Anderson9f944592009-08-11 20:47:22 +0000168 setOperationAction(ISD::FSIN , MVT::f32, Expand);
169 setOperationAction(ISD::FCOS , MVT::f32, Expand);
Evan Cheng0e88c7d2013-01-29 02:32:37 +0000170 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
Owen Anderson9f944592009-08-11 20:47:22 +0000171 setOperationAction(ISD::FREM , MVT::f32, Expand);
172 setOperationAction(ISD::FPOW , MVT::f32, Expand);
Hal Finkel0a479ae2012-06-22 00:49:52 +0000173 setOperationAction(ISD::FMA , MVT::f32, Legal);
Dale Johannesen5c94cb32008-01-18 19:55:37 +0000174
Owen Anderson9f944592009-08-11 20:47:22 +0000175 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000176
Chris Lattnerf22556d2005-08-16 17:14:42 +0000177 // If we're enabling GP optimizations, use hardware square root
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000178 if (!Subtarget.hasFSQRT() &&
Eric Christophercccae792015-01-30 22:02:31 +0000179 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTE() &&
180 Subtarget.hasFRE()))
Owen Anderson9f944592009-08-11 20:47:22 +0000181 setOperationAction(ISD::FSQRT, MVT::f64, Expand);
Hal Finkel2e103312013-04-03 04:01:11 +0000182
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000183 if (!Subtarget.hasFSQRT() &&
Eric Christophercccae792015-01-30 22:02:31 +0000184 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTES() &&
185 Subtarget.hasFRES()))
Owen Anderson9f944592009-08-11 20:47:22 +0000186 setOperationAction(ISD::FSQRT, MVT::f32, Expand);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000187
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000188 if (Subtarget.hasFCPSGN()) {
Hal Finkeldbc78e12013-08-19 05:01:02 +0000189 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Legal);
190 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Legal);
191 } else {
192 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
193 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
194 }
Scott Michelcf0da6c2009-02-17 22:15:04 +0000195
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000196 if (Subtarget.hasFPRND()) {
Hal Finkelc20a08d2013-03-29 08:57:48 +0000197 setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
198 setOperationAction(ISD::FCEIL, MVT::f64, Legal);
199 setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
Hal Finkel2b7b2f32013-08-08 04:31:34 +0000200 setOperationAction(ISD::FROUND, MVT::f64, Legal);
Hal Finkelc20a08d2013-03-29 08:57:48 +0000201
202 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
203 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
204 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
Hal Finkel2b7b2f32013-08-08 04:31:34 +0000205 setOperationAction(ISD::FROUND, MVT::f32, Legal);
Hal Finkelc20a08d2013-03-29 08:57:48 +0000206 }
207
Nate Begeman2fba8a32006-01-14 03:14:10 +0000208 // PowerPC does not have BSWAP, CTPOP or CTTZ
Owen Anderson9f944592009-08-11 20:47:22 +0000209 setOperationAction(ISD::BSWAP, MVT::i32 , Expand);
Owen Anderson9f944592009-08-11 20:47:22 +0000210 setOperationAction(ISD::CTTZ , MVT::i32 , Expand);
Chandler Carruth637cc6a2011-12-13 01:56:10 +0000211 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand);
212 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand);
Owen Anderson9f944592009-08-11 20:47:22 +0000213 setOperationAction(ISD::BSWAP, MVT::i64 , Expand);
Owen Anderson9f944592009-08-11 20:47:22 +0000214 setOperationAction(ISD::CTTZ , MVT::i64 , Expand);
Chandler Carruth637cc6a2011-12-13 01:56:10 +0000215 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand);
216 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000217
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000218 if (Subtarget.hasPOPCNTD()) {
Hal Finkel290376d2013-04-01 15:58:15 +0000219 setOperationAction(ISD::CTPOP, MVT::i32 , Legal);
Hal Finkela4d07482013-03-28 13:29:47 +0000220 setOperationAction(ISD::CTPOP, MVT::i64 , Legal);
221 } else {
222 setOperationAction(ISD::CTPOP, MVT::i32 , Expand);
223 setOperationAction(ISD::CTPOP, MVT::i64 , Expand);
224 }
225
Nate Begeman1b8121b2006-01-11 21:21:00 +0000226 // PowerPC does not have ROTR
Owen Anderson9f944592009-08-11 20:47:22 +0000227 setOperationAction(ISD::ROTR, MVT::i32 , Expand);
228 setOperationAction(ISD::ROTR, MVT::i64 , Expand);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000229
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000230 if (!Subtarget.useCRBits()) {
Hal Finkel940ab932014-02-28 00:27:01 +0000231 // PowerPC does not have Select
232 setOperationAction(ISD::SELECT, MVT::i32, Expand);
233 setOperationAction(ISD::SELECT, MVT::i64, Expand);
234 setOperationAction(ISD::SELECT, MVT::f32, Expand);
235 setOperationAction(ISD::SELECT, MVT::f64, Expand);
236 }
Scott Michelcf0da6c2009-02-17 22:15:04 +0000237
Chris Lattner7f1fa8e2005-08-26 17:36:52 +0000238 // PowerPC wants to turn select_cc of FP into fsel when possible.
Owen Anderson9f944592009-08-11 20:47:22 +0000239 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
240 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
Nate Begemana162f202006-01-31 08:17:29 +0000241
Nate Begeman7e7f4392006-02-01 07:19:44 +0000242 // PowerPC wants to optimize integer setcc a bit
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000243 if (!Subtarget.useCRBits())
Hal Finkel940ab932014-02-28 00:27:01 +0000244 setOperationAction(ISD::SETCC, MVT::i32, Custom);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000245
Nate Begemanbb01d4f2006-03-17 01:40:33 +0000246 // PowerPC does not have BRCOND which requires SetCC
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000247 if (!Subtarget.useCRBits())
Hal Finkel940ab932014-02-28 00:27:01 +0000248 setOperationAction(ISD::BRCOND, MVT::Other, Expand);
Evan Cheng0d41d192006-10-30 08:02:39 +0000249
Owen Anderson9f944592009-08-11 20:47:22 +0000250 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000251
Chris Lattnerda2e04c2005-08-31 21:09:52 +0000252 // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores.
Owen Anderson9f944592009-08-11 20:47:22 +0000253 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
Nate Begeman60952142005-09-06 22:03:27 +0000254
Jim Laskey6267b2c2005-08-17 00:40:22 +0000255 // PowerPC does not have [U|S]INT_TO_FP
Owen Anderson9f944592009-08-11 20:47:22 +0000256 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand);
257 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand);
Jim Laskey6267b2c2005-08-17 00:40:22 +0000258
Wesley Peck527da1b2010-11-23 03:31:01 +0000259 setOperationAction(ISD::BITCAST, MVT::f32, Expand);
260 setOperationAction(ISD::BITCAST, MVT::i32, Expand);
261 setOperationAction(ISD::BITCAST, MVT::i64, Expand);
262 setOperationAction(ISD::BITCAST, MVT::f64, Expand);
Chris Lattnerc46fc242005-12-23 05:13:35 +0000263
Chris Lattner84b49d52006-04-28 21:56:10 +0000264 // We cannot sextinreg(i1). Expand to shifts.
Owen Anderson9f944592009-08-11 20:47:22 +0000265 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
Jim Laskeye0008e22007-02-22 14:56:36 +0000266
Hal Finkel1996f3d2013-03-27 19:10:42 +0000267 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support
Hal Finkel756810f2013-03-21 21:37:52 +0000268 // SjLj exception handling but a light-weight setjmp/longjmp replacement to
269 // support continuation, user-level threading, and etc.. As a result, no
270 // other SjLj exception interfaces are implemented and please don't build
271 // your own exception handling based on them.
272 // LLVM/Clang supports zero-cost DWARF exception handling.
273 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
274 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000275
276 // We want to legalize GlobalAddress and ConstantPool nodes into the
Nate Begeman4e56db62005-12-10 02:36:00 +0000277 // appropriate instructions to materialize the address.
Owen Anderson9f944592009-08-11 20:47:22 +0000278 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
279 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
Bob Wilsonf84f7102009-11-04 21:31:18 +0000280 setOperationAction(ISD::BlockAddress, MVT::i32, Custom);
Owen Anderson9f944592009-08-11 20:47:22 +0000281 setOperationAction(ISD::ConstantPool, MVT::i32, Custom);
282 setOperationAction(ISD::JumpTable, MVT::i32, Custom);
283 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
284 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
Bob Wilsonf84f7102009-11-04 21:31:18 +0000285 setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
Owen Anderson9f944592009-08-11 20:47:22 +0000286 setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
287 setOperationAction(ISD::JumpTable, MVT::i64, Custom);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000288
Nate Begemanf69d13b2008-08-11 17:36:31 +0000289 // TRAP is legal.
Owen Anderson9f944592009-08-11 20:47:22 +0000290 setOperationAction(ISD::TRAP, MVT::Other, Legal);
Bill Wendling95e1af22008-09-17 00:30:57 +0000291
292 // TRAMPOLINE is custom lowered.
Duncan Sandsa0984362011-09-06 13:37:06 +0000293 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
294 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
Bill Wendling95e1af22008-09-17 00:30:57 +0000295
Nate Begemane74795c2006-01-25 18:21:52 +0000296 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
Owen Anderson9f944592009-08-11 20:47:22 +0000297 setOperationAction(ISD::VASTART , MVT::Other, Custom);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000298
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000299 if (Subtarget.isSVR4ABI()) {
Evan Cheng39e90022012-07-02 22:39:56 +0000300 if (isPPC64) {
Hal Finkele44eb282012-03-24 03:53:55 +0000301 // VAARG always uses double-word chunks, so promote anything smaller.
302 setOperationAction(ISD::VAARG, MVT::i1, Promote);
303 AddPromotedToType (ISD::VAARG, MVT::i1, MVT::i64);
304 setOperationAction(ISD::VAARG, MVT::i8, Promote);
305 AddPromotedToType (ISD::VAARG, MVT::i8, MVT::i64);
306 setOperationAction(ISD::VAARG, MVT::i16, Promote);
307 AddPromotedToType (ISD::VAARG, MVT::i16, MVT::i64);
308 setOperationAction(ISD::VAARG, MVT::i32, Promote);
309 AddPromotedToType (ISD::VAARG, MVT::i32, MVT::i64);
310 setOperationAction(ISD::VAARG, MVT::Other, Expand);
311 } else {
312 // VAARG is custom lowered with the 32-bit SVR4 ABI.
313 setOperationAction(ISD::VAARG, MVT::Other, Custom);
314 setOperationAction(ISD::VAARG, MVT::i64, Custom);
315 }
Roman Divacky4394e682011-06-28 15:30:42 +0000316 } else
Owen Anderson9f944592009-08-11 20:47:22 +0000317 setOperationAction(ISD::VAARG, MVT::Other, Expand);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000318
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000319 if (Subtarget.isSVR4ABI() && !isPPC64)
Roman Divackyc3825df2013-07-25 21:36:47 +0000320 // VACOPY is custom lowered with the 32-bit SVR4 ABI.
321 setOperationAction(ISD::VACOPY , MVT::Other, Custom);
322 else
323 setOperationAction(ISD::VACOPY , MVT::Other, Expand);
324
Chris Lattner5bd514d2006-01-15 09:02:48 +0000325 // Use the default implementation.
Owen Anderson9f944592009-08-11 20:47:22 +0000326 setOperationAction(ISD::VAEND , MVT::Other, Expand);
327 setOperationAction(ISD::STACKSAVE , MVT::Other, Expand);
328 setOperationAction(ISD::STACKRESTORE , MVT::Other, Custom);
329 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Custom);
330 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64 , Custom);
Chris Lattnerab4df8342006-10-18 01:18:48 +0000331
Chris Lattner6961fc72006-03-26 10:06:40 +0000332 // We want to custom lower some of our intrinsics.
Owen Anderson9f944592009-08-11 20:47:22 +0000333 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000334
Hal Finkel25c19922013-05-15 21:37:41 +0000335 // To handle counter-based loop conditions.
336 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i1, Custom);
337
Dale Johannesen160be0f2008-11-07 22:54:33 +0000338 // Comparisons that require checking two conditions.
Owen Anderson9f944592009-08-11 20:47:22 +0000339 setCondCodeAction(ISD::SETULT, MVT::f32, Expand);
340 setCondCodeAction(ISD::SETULT, MVT::f64, Expand);
341 setCondCodeAction(ISD::SETUGT, MVT::f32, Expand);
342 setCondCodeAction(ISD::SETUGT, MVT::f64, Expand);
343 setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand);
344 setCondCodeAction(ISD::SETUEQ, MVT::f64, Expand);
345 setCondCodeAction(ISD::SETOGE, MVT::f32, Expand);
346 setCondCodeAction(ISD::SETOGE, MVT::f64, Expand);
347 setCondCodeAction(ISD::SETOLE, MVT::f32, Expand);
348 setCondCodeAction(ISD::SETOLE, MVT::f64, Expand);
349 setCondCodeAction(ISD::SETONE, MVT::f32, Expand);
350 setCondCodeAction(ISD::SETONE, MVT::f64, Expand);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000351
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000352 if (Subtarget.has64BitSupport()) {
Nate Begeman0b71e002005-10-18 00:28:58 +0000353 // They also have instructions for converting between i64 and fp.
Owen Anderson9f944592009-08-11 20:47:22 +0000354 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
355 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Expand);
356 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
357 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand);
Dale Johannesen37bc85f2009-06-04 20:53:52 +0000358 // This is just the low 32 bits of a (signed) fp->i64 conversion.
359 // We cannot do this with Promote because i64 is not a legal type.
Owen Anderson9f944592009-08-11 20:47:22 +0000360 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000361
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000362 if (Subtarget.hasLFIWAX() || Subtarget.isPPC64())
Hal Finkele53429a2013-03-31 01:58:02 +0000363 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
Nate Begeman762bf802005-10-25 23:48:36 +0000364 } else {
Chris Lattner595088a2005-11-17 07:30:41 +0000365 // PowerPC does not have FP_TO_UINT on 32-bit implementations.
Owen Anderson9f944592009-08-11 20:47:22 +0000366 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand);
Nate Begemane74dfbb2005-10-18 00:56:42 +0000367 }
368
Hal Finkelf6d45f22013-04-01 17:52:07 +0000369 // With the instructions enabled under FPCVT, we can do everything.
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000370 if (Subtarget.hasFPCVT()) {
371 if (Subtarget.has64BitSupport()) {
Hal Finkelf6d45f22013-04-01 17:52:07 +0000372 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
373 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
374 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
375 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
376 }
377
378 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
379 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
380 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
381 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
382 }
383
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000384 if (Subtarget.use64BitRegs()) {
Chris Lattnerb1935762007-10-19 04:08:28 +0000385 // 64-bit PowerPC implementations can support i64 types directly
Craig Topperabadc662012-04-20 06:31:50 +0000386 addRegisterClass(MVT::i64, &PPC::G8RCRegClass);
Nate Begeman0b71e002005-10-18 00:28:58 +0000387 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or
Owen Anderson9f944592009-08-11 20:47:22 +0000388 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
Dan Gohman8d2ead22008-03-07 20:36:53 +0000389 // 64-bit PowerPC wants to expand i128 shifts itself.
Owen Anderson9f944592009-08-11 20:47:22 +0000390 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
391 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
392 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
Nate Begeman0b71e002005-10-18 00:28:58 +0000393 } else {
Chris Lattnerb1935762007-10-19 04:08:28 +0000394 // 32-bit PowerPC wants to expand i64 shifts itself.
Owen Anderson9f944592009-08-11 20:47:22 +0000395 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
396 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom);
397 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom);
Nate Begeman60952142005-09-06 22:03:27 +0000398 }
Evan Cheng19264272006-03-01 01:11:20 +0000399
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000400 if (Subtarget.hasAltivec()) {
Chris Lattnerbaa73e02006-03-31 19:52:36 +0000401 // First set operation action for all vector types to expand. Then we
402 // will selectively turn on ones that can be effectively codegen'd.
Ahmed Bougacha67dd2d22015-01-07 21:27:10 +0000403 for (MVT VT : MVT::vector_valuetypes()) {
Chris Lattner06a21ba2006-04-16 01:37:57 +0000404 // add/sub are legal for all supported vector VT's.
Duncan Sands13237ac2008-06-06 12:08:01 +0000405 setOperationAction(ISD::ADD , VT, Legal);
406 setOperationAction(ISD::SUB , VT, Legal);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000407
Bill Schmidt433b1c32015-02-05 15:24:47 +0000408 // Vector instructions introduced in P8
409 if (Subtarget.hasP8Altivec()) {
Bill Schmidtfe88b182015-02-03 21:58:23 +0000410 setOperationAction(ISD::CTPOP, VT, Legal);
Bill Schmidt433b1c32015-02-05 15:24:47 +0000411 setOperationAction(ISD::CTLZ, VT, Legal);
412 }
413 else {
Bill Schmidtfe88b182015-02-03 21:58:23 +0000414 setOperationAction(ISD::CTPOP, VT, Expand);
Bill Schmidt433b1c32015-02-05 15:24:47 +0000415 setOperationAction(ISD::CTLZ, VT, Expand);
416 }
Bill Schmidtfe88b182015-02-03 21:58:23 +0000417
Chris Lattner95c7adc2006-04-04 17:25:31 +0000418 // We promote all shuffles to v16i8.
Duncan Sands13237ac2008-06-06 12:08:01 +0000419 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Promote);
Owen Anderson9f944592009-08-11 20:47:22 +0000420 AddPromotedToType (ISD::VECTOR_SHUFFLE, VT, MVT::v16i8);
Chris Lattner06a21ba2006-04-16 01:37:57 +0000421
422 // We promote all non-typed operations to v4i32.
Duncan Sands13237ac2008-06-06 12:08:01 +0000423 setOperationAction(ISD::AND , VT, Promote);
Owen Anderson9f944592009-08-11 20:47:22 +0000424 AddPromotedToType (ISD::AND , VT, MVT::v4i32);
Duncan Sands13237ac2008-06-06 12:08:01 +0000425 setOperationAction(ISD::OR , VT, Promote);
Owen Anderson9f944592009-08-11 20:47:22 +0000426 AddPromotedToType (ISD::OR , VT, MVT::v4i32);
Duncan Sands13237ac2008-06-06 12:08:01 +0000427 setOperationAction(ISD::XOR , VT, Promote);
Owen Anderson9f944592009-08-11 20:47:22 +0000428 AddPromotedToType (ISD::XOR , VT, MVT::v4i32);
Duncan Sands13237ac2008-06-06 12:08:01 +0000429 setOperationAction(ISD::LOAD , VT, Promote);
Owen Anderson9f944592009-08-11 20:47:22 +0000430 AddPromotedToType (ISD::LOAD , VT, MVT::v4i32);
Duncan Sands13237ac2008-06-06 12:08:01 +0000431 setOperationAction(ISD::SELECT, VT, Promote);
Owen Anderson9f944592009-08-11 20:47:22 +0000432 AddPromotedToType (ISD::SELECT, VT, MVT::v4i32);
Duncan Sands13237ac2008-06-06 12:08:01 +0000433 setOperationAction(ISD::STORE, VT, Promote);
Owen Anderson9f944592009-08-11 20:47:22 +0000434 AddPromotedToType (ISD::STORE, VT, MVT::v4i32);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000435
Chris Lattner06a21ba2006-04-16 01:37:57 +0000436 // No other operations are legal.
Duncan Sands13237ac2008-06-06 12:08:01 +0000437 setOperationAction(ISD::MUL , VT, Expand);
438 setOperationAction(ISD::SDIV, VT, Expand);
439 setOperationAction(ISD::SREM, VT, Expand);
440 setOperationAction(ISD::UDIV, VT, Expand);
441 setOperationAction(ISD::UREM, VT, Expand);
442 setOperationAction(ISD::FDIV, VT, Expand);
Hal Finkele3930222013-07-08 17:30:25 +0000443 setOperationAction(ISD::FREM, VT, Expand);
Duncan Sands13237ac2008-06-06 12:08:01 +0000444 setOperationAction(ISD::FNEG, VT, Expand);
Craig Topperc8a2adf2012-11-15 08:02:19 +0000445 setOperationAction(ISD::FSQRT, VT, Expand);
446 setOperationAction(ISD::FLOG, VT, Expand);
447 setOperationAction(ISD::FLOG10, VT, Expand);
448 setOperationAction(ISD::FLOG2, VT, Expand);
449 setOperationAction(ISD::FEXP, VT, Expand);
450 setOperationAction(ISD::FEXP2, VT, Expand);
451 setOperationAction(ISD::FSIN, VT, Expand);
452 setOperationAction(ISD::FCOS, VT, Expand);
453 setOperationAction(ISD::FABS, VT, Expand);
454 setOperationAction(ISD::FPOWI, VT, Expand);
Craig Topperc4343f22012-11-14 08:11:25 +0000455 setOperationAction(ISD::FFLOOR, VT, Expand);
Craig Topper61d04572012-11-15 06:51:10 +0000456 setOperationAction(ISD::FCEIL, VT, Expand);
457 setOperationAction(ISD::FTRUNC, VT, Expand);
458 setOperationAction(ISD::FRINT, VT, Expand);
459 setOperationAction(ISD::FNEARBYINT, VT, Expand);
Duncan Sands13237ac2008-06-06 12:08:01 +0000460 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand);
461 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
462 setOperationAction(ISD::BUILD_VECTOR, VT, Expand);
Ulrich Weigand51eccec2014-08-04 13:27:12 +0000463 setOperationAction(ISD::MULHU, VT, Expand);
464 setOperationAction(ISD::MULHS, VT, Expand);
Duncan Sands13237ac2008-06-06 12:08:01 +0000465 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
466 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
467 setOperationAction(ISD::UDIVREM, VT, Expand);
468 setOperationAction(ISD::SDIVREM, VT, Expand);
469 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand);
470 setOperationAction(ISD::FPOW, VT, Expand);
Benjamin Kramerf3ad2352014-05-19 13:12:38 +0000471 setOperationAction(ISD::BSWAP, VT, Expand);
Chandler Carruth637cc6a2011-12-13 01:56:10 +0000472 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand);
Duncan Sands13237ac2008-06-06 12:08:01 +0000473 setOperationAction(ISD::CTTZ, VT, Expand);
Chandler Carruth637cc6a2011-12-13 01:56:10 +0000474 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand);
Benjamin Kramerc5071462012-12-19 15:49:14 +0000475 setOperationAction(ISD::VSELECT, VT, Expand);
Adhemerval Zanellac4182d12012-11-05 17:15:56 +0000476 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
477
Ahmed Bougacha2b6917b2015-01-08 00:51:32 +0000478 for (MVT InnerVT : MVT::vector_valuetypes()) {
Adhemerval Zanellac4182d12012-11-05 17:15:56 +0000479 setTruncStoreAction(VT, InnerVT, Expand);
Ahmed Bougacha2b6917b2015-01-08 00:51:32 +0000480 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
481 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
482 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
483 }
Chris Lattnerbaa73e02006-03-31 19:52:36 +0000484 }
485
Chris Lattner95c7adc2006-04-04 17:25:31 +0000486 // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle
487 // with merges, splats, etc.
Owen Anderson9f944592009-08-11 20:47:22 +0000488 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom);
Chris Lattner95c7adc2006-04-04 17:25:31 +0000489
Owen Anderson9f944592009-08-11 20:47:22 +0000490 setOperationAction(ISD::AND , MVT::v4i32, Legal);
491 setOperationAction(ISD::OR , MVT::v4i32, Legal);
492 setOperationAction(ISD::XOR , MVT::v4i32, Legal);
493 setOperationAction(ISD::LOAD , MVT::v4i32, Legal);
Hal Finkel940ab932014-02-28 00:27:01 +0000494 setOperationAction(ISD::SELECT, MVT::v4i32,
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000495 Subtarget.useCRBits() ? Legal : Expand);
Owen Anderson9f944592009-08-11 20:47:22 +0000496 setOperationAction(ISD::STORE , MVT::v4i32, Legal);
Adhemerval Zanella5c6e0842012-10-08 17:27:24 +0000497 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal);
498 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal);
499 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal);
500 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal);
Adhemerval Zanellabdface52012-11-15 20:56:03 +0000501 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
502 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
503 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
504 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000505
Craig Topperabadc662012-04-20 06:31:50 +0000506 addRegisterClass(MVT::v4f32, &PPC::VRRCRegClass);
507 addRegisterClass(MVT::v4i32, &PPC::VRRCRegClass);
508 addRegisterClass(MVT::v8i16, &PPC::VRRCRegClass);
509 addRegisterClass(MVT::v16i8, &PPC::VRRCRegClass);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000510
Owen Anderson9f944592009-08-11 20:47:22 +0000511 setOperationAction(ISD::MUL, MVT::v4f32, Legal);
Hal Finkel0a479ae2012-06-22 00:49:52 +0000512 setOperationAction(ISD::FMA, MVT::v4f32, Legal);
Hal Finkel2e103312013-04-03 04:01:11 +0000513
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000514 if (TM.Options.UnsafeFPMath || Subtarget.hasVSX()) {
Hal Finkel2e103312013-04-03 04:01:11 +0000515 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
516 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
517 }
518
Kit Barton20d39812015-03-10 19:49:38 +0000519
520 if (Subtarget.hasP8Altivec())
521 setOperationAction(ISD::MUL, MVT::v4i32, Legal);
522 else
523 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
524
Owen Anderson9f944592009-08-11 20:47:22 +0000525 setOperationAction(ISD::MUL, MVT::v8i16, Custom);
526 setOperationAction(ISD::MUL, MVT::v16i8, Custom);
Chris Lattnera8713b12006-03-20 01:53:53 +0000527
Owen Anderson9f944592009-08-11 20:47:22 +0000528 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom);
529 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000530
Owen Anderson9f944592009-08-11 20:47:22 +0000531 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Custom);
532 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom);
533 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Custom);
534 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
Adhemerval Zanella56775e02012-10-30 13:50:19 +0000535
536 // Altivec does not contain unordered floating-point compare instructions
537 setCondCodeAction(ISD::SETUO, MVT::v4f32, Expand);
538 setCondCodeAction(ISD::SETUEQ, MVT::v4f32, Expand);
Hal Finkel21ada792013-07-08 20:00:03 +0000539 setCondCodeAction(ISD::SETO, MVT::v4f32, Expand);
540 setCondCodeAction(ISD::SETONE, MVT::v4f32, Expand);
Hal Finkel27774d92014-03-13 07:58:58 +0000541
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000542 if (Subtarget.hasVSX()) {
Hal Finkel27774d92014-03-13 07:58:58 +0000543 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal);
Hal Finkel82569b62014-03-27 22:22:48 +0000544 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal);
Hal Finkel27774d92014-03-13 07:58:58 +0000545
546 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal);
547 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal);
548 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal);
549 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal);
550 setOperationAction(ISD::FROUND, MVT::v2f64, Legal);
551
552 setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
553
554 setOperationAction(ISD::MUL, MVT::v2f64, Legal);
555 setOperationAction(ISD::FMA, MVT::v2f64, Legal);
556
557 setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
558 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal);
559
Hal Finkel732f0f72014-03-26 12:49:28 +0000560 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
561 setOperationAction(ISD::VSELECT, MVT::v8i16, Legal);
562 setOperationAction(ISD::VSELECT, MVT::v4i32, Legal);
563 setOperationAction(ISD::VSELECT, MVT::v4f32, Legal);
564 setOperationAction(ISD::VSELECT, MVT::v2f64, Legal);
565
Hal Finkel27774d92014-03-13 07:58:58 +0000566 // Share the Altivec comparison restrictions.
567 setCondCodeAction(ISD::SETUO, MVT::v2f64, Expand);
568 setCondCodeAction(ISD::SETUEQ, MVT::v2f64, Expand);
Hal Finkel27774d92014-03-13 07:58:58 +0000569 setCondCodeAction(ISD::SETO, MVT::v2f64, Expand);
570 setCondCodeAction(ISD::SETONE, MVT::v2f64, Expand);
571
Hal Finkel9281c9a2014-03-26 18:26:30 +0000572 setOperationAction(ISD::LOAD, MVT::v2f64, Legal);
573 setOperationAction(ISD::STORE, MVT::v2f64, Legal);
574
Hal Finkeldf3e34d2014-03-26 22:58:37 +0000575 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Legal);
576
Hal Finkel19be5062014-03-29 05:29:01 +0000577 addRegisterClass(MVT::f64, &PPC::VSFRCRegClass);
Hal Finkel27774d92014-03-13 07:58:58 +0000578
579 addRegisterClass(MVT::v4f32, &PPC::VSRCRegClass);
580 addRegisterClass(MVT::v2f64, &PPC::VSRCRegClass);
Hal Finkela6c8b512014-03-26 16:12:58 +0000581
Kit Barton0cfa7b72015-03-03 19:55:45 +0000582 if (Subtarget.hasP8Altivec()) {
Kit Bartone48b1e12015-03-05 16:24:38 +0000583 setOperationAction(ISD::SHL, MVT::v2i64, Legal);
584 setOperationAction(ISD::SRA, MVT::v2i64, Legal);
585 setOperationAction(ISD::SRL, MVT::v2i64, Legal);
586
Kit Barton0cfa7b72015-03-03 19:55:45 +0000587 setOperationAction(ISD::SETCC, MVT::v2i64, Legal);
588 }
589 else {
Kit Bartone48b1e12015-03-05 16:24:38 +0000590 setOperationAction(ISD::SHL, MVT::v2i64, Expand);
591 setOperationAction(ISD::SRA, MVT::v2i64, Expand);
592 setOperationAction(ISD::SRL, MVT::v2i64, Expand);
593
Kit Barton0cfa7b72015-03-03 19:55:45 +0000594 setOperationAction(ISD::SETCC, MVT::v2i64, Custom);
595
596 // VSX v2i64 only supports non-arithmetic operations.
597 setOperationAction(ISD::ADD, MVT::v2i64, Expand);
598 setOperationAction(ISD::SUB, MVT::v2i64, Expand);
599 }
Hal Finkel777c9dd2014-03-29 16:04:40 +0000600
Hal Finkel9281c9a2014-03-26 18:26:30 +0000601 setOperationAction(ISD::LOAD, MVT::v2i64, Promote);
602 AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64);
603 setOperationAction(ISD::STORE, MVT::v2i64, Promote);
604 AddPromotedToType (ISD::STORE, MVT::v2i64, MVT::v2f64);
605
Hal Finkeldf3e34d2014-03-26 22:58:37 +0000606 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Legal);
607
Hal Finkel7279f4b2014-03-26 19:13:54 +0000608 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal);
609 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal);
610 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal);
611 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal);
612
Hal Finkel5c0d1452014-03-30 13:22:59 +0000613 // Vector operation legalization checks the result type of
614 // SIGN_EXTEND_INREG, overall legalization checks the inner type.
615 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i64, Legal);
616 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal);
617 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Custom);
618 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Custom);
619
Hal Finkela6c8b512014-03-26 16:12:58 +0000620 addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass);
Hal Finkel27774d92014-03-13 07:58:58 +0000621 }
Bill Schmidtfe88b182015-02-03 21:58:23 +0000622
623 if (Subtarget.hasP8Altivec())
624 addRegisterClass(MVT::v2i64, &PPC::VRRCRegClass);
Nate Begeman3e7db9c2005-11-29 08:17:20 +0000625 }
Scott Michelcf0da6c2009-02-17 22:15:04 +0000626
Hal Finkelc93a9a22015-02-25 01:06:45 +0000627 if (Subtarget.hasQPX()) {
628 setOperationAction(ISD::FADD, MVT::v4f64, Legal);
629 setOperationAction(ISD::FSUB, MVT::v4f64, Legal);
630 setOperationAction(ISD::FMUL, MVT::v4f64, Legal);
631 setOperationAction(ISD::FREM, MVT::v4f64, Expand);
632
633 setOperationAction(ISD::FCOPYSIGN, MVT::v4f64, Legal);
634 setOperationAction(ISD::FGETSIGN, MVT::v4f64, Expand);
635
636 setOperationAction(ISD::LOAD , MVT::v4f64, Custom);
637 setOperationAction(ISD::STORE , MVT::v4f64, Custom);
638
639 setTruncStoreAction(MVT::v4f64, MVT::v4f32, Custom);
640 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Custom);
641
642 if (!Subtarget.useCRBits())
643 setOperationAction(ISD::SELECT, MVT::v4f64, Expand);
644 setOperationAction(ISD::VSELECT, MVT::v4f64, Legal);
645
646 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f64, Legal);
647 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f64, Expand);
648 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f64, Expand);
649 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f64, Expand);
650 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f64, Custom);
651 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f64, Legal);
652 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom);
653
654 setOperationAction(ISD::FP_TO_SINT , MVT::v4f64, Legal);
655 setOperationAction(ISD::FP_TO_UINT , MVT::v4f64, Expand);
656
657 setOperationAction(ISD::FP_ROUND , MVT::v4f32, Legal);
658 setOperationAction(ISD::FP_ROUND_INREG , MVT::v4f32, Expand);
659 setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Legal);
660
661 setOperationAction(ISD::FNEG , MVT::v4f64, Legal);
662 setOperationAction(ISD::FABS , MVT::v4f64, Legal);
663 setOperationAction(ISD::FSIN , MVT::v4f64, Expand);
664 setOperationAction(ISD::FCOS , MVT::v4f64, Expand);
665 setOperationAction(ISD::FPOWI , MVT::v4f64, Expand);
666 setOperationAction(ISD::FPOW , MVT::v4f64, Expand);
667 setOperationAction(ISD::FLOG , MVT::v4f64, Expand);
668 setOperationAction(ISD::FLOG2 , MVT::v4f64, Expand);
669 setOperationAction(ISD::FLOG10 , MVT::v4f64, Expand);
670 setOperationAction(ISD::FEXP , MVT::v4f64, Expand);
671 setOperationAction(ISD::FEXP2 , MVT::v4f64, Expand);
672
673 setOperationAction(ISD::FMINNUM, MVT::v4f64, Legal);
674 setOperationAction(ISD::FMAXNUM, MVT::v4f64, Legal);
675
676 setIndexedLoadAction(ISD::PRE_INC, MVT::v4f64, Legal);
677 setIndexedStoreAction(ISD::PRE_INC, MVT::v4f64, Legal);
678
679 addRegisterClass(MVT::v4f64, &PPC::QFRCRegClass);
680
681 setOperationAction(ISD::FADD, MVT::v4f32, Legal);
682 setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
683 setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
684 setOperationAction(ISD::FREM, MVT::v4f32, Expand);
685
686 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal);
687 setOperationAction(ISD::FGETSIGN, MVT::v4f32, Expand);
688
689 setOperationAction(ISD::LOAD , MVT::v4f32, Custom);
690 setOperationAction(ISD::STORE , MVT::v4f32, Custom);
691
692 if (!Subtarget.useCRBits())
693 setOperationAction(ISD::SELECT, MVT::v4f32, Expand);
694 setOperationAction(ISD::VSELECT, MVT::v4f32, Legal);
695
696 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f32, Legal);
697 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f32, Expand);
698 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f32, Expand);
699 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f32, Expand);
700 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f32, Custom);
701 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal);
702 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
703
704 setOperationAction(ISD::FP_TO_SINT , MVT::v4f32, Legal);
705 setOperationAction(ISD::FP_TO_UINT , MVT::v4f32, Expand);
706
707 setOperationAction(ISD::FNEG , MVT::v4f32, Legal);
708 setOperationAction(ISD::FABS , MVT::v4f32, Legal);
709 setOperationAction(ISD::FSIN , MVT::v4f32, Expand);
710 setOperationAction(ISD::FCOS , MVT::v4f32, Expand);
711 setOperationAction(ISD::FPOWI , MVT::v4f32, Expand);
712 setOperationAction(ISD::FPOW , MVT::v4f32, Expand);
713 setOperationAction(ISD::FLOG , MVT::v4f32, Expand);
714 setOperationAction(ISD::FLOG2 , MVT::v4f32, Expand);
715 setOperationAction(ISD::FLOG10 , MVT::v4f32, Expand);
716 setOperationAction(ISD::FEXP , MVT::v4f32, Expand);
717 setOperationAction(ISD::FEXP2 , MVT::v4f32, Expand);
718
719 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
720 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
721
722 setIndexedLoadAction(ISD::PRE_INC, MVT::v4f32, Legal);
723 setIndexedStoreAction(ISD::PRE_INC, MVT::v4f32, Legal);
724
725 addRegisterClass(MVT::v4f32, &PPC::QSRCRegClass);
726
727 setOperationAction(ISD::AND , MVT::v4i1, Legal);
728 setOperationAction(ISD::OR , MVT::v4i1, Legal);
729 setOperationAction(ISD::XOR , MVT::v4i1, Legal);
730
731 if (!Subtarget.useCRBits())
732 setOperationAction(ISD::SELECT, MVT::v4i1, Expand);
733 setOperationAction(ISD::VSELECT, MVT::v4i1, Legal);
734
735 setOperationAction(ISD::LOAD , MVT::v4i1, Custom);
736 setOperationAction(ISD::STORE , MVT::v4i1, Custom);
737
738 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4i1, Custom);
739 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4i1, Expand);
740 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4i1, Expand);
741 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4i1, Expand);
742 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4i1, Custom);
743 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i1, Expand);
744 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom);
745
746 setOperationAction(ISD::SINT_TO_FP, MVT::v4i1, Custom);
747 setOperationAction(ISD::UINT_TO_FP, MVT::v4i1, Custom);
748
749 addRegisterClass(MVT::v4i1, &PPC::QBRCRegClass);
750
751 setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal);
752 setOperationAction(ISD::FCEIL, MVT::v4f64, Legal);
753 setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal);
754 setOperationAction(ISD::FROUND, MVT::v4f64, Legal);
755
756 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
757 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
758 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
759 setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
760
761 setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Expand);
762 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand);
763
764 // These need to set FE_INEXACT, and so cannot be vectorized here.
765 setOperationAction(ISD::FRINT, MVT::v4f64, Expand);
766 setOperationAction(ISD::FRINT, MVT::v4f32, Expand);
767
768 if (TM.Options.UnsafeFPMath) {
769 setOperationAction(ISD::FDIV, MVT::v4f64, Legal);
770 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal);
771
772 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
773 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
774 } else {
775 setOperationAction(ISD::FDIV, MVT::v4f64, Expand);
776 setOperationAction(ISD::FSQRT, MVT::v4f64, Expand);
777
778 setOperationAction(ISD::FDIV, MVT::v4f32, Expand);
779 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
780 }
781 }
782
Hal Finkel01fa7702014-12-03 00:19:17 +0000783 if (Subtarget.has64BitSupport())
Hal Finkel322e41a2012-04-01 20:08:17 +0000784 setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
Hal Finkel01fa7702014-12-03 00:19:17 +0000785
786 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom);
Hal Finkel322e41a2012-04-01 20:08:17 +0000787
Robin Morissete1ca44b2014-10-02 22:27:07 +0000788 if (!isPPC64) {
789 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Expand);
790 setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand);
791 }
Eli Friedman7dfa7912011-08-29 18:23:02 +0000792
Duncan Sands8d6e2e12008-11-23 15:47:28 +0000793 setBooleanContents(ZeroOrOneBooleanContent);
Hal Finkelc93a9a22015-02-25 01:06:45 +0000794
795 if (Subtarget.hasAltivec()) {
796 // Altivec instructions set fields to all zeros or all ones.
797 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
798 }
Scott Michelcf0da6c2009-02-17 22:15:04 +0000799
Joerg Sonnenbergerb5459e62014-07-24 22:20:10 +0000800 if (!isPPC64) {
801 // These libcalls are not available in 32-bit.
802 setLibcallName(RTLIB::SHL_I128, nullptr);
803 setLibcallName(RTLIB::SRL_I128, nullptr);
804 setLibcallName(RTLIB::SRA_I128, nullptr);
805 }
806
Evan Cheng39e90022012-07-02 22:39:56 +0000807 if (isPPC64) {
Chris Lattner454436d2006-10-18 01:20:43 +0000808 setStackPointerRegisterToSaveRestore(PPC::X1);
Jim Laskeye0008e22007-02-22 14:56:36 +0000809 setExceptionPointerRegister(PPC::X3);
810 setExceptionSelectorRegister(PPC::X4);
811 } else {
Chris Lattner454436d2006-10-18 01:20:43 +0000812 setStackPointerRegisterToSaveRestore(PPC::R1);
Jim Laskeye0008e22007-02-22 14:56:36 +0000813 setExceptionPointerRegister(PPC::R3);
814 setExceptionSelectorRegister(PPC::R4);
815 }
Scott Michelcf0da6c2009-02-17 22:15:04 +0000816
Chris Lattnerf4184352006-03-01 04:57:39 +0000817 // We have target-specific dag combine patterns for the following nodes:
818 setTargetDAGCombine(ISD::SINT_TO_FP);
Hal Finkel5efb9182015-01-06 06:01:57 +0000819 if (Subtarget.hasFPCVT())
820 setTargetDAGCombine(ISD::UINT_TO_FP);
Hal Finkelcf2e9082013-05-24 23:00:14 +0000821 setTargetDAGCombine(ISD::LOAD);
Chris Lattner27f53452006-03-01 05:50:56 +0000822 setTargetDAGCombine(ISD::STORE);
Chris Lattner9754d142006-04-18 17:59:36 +0000823 setTargetDAGCombine(ISD::BR_CC);
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000824 if (Subtarget.useCRBits())
Hal Finkel940ab932014-02-28 00:27:01 +0000825 setTargetDAGCombine(ISD::BRCOND);
Chris Lattnera7976d32006-07-10 20:56:58 +0000826 setTargetDAGCombine(ISD::BSWAP);
Hal Finkelbc2ee4c2013-05-25 04:05:05 +0000827 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
Bill Schmidtfae5d712014-12-09 16:35:51 +0000828 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
829 setTargetDAGCombine(ISD::INTRINSIC_VOID);
Scott Michelcf0da6c2009-02-17 22:15:04 +0000830
Hal Finkel46043ed2014-03-01 21:36:57 +0000831 setTargetDAGCombine(ISD::SIGN_EXTEND);
832 setTargetDAGCombine(ISD::ZERO_EXTEND);
833 setTargetDAGCombine(ISD::ANY_EXTEND);
834
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000835 if (Subtarget.useCRBits()) {
Hal Finkel940ab932014-02-28 00:27:01 +0000836 setTargetDAGCombine(ISD::TRUNCATE);
837 setTargetDAGCombine(ISD::SETCC);
838 setTargetDAGCombine(ISD::SELECT_CC);
839 }
840
Hal Finkel2e103312013-04-03 04:01:11 +0000841 // Use reciprocal estimates.
842 if (TM.Options.UnsafeFPMath) {
843 setTargetDAGCombine(ISD::FDIV);
844 setTargetDAGCombine(ISD::FSQRT);
845 }
846
Dale Johannesen10432e52007-10-19 00:59:18 +0000847 // Darwin long double math library functions have $LDBL128 appended.
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000848 if (Subtarget.isDarwin()) {
Duncan Sands53c954f2008-01-10 10:28:30 +0000849 setLibcallName(RTLIB::COS_PPCF128, "cosl$LDBL128");
Dale Johannesen10432e52007-10-19 00:59:18 +0000850 setLibcallName(RTLIB::POW_PPCF128, "powl$LDBL128");
851 setLibcallName(RTLIB::REM_PPCF128, "fmodl$LDBL128");
Duncan Sands53c954f2008-01-10 10:28:30 +0000852 setLibcallName(RTLIB::SIN_PPCF128, "sinl$LDBL128");
853 setLibcallName(RTLIB::SQRT_PPCF128, "sqrtl$LDBL128");
Dale Johannesenda2d8062008-09-04 00:47:13 +0000854 setLibcallName(RTLIB::LOG_PPCF128, "logl$LDBL128");
855 setLibcallName(RTLIB::LOG2_PPCF128, "log2l$LDBL128");
856 setLibcallName(RTLIB::LOG10_PPCF128, "log10l$LDBL128");
857 setLibcallName(RTLIB::EXP_PPCF128, "expl$LDBL128");
858 setLibcallName(RTLIB::EXP2_PPCF128, "exp2l$LDBL128");
Dale Johannesen10432e52007-10-19 00:59:18 +0000859 }
860
Hal Finkel940ab932014-02-28 00:27:01 +0000861 // With 32 condition bits, we don't need to sink (and duplicate) compares
862 // aggressively in CodeGenPrep.
Hal Finkel7a0516e2015-02-12 01:02:52 +0000863 if (Subtarget.useCRBits()) {
Hal Finkel940ab932014-02-28 00:27:01 +0000864 setHasMultipleConditionRegisters();
Hal Finkel7a0516e2015-02-12 01:02:52 +0000865 setJumpIsExpensive();
866 }
Hal Finkel940ab932014-02-28 00:27:01 +0000867
Hal Finkel65298572011-10-17 18:53:03 +0000868 setMinFunctionAlignment(2);
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000869 if (Subtarget.isDarwin())
Hal Finkel65298572011-10-17 18:53:03 +0000870 setPrefFunctionAlignment(4);
Eli Friedman2518f832011-05-06 20:34:06 +0000871
Hal Finkeld73bfba2015-01-03 14:58:25 +0000872 switch (Subtarget.getDarwinDirective()) {
873 default: break;
874 case PPC::DIR_970:
875 case PPC::DIR_A2:
876 case PPC::DIR_E500mc:
877 case PPC::DIR_E5500:
878 case PPC::DIR_PWR4:
879 case PPC::DIR_PWR5:
880 case PPC::DIR_PWR5X:
881 case PPC::DIR_PWR6:
882 case PPC::DIR_PWR6X:
883 case PPC::DIR_PWR7:
884 case PPC::DIR_PWR8:
885 setPrefFunctionAlignment(4);
886 setPrefLoopAlignment(4);
887 break;
888 }
889
Eli Friedman30a49e92011-08-03 21:06:02 +0000890 setInsertFencesForAtomic(true);
891
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000892 if (Subtarget.enableMachineScheduler())
Hal Finkel21442b22013-09-11 23:05:25 +0000893 setSchedulingPreference(Sched::Source);
894 else
895 setSchedulingPreference(Sched::Hybrid);
Hal Finkel6f0ae782011-11-22 16:21:04 +0000896
Eric Christopher23a3a7c2015-02-26 00:00:24 +0000897 computeRegisterProperties(STI.getRegisterInfo());
Hal Finkel742b5352012-08-28 16:12:39 +0000898
Hal Finkeld73bfba2015-01-03 14:58:25 +0000899 // The Freescale cores do better with aggressive inlining of memcpy and
900 // friends. GCC uses same threshold of 128 bytes (= 32 word stores).
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000901 if (Subtarget.getDarwinDirective() == PPC::DIR_E500mc ||
902 Subtarget.getDarwinDirective() == PPC::DIR_E5500) {
Jim Grosbach341ad3e2013-02-20 21:13:59 +0000903 MaxStoresPerMemset = 32;
904 MaxStoresPerMemsetOptSize = 16;
905 MaxStoresPerMemcpy = 32;
906 MaxStoresPerMemcpyOptSize = 8;
907 MaxStoresPerMemmove = 32;
908 MaxStoresPerMemmoveOptSize = 8;
Hal Finkel5c3cacf2015-02-27 19:58:28 +0000909 } else if (Subtarget.getDarwinDirective() == PPC::DIR_A2) {
910 // The A2 also benefits from (very) aggressive inlining of memcpy and
911 // friends. The overhead of a the function call, even when warm, can be
912 // over one hundred cycles.
913 MaxStoresPerMemset = 128;
914 MaxStoresPerMemcpy = 128;
915 MaxStoresPerMemmove = 128;
Hal Finkel742b5352012-08-28 16:12:39 +0000916 }
Chris Lattnerf22556d2005-08-16 17:14:42 +0000917}
918
Hal Finkel262a2242013-09-12 23:20:06 +0000919/// getMaxByValAlign - Helper for getByValTypeAlignment to determine
920/// the desired ByVal argument alignment.
921static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign,
922 unsigned MaxMaxAlign) {
923 if (MaxAlign == MaxMaxAlign)
924 return;
925 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
926 if (MaxMaxAlign >= 32 && VTy->getBitWidth() >= 256)
927 MaxAlign = 32;
928 else if (VTy->getBitWidth() >= 128 && MaxAlign < 16)
929 MaxAlign = 16;
930 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
931 unsigned EltAlign = 0;
932 getMaxByValAlign(ATy->getElementType(), EltAlign, MaxMaxAlign);
933 if (EltAlign > MaxAlign)
934 MaxAlign = EltAlign;
935 } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
936 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
937 unsigned EltAlign = 0;
938 getMaxByValAlign(STy->getElementType(i), EltAlign, MaxMaxAlign);
939 if (EltAlign > MaxAlign)
940 MaxAlign = EltAlign;
941 if (MaxAlign == MaxMaxAlign)
942 break;
943 }
944 }
945}
946
Dale Johannesencbde4c22008-02-28 22:31:51 +0000947/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
948/// function arguments in the caller parameter area.
Chris Lattner229907c2011-07-18 04:54:35 +0000949unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty) const {
Dale Johannesencbde4c22008-02-28 22:31:51 +0000950 // Darwin passes everything on 4 byte boundary.
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000951 if (Subtarget.isDarwin())
Dale Johannesencbde4c22008-02-28 22:31:51 +0000952 return 4;
Roman Divackyb9663cc2012-04-02 15:49:30 +0000953
954 // 16byte and wider vectors are passed on 16byte boundary.
Roman Divackyb9663cc2012-04-02 15:49:30 +0000955 // The rest is 8 on PPC64 and 4 on PPC32 boundary.
Eric Christopherb1aaebe2014-06-12 22:38:18 +0000956 unsigned Align = Subtarget.isPPC64() ? 8 : 4;
957 if (Subtarget.hasAltivec() || Subtarget.hasQPX())
958 getMaxByValAlign(Ty, Align, Subtarget.hasQPX() ? 32 : 16);
Hal Finkel262a2242013-09-12 23:20:06 +0000959 return Align;
Dale Johannesencbde4c22008-02-28 22:31:51 +0000960}
961
Chris Lattner347ed8a2006-01-09 23:52:17 +0000962const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
963 switch (Opcode) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000964 default: return nullptr;
Evan Cheng32e376f2008-07-12 02:23:19 +0000965 case PPCISD::FSEL: return "PPCISD::FSEL";
966 case PPCISD::FCFID: return "PPCISD::FCFID";
Hal Finkel3fe09ea2015-01-06 07:02:15 +0000967 case PPCISD::FCFIDU: return "PPCISD::FCFIDU";
968 case PPCISD::FCFIDS: return "PPCISD::FCFIDS";
969 case PPCISD::FCFIDUS: return "PPCISD::FCFIDUS";
Evan Cheng32e376f2008-07-12 02:23:19 +0000970 case PPCISD::FCTIDZ: return "PPCISD::FCTIDZ";
971 case PPCISD::FCTIWZ: return "PPCISD::FCTIWZ";
Hal Finkel3fe09ea2015-01-06 07:02:15 +0000972 case PPCISD::FCTIDUZ: return "PPCISD::FCTIDUZ";
973 case PPCISD::FCTIWUZ: return "PPCISD::FCTIWUZ";
Hal Finkel2e103312013-04-03 04:01:11 +0000974 case PPCISD::FRE: return "PPCISD::FRE";
975 case PPCISD::FRSQRTE: return "PPCISD::FRSQRTE";
Evan Cheng32e376f2008-07-12 02:23:19 +0000976 case PPCISD::STFIWX: return "PPCISD::STFIWX";
977 case PPCISD::VMADDFP: return "PPCISD::VMADDFP";
978 case PPCISD::VNMSUBFP: return "PPCISD::VNMSUBFP";
979 case PPCISD::VPERM: return "PPCISD::VPERM";
Hal Finkel4edc66b2015-01-03 01:16:37 +0000980 case PPCISD::CMPB: return "PPCISD::CMPB";
Evan Cheng32e376f2008-07-12 02:23:19 +0000981 case PPCISD::Hi: return "PPCISD::Hi";
982 case PPCISD::Lo: return "PPCISD::Lo";
Tilmann Schellerd1aaa322009-08-15 11:54:46 +0000983 case PPCISD::TOC_ENTRY: return "PPCISD::TOC_ENTRY";
Evan Cheng32e376f2008-07-12 02:23:19 +0000984 case PPCISD::DYNALLOC: return "PPCISD::DYNALLOC";
985 case PPCISD::GlobalBaseReg: return "PPCISD::GlobalBaseReg";
986 case PPCISD::SRL: return "PPCISD::SRL";
987 case PPCISD::SRA: return "PPCISD::SRA";
988 case PPCISD::SHL: return "PPCISD::SHL";
Ulrich Weigandf62e83f2013-03-22 15:24:13 +0000989 case PPCISD::CALL: return "PPCISD::CALL";
990 case PPCISD::CALL_NOP: return "PPCISD::CALL_NOP";
Evan Cheng32e376f2008-07-12 02:23:19 +0000991 case PPCISD::MTCTR: return "PPCISD::MTCTR";
Ulrich Weigandf62e83f2013-03-22 15:24:13 +0000992 case PPCISD::BCTRL: return "PPCISD::BCTRL";
Hal Finkelfc096c92014-12-23 22:29:40 +0000993 case PPCISD::BCTRL_LOAD_TOC: return "PPCISD::BCTRL_LOAD_TOC";
Evan Cheng32e376f2008-07-12 02:23:19 +0000994 case PPCISD::RET_FLAG: return "PPCISD::RET_FLAG";
Hal Finkelbbdee932014-12-02 22:01:00 +0000995 case PPCISD::READ_TIME_BASE: return "PPCISD::READ_TIME_BASE";
Hal Finkel756810f2013-03-21 21:37:52 +0000996 case PPCISD::EH_SJLJ_SETJMP: return "PPCISD::EH_SJLJ_SETJMP";
997 case PPCISD::EH_SJLJ_LONGJMP: return "PPCISD::EH_SJLJ_LONGJMP";
Ulrich Weigandd5ebc622013-07-03 17:05:42 +0000998 case PPCISD::MFOCRF: return "PPCISD::MFOCRF";
Nemanja Ivanovicc38b5312015-04-11 10:40:42 +0000999 case PPCISD::MFVSR: return "PPCISD::MFVSR";
1000 case PPCISD::MTVSRA: return "PPCISD::MTVSRA";
1001 case PPCISD::MTVSRZ: return "PPCISD::MTVSRZ";
Evan Cheng32e376f2008-07-12 02:23:19 +00001002 case PPCISD::VCMP: return "PPCISD::VCMP";
1003 case PPCISD::VCMPo: return "PPCISD::VCMPo";
1004 case PPCISD::LBRX: return "PPCISD::LBRX";
1005 case PPCISD::STBRX: return "PPCISD::STBRX";
Hal Finkel3fe09ea2015-01-06 07:02:15 +00001006 case PPCISD::LFIWAX: return "PPCISD::LFIWAX";
1007 case PPCISD::LFIWZX: return "PPCISD::LFIWZX";
Evan Cheng32e376f2008-07-12 02:23:19 +00001008 case PPCISD::COND_BRANCH: return "PPCISD::COND_BRANCH";
Hal Finkel25c19922013-05-15 21:37:41 +00001009 case PPCISD::BDNZ: return "PPCISD::BDNZ";
1010 case PPCISD::BDZ: return "PPCISD::BDZ";
Evan Cheng32e376f2008-07-12 02:23:19 +00001011 case PPCISD::MFFS: return "PPCISD::MFFS";
Evan Cheng32e376f2008-07-12 02:23:19 +00001012 case PPCISD::FADDRTZ: return "PPCISD::FADDRTZ";
Evan Cheng32e376f2008-07-12 02:23:19 +00001013 case PPCISD::TC_RETURN: return "PPCISD::TC_RETURN";
Hal Finkel5ab37802012-08-28 02:10:27 +00001014 case PPCISD::CR6SET: return "PPCISD::CR6SET";
1015 case PPCISD::CR6UNSET: return "PPCISD::CR6UNSET";
Roman Divacky32143e22013-12-20 18:08:54 +00001016 case PPCISD::PPC32_GOT: return "PPCISD::PPC32_GOT";
Bill Schmidt9f0b4ec2012-12-14 17:02:38 +00001017 case PPCISD::ADDIS_GOT_TPREL_HA: return "PPCISD::ADDIS_GOT_TPREL_HA";
1018 case PPCISD::LD_GOT_TPREL_L: return "PPCISD::LD_GOT_TPREL_L";
Bill Schmidtca4a0c92012-12-04 16:18:08 +00001019 case PPCISD::ADD_TLS: return "PPCISD::ADD_TLS";
Bill Schmidtc56f1d32012-12-11 20:30:11 +00001020 case PPCISD::ADDIS_TLSGD_HA: return "PPCISD::ADDIS_TLSGD_HA";
1021 case PPCISD::ADDI_TLSGD_L: return "PPCISD::ADDI_TLSGD_L";
Bill Schmidt82f1c772015-02-10 19:09:05 +00001022 case PPCISD::GET_TLS_ADDR: return "PPCISD::GET_TLS_ADDR";
1023 case PPCISD::ADDI_TLSGD_L_ADDR: return "PPCISD::ADDI_TLSGD_L_ADDR";
Bill Schmidt24b8dd62012-12-12 19:29:35 +00001024 case PPCISD::ADDIS_TLSLD_HA: return "PPCISD::ADDIS_TLSLD_HA";
1025 case PPCISD::ADDI_TLSLD_L: return "PPCISD::ADDI_TLSLD_L";
Bill Schmidt82f1c772015-02-10 19:09:05 +00001026 case PPCISD::GET_TLSLD_ADDR: return "PPCISD::GET_TLSLD_ADDR";
1027 case PPCISD::ADDI_TLSLD_L_ADDR: return "PPCISD::ADDI_TLSLD_L_ADDR";
Bill Schmidt24b8dd62012-12-12 19:29:35 +00001028 case PPCISD::ADDIS_DTPREL_HA: return "PPCISD::ADDIS_DTPREL_HA";
1029 case PPCISD::ADDI_DTPREL_L: return "PPCISD::ADDI_DTPREL_L";
Bill Schmidt51e79512013-02-20 15:50:31 +00001030 case PPCISD::VADD_SPLAT: return "PPCISD::VADD_SPLAT";
Bill Schmidta87a7e22013-05-14 19:35:45 +00001031 case PPCISD::SC: return "PPCISD::SC";
Hal Finkelc93a9a22015-02-25 01:06:45 +00001032 case PPCISD::QVFPERM: return "PPCISD::QVFPERM";
1033 case PPCISD::QVGPCI: return "PPCISD::QVGPCI";
1034 case PPCISD::QVALIGNI: return "PPCISD::QVALIGNI";
1035 case PPCISD::QVESPLATI: return "PPCISD::QVESPLATI";
1036 case PPCISD::QBFLT: return "PPCISD::QBFLT";
1037 case PPCISD::QVLFSb: return "PPCISD::QVLFSb";
Chris Lattner347ed8a2006-01-09 23:52:17 +00001038 }
1039}
1040
Hal Finkelc93a9a22015-02-25 01:06:45 +00001041EVT PPCTargetLowering::getSetCCResultType(LLVMContext &C, EVT VT) const {
Adhemerval Zanellafe3f7932012-10-08 18:59:53 +00001042 if (!VT.isVector())
Eric Christopherb1aaebe2014-06-12 22:38:18 +00001043 return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
Hal Finkelc93a9a22015-02-25 01:06:45 +00001044
1045 if (Subtarget.hasQPX())
1046 return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements());
1047
Adhemerval Zanellafe3f7932012-10-08 18:59:53 +00001048 return VT.changeVectorElementTypeToInteger();
Scott Michela6729e82008-03-10 15:42:14 +00001049}
1050
Hal Finkel62ac7362014-09-19 11:42:56 +00001051bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const {
1052 assert(VT.isFloatingPoint() && "Non-floating-point FMA?");
1053 return true;
1054}
1055
Chris Lattner4211ca92006-04-14 06:01:58 +00001056//===----------------------------------------------------------------------===//
1057// Node matching predicates, for use by the tblgen matching code.
1058//===----------------------------------------------------------------------===//
1059
Chris Lattner7f1fa8e2005-08-26 17:36:52 +00001060/// isFloatingPointZero - Return true if this is 0.0 or -0.0.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001061static bool isFloatingPointZero(SDValue Op) {
Chris Lattner7f1fa8e2005-08-26 17:36:52 +00001062 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op))
Dale Johannesen3cf889f2007-08-31 04:03:46 +00001063 return CFP->getValueAPF().isZero();
Gabor Greiff304a7a2008-08-28 21:40:38 +00001064 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) {
Chris Lattner7f1fa8e2005-08-26 17:36:52 +00001065 // Maybe this has already been legalized into the constant pool?
1066 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op.getOperand(1)))
Dan Gohmanbcaf6812010-04-15 01:51:59 +00001067 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
Dale Johannesen3cf889f2007-08-31 04:03:46 +00001068 return CFP->getValueAPF().isZero();
Chris Lattner7f1fa8e2005-08-26 17:36:52 +00001069 }
1070 return false;
1071}
1072
Chris Lattnere8b83b42006-04-06 17:23:16 +00001073/// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return
1074/// true if Op is undef or if it matches the specified value.
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001075static bool isConstantOrUndef(int Op, int Val) {
1076 return Op < 0 || Op == Val;
Chris Lattnere8b83b42006-04-06 17:23:16 +00001077}
1078
1079/// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a
1080/// VPKUHUM instruction.
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001081/// The ShuffleKind distinguishes between big-endian operations with
1082/// two different inputs (0), either-endian operations with two identical
1083/// inputs (1), and little-endian operantion with two different inputs (2).
1084/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1085bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
Bill Schmidtf910a062014-06-10 14:35:01 +00001086 SelectionDAG &DAG) {
Eric Christopher8b770652015-01-26 19:03:15 +00001087 bool IsLE = DAG.getTarget().getDataLayout()->isLittleEndian();
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001088 if (ShuffleKind == 0) {
Eric Christopherd9134482014-08-04 21:25:23 +00001089 if (IsLE)
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001090 return false;
Chris Lattnera4bbfae2006-04-06 22:28:36 +00001091 for (unsigned i = 0; i != 16; ++i)
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001092 if (!isConstantOrUndef(N->getMaskElt(i), i*2+1))
Chris Lattnera4bbfae2006-04-06 22:28:36 +00001093 return false;
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001094 } else if (ShuffleKind == 2) {
Eric Christopherd9134482014-08-04 21:25:23 +00001095 if (!IsLE)
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001096 return false;
1097 for (unsigned i = 0; i != 16; ++i)
1098 if (!isConstantOrUndef(N->getMaskElt(i), i*2))
1099 return false;
1100 } else if (ShuffleKind == 1) {
Eric Christopherd9134482014-08-04 21:25:23 +00001101 unsigned j = IsLE ? 0 : 1;
Chris Lattnera4bbfae2006-04-06 22:28:36 +00001102 for (unsigned i = 0; i != 8; ++i)
Bill Schmidtf910a062014-06-10 14:35:01 +00001103 if (!isConstantOrUndef(N->getMaskElt(i), i*2+j) ||
1104 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j))
Chris Lattnera4bbfae2006-04-06 22:28:36 +00001105 return false;
1106 }
Chris Lattner1d338192006-04-06 18:26:28 +00001107 return true;
Chris Lattnere8b83b42006-04-06 17:23:16 +00001108}
1109
1110/// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a
1111/// VPKUWUM instruction.
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001112/// The ShuffleKind distinguishes between big-endian operations with
1113/// two different inputs (0), either-endian operations with two identical
1114/// inputs (1), and little-endian operantion with two different inputs (2).
1115/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1116bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
Bill Schmidtf910a062014-06-10 14:35:01 +00001117 SelectionDAG &DAG) {
Eric Christopher8b770652015-01-26 19:03:15 +00001118 bool IsLE = DAG.getTarget().getDataLayout()->isLittleEndian();
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001119 if (ShuffleKind == 0) {
Eric Christopherd9134482014-08-04 21:25:23 +00001120 if (IsLE)
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001121 return false;
Chris Lattnera4bbfae2006-04-06 22:28:36 +00001122 for (unsigned i = 0; i != 16; i += 2)
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001123 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) ||
1124 !isConstantOrUndef(N->getMaskElt(i+1), i*2+3))
Chris Lattnera4bbfae2006-04-06 22:28:36 +00001125 return false;
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001126 } else if (ShuffleKind == 2) {
Eric Christopherd9134482014-08-04 21:25:23 +00001127 if (!IsLE)
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001128 return false;
1129 for (unsigned i = 0; i != 16; i += 2)
1130 if (!isConstantOrUndef(N->getMaskElt(i ), i*2) ||
1131 !isConstantOrUndef(N->getMaskElt(i+1), i*2+1))
1132 return false;
1133 } else if (ShuffleKind == 1) {
Eric Christopherd9134482014-08-04 21:25:23 +00001134 unsigned j = IsLE ? 0 : 2;
Chris Lattnera4bbfae2006-04-06 22:28:36 +00001135 for (unsigned i = 0; i != 8; i += 2)
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00001136 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) ||
1137 !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) ||
1138 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) ||
1139 !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1))
Chris Lattnera4bbfae2006-04-06 22:28:36 +00001140 return false;
1141 }
Chris Lattner1d338192006-04-06 18:26:28 +00001142 return true;
Chris Lattnere8b83b42006-04-06 17:23:16 +00001143}
1144
Chris Lattnerf38e0332006-04-06 22:02:42 +00001145/// isVMerge - Common function, used to match vmrg* shuffles.
1146///
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001147static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize,
Chris Lattnerf38e0332006-04-06 22:02:42 +00001148 unsigned LHSStart, unsigned RHSStart) {
Hal Finkeldf3e34d2014-03-26 22:58:37 +00001149 if (N->getValueType(0) != MVT::v16i8)
1150 return false;
Chris Lattnerd1dcb522006-04-06 21:11:54 +00001151 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) &&
1152 "Unsupported merge size!");
Scott Michelcf0da6c2009-02-17 22:15:04 +00001153
Chris Lattnerd1dcb522006-04-06 21:11:54 +00001154 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units
1155 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001156 if (!isConstantOrUndef(N->getMaskElt(i*UnitSize*2+j),
Chris Lattnerf38e0332006-04-06 22:02:42 +00001157 LHSStart+j+i*UnitSize) ||
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001158 !isConstantOrUndef(N->getMaskElt(i*UnitSize*2+UnitSize+j),
Chris Lattnerf38e0332006-04-06 22:02:42 +00001159 RHSStart+j+i*UnitSize))
Chris Lattnerd1dcb522006-04-06 21:11:54 +00001160 return false;
1161 }
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001162 return true;
Chris Lattnerf38e0332006-04-06 22:02:42 +00001163}
1164
1165/// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for
Bill Schmidtf910a062014-06-10 14:35:01 +00001166/// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes).
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001167/// The ShuffleKind distinguishes between big-endian merges with two
1168/// different inputs (0), either-endian merges with two identical inputs (1),
1169/// and little-endian merges with two different inputs (2). For the latter,
1170/// the input operands are swapped (see PPCInstrAltivec.td).
Wesley Peck527da1b2010-11-23 03:31:01 +00001171bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001172 unsigned ShuffleKind, SelectionDAG &DAG) {
Eric Christopher8b770652015-01-26 19:03:15 +00001173 if (DAG.getTarget().getDataLayout()->isLittleEndian()) {
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001174 if (ShuffleKind == 1) // unary
1175 return isVMerge(N, UnitSize, 0, 0);
1176 else if (ShuffleKind == 2) // swapped
Bill Schmidtf910a062014-06-10 14:35:01 +00001177 return isVMerge(N, UnitSize, 0, 16);
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001178 else
1179 return false;
Bill Schmidtf910a062014-06-10 14:35:01 +00001180 } else {
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001181 if (ShuffleKind == 1) // unary
1182 return isVMerge(N, UnitSize, 8, 8);
1183 else if (ShuffleKind == 0) // normal
Bill Schmidtf910a062014-06-10 14:35:01 +00001184 return isVMerge(N, UnitSize, 8, 24);
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001185 else
1186 return false;
Bill Schmidtf910a062014-06-10 14:35:01 +00001187 }
Chris Lattnerd1dcb522006-04-06 21:11:54 +00001188}
1189
1190/// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for
Bill Schmidtf910a062014-06-10 14:35:01 +00001191/// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes).
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001192/// The ShuffleKind distinguishes between big-endian merges with two
1193/// different inputs (0), either-endian merges with two identical inputs (1),
1194/// and little-endian merges with two different inputs (2). For the latter,
1195/// the input operands are swapped (see PPCInstrAltivec.td).
Wesley Peck527da1b2010-11-23 03:31:01 +00001196bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001197 unsigned ShuffleKind, SelectionDAG &DAG) {
Eric Christopher8b770652015-01-26 19:03:15 +00001198 if (DAG.getTarget().getDataLayout()->isLittleEndian()) {
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001199 if (ShuffleKind == 1) // unary
1200 return isVMerge(N, UnitSize, 8, 8);
1201 else if (ShuffleKind == 2) // swapped
Bill Schmidtf910a062014-06-10 14:35:01 +00001202 return isVMerge(N, UnitSize, 8, 24);
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001203 else
1204 return false;
Bill Schmidtf910a062014-06-10 14:35:01 +00001205 } else {
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001206 if (ShuffleKind == 1) // unary
1207 return isVMerge(N, UnitSize, 0, 0);
1208 else if (ShuffleKind == 0) // normal
Bill Schmidtf910a062014-06-10 14:35:01 +00001209 return isVMerge(N, UnitSize, 0, 16);
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00001210 else
1211 return false;
Bill Schmidtf910a062014-06-10 14:35:01 +00001212 }
Chris Lattnerd1dcb522006-04-06 21:11:54 +00001213}
1214
1215
Chris Lattner1d338192006-04-06 18:26:28 +00001216/// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift
1217/// amount, otherwise return -1.
Bill Schmidt42a69362014-08-05 20:47:25 +00001218/// The ShuffleKind distinguishes between big-endian operations with two
1219/// different inputs (0), either-endian operations with two identical inputs
1220/// (1), and little-endian operations with two different inputs (2). For the
1221/// latter, the input operands are swapped (see PPCInstrAltivec.td).
1222int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind,
1223 SelectionDAG &DAG) {
Hal Finkeldf3e34d2014-03-26 22:58:37 +00001224 if (N->getValueType(0) != MVT::v16i8)
Hal Finkela775e512014-04-08 19:00:27 +00001225 return -1;
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001226
1227 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
Wesley Peck527da1b2010-11-23 03:31:01 +00001228
Chris Lattner1d338192006-04-06 18:26:28 +00001229 // Find the first non-undef value in the shuffle mask.
1230 unsigned i;
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001231 for (i = 0; i != 16 && SVOp->getMaskElt(i) < 0; ++i)
Chris Lattner1d338192006-04-06 18:26:28 +00001232 /*search*/;
Scott Michelcf0da6c2009-02-17 22:15:04 +00001233
Chris Lattner1d338192006-04-06 18:26:28 +00001234 if (i == 16) return -1; // all undef.
Scott Michelcf0da6c2009-02-17 22:15:04 +00001235
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001236 // Otherwise, check to see if the rest of the elements are consecutively
Chris Lattner1d338192006-04-06 18:26:28 +00001237 // numbered from this value.
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001238 unsigned ShiftAmt = SVOp->getMaskElt(i);
Chris Lattner1d338192006-04-06 18:26:28 +00001239 if (ShiftAmt < i) return -1;
Chris Lattnere8b83b42006-04-06 17:23:16 +00001240
Bill Schmidtf04e9982014-08-04 23:21:01 +00001241 ShiftAmt -= i;
Eric Christopher8b770652015-01-26 19:03:15 +00001242 bool isLE = DAG.getTarget().getDataLayout()->isLittleEndian();
Bill Schmidtf910a062014-06-10 14:35:01 +00001243
Bill Schmidt42a69362014-08-05 20:47:25 +00001244 if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) {
Bill Schmidtf04e9982014-08-04 23:21:01 +00001245 // Check the rest of the elements to see if they are consecutive.
1246 for (++i; i != 16; ++i)
1247 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i))
1248 return -1;
Bill Schmidt42a69362014-08-05 20:47:25 +00001249 } else if (ShuffleKind == 1) {
Bill Schmidtf04e9982014-08-04 23:21:01 +00001250 // Check the rest of the elements to see if they are consecutive.
1251 for (++i; i != 16; ++i)
1252 if (!isConstantOrUndef(SVOp->getMaskElt(i), (ShiftAmt+i) & 15))
1253 return -1;
Bill Schmidt42a69362014-08-05 20:47:25 +00001254 } else
1255 return -1;
1256
1257 if (ShuffleKind == 2 && isLE)
1258 ShiftAmt = 16 - ShiftAmt;
Bill Schmidtf04e9982014-08-04 23:21:01 +00001259
Chris Lattner1d338192006-04-06 18:26:28 +00001260 return ShiftAmt;
1261}
Chris Lattnerffc47562006-03-20 06:33:01 +00001262
1263/// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand
1264/// specifies a splat of a single element that is suitable for input to
1265/// VSPLTB/VSPLTH/VSPLTW.
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001266bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) {
Owen Anderson9f944592009-08-11 20:47:22 +00001267 assert(N->getValueType(0) == MVT::v16i8 &&
Chris Lattner95c7adc2006-04-04 17:25:31 +00001268 (EltSize == 1 || EltSize == 2 || EltSize == 4));
Scott Michelcf0da6c2009-02-17 22:15:04 +00001269
Chris Lattnera8fbb6d2006-03-20 06:37:44 +00001270 // This is a splat operation if each element of the permute is the same, and
1271 // if the value doesn't reference the second vector.
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001272 unsigned ElementBase = N->getMaskElt(0);
Wesley Peck527da1b2010-11-23 03:31:01 +00001273
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001274 // FIXME: Handle UNDEF elements too!
1275 if (ElementBase >= 16)
Chris Lattner95c7adc2006-04-04 17:25:31 +00001276 return false;
Scott Michelcf0da6c2009-02-17 22:15:04 +00001277
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001278 // Check that the indices are consecutive, in the case of a multi-byte element
1279 // splatted with a v16i8 mask.
1280 for (unsigned i = 1; i != EltSize; ++i)
1281 if (N->getMaskElt(i) < 0 || N->getMaskElt(i) != (int)(i+ElementBase))
Chris Lattner95c7adc2006-04-04 17:25:31 +00001282 return false;
Scott Michelcf0da6c2009-02-17 22:15:04 +00001283
Chris Lattner95c7adc2006-04-04 17:25:31 +00001284 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) {
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001285 if (N->getMaskElt(i) < 0) continue;
Chris Lattner95c7adc2006-04-04 17:25:31 +00001286 for (unsigned j = 0; j != EltSize; ++j)
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001287 if (N->getMaskElt(i+j) != N->getMaskElt(j))
Chris Lattner95c7adc2006-04-04 17:25:31 +00001288 return false;
Chris Lattnera8fbb6d2006-03-20 06:37:44 +00001289 }
Chris Lattner95c7adc2006-04-04 17:25:31 +00001290 return true;
Chris Lattnerffc47562006-03-20 06:33:01 +00001291}
1292
1293/// getVSPLTImmediate - Return the appropriate VSPLT* immediate to splat the
1294/// specified isSplatShuffleMask VECTOR_SHUFFLE mask.
Bill Schmidtf910a062014-06-10 14:35:01 +00001295unsigned PPC::getVSPLTImmediate(SDNode *N, unsigned EltSize,
1296 SelectionDAG &DAG) {
Nate Begeman8d6d4b92009-04-27 18:41:29 +00001297 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
1298 assert(isSplatShuffleMask(SVOp, EltSize));
Eric Christopher8b770652015-01-26 19:03:15 +00001299 if (DAG.getTarget().getDataLayout()->isLittleEndian())
Bill Schmidtf910a062014-06-10 14:35:01 +00001300 return (16 / EltSize) - 1 - (SVOp->getMaskElt(0) / EltSize);
1301 else
1302 return SVOp->getMaskElt(0) / EltSize;
Chris Lattnerffc47562006-03-20 06:33:01 +00001303}
1304
Chris Lattner74cf9ff2006-04-12 17:37:20 +00001305/// get_VSPLTI_elt - If this is a build_vector of constants which can be formed
Chris Lattnerd71a1f92006-04-08 06:46:53 +00001306/// by using a vspltis[bhw] instruction of the specified element size, return
1307/// the constant being splatted. The ByteSize field indicates the number of
1308/// bytes of each element [124] -> [bhw].
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001309SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
Craig Topper062a2ba2014-04-25 05:30:21 +00001310 SDValue OpVal(nullptr, 0);
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001311
1312 // If ByteSize of the splat is bigger than the element size of the
1313 // build_vector, then we have a case where we are checking for a splat where
1314 // multiple elements of the buildvector are folded together into a single
1315 // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8).
1316 unsigned EltSize = 16/N->getNumOperands();
1317 if (EltSize < ByteSize) {
1318 unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001319 SDValue UniquedVals[4];
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001320 assert(Multiple > 1 && Multiple <= 4 && "How can this happen?");
Scott Michelcf0da6c2009-02-17 22:15:04 +00001321
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001322 // See if all of the elements in the buildvector agree across.
1323 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
1324 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue;
1325 // If the element isn't a constant, bail fully out.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001326 if (!isa<ConstantSDNode>(N->getOperand(i))) return SDValue();
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001327
Scott Michelcf0da6c2009-02-17 22:15:04 +00001328
Craig Topper062a2ba2014-04-25 05:30:21 +00001329 if (!UniquedVals[i&(Multiple-1)].getNode())
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001330 UniquedVals[i&(Multiple-1)] = N->getOperand(i);
1331 else if (UniquedVals[i&(Multiple-1)] != N->getOperand(i))
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001332 return SDValue(); // no match.
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001333 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00001334
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001335 // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains
1336 // either constant or undef values that are identical for each chunk. See
1337 // if these chunks can form into a larger vspltis*.
Scott Michelcf0da6c2009-02-17 22:15:04 +00001338
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001339 // Check to see if all of the leading entries are either 0 or -1. If
1340 // neither, then this won't fit into the immediate field.
1341 bool LeadingZero = true;
1342 bool LeadingOnes = true;
1343 for (unsigned i = 0; i != Multiple-1; ++i) {
Craig Topper062a2ba2014-04-25 05:30:21 +00001344 if (!UniquedVals[i].getNode()) continue; // Must have been undefs.
Scott Michelcf0da6c2009-02-17 22:15:04 +00001345
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001346 LeadingZero &= cast<ConstantSDNode>(UniquedVals[i])->isNullValue();
1347 LeadingOnes &= cast<ConstantSDNode>(UniquedVals[i])->isAllOnesValue();
1348 }
1349 // Finally, check the least significant entry.
1350 if (LeadingZero) {
Craig Topper062a2ba2014-04-25 05:30:21 +00001351 if (!UniquedVals[Multiple-1].getNode())
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001352 return DAG.getTargetConstant(0, SDLoc(N), MVT::i32); // 0,0,0,undef
Dan Gohmaneffb8942008-09-12 16:56:44 +00001353 int Val = cast<ConstantSDNode>(UniquedVals[Multiple-1])->getZExtValue();
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001354 if (Val < 16) // 0,0,0,4 -> vspltisw(4)
1355 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32);
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001356 }
1357 if (LeadingOnes) {
Craig Topper062a2ba2014-04-25 05:30:21 +00001358 if (!UniquedVals[Multiple-1].getNode())
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001359 return DAG.getTargetConstant(~0U, SDLoc(N), MVT::i32); // -1,-1,-1,undef
Dan Gohman6e054832008-09-26 21:54:37 +00001360 int Val =cast<ConstantSDNode>(UniquedVals[Multiple-1])->getSExtValue();
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001361 if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2)
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001362 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32);
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001363 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00001364
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001365 return SDValue();
Chris Lattnerd9e80f42006-04-08 07:14:26 +00001366 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00001367
Chris Lattner2771e2c2006-03-25 06:12:06 +00001368 // Check to see if this buildvec has a single non-undef value in its elements.
1369 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
1370 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue;
Craig Topper062a2ba2014-04-25 05:30:21 +00001371 if (!OpVal.getNode())
Chris Lattner2771e2c2006-03-25 06:12:06 +00001372 OpVal = N->getOperand(i);
1373 else if (OpVal != N->getOperand(i))
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001374 return SDValue();
Chris Lattner2771e2c2006-03-25 06:12:06 +00001375 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00001376
Craig Topper062a2ba2014-04-25 05:30:21 +00001377 if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def.
Scott Michelcf0da6c2009-02-17 22:15:04 +00001378
Eli Friedman9c6ab1a2009-05-24 02:03:36 +00001379 unsigned ValSizeInBytes = EltSize;
Nate Begeman1b392872006-03-28 04:15:58 +00001380 uint64_t Value = 0;
Chris Lattner2771e2c2006-03-25 06:12:06 +00001381 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) {
Dan Gohmaneffb8942008-09-12 16:56:44 +00001382 Value = CN->getZExtValue();
Chris Lattner2771e2c2006-03-25 06:12:06 +00001383 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) {
Owen Anderson9f944592009-08-11 20:47:22 +00001384 assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!");
Dale Johannesen3cf889f2007-08-31 04:03:46 +00001385 Value = FloatToBits(CN->getValueAPF().convertToFloat());
Chris Lattner2771e2c2006-03-25 06:12:06 +00001386 }
1387
1388 // If the splat value is larger than the element value, then we can never do
1389 // this splat. The only case that we could fit the replicated bits into our
1390 // immediate field for would be zero, and we prefer to use vxor for it.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001391 if (ValSizeInBytes < ByteSize) return SDValue();
Scott Michelcf0da6c2009-02-17 22:15:04 +00001392
Benjamin Kramerb4b51502015-03-25 16:49:59 +00001393 // If the element value is larger than the splat value, check if it consists
1394 // of a repeated bit pattern of size ByteSize.
1395 if (!APInt(ValSizeInBytes * 8, Value).isSplat(ByteSize * 8))
1396 return SDValue();
Chris Lattner2771e2c2006-03-25 06:12:06 +00001397
1398 // Properly sign extend the value.
Richard Smith228e6d42012-08-24 23:29:28 +00001399 int MaskVal = SignExtend32(Value, ByteSize * 8);
Scott Michelcf0da6c2009-02-17 22:15:04 +00001400
Evan Chengb1ddc982006-03-26 09:52:32 +00001401 // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001402 if (MaskVal == 0) return SDValue();
Chris Lattner2771e2c2006-03-25 06:12:06 +00001403
Chris Lattnerd71a1f92006-04-08 06:46:53 +00001404 // Finally, if this value fits in a 5 bit sext field, return it
Richard Smith228e6d42012-08-24 23:29:28 +00001405 if (SignExtend32<5>(MaskVal) == MaskVal)
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001406 return DAG.getTargetConstant(MaskVal, SDLoc(N), MVT::i32);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001407 return SDValue();
Chris Lattner2771e2c2006-03-25 06:12:06 +00001408}
1409
Hal Finkelc93a9a22015-02-25 01:06:45 +00001410/// isQVALIGNIShuffleMask - If this is a qvaligni shuffle mask, return the shift
1411/// amount, otherwise return -1.
1412int PPC::isQVALIGNIShuffleMask(SDNode *N) {
1413 EVT VT = N->getValueType(0);
1414 if (VT != MVT::v4f64 && VT != MVT::v4f32 && VT != MVT::v4i1)
1415 return -1;
1416
1417 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
1418
1419 // Find the first non-undef value in the shuffle mask.
1420 unsigned i;
1421 for (i = 0; i != 4 && SVOp->getMaskElt(i) < 0; ++i)
1422 /*search*/;
1423
1424 if (i == 4) return -1; // all undef.
1425
1426 // Otherwise, check to see if the rest of the elements are consecutively
1427 // numbered from this value.
1428 unsigned ShiftAmt = SVOp->getMaskElt(i);
1429 if (ShiftAmt < i) return -1;
1430 ShiftAmt -= i;
1431
1432 // Check the rest of the elements to see if they are consecutive.
1433 for (++i; i != 4; ++i)
1434 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i))
1435 return -1;
1436
1437 return ShiftAmt;
1438}
1439
Chris Lattner4211ca92006-04-14 06:01:58 +00001440//===----------------------------------------------------------------------===//
Chris Lattnera801fced2006-11-08 02:15:41 +00001441// Addressing Mode Selection
1442//===----------------------------------------------------------------------===//
1443
1444/// isIntS16Immediate - This method tests to see if the node is either a 32-bit
1445/// or 64-bit immediate, and if the value can be accurately represented as a
1446/// sign extension from a 16-bit value. If so, this returns true and the
1447/// immediate.
1448static bool isIntS16Immediate(SDNode *N, short &Imm) {
Adam Nemet571eb5f2014-05-20 17:20:34 +00001449 if (!isa<ConstantSDNode>(N))
Chris Lattnera801fced2006-11-08 02:15:41 +00001450 return false;
Scott Michelcf0da6c2009-02-17 22:15:04 +00001451
Dan Gohmaneffb8942008-09-12 16:56:44 +00001452 Imm = (short)cast<ConstantSDNode>(N)->getZExtValue();
Owen Anderson9f944592009-08-11 20:47:22 +00001453 if (N->getValueType(0) == MVT::i32)
Dan Gohmaneffb8942008-09-12 16:56:44 +00001454 return Imm == (int32_t)cast<ConstantSDNode>(N)->getZExtValue();
Chris Lattnera801fced2006-11-08 02:15:41 +00001455 else
Dan Gohmaneffb8942008-09-12 16:56:44 +00001456 return Imm == (int64_t)cast<ConstantSDNode>(N)->getZExtValue();
Chris Lattnera801fced2006-11-08 02:15:41 +00001457}
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001458static bool isIntS16Immediate(SDValue Op, short &Imm) {
Gabor Greiff304a7a2008-08-28 21:40:38 +00001459 return isIntS16Immediate(Op.getNode(), Imm);
Chris Lattnera801fced2006-11-08 02:15:41 +00001460}
1461
1462
1463/// SelectAddressRegReg - Given the specified addressed, check to see if it
1464/// can be represented as an indexed [r+r] operation. Returns false if it
1465/// can be more efficiently represented with [r+imm].
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001466bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base,
1467 SDValue &Index,
Dan Gohman02b93132009-01-15 16:29:45 +00001468 SelectionDAG &DAG) const {
Chris Lattnera801fced2006-11-08 02:15:41 +00001469 short imm = 0;
1470 if (N.getOpcode() == ISD::ADD) {
1471 if (isIntS16Immediate(N.getOperand(1), imm))
1472 return false; // r+i
1473 if (N.getOperand(1).getOpcode() == PPCISD::Lo)
1474 return false; // r+i
Scott Michelcf0da6c2009-02-17 22:15:04 +00001475
Chris Lattnera801fced2006-11-08 02:15:41 +00001476 Base = N.getOperand(0);
1477 Index = N.getOperand(1);
1478 return true;
1479 } else if (N.getOpcode() == ISD::OR) {
1480 if (isIntS16Immediate(N.getOperand(1), imm))
1481 return false; // r+i can fold it if we can.
Scott Michelcf0da6c2009-02-17 22:15:04 +00001482
Chris Lattnera801fced2006-11-08 02:15:41 +00001483 // If this is an or of disjoint bitfields, we can codegen this as an add
1484 // (for better address arithmetic) if the LHS and RHS of the OR are provably
1485 // disjoint.
Dan Gohmanf19609a2008-02-27 01:23:58 +00001486 APInt LHSKnownZero, LHSKnownOne;
1487 APInt RHSKnownZero, RHSKnownOne;
Jay Foada0653a32014-05-14 21:14:37 +00001488 DAG.computeKnownBits(N.getOperand(0),
1489 LHSKnownZero, LHSKnownOne);
Scott Michelcf0da6c2009-02-17 22:15:04 +00001490
Dan Gohmanf19609a2008-02-27 01:23:58 +00001491 if (LHSKnownZero.getBoolValue()) {
Jay Foada0653a32014-05-14 21:14:37 +00001492 DAG.computeKnownBits(N.getOperand(1),
1493 RHSKnownZero, RHSKnownOne);
Chris Lattnera801fced2006-11-08 02:15:41 +00001494 // If all of the bits are known zero on the LHS or RHS, the add won't
1495 // carry.
Dan Gohman26854f22008-02-27 21:12:32 +00001496 if (~(LHSKnownZero | RHSKnownZero) == 0) {
Chris Lattnera801fced2006-11-08 02:15:41 +00001497 Base = N.getOperand(0);
1498 Index = N.getOperand(1);
1499 return true;
1500 }
1501 }
1502 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00001503
Chris Lattnera801fced2006-11-08 02:15:41 +00001504 return false;
1505}
1506
Hal Finkeldbbf09b2013-07-09 06:34:51 +00001507// If we happen to be doing an i64 load or store into a stack slot that has
1508// less than a 4-byte alignment, then the frame-index elimination may need to
1509// use an indexed load or store instruction (because the offset may not be a
1510// multiple of 4). The extra register needed to hold the offset comes from the
1511// register scavenger, and it is possible that the scavenger will need to use
1512// an emergency spill slot. As a result, we need to make sure that a spill slot
1513// is allocated when doing an i64 load/store into a less-than-4-byte-aligned
1514// stack slot.
1515static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) {
1516 // FIXME: This does not handle the LWA case.
1517 if (VT != MVT::i64)
1518 return;
1519
Hal Finkel7ab3db52013-07-10 15:29:01 +00001520 // NOTE: We'll exclude negative FIs here, which come from argument
1521 // lowering, because there are no known test cases triggering this problem
1522 // using packed structures (or similar). We can remove this exclusion if
1523 // we find such a test case. The reason why this is so test-case driven is
1524 // because this entire 'fixup' is only to prevent crashes (from the
1525 // register scavenger) on not-really-valid inputs. For example, if we have:
1526 // %a = alloca i1
1527 // %b = bitcast i1* %a to i64*
1528 // store i64* a, i64 b
1529 // then the store should really be marked as 'align 1', but is not. If it
1530 // were marked as 'align 1' then the indexed form would have been
1531 // instruction-selected initially, and the problem this 'fixup' is preventing
1532 // won't happen regardless.
Hal Finkeldbbf09b2013-07-09 06:34:51 +00001533 if (FrameIdx < 0)
1534 return;
1535
1536 MachineFunction &MF = DAG.getMachineFunction();
1537 MachineFrameInfo *MFI = MF.getFrameInfo();
1538
1539 unsigned Align = MFI->getObjectAlignment(FrameIdx);
1540 if (Align >= 4)
1541 return;
1542
1543 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1544 FuncInfo->setHasNonRISpills();
1545}
1546
Chris Lattnera801fced2006-11-08 02:15:41 +00001547/// Returns true if the address N can be represented by a base register plus
1548/// a signed 16-bit displacement [r+imm], and if it is not better
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00001549/// represented as reg+reg. If Aligned is true, only accept displacements
1550/// suitable for STD and friends, i.e. multiples of 4.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001551bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
Dan Gohman02b93132009-01-15 16:29:45 +00001552 SDValue &Base,
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00001553 SelectionDAG &DAG,
1554 bool Aligned) const {
Dale Johannesenab8e4422009-02-06 19:16:40 +00001555 // FIXME dl should come from parent load or store, not from address
Andrew Trickef9de2a2013-05-25 02:42:55 +00001556 SDLoc dl(N);
Chris Lattnera801fced2006-11-08 02:15:41 +00001557 // If this can be more profitably realized as r+r, fail.
1558 if (SelectAddressRegReg(N, Disp, Base, DAG))
1559 return false;
Scott Michelcf0da6c2009-02-17 22:15:04 +00001560
Chris Lattnera801fced2006-11-08 02:15:41 +00001561 if (N.getOpcode() == ISD::ADD) {
1562 short imm = 0;
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00001563 if (isIntS16Immediate(N.getOperand(1), imm) &&
1564 (!Aligned || (imm & 3) == 0)) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001565 Disp = DAG.getTargetConstant(imm, dl, N.getValueType());
Chris Lattnera801fced2006-11-08 02:15:41 +00001566 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) {
1567 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
Hal Finkeldbbf09b2013-07-09 06:34:51 +00001568 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
Chris Lattnera801fced2006-11-08 02:15:41 +00001569 } else {
1570 Base = N.getOperand(0);
1571 }
1572 return true; // [r+i]
1573 } else if (N.getOperand(1).getOpcode() == PPCISD::Lo) {
1574 // Match LOAD (ADD (X, Lo(G))).
Gabor Greifc8a9abe2012-04-20 11:41:38 +00001575 assert(!cast<ConstantSDNode>(N.getOperand(1).getOperand(1))->getZExtValue()
Chris Lattnera801fced2006-11-08 02:15:41 +00001576 && "Cannot handle constant offsets yet!");
1577 Disp = N.getOperand(1).getOperand(0); // The global address.
1578 assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||
Roman Divackye3f15c982012-06-04 17:36:38 +00001579 Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||
Chris Lattnera801fced2006-11-08 02:15:41 +00001580 Disp.getOpcode() == ISD::TargetConstantPool ||
1581 Disp.getOpcode() == ISD::TargetJumpTable);
1582 Base = N.getOperand(0);
1583 return true; // [&g+r]
1584 }
1585 } else if (N.getOpcode() == ISD::OR) {
1586 short imm = 0;
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00001587 if (isIntS16Immediate(N.getOperand(1), imm) &&
1588 (!Aligned || (imm & 3) == 0)) {
Chris Lattnera801fced2006-11-08 02:15:41 +00001589 // If this is an or of disjoint bitfields, we can codegen this as an add
1590 // (for better address arithmetic) if the LHS and RHS of the OR are
1591 // provably disjoint.
Dan Gohmanf19609a2008-02-27 01:23:58 +00001592 APInt LHSKnownZero, LHSKnownOne;
Jay Foada0653a32014-05-14 21:14:37 +00001593 DAG.computeKnownBits(N.getOperand(0), LHSKnownZero, LHSKnownOne);
Bill Wendling63061832008-03-24 23:16:37 +00001594
Dan Gohmanf19609a2008-02-27 01:23:58 +00001595 if ((LHSKnownZero.getZExtValue()|~(uint64_t)imm) == ~0ULL) {
Chris Lattnera801fced2006-11-08 02:15:41 +00001596 // If all of the bits are known zero on the LHS or RHS, the add won't
1597 // carry.
Ulrich Weigand55a96652014-07-20 22:26:40 +00001598 if (FrameIndexSDNode *FI =
1599 dyn_cast<FrameIndexSDNode>(N.getOperand(0))) {
1600 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
1601 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
1602 } else {
1603 Base = N.getOperand(0);
1604 }
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001605 Disp = DAG.getTargetConstant(imm, dl, N.getValueType());
Chris Lattnera801fced2006-11-08 02:15:41 +00001606 return true;
1607 }
1608 }
1609 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) {
1610 // Loading from a constant address.
Scott Michelcf0da6c2009-02-17 22:15:04 +00001611
Chris Lattnera801fced2006-11-08 02:15:41 +00001612 // If this address fits entirely in a 16-bit sext immediate field, codegen
1613 // this as "d, 0"
1614 short Imm;
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00001615 if (isIntS16Immediate(CN, Imm) && (!Aligned || (Imm & 3) == 0)) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001616 Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0));
Eric Christopherb1aaebe2014-06-12 22:38:18 +00001617 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
Hal Finkelf70c41e2013-03-21 23:45:03 +00001618 CN->getValueType(0));
Chris Lattnera801fced2006-11-08 02:15:41 +00001619 return true;
1620 }
Chris Lattner4a9c0bb2007-02-17 06:44:03 +00001621
1622 // Handle 32-bit sext immediates with LIS + addr mode.
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00001623 if ((CN->getValueType(0) == MVT::i32 ||
1624 (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) &&
1625 (!Aligned || (CN->getZExtValue() & 3) == 0)) {
Dan Gohmaneffb8942008-09-12 16:56:44 +00001626 int Addr = (int)CN->getZExtValue();
Scott Michelcf0da6c2009-02-17 22:15:04 +00001627
Chris Lattnera801fced2006-11-08 02:15:41 +00001628 // Otherwise, break this down into an LIS + disp.
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001629 Disp = DAG.getTargetConstant((short)Addr, dl, MVT::i32);
Scott Michelcf0da6c2009-02-17 22:15:04 +00001630
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001631 Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, dl,
1632 MVT::i32);
Owen Anderson9f944592009-08-11 20:47:22 +00001633 unsigned Opc = CN->getValueType(0) == MVT::i32 ? PPC::LIS : PPC::LIS8;
Dan Gohman32f71d72009-09-25 18:54:59 +00001634 Base = SDValue(DAG.getMachineNode(Opc, dl, CN->getValueType(0), Base), 0);
Chris Lattnera801fced2006-11-08 02:15:41 +00001635 return true;
1636 }
1637 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00001638
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001639 Disp = DAG.getTargetConstant(0, dl, getPointerTy());
Hal Finkeldbbf09b2013-07-09 06:34:51 +00001640 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) {
Chris Lattnera801fced2006-11-08 02:15:41 +00001641 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
Hal Finkeldbbf09b2013-07-09 06:34:51 +00001642 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
1643 } else
Chris Lattnera801fced2006-11-08 02:15:41 +00001644 Base = N;
1645 return true; // [r+0]
1646}
1647
1648/// SelectAddressRegRegOnly - Given the specified addressed, force it to be
1649/// represented as an indexed [r+r] operation.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001650bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base,
1651 SDValue &Index,
Dan Gohman02b93132009-01-15 16:29:45 +00001652 SelectionDAG &DAG) const {
Chris Lattnera801fced2006-11-08 02:15:41 +00001653 // Check to see if we can easily represent this as an [r+r] address. This
1654 // will fail if it thinks that the address is more profitably represented as
1655 // reg+imm, e.g. where imm = 0.
1656 if (SelectAddressRegReg(N, Base, Index, DAG))
1657 return true;
Scott Michelcf0da6c2009-02-17 22:15:04 +00001658
Chris Lattnera801fced2006-11-08 02:15:41 +00001659 // If the operand is an addition, always emit this as [r+r], since this is
1660 // better (for code size, and execution, as the memop does the add for free)
1661 // than emitting an explicit add.
1662 if (N.getOpcode() == ISD::ADD) {
1663 Base = N.getOperand(0);
1664 Index = N.getOperand(1);
1665 return true;
1666 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00001667
Chris Lattnera801fced2006-11-08 02:15:41 +00001668 // Otherwise, do it the hard way, using R0 as the base register.
Eric Christopherb1aaebe2014-06-12 22:38:18 +00001669 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
Hal Finkelf70c41e2013-03-21 23:45:03 +00001670 N.getValueType());
Chris Lattnera801fced2006-11-08 02:15:41 +00001671 Index = N;
1672 return true;
1673}
1674
Chris Lattnera801fced2006-11-08 02:15:41 +00001675/// getPreIndexedAddressParts - returns true by value, base pointer and
1676/// offset pointer and addressing mode by reference if the node's address
1677/// can be legally represented as pre-indexed load / store address.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001678bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
1679 SDValue &Offset,
Evan Chengb1500072006-11-09 17:55:04 +00001680 ISD::MemIndexedMode &AM,
Dan Gohman02b93132009-01-15 16:29:45 +00001681 SelectionDAG &DAG) const {
Hal Finkel595817e2012-06-04 02:21:00 +00001682 if (DisablePPCPreinc) return false;
Scott Michelcf0da6c2009-02-17 22:15:04 +00001683
Ulrich Weigande90b0222013-03-22 14:58:48 +00001684 bool isLoad = true;
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001685 SDValue Ptr;
Owen Anderson53aa7a92009-08-10 22:56:29 +00001686 EVT VT;
Hal Finkelb09680b2013-03-18 23:00:58 +00001687 unsigned Alignment;
Chris Lattnera801fced2006-11-08 02:15:41 +00001688 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
1689 Ptr = LD->getBasePtr();
Dan Gohman47a7d6f2008-01-30 00:15:11 +00001690 VT = LD->getMemoryVT();
Hal Finkelb09680b2013-03-18 23:00:58 +00001691 Alignment = LD->getAlignment();
Chris Lattnera801fced2006-11-08 02:15:41 +00001692 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
Chris Lattner68371252006-11-14 01:38:31 +00001693 Ptr = ST->getBasePtr();
Dan Gohman47a7d6f2008-01-30 00:15:11 +00001694 VT = ST->getMemoryVT();
Hal Finkelb09680b2013-03-18 23:00:58 +00001695 Alignment = ST->getAlignment();
Ulrich Weigande90b0222013-03-22 14:58:48 +00001696 isLoad = false;
Chris Lattnera801fced2006-11-08 02:15:41 +00001697 } else
1698 return false;
1699
Hal Finkelc93a9a22015-02-25 01:06:45 +00001700 // PowerPC doesn't have preinc load/store instructions for vectors (except
1701 // for QPX, which does have preinc r+r forms).
1702 if (VT.isVector()) {
1703 if (!Subtarget.hasQPX() || (VT != MVT::v4f64 && VT != MVT::v4f32)) {
1704 return false;
1705 } else if (SelectAddressRegRegOnly(Ptr, Offset, Base, DAG)) {
1706 AM = ISD::PRE_INC;
1707 return true;
1708 }
1709 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00001710
Ulrich Weigande90b0222013-03-22 14:58:48 +00001711 if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) {
1712
1713 // Common code will reject creating a pre-inc form if the base pointer
1714 // is a frame index, or if N is a store and the base pointer is either
1715 // the same as or a predecessor of the value being stored. Check for
1716 // those situations here, and try with swapped Base/Offset instead.
1717 bool Swap = false;
1718
1719 if (isa<FrameIndexSDNode>(Base) || isa<RegisterSDNode>(Base))
1720 Swap = true;
1721 else if (!isLoad) {
1722 SDValue Val = cast<StoreSDNode>(N)->getValue();
1723 if (Val == Base || Base.getNode()->isPredecessorOf(Val.getNode()))
1724 Swap = true;
1725 }
1726
1727 if (Swap)
1728 std::swap(Base, Offset);
1729
Hal Finkelca542be2012-06-20 15:43:03 +00001730 AM = ISD::PRE_INC;
1731 return true;
Hal Finkel1cc27e42012-06-19 02:34:32 +00001732 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00001733
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00001734 // LDU/STU can only handle immediates that are a multiple of 4.
Owen Anderson9f944592009-08-11 20:47:22 +00001735 if (VT != MVT::i64) {
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00001736 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, false))
Chris Lattner474b5b72006-11-15 19:55:13 +00001737 return false;
1738 } else {
Hal Finkelb09680b2013-03-18 23:00:58 +00001739 // LDU/STU need an address with at least 4-byte alignment.
1740 if (Alignment < 4)
1741 return false;
1742
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00001743 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, true))
Chris Lattner474b5b72006-11-15 19:55:13 +00001744 return false;
1745 }
Chris Lattnerb314b152006-11-11 00:08:42 +00001746
Chris Lattnerb314b152006-11-11 00:08:42 +00001747 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
Chris Lattner474b5b72006-11-15 19:55:13 +00001748 // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of
1749 // sext i32 to i64 when addr mode is r+i.
Owen Anderson9f944592009-08-11 20:47:22 +00001750 if (LD->getValueType(0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 &&
Chris Lattnerb314b152006-11-11 00:08:42 +00001751 LD->getExtensionType() == ISD::SEXTLOAD &&
1752 isa<ConstantSDNode>(Offset))
1753 return false;
Scott Michelcf0da6c2009-02-17 22:15:04 +00001754 }
1755
Chris Lattnerce645542006-11-10 02:08:47 +00001756 AM = ISD::PRE_INC;
1757 return true;
Chris Lattnera801fced2006-11-08 02:15:41 +00001758}
1759
1760//===----------------------------------------------------------------------===//
Chris Lattner4211ca92006-04-14 06:01:58 +00001761// LowerOperation implementation
1762//===----------------------------------------------------------------------===//
1763
Chris Lattneredb9d842010-11-15 02:46:57 +00001764/// GetLabelAccessInfo - Return true if we should reference labels using a
1765/// PICBase, set the HiOpFlags and LoOpFlags to the target MO flags.
Eric Christophercccae792015-01-30 22:02:31 +00001766static bool GetLabelAccessInfo(const TargetMachine &TM,
1767 const PPCSubtarget &Subtarget,
1768 unsigned &HiOpFlags, unsigned &LoOpFlags,
Craig Topper062a2ba2014-04-25 05:30:21 +00001769 const GlobalValue *GV = nullptr) {
Ulrich Weigandd51c09f2013-06-21 14:42:20 +00001770 HiOpFlags = PPCII::MO_HA;
1771 LoOpFlags = PPCII::MO_LO;
Wesley Peck527da1b2010-11-23 03:31:01 +00001772
Hal Finkel3ee2af72014-07-18 23:29:49 +00001773 // Don't use the pic base if not in PIC relocation model.
1774 bool isPIC = TM.getRelocationModel() == Reloc::PIC_;
1775
Chris Lattnerdd6df842010-11-15 03:13:19 +00001776 if (isPIC) {
1777 HiOpFlags |= PPCII::MO_PIC_FLAG;
1778 LoOpFlags |= PPCII::MO_PIC_FLAG;
1779 }
1780
1781 // If this is a reference to a global value that requires a non-lazy-ptr, make
1782 // sure that instruction lowering adds it.
Eric Christophere8dbfe12015-02-13 22:23:04 +00001783 if (GV && Subtarget.hasLazyResolverStub(GV)) {
Chris Lattnerdd6df842010-11-15 03:13:19 +00001784 HiOpFlags |= PPCII::MO_NLP_FLAG;
1785 LoOpFlags |= PPCII::MO_NLP_FLAG;
Wesley Peck527da1b2010-11-23 03:31:01 +00001786
Chris Lattnerdd6df842010-11-15 03:13:19 +00001787 if (GV->hasHiddenVisibility()) {
1788 HiOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG;
1789 LoOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG;
1790 }
1791 }
Wesley Peck527da1b2010-11-23 03:31:01 +00001792
Chris Lattneredb9d842010-11-15 02:46:57 +00001793 return isPIC;
1794}
1795
1796static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC,
1797 SelectionDAG &DAG) {
Andrew Trickef9de2a2013-05-25 02:42:55 +00001798 SDLoc DL(HiPart);
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00001799 EVT PtrVT = HiPart.getValueType();
1800 SDValue Zero = DAG.getConstant(0, DL, PtrVT);
Chris Lattneredb9d842010-11-15 02:46:57 +00001801
1802 SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero);
1803 SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero);
Wesley Peck527da1b2010-11-23 03:31:01 +00001804
Chris Lattneredb9d842010-11-15 02:46:57 +00001805 // With PIC, the first instruction is actually "GR+hi(&G)".
1806 if (isPIC)
1807 Hi = DAG.getNode(ISD::ADD, DL, PtrVT,
1808 DAG.getNode(PPCISD::GlobalBaseReg, DL, PtrVT), Hi);
Wesley Peck527da1b2010-11-23 03:31:01 +00001809
Chris Lattneredb9d842010-11-15 02:46:57 +00001810 // Generate non-pic code that has direct accesses to the constant pool.
1811 // The address of the global is just (hi(&g)+lo(&g)).
1812 return DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo);
1813}
1814
Hal Finkele6698d52015-02-01 15:03:28 +00001815static void setUsesTOCBasePtr(MachineFunction &MF) {
1816 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1817 FuncInfo->setUsesTOCBasePtr();
1818}
1819
1820static void setUsesTOCBasePtr(SelectionDAG &DAG) {
1821 setUsesTOCBasePtr(DAG.getMachineFunction());
1822}
1823
Hal Finkelcf599212015-02-25 21:36:59 +00001824static SDValue getTOCEntry(SelectionDAG &DAG, SDLoc dl, bool Is64Bit,
1825 SDValue GA) {
1826 EVT VT = Is64Bit ? MVT::i64 : MVT::i32;
1827 SDValue Reg = Is64Bit ? DAG.getRegister(PPC::X2, VT) :
1828 DAG.getNode(PPCISD::GlobalBaseReg, dl, VT);
1829
1830 SDValue Ops[] = { GA, Reg };
1831 return DAG.getMemIntrinsicNode(PPCISD::TOC_ENTRY, dl,
1832 DAG.getVTList(VT, MVT::Other), Ops, VT,
1833 MachinePointerInfo::getGOT(), 0, false, true,
1834 false, 0);
1835}
1836
Scott Michelcf0da6c2009-02-17 22:15:04 +00001837SDValue PPCTargetLowering::LowerConstantPool(SDValue Op,
Dan Gohman21cea8a2010-04-17 15:26:15 +00001838 SelectionDAG &DAG) const {
Owen Anderson53aa7a92009-08-10 22:56:29 +00001839 EVT PtrVT = Op.getValueType();
Chris Lattner4211ca92006-04-14 06:01:58 +00001840 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
Dan Gohmanbcaf6812010-04-15 01:51:59 +00001841 const Constant *C = CP->getConstVal();
Chris Lattner4211ca92006-04-14 06:01:58 +00001842
Roman Divackyace47072012-08-24 16:26:02 +00001843 // 64-bit SVR4 ABI code is always position-independent.
1844 // The actual address of the GlobalValue is stored in the TOC.
Eric Christopherb1aaebe2014-06-12 22:38:18 +00001845 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) {
Hal Finkele6698d52015-02-01 15:03:28 +00001846 setUsesTOCBasePtr(DAG);
Roman Divackyace47072012-08-24 16:26:02 +00001847 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0);
Hal Finkelcf599212015-02-25 21:36:59 +00001848 return getTOCEntry(DAG, SDLoc(CP), true, GA);
Roman Divackyace47072012-08-24 16:26:02 +00001849 }
1850
Chris Lattneredb9d842010-11-15 02:46:57 +00001851 unsigned MOHiFlag, MOLoFlag;
Eric Christophercccae792015-01-30 22:02:31 +00001852 bool isPIC =
1853 GetLabelAccessInfo(DAG.getTarget(), Subtarget, MOHiFlag, MOLoFlag);
Hal Finkel3ee2af72014-07-18 23:29:49 +00001854
1855 if (isPIC && Subtarget.isSVR4ABI()) {
1856 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(),
1857 PPCII::MO_PIC_FLAG);
Hal Finkelcf599212015-02-25 21:36:59 +00001858 return getTOCEntry(DAG, SDLoc(CP), false, GA);
Hal Finkel3ee2af72014-07-18 23:29:49 +00001859 }
1860
Chris Lattneredb9d842010-11-15 02:46:57 +00001861 SDValue CPIHi =
1862 DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOHiFlag);
1863 SDValue CPILo =
1864 DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOLoFlag);
1865 return LowerLabelRef(CPIHi, CPILo, isPIC, DAG);
Chris Lattner4211ca92006-04-14 06:01:58 +00001866}
1867
Dan Gohman21cea8a2010-04-17 15:26:15 +00001868SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
Owen Anderson53aa7a92009-08-10 22:56:29 +00001869 EVT PtrVT = Op.getValueType();
Nate Begeman4ca2ea52006-04-22 18:53:45 +00001870 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
Wesley Peck527da1b2010-11-23 03:31:01 +00001871
Roman Divackyace47072012-08-24 16:26:02 +00001872 // 64-bit SVR4 ABI code is always position-independent.
1873 // The actual address of the GlobalValue is stored in the TOC.
Eric Christopherb1aaebe2014-06-12 22:38:18 +00001874 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) {
Hal Finkele6698d52015-02-01 15:03:28 +00001875 setUsesTOCBasePtr(DAG);
Roman Divackyace47072012-08-24 16:26:02 +00001876 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
Hal Finkelcf599212015-02-25 21:36:59 +00001877 return getTOCEntry(DAG, SDLoc(JT), true, GA);
Roman Divackyace47072012-08-24 16:26:02 +00001878 }
1879
Chris Lattneredb9d842010-11-15 02:46:57 +00001880 unsigned MOHiFlag, MOLoFlag;
Eric Christophercccae792015-01-30 22:02:31 +00001881 bool isPIC =
1882 GetLabelAccessInfo(DAG.getTarget(), Subtarget, MOHiFlag, MOLoFlag);
Hal Finkel3ee2af72014-07-18 23:29:49 +00001883
1884 if (isPIC && Subtarget.isSVR4ABI()) {
1885 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT,
1886 PPCII::MO_PIC_FLAG);
Hal Finkelcf599212015-02-25 21:36:59 +00001887 return getTOCEntry(DAG, SDLoc(GA), false, GA);
Hal Finkel3ee2af72014-07-18 23:29:49 +00001888 }
1889
Chris Lattneredb9d842010-11-15 02:46:57 +00001890 SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag);
1891 SDValue JTILo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOLoFlag);
1892 return LowerLabelRef(JTIHi, JTILo, isPIC, DAG);
Lauro Ramos Venancio09d73c02007-07-11 17:19:51 +00001893}
1894
Dan Gohman21cea8a2010-04-17 15:26:15 +00001895SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op,
1896 SelectionDAG &DAG) const {
Bob Wilsonf84f7102009-11-04 21:31:18 +00001897 EVT PtrVT = Op.getValueType();
Ulrich Weigandc8c2ea22014-10-31 10:33:14 +00001898 BlockAddressSDNode *BASDN = cast<BlockAddressSDNode>(Op);
1899 const BlockAddress *BA = BASDN->getBlockAddress();
Bob Wilsonf84f7102009-11-04 21:31:18 +00001900
Ulrich Weigandc8c2ea22014-10-31 10:33:14 +00001901 // 64-bit SVR4 ABI code is always position-independent.
1902 // The actual BlockAddress is stored in the TOC.
1903 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) {
Hal Finkele6698d52015-02-01 15:03:28 +00001904 setUsesTOCBasePtr(DAG);
Ulrich Weigandc8c2ea22014-10-31 10:33:14 +00001905 SDValue GA = DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset());
Hal Finkelcf599212015-02-25 21:36:59 +00001906 return getTOCEntry(DAG, SDLoc(BASDN), true, GA);
Ulrich Weigandc8c2ea22014-10-31 10:33:14 +00001907 }
Wesley Peck527da1b2010-11-23 03:31:01 +00001908
Chris Lattneredb9d842010-11-15 02:46:57 +00001909 unsigned MOHiFlag, MOLoFlag;
Eric Christophercccae792015-01-30 22:02:31 +00001910 bool isPIC =
1911 GetLabelAccessInfo(DAG.getTarget(), Subtarget, MOHiFlag, MOLoFlag);
Michael Liaoabb87d42012-09-12 21:43:09 +00001912 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag);
1913 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag);
Chris Lattneredb9d842010-11-15 02:46:57 +00001914 return LowerLabelRef(TgtBAHi, TgtBALo, isPIC, DAG);
1915}
1916
Roman Divackye3f15c982012-06-04 17:36:38 +00001917SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1918 SelectionDAG &DAG) const {
1919
Bill Schmidtbdae03f2013-09-17 20:22:05 +00001920 // FIXME: TLS addresses currently use medium model code sequences,
1921 // which is the most useful form. Eventually support for small and
1922 // large models could be added if users need it, at the cost of
1923 // additional complexity.
Roman Divackye3f15c982012-06-04 17:36:38 +00001924 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
Andrew Trickef9de2a2013-05-25 02:42:55 +00001925 SDLoc dl(GA);
Roman Divackye3f15c982012-06-04 17:36:38 +00001926 const GlobalValue *GV = GA->getGlobal();
1927 EVT PtrVT = getPointerTy();
Eric Christopherb1aaebe2014-06-12 22:38:18 +00001928 bool is64bit = Subtarget.isPPC64();
Justin Hibbitsa88b6052014-11-12 15:16:30 +00001929 const Module *M = DAG.getMachineFunction().getFunction()->getParent();
1930 PICLevel::Level picLevel = M->getPICLevel();
Roman Divackye3f15c982012-06-04 17:36:38 +00001931
Bill Schmidtca4a0c92012-12-04 16:18:08 +00001932 TLSModel::Model Model = getTargetMachine().getTLSModel(GV);
Roman Divackye3f15c982012-06-04 17:36:38 +00001933
Bill Schmidtca4a0c92012-12-04 16:18:08 +00001934 if (Model == TLSModel::LocalExec) {
1935 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
Ulrich Weigandd51c09f2013-06-21 14:42:20 +00001936 PPCII::MO_TPREL_HA);
Bill Schmidtca4a0c92012-12-04 16:18:08 +00001937 SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
Ulrich Weigandd51c09f2013-06-21 14:42:20 +00001938 PPCII::MO_TPREL_LO);
Bill Schmidtca4a0c92012-12-04 16:18:08 +00001939 SDValue TLSReg = DAG.getRegister(is64bit ? PPC::X13 : PPC::R2,
1940 is64bit ? MVT::i64 : MVT::i32);
1941 SDValue Hi = DAG.getNode(PPCISD::Hi, dl, PtrVT, TGAHi, TLSReg);
1942 return DAG.getNode(PPCISD::Lo, dl, PtrVT, TGALo, Hi);
1943 }
Roman Divackye3f15c982012-06-04 17:36:38 +00001944
Bill Schmidtc56f1d32012-12-11 20:30:11 +00001945 if (Model == TLSModel::InitialExec) {
Bill Schmidt732eb912012-12-13 18:45:54 +00001946 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0);
Ulrich Weigand5b427592013-07-05 12:22:36 +00001947 SDValue TGATLS = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
1948 PPCII::MO_TLS);
Roman Divacky32143e22013-12-20 18:08:54 +00001949 SDValue GOTPtr;
1950 if (is64bit) {
Hal Finkele6698d52015-02-01 15:03:28 +00001951 setUsesTOCBasePtr(DAG);
Roman Divacky32143e22013-12-20 18:08:54 +00001952 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
1953 GOTPtr = DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl,
1954 PtrVT, GOTReg, TGA);
1955 } else
1956 GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT);
Bill Schmidt9f0b4ec2012-12-14 17:02:38 +00001957 SDValue TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl,
Roman Divacky32143e22013-12-20 18:08:54 +00001958 PtrVT, TGA, GOTPtr);
Ulrich Weigand5b427592013-07-05 12:22:36 +00001959 return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TPOffset, TGATLS);
Bill Schmidtc56f1d32012-12-11 20:30:11 +00001960 }
Bill Schmidtca4a0c92012-12-04 16:18:08 +00001961
Bill Schmidtc56f1d32012-12-11 20:30:11 +00001962 if (Model == TLSModel::GeneralDynamic) {
Bill Schmidt82f1c772015-02-10 19:09:05 +00001963 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0);
Hal Finkel7c8ae532014-07-25 17:47:22 +00001964 SDValue GOTPtr;
1965 if (is64bit) {
Hal Finkele6698d52015-02-01 15:03:28 +00001966 setUsesTOCBasePtr(DAG);
Hal Finkel7c8ae532014-07-25 17:47:22 +00001967 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
1968 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSGD_HA, dl, PtrVT,
1969 GOTReg, TGA);
1970 } else {
Justin Hibbitsa88b6052014-11-12 15:16:30 +00001971 if (picLevel == PICLevel::Small)
1972 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT);
1973 else
1974 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT);
Hal Finkel7c8ae532014-07-25 17:47:22 +00001975 }
Bill Schmidt82f1c772015-02-10 19:09:05 +00001976 return DAG.getNode(PPCISD::ADDI_TLSGD_L_ADDR, dl, PtrVT,
1977 GOTPtr, TGA, TGA);
Bill Schmidtc56f1d32012-12-11 20:30:11 +00001978 }
1979
Bill Schmidt24b8dd62012-12-12 19:29:35 +00001980 if (Model == TLSModel::LocalDynamic) {
Bill Schmidt82f1c772015-02-10 19:09:05 +00001981 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0);
Hal Finkel7c8ae532014-07-25 17:47:22 +00001982 SDValue GOTPtr;
1983 if (is64bit) {
Hal Finkele6698d52015-02-01 15:03:28 +00001984 setUsesTOCBasePtr(DAG);
Hal Finkel7c8ae532014-07-25 17:47:22 +00001985 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
1986 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSLD_HA, dl, PtrVT,
1987 GOTReg, TGA);
1988 } else {
Justin Hibbitsa88b6052014-11-12 15:16:30 +00001989 if (picLevel == PICLevel::Small)
1990 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT);
1991 else
1992 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT);
Hal Finkel7c8ae532014-07-25 17:47:22 +00001993 }
Bill Schmidt82f1c772015-02-10 19:09:05 +00001994 SDValue TLSAddr = DAG.getNode(PPCISD::ADDI_TLSLD_L_ADDR, dl,
1995 PtrVT, GOTPtr, TGA, TGA);
1996 SDValue DtvOffsetHi = DAG.getNode(PPCISD::ADDIS_DTPREL_HA, dl,
1997 PtrVT, TLSAddr, TGA);
Bill Schmidt24b8dd62012-12-12 19:29:35 +00001998 return DAG.getNode(PPCISD::ADDI_DTPREL_L, dl, PtrVT, DtvOffsetHi, TGA);
1999 }
2000
2001 llvm_unreachable("Unknown TLS model!");
Roman Divackye3f15c982012-06-04 17:36:38 +00002002}
2003
Chris Lattneredb9d842010-11-15 02:46:57 +00002004SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op,
2005 SelectionDAG &DAG) const {
2006 EVT PtrVT = Op.getValueType();
2007 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op);
Andrew Trickef9de2a2013-05-25 02:42:55 +00002008 SDLoc DL(GSDN);
Chris Lattneredb9d842010-11-15 02:46:57 +00002009 const GlobalValue *GV = GSDN->getGlobal();
2010
Chris Lattneredb9d842010-11-15 02:46:57 +00002011 // 64-bit SVR4 ABI code is always position-independent.
2012 // The actual address of the GlobalValue is stored in the TOC.
Eric Christopherb1aaebe2014-06-12 22:38:18 +00002013 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) {
Hal Finkele6698d52015-02-01 15:03:28 +00002014 setUsesTOCBasePtr(DAG);
Chris Lattneredb9d842010-11-15 02:46:57 +00002015 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset());
Hal Finkelcf599212015-02-25 21:36:59 +00002016 return getTOCEntry(DAG, DL, true, GA);
Chris Lattneredb9d842010-11-15 02:46:57 +00002017 }
2018
Chris Lattnerdd6df842010-11-15 03:13:19 +00002019 unsigned MOHiFlag, MOLoFlag;
Eric Christophercccae792015-01-30 22:02:31 +00002020 bool isPIC =
2021 GetLabelAccessInfo(DAG.getTarget(), Subtarget, MOHiFlag, MOLoFlag, GV);
Chris Lattneredb9d842010-11-15 02:46:57 +00002022
Hal Finkel3ee2af72014-07-18 23:29:49 +00002023 if (isPIC && Subtarget.isSVR4ABI()) {
2024 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT,
2025 GSDN->getOffset(),
2026 PPCII::MO_PIC_FLAG);
Hal Finkelcf599212015-02-25 21:36:59 +00002027 return getTOCEntry(DAG, DL, false, GA);
Hal Finkel3ee2af72014-07-18 23:29:49 +00002028 }
2029
Chris Lattnerdd6df842010-11-15 03:13:19 +00002030 SDValue GAHi =
2031 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOHiFlag);
2032 SDValue GALo =
2033 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOLoFlag);
Wesley Peck527da1b2010-11-23 03:31:01 +00002034
Chris Lattnerdd6df842010-11-15 03:13:19 +00002035 SDValue Ptr = LowerLabelRef(GAHi, GALo, isPIC, DAG);
Bob Wilsonf84f7102009-11-04 21:31:18 +00002036
Chris Lattnerdd6df842010-11-15 03:13:19 +00002037 // If the global reference is actually to a non-lazy-pointer, we have to do an
2038 // extra load to get the address of the global.
2039 if (MOHiFlag & PPCII::MO_NLP_FLAG)
2040 Ptr = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00002041 false, false, false, 0);
Chris Lattnerdd6df842010-11-15 03:13:19 +00002042 return Ptr;
Chris Lattner4211ca92006-04-14 06:01:58 +00002043}
2044
Dan Gohman21cea8a2010-04-17 15:26:15 +00002045SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
Chris Lattner4211ca92006-04-14 06:01:58 +00002046 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
Andrew Trickef9de2a2013-05-25 02:42:55 +00002047 SDLoc dl(Op);
Scott Michelcf0da6c2009-02-17 22:15:04 +00002048
Hal Finkel777c9dd2014-03-29 16:04:40 +00002049 if (Op.getValueType() == MVT::v2i64) {
2050 // When the operands themselves are v2i64 values, we need to do something
2051 // special because VSX has no underlying comparison operations for these.
2052 if (Op.getOperand(0).getValueType() == MVT::v2i64) {
2053 // Equality can be handled by casting to the legal type for Altivec
2054 // comparisons, everything else needs to be expanded.
2055 if (CC == ISD::SETEQ || CC == ISD::SETNE) {
2056 return DAG.getNode(ISD::BITCAST, dl, MVT::v2i64,
2057 DAG.getSetCC(dl, MVT::v4i32,
2058 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(0)),
2059 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(1)),
2060 CC));
2061 }
2062
2063 return SDValue();
2064 }
2065
2066 // We handle most of these in the usual way.
2067 return Op;
2068 }
2069
Chris Lattner4211ca92006-04-14 06:01:58 +00002070 // If we're comparing for equality to zero, expose the fact that this is
2071 // implented as a ctlz/srl pair on ppc, so that the dag combiner can
2072 // fold the new nodes.
2073 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2074 if (C->isNullValue() && CC == ISD::SETEQ) {
Owen Anderson53aa7a92009-08-10 22:56:29 +00002075 EVT VT = Op.getOperand(0).getValueType();
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002076 SDValue Zext = Op.getOperand(0);
Owen Anderson9f944592009-08-11 20:47:22 +00002077 if (VT.bitsLT(MVT::i32)) {
2078 VT = MVT::i32;
Dale Johannesenf2bb6f02009-02-04 01:48:28 +00002079 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
Scott Michelcf0da6c2009-02-17 22:15:04 +00002080 }
Duncan Sands13237ac2008-06-06 12:08:01 +00002081 unsigned Log2b = Log2_32(VT.getSizeInBits());
Dale Johannesenf2bb6f02009-02-04 01:48:28 +00002082 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
2083 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002084 DAG.getConstant(Log2b, dl, MVT::i32));
Owen Anderson9f944592009-08-11 20:47:22 +00002085 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
Chris Lattner4211ca92006-04-14 06:01:58 +00002086 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00002087 // Leave comparisons against 0 and -1 alone for now, since they're usually
Chris Lattner4211ca92006-04-14 06:01:58 +00002088 // optimized. FIXME: revisit this when we can custom lower all setcc
2089 // optimizations.
2090 if (C->isAllOnesValue() || C->isNullValue())
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002091 return SDValue();
Chris Lattner4211ca92006-04-14 06:01:58 +00002092 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00002093
Chris Lattner4211ca92006-04-14 06:01:58 +00002094 // If we have an integer seteq/setne, turn it into a compare against zero
Chris Lattner97ff46b2006-11-14 05:28:08 +00002095 // by xor'ing the rhs with the lhs, which is faster than setting a
2096 // condition register, reading it back out, and masking the correct bit. The
2097 // normal approach here uses sub to do this instead of xor. Using xor exposes
2098 // the result to other bit-twiddling opportunities.
Owen Anderson53aa7a92009-08-10 22:56:29 +00002099 EVT LHSVT = Op.getOperand(0).getValueType();
Duncan Sands13237ac2008-06-06 12:08:01 +00002100 if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
Owen Anderson53aa7a92009-08-10 22:56:29 +00002101 EVT VT = Op.getValueType();
Scott Michelcf0da6c2009-02-17 22:15:04 +00002102 SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, Op.getOperand(0),
Chris Lattner4211ca92006-04-14 06:01:58 +00002103 Op.getOperand(1));
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002104 return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, dl, LHSVT), CC);
Chris Lattner4211ca92006-04-14 06:01:58 +00002105 }
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002106 return SDValue();
Chris Lattner4211ca92006-04-14 06:01:58 +00002107}
2108
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002109SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG,
Dan Gohman21cea8a2010-04-17 15:26:15 +00002110 const PPCSubtarget &Subtarget) const {
Roman Divacky4394e682011-06-28 15:30:42 +00002111 SDNode *Node = Op.getNode();
2112 EVT VT = Node->getValueType(0);
2113 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
2114 SDValue InChain = Node->getOperand(0);
2115 SDValue VAListPtr = Node->getOperand(1);
2116 const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
Andrew Trickef9de2a2013-05-25 02:42:55 +00002117 SDLoc dl(Node);
Scott Michelcf0da6c2009-02-17 22:15:04 +00002118
Roman Divacky4394e682011-06-28 15:30:42 +00002119 assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only");
2120
2121 // gpr_index
2122 SDValue GprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain,
2123 VAListPtr, MachinePointerInfo(SV), MVT::i8,
Louis Gerbarg67474e32014-07-31 21:45:05 +00002124 false, false, false, 0);
Roman Divacky4394e682011-06-28 15:30:42 +00002125 InChain = GprIndex.getValue(1);
2126
2127 if (VT == MVT::i64) {
2128 // Check if GprIndex is even
2129 SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002130 DAG.getConstant(1, dl, MVT::i32));
Roman Divacky4394e682011-06-28 15:30:42 +00002131 SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002132 DAG.getConstant(0, dl, MVT::i32), ISD::SETNE);
Roman Divacky4394e682011-06-28 15:30:42 +00002133 SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002134 DAG.getConstant(1, dl, MVT::i32));
Roman Divacky4394e682011-06-28 15:30:42 +00002135 // Align GprIndex to be even if it isn't
2136 GprIndex = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC64, GprIndexPlusOne,
2137 GprIndex);
2138 }
2139
2140 // fpr index is 1 byte after gpr
2141 SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002142 DAG.getConstant(1, dl, MVT::i32));
Roman Divacky4394e682011-06-28 15:30:42 +00002143
2144 // fpr
2145 SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain,
2146 FprPtr, MachinePointerInfo(SV), MVT::i8,
Louis Gerbarg67474e32014-07-31 21:45:05 +00002147 false, false, false, 0);
Roman Divacky4394e682011-06-28 15:30:42 +00002148 InChain = FprIndex.getValue(1);
2149
2150 SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002151 DAG.getConstant(8, dl, MVT::i32));
Roman Divacky4394e682011-06-28 15:30:42 +00002152
2153 SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002154 DAG.getConstant(4, dl, MVT::i32));
Roman Divacky4394e682011-06-28 15:30:42 +00002155
2156 // areas
2157 SDValue OverflowArea = DAG.getLoad(MVT::i32, dl, InChain, OverflowAreaPtr,
Pete Cooper82cd9e82011-11-08 18:42:53 +00002158 MachinePointerInfo(), false, false,
2159 false, 0);
Roman Divacky4394e682011-06-28 15:30:42 +00002160 InChain = OverflowArea.getValue(1);
2161
2162 SDValue RegSaveArea = DAG.getLoad(MVT::i32, dl, InChain, RegSaveAreaPtr,
Pete Cooper82cd9e82011-11-08 18:42:53 +00002163 MachinePointerInfo(), false, false,
2164 false, 0);
Roman Divacky4394e682011-06-28 15:30:42 +00002165 InChain = RegSaveArea.getValue(1);
2166
2167 // select overflow_area if index > 8
2168 SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002169 DAG.getConstant(8, dl, MVT::i32), ISD::SETLT);
Roman Divacky4394e682011-06-28 15:30:42 +00002170
Roman Divacky4394e682011-06-28 15:30:42 +00002171 // adjustment constant gpr_index * 4/8
2172 SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32,
2173 VT.isInteger() ? GprIndex : FprIndex,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002174 DAG.getConstant(VT.isInteger() ? 4 : 8, dl,
Roman Divacky4394e682011-06-28 15:30:42 +00002175 MVT::i32));
2176
2177 // OurReg = RegSaveArea + RegConstant
2178 SDValue OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, RegSaveArea,
2179 RegConstant);
2180
2181 // Floating types are 32 bytes into RegSaveArea
2182 if (VT.isFloatingPoint())
2183 OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, OurReg,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002184 DAG.getConstant(32, dl, MVT::i32));
Roman Divacky4394e682011-06-28 15:30:42 +00002185
2186 // increase {f,g}pr_index by 1 (or 2 if VT is i64)
2187 SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32,
2188 VT.isInteger() ? GprIndex : FprIndex,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002189 DAG.getConstant(VT == MVT::i64 ? 2 : 1, dl,
Roman Divacky4394e682011-06-28 15:30:42 +00002190 MVT::i32));
2191
2192 InChain = DAG.getTruncStore(InChain, dl, IndexPlus1,
2193 VT.isInteger() ? VAListPtr : FprPtr,
2194 MachinePointerInfo(SV),
2195 MVT::i8, false, false, 0);
2196
2197 // determine if we should load from reg_save_area or overflow_area
2198 SDValue Result = DAG.getNode(ISD::SELECT, dl, PtrVT, CC, OurReg, OverflowArea);
2199
2200 // increase overflow_area by 4/8 if gpr/fpr > 8
2201 SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea,
2202 DAG.getConstant(VT.isInteger() ? 4 : 8,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002203 dl, MVT::i32));
Roman Divacky4394e682011-06-28 15:30:42 +00002204
2205 OverflowArea = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC, OverflowArea,
2206 OverflowAreaPlusN);
2207
2208 InChain = DAG.getTruncStore(InChain, dl, OverflowArea,
2209 OverflowAreaPtr,
2210 MachinePointerInfo(),
2211 MVT::i32, false, false, 0);
2212
NAKAMURA Takumi8ad54e02012-08-30 15:52:23 +00002213 return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00002214 false, false, false, 0);
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002215}
2216
Roman Divackyc3825df2013-07-25 21:36:47 +00002217SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG,
2218 const PPCSubtarget &Subtarget) const {
2219 assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only");
2220
2221 // We have to copy the entire va_list struct:
2222 // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte
2223 return DAG.getMemcpy(Op.getOperand(0), Op,
2224 Op.getOperand(1), Op.getOperand(2),
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002225 DAG.getConstant(12, SDLoc(Op), MVT::i32), 8, false, true,
2226 false, MachinePointerInfo(), MachinePointerInfo());
Roman Divackyc3825df2013-07-25 21:36:47 +00002227}
2228
Duncan Sandsa0984362011-09-06 13:37:06 +00002229SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
2230 SelectionDAG &DAG) const {
2231 return Op.getOperand(0);
2232}
2233
2234SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
2235 SelectionDAG &DAG) const {
Bill Wendling95e1af22008-09-17 00:30:57 +00002236 SDValue Chain = Op.getOperand(0);
2237 SDValue Trmp = Op.getOperand(1); // trampoline
2238 SDValue FPtr = Op.getOperand(2); // nested function
2239 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
Andrew Trickef9de2a2013-05-25 02:42:55 +00002240 SDLoc dl(Op);
Bill Wendling95e1af22008-09-17 00:30:57 +00002241
Owen Anderson53aa7a92009-08-10 22:56:29 +00002242 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Owen Anderson9f944592009-08-11 20:47:22 +00002243 bool isPPC64 = (PtrVT == MVT::i64);
Chris Lattner229907c2011-07-18 04:54:35 +00002244 Type *IntPtrTy =
Micah Villmowcdfe20b2012-10-08 16:38:25 +00002245 DAG.getTargetLoweringInfo().getDataLayout()->getIntPtrType(
Chandler Carruth7ec50852012-11-01 08:07:29 +00002246 *DAG.getContext());
Bill Wendling95e1af22008-09-17 00:30:57 +00002247
Scott Michelcf0da6c2009-02-17 22:15:04 +00002248 TargetLowering::ArgListTy Args;
Bill Wendling95e1af22008-09-17 00:30:57 +00002249 TargetLowering::ArgListEntry Entry;
2250
2251 Entry.Ty = IntPtrTy;
2252 Entry.Node = Trmp; Args.push_back(Entry);
2253
2254 // TrampSize == (isPPC64 ? 48 : 40);
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002255 Entry.Node = DAG.getConstant(isPPC64 ? 48 : 40, dl,
Owen Anderson9f944592009-08-11 20:47:22 +00002256 isPPC64 ? MVT::i64 : MVT::i32);
Bill Wendling95e1af22008-09-17 00:30:57 +00002257 Args.push_back(Entry);
2258
2259 Entry.Node = FPtr; Args.push_back(Entry);
2260 Entry.Node = Nest; Args.push_back(Entry);
Scott Michelcf0da6c2009-02-17 22:15:04 +00002261
Bill Wendling95e1af22008-09-17 00:30:57 +00002262 // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg)
Saleem Abdulrasoolf3a5a5c2014-05-17 21:50:17 +00002263 TargetLowering::CallLoweringInfo CLI(DAG);
2264 CLI.setDebugLoc(dl).setChain(Chain)
2265 .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
Juergen Ributzka3bd03c72014-07-01 22:01:54 +00002266 DAG.getExternalSymbol("__trampoline_setup", PtrVT),
2267 std::move(Args), 0);
Bill Wendling95e1af22008-09-17 00:30:57 +00002268
Saleem Abdulrasoolf3a5a5c2014-05-17 21:50:17 +00002269 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
Duncan Sandsa0984362011-09-06 13:37:06 +00002270 return CallResult.second;
Bill Wendling95e1af22008-09-17 00:30:57 +00002271}
2272
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002273SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG,
Dan Gohman21cea8a2010-04-17 15:26:15 +00002274 const PPCSubtarget &Subtarget) const {
Dan Gohman31ae5862010-04-17 14:41:14 +00002275 MachineFunction &MF = DAG.getMachineFunction();
2276 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2277
Andrew Trickef9de2a2013-05-25 02:42:55 +00002278 SDLoc dl(Op);
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002279
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00002280 if (Subtarget.isDarwinABI() || Subtarget.isPPC64()) {
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002281 // vastart just stores the address of the VarArgsFrameIndex slot into the
2282 // memory location argument.
Owen Anderson53aa7a92009-08-10 22:56:29 +00002283 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Dan Gohman31ae5862010-04-17 14:41:14 +00002284 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
Dan Gohman2d489b52008-02-06 22:27:42 +00002285 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
Chris Lattner676c61d2010-09-21 18:41:36 +00002286 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
2287 MachinePointerInfo(SV),
David Greene87a5abe2010-02-15 16:56:53 +00002288 false, false, 0);
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002289 }
2290
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00002291 // For the 32-bit SVR4 ABI we follow the layout of the va_list struct.
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002292 // We suppose the given va_list is already allocated.
2293 //
2294 // typedef struct {
2295 // char gpr; /* index into the array of 8 GPRs
2296 // * stored in the register save area
2297 // * gpr=0 corresponds to r3,
2298 // * gpr=1 to r4, etc.
2299 // */
2300 // char fpr; /* index into the array of 8 FPRs
2301 // * stored in the register save area
2302 // * fpr=0 corresponds to f1,
2303 // * fpr=1 to f2, etc.
2304 // */
2305 // char *overflow_arg_area;
2306 // /* location on stack that holds
2307 // * the next overflow argument
2308 // */
2309 // char *reg_save_area;
2310 // /* where r3:r10 and f1:f8 (if saved)
2311 // * are stored
2312 // */
2313 // } va_list[1];
2314
2315
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002316 SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), dl, MVT::i32);
2317 SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), dl, MVT::i32);
Scott Michelcf0da6c2009-02-17 22:15:04 +00002318
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002319
Owen Anderson53aa7a92009-08-10 22:56:29 +00002320 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Scott Michelcf0da6c2009-02-17 22:15:04 +00002321
Dan Gohman31ae5862010-04-17 14:41:14 +00002322 SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(),
2323 PtrVT);
2324 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
2325 PtrVT);
Scott Michelcf0da6c2009-02-17 22:15:04 +00002326
Duncan Sands13237ac2008-06-06 12:08:01 +00002327 uint64_t FrameOffset = PtrVT.getSizeInBits()/8;
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002328 SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, dl, PtrVT);
Dan Gohman2d489b52008-02-06 22:27:42 +00002329
Duncan Sands13237ac2008-06-06 12:08:01 +00002330 uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1;
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002331 SDValue ConstStackOffset = DAG.getConstant(StackOffset, dl, PtrVT);
Dan Gohman2d489b52008-02-06 22:27:42 +00002332
2333 uint64_t FPROffset = 1;
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002334 SDValue ConstFPROffset = DAG.getConstant(FPROffset, dl, PtrVT);
Scott Michelcf0da6c2009-02-17 22:15:04 +00002335
Dan Gohman2d489b52008-02-06 22:27:42 +00002336 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
Scott Michelcf0da6c2009-02-17 22:15:04 +00002337
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002338 // Store first byte : number of int regs
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002339 SDValue firstStore = DAG.getTruncStore(Op.getOperand(0), dl, ArgGPR,
Chris Lattner6963c1f2010-09-21 17:42:31 +00002340 Op.getOperand(1),
2341 MachinePointerInfo(SV),
2342 MVT::i8, false, false, 0);
Dan Gohman2d489b52008-02-06 22:27:42 +00002343 uint64_t nextOffset = FPROffset;
Dale Johannesen021052a2009-02-04 20:06:27 +00002344 SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, Op.getOperand(1),
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002345 ConstFPROffset);
Scott Michelcf0da6c2009-02-17 22:15:04 +00002346
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002347 // Store second byte : number of float regs
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002348 SDValue secondStore =
Chris Lattner6963c1f2010-09-21 17:42:31 +00002349 DAG.getTruncStore(firstStore, dl, ArgFPR, nextPtr,
2350 MachinePointerInfo(SV, nextOffset), MVT::i8,
David Greene87a5abe2010-02-15 16:56:53 +00002351 false, false, 0);
Dan Gohman2d489b52008-02-06 22:27:42 +00002352 nextOffset += StackOffset;
Dale Johannesen021052a2009-02-04 20:06:27 +00002353 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstStackOffset);
Scott Michelcf0da6c2009-02-17 22:15:04 +00002354
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002355 // Store second word : arguments given on stack
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002356 SDValue thirdStore =
Chris Lattner676c61d2010-09-21 18:41:36 +00002357 DAG.getStore(secondStore, dl, StackOffsetFI, nextPtr,
2358 MachinePointerInfo(SV, nextOffset),
David Greene87a5abe2010-02-15 16:56:53 +00002359 false, false, 0);
Dan Gohman2d489b52008-02-06 22:27:42 +00002360 nextOffset += FrameOffset;
Dale Johannesen021052a2009-02-04 20:06:27 +00002361 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstFrameOffset);
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002362
2363 // Store third word : arguments given in registers
Chris Lattner676c61d2010-09-21 18:41:36 +00002364 return DAG.getStore(thirdStore, dl, FR, nextPtr,
2365 MachinePointerInfo(SV, nextOffset),
David Greene87a5abe2010-02-15 16:56:53 +00002366 false, false, 0);
Nicolas Geoffray23710a72007-04-03 13:59:52 +00002367
Chris Lattner4211ca92006-04-14 06:01:58 +00002368}
2369
Chris Lattner4f2e4e02007-03-06 00:59:59 +00002370#include "PPCGenCallingConv.inc"
2371
Bill Schmidt8c3976e2013-08-26 20:11:46 +00002372// Function whose sole purpose is to kill compiler warnings
2373// stemming from unused functions included from PPCGenCallingConv.inc.
2374CCAssignFn *PPCTargetLowering::useFastISelCCs(unsigned Flag) const {
Bill Schmidt8470b0f2013-08-30 22:18:55 +00002375 return Flag ? CC_PPC64_ELF_FIS : RetCC_PPC64_ELF_FIS;
Bill Schmidt8c3976e2013-08-26 20:11:46 +00002376}
2377
Bill Schmidt230b4512013-06-12 16:39:22 +00002378bool llvm::CC_PPC32_SVR4_Custom_Dummy(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
2379 CCValAssign::LocInfo &LocInfo,
2380 ISD::ArgFlagsTy &ArgFlags,
2381 CCState &State) {
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002382 return true;
2383}
2384
Bill Schmidt230b4512013-06-12 16:39:22 +00002385bool llvm::CC_PPC32_SVR4_Custom_AlignArgRegs(unsigned &ValNo, MVT &ValVT,
2386 MVT &LocVT,
2387 CCValAssign::LocInfo &LocInfo,
2388 ISD::ArgFlagsTy &ArgFlags,
2389 CCState &State) {
Craig Topper840beec2014-04-04 05:16:06 +00002390 static const MCPhysReg ArgRegs[] = {
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002391 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
2392 PPC::R7, PPC::R8, PPC::R9, PPC::R10,
2393 };
2394 const unsigned NumArgRegs = array_lengthof(ArgRegs);
Wesley Peck527da1b2010-11-23 03:31:01 +00002395
Tim Northover3b6b7ca2015-02-21 02:11:17 +00002396 unsigned RegNum = State.getFirstUnallocated(ArgRegs);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002397
2398 // Skip one register if the first unallocated register has an even register
2399 // number and there are still argument registers available which have not been
2400 // allocated yet. RegNum is actually an index into ArgRegs, which means we
2401 // need to skip a register if RegNum is odd.
2402 if (RegNum != NumArgRegs && RegNum % 2 == 1) {
2403 State.AllocateReg(ArgRegs[RegNum]);
2404 }
Wesley Peck527da1b2010-11-23 03:31:01 +00002405
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002406 // Always return false here, as this function only makes sure that the first
2407 // unallocated register has an odd register number and does not actually
2408 // allocate a register for the current argument.
2409 return false;
2410}
2411
Bill Schmidt230b4512013-06-12 16:39:22 +00002412bool llvm::CC_PPC32_SVR4_Custom_AlignFPArgRegs(unsigned &ValNo, MVT &ValVT,
2413 MVT &LocVT,
2414 CCValAssign::LocInfo &LocInfo,
2415 ISD::ArgFlagsTy &ArgFlags,
2416 CCState &State) {
Craig Topper840beec2014-04-04 05:16:06 +00002417 static const MCPhysReg ArgRegs[] = {
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002418 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7,
2419 PPC::F8
2420 };
2421
2422 const unsigned NumArgRegs = array_lengthof(ArgRegs);
Wesley Peck527da1b2010-11-23 03:31:01 +00002423
Tim Northover3b6b7ca2015-02-21 02:11:17 +00002424 unsigned RegNum = State.getFirstUnallocated(ArgRegs);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002425
2426 // If there is only one Floating-point register left we need to put both f64
2427 // values of a split ppc_fp128 value on the stack.
2428 if (RegNum != NumArgRegs && ArgRegs[RegNum] == PPC::F8) {
2429 State.AllocateReg(ArgRegs[RegNum]);
2430 }
Wesley Peck527da1b2010-11-23 03:31:01 +00002431
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002432 // Always return false here, as this function only makes sure that the two f64
2433 // values a ppc_fp128 value is split into are both passed in registers or both
2434 // passed on the stack and does not actually allocate a register for the
2435 // current argument.
2436 return false;
2437}
2438
Benjamin Kramer7149aab2015-03-01 18:09:56 +00002439/// FPR - The set of FP registers that should be allocated for arguments,
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00002440/// on Darwin.
Benjamin Kramer7149aab2015-03-01 18:09:56 +00002441static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5,
2442 PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10,
2443 PPC::F11, PPC::F12, PPC::F13};
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00002444
Benjamin Kramer7149aab2015-03-01 18:09:56 +00002445/// QFPR - The set of QPX registers that should be allocated for arguments.
2446static const MCPhysReg QFPR[] = {
2447 PPC::QF1, PPC::QF2, PPC::QF3, PPC::QF4, PPC::QF5, PPC::QF6, PPC::QF7,
2448 PPC::QF8, PPC::QF9, PPC::QF10, PPC::QF11, PPC::QF12, PPC::QF13};
Hal Finkelc93a9a22015-02-25 01:06:45 +00002449
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00002450/// CalculateStackSlotSize - Calculates the size reserved for this argument on
2451/// the stack.
Owen Anderson53aa7a92009-08-10 22:56:29 +00002452static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags,
Tilmann Scheller98bdaaa2009-07-03 06:43:35 +00002453 unsigned PtrByteSize) {
Hal Finkel940ab932014-02-28 00:27:01 +00002454 unsigned ArgSize = ArgVT.getStoreSize();
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00002455 if (Flags.isByVal())
2456 ArgSize = Flags.getByValSize();
Ulrich Weigand85d5df22014-07-21 00:13:26 +00002457
2458 // Round up to multiples of the pointer size, except for array members,
2459 // which are always packed.
2460 if (!Flags.isInConsecutiveRegs())
2461 ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00002462
2463 return ArgSize;
2464}
Ulrich Weigandec2bf932014-07-07 19:26:41 +00002465
2466/// CalculateStackSlotAlignment - Calculates the alignment of this argument
2467/// on the stack.
Ulrich Weigand85d5df22014-07-21 00:13:26 +00002468static unsigned CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
2469 ISD::ArgFlagsTy Flags,
Ulrich Weigandec2bf932014-07-07 19:26:41 +00002470 unsigned PtrByteSize) {
2471 unsigned Align = PtrByteSize;
2472
2473 // Altivec parameters are padded to a 16 byte boundary.
2474 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
2475 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
2476 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64)
2477 Align = 16;
Hal Finkelc93a9a22015-02-25 01:06:45 +00002478 // QPX vector types stored in double-precision are padded to a 32 byte
2479 // boundary.
2480 else if (ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1)
2481 Align = 32;
Ulrich Weigandec2bf932014-07-07 19:26:41 +00002482
2483 // ByVal parameters are aligned as requested.
2484 if (Flags.isByVal()) {
2485 unsigned BVAlign = Flags.getByValAlign();
2486 if (BVAlign > PtrByteSize) {
2487 if (BVAlign % PtrByteSize != 0)
2488 llvm_unreachable(
2489 "ByVal alignment is not a multiple of the pointer size");
2490
2491 Align = BVAlign;
2492 }
2493 }
2494
Ulrich Weigand85d5df22014-07-21 00:13:26 +00002495 // Array members are always packed to their original alignment.
2496 if (Flags.isInConsecutiveRegs()) {
2497 // If the array member was split into multiple registers, the first
2498 // needs to be aligned to the size of the full type. (Except for
2499 // ppcf128, which is only aligned as its f64 components.)
2500 if (Flags.isSplit() && OrigVT != MVT::ppcf128)
2501 Align = OrigVT.getStoreSize();
2502 else
2503 Align = ArgVT.getStoreSize();
2504 }
2505
Ulrich Weigandec2bf932014-07-07 19:26:41 +00002506 return Align;
2507}
2508
Ulrich Weigand8658f172014-07-20 23:43:15 +00002509/// CalculateStackSlotUsed - Return whether this argument will use its
2510/// stack slot (instead of being passed in registers). ArgOffset,
2511/// AvailableFPRs, and AvailableVRs must hold the current argument
2512/// position, and will be updated to account for this argument.
Ulrich Weigand85d5df22014-07-21 00:13:26 +00002513static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT,
2514 ISD::ArgFlagsTy Flags,
Ulrich Weigand8658f172014-07-20 23:43:15 +00002515 unsigned PtrByteSize,
2516 unsigned LinkageSize,
2517 unsigned ParamAreaSize,
2518 unsigned &ArgOffset,
2519 unsigned &AvailableFPRs,
Hal Finkelc93a9a22015-02-25 01:06:45 +00002520 unsigned &AvailableVRs, bool HasQPX) {
Ulrich Weigand8658f172014-07-20 23:43:15 +00002521 bool UseMemory = false;
2522
2523 // Respect alignment of argument on the stack.
Ulrich Weigand85d5df22014-07-21 00:13:26 +00002524 unsigned Align =
2525 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
Ulrich Weigand8658f172014-07-20 23:43:15 +00002526 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align;
2527 // If there's no space left in the argument save area, we must
2528 // use memory (this check also catches zero-sized arguments).
2529 if (ArgOffset >= LinkageSize + ParamAreaSize)
2530 UseMemory = true;
2531
2532 // Allocate argument on the stack.
2533 ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
Ulrich Weigand85d5df22014-07-21 00:13:26 +00002534 if (Flags.isInConsecutiveRegsLast())
2535 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
Ulrich Weigand8658f172014-07-20 23:43:15 +00002536 // If we overran the argument save area, we must use memory
2537 // (this check catches arguments passed partially in memory)
2538 if (ArgOffset > LinkageSize + ParamAreaSize)
2539 UseMemory = true;
2540
2541 // However, if the argument is actually passed in an FPR or a VR,
2542 // we don't use memory after all.
2543 if (!Flags.isByVal()) {
Hal Finkelc93a9a22015-02-25 01:06:45 +00002544 if (ArgVT == MVT::f32 || ArgVT == MVT::f64 ||
2545 // QPX registers overlap with the scalar FP registers.
2546 (HasQPX && (ArgVT == MVT::v4f32 ||
2547 ArgVT == MVT::v4f64 ||
2548 ArgVT == MVT::v4i1)))
Ulrich Weigand8658f172014-07-20 23:43:15 +00002549 if (AvailableFPRs > 0) {
2550 --AvailableFPRs;
2551 return false;
2552 }
2553 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
2554 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
2555 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64)
2556 if (AvailableVRs > 0) {
2557 --AvailableVRs;
2558 return false;
2559 }
2560 }
2561
2562 return UseMemory;
2563}
2564
Ulrich Weigand2bffb952014-06-23 13:08:27 +00002565/// EnsureStackAlignment - Round stack frame size up from NumBytes to
2566/// ensure minimum alignment required for target.
Eric Christophercccae792015-01-30 22:02:31 +00002567static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering,
Ulrich Weigand2bffb952014-06-23 13:08:27 +00002568 unsigned NumBytes) {
Eric Christophercccae792015-01-30 22:02:31 +00002569 unsigned TargetAlign = Lowering->getStackAlignment();
Ulrich Weigand2bffb952014-06-23 13:08:27 +00002570 unsigned AlignMask = TargetAlign - 1;
2571 NumBytes = (NumBytes + AlignMask) & ~AlignMask;
2572 return NumBytes;
2573}
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00002574
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002575SDValue
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002576PPCTargetLowering::LowerFormalArguments(SDValue Chain,
Sandeep Patel68c5f472009-09-02 08:44:58 +00002577 CallingConv::ID CallConv, bool isVarArg,
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002578 const SmallVectorImpl<ISD::InputArg>
2579 &Ins,
Andrew Trickef9de2a2013-05-25 02:42:55 +00002580 SDLoc dl, SelectionDAG &DAG,
Dan Gohman21cea8a2010-04-17 15:26:15 +00002581 SmallVectorImpl<SDValue> &InVals)
2582 const {
Eric Christopherb1aaebe2014-06-12 22:38:18 +00002583 if (Subtarget.isSVR4ABI()) {
2584 if (Subtarget.isPPC64())
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002585 return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins,
2586 dl, DAG, InVals);
2587 else
2588 return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins,
2589 dl, DAG, InVals);
Bill Schmidt019cc6f2012-09-19 15:42:13 +00002590 } else {
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002591 return LowerFormalArguments_Darwin(Chain, CallConv, isVarArg, Ins,
2592 dl, DAG, InVals);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002593 }
2594}
2595
2596SDValue
Bill Schmidt019cc6f2012-09-19 15:42:13 +00002597PPCTargetLowering::LowerFormalArguments_32SVR4(
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002598 SDValue Chain,
Sandeep Patel68c5f472009-09-02 08:44:58 +00002599 CallingConv::ID CallConv, bool isVarArg,
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002600 const SmallVectorImpl<ISD::InputArg>
2601 &Ins,
Andrew Trickef9de2a2013-05-25 02:42:55 +00002602 SDLoc dl, SelectionDAG &DAG,
Dan Gohman21cea8a2010-04-17 15:26:15 +00002603 SmallVectorImpl<SDValue> &InVals) const {
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002604
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00002605 // 32-bit SVR4 ABI Stack Frame Layout:
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002606 // +-----------------------------------+
2607 // +--> | Back chain |
2608 // | +-----------------------------------+
2609 // | | Floating-point register save area |
2610 // | +-----------------------------------+
2611 // | | General register save area |
2612 // | +-----------------------------------+
2613 // | | CR save word |
2614 // | +-----------------------------------+
2615 // | | VRSAVE save word |
2616 // | +-----------------------------------+
2617 // | | Alignment padding |
2618 // | +-----------------------------------+
2619 // | | Vector register save area |
2620 // | +-----------------------------------+
2621 // | | Local variable space |
2622 // | +-----------------------------------+
2623 // | | Parameter list area |
2624 // | +-----------------------------------+
2625 // | | LR save word |
2626 // | +-----------------------------------+
2627 // SP--> +--- | Back chain |
2628 // +-----------------------------------+
2629 //
2630 // Specifications:
2631 // System V Application Binary Interface PowerPC Processor Supplement
2632 // AltiVec Technology Programming Interface Manual
Wesley Peck527da1b2010-11-23 03:31:01 +00002633
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002634 MachineFunction &MF = DAG.getMachineFunction();
2635 MachineFrameInfo *MFI = MF.getFrameInfo();
Dan Gohman31ae5862010-04-17 14:41:14 +00002636 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002637
Owen Anderson53aa7a92009-08-10 22:56:29 +00002638 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002639 // Potential tail calls could cause overwriting of argument stack slots.
Nick Lewycky50f02cb2011-12-02 22:16:29 +00002640 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
2641 (CallConv == CallingConv::Fast));
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002642 unsigned PtrByteSize = 4;
2643
2644 // Assign locations to all of the incoming arguments.
2645 SmallVector<CCValAssign, 16> ArgLocs;
Eric Christopherb5217502014-08-06 18:45:26 +00002646 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2647 *DAG.getContext());
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002648
2649 // Reserve space for the linkage area on the stack.
Eric Christophera4ae2132015-02-13 22:22:57 +00002650 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
Ulrich Weigand8ca988f2014-06-23 14:15:53 +00002651 CCInfo.AllocateStack(LinkageSize, PtrByteSize);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002652
Bill Schmidtef17c142013-02-06 17:33:58 +00002653 CCInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4);
Wesley Peck527da1b2010-11-23 03:31:01 +00002654
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002655 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
2656 CCValAssign &VA = ArgLocs[i];
Wesley Peck527da1b2010-11-23 03:31:01 +00002657
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002658 // Arguments stored in registers.
2659 if (VA.isRegLoc()) {
Craig Topper760b1342012-02-22 05:59:10 +00002660 const TargetRegisterClass *RC;
Owen Anderson53aa7a92009-08-10 22:56:29 +00002661 EVT ValVT = VA.getValVT();
Wesley Peck527da1b2010-11-23 03:31:01 +00002662
Owen Anderson9f944592009-08-11 20:47:22 +00002663 switch (ValVT.getSimpleVT().SimpleTy) {
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002664 default:
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002665 llvm_unreachable("ValVT not supported by formal arguments Lowering");
Hal Finkel940ab932014-02-28 00:27:01 +00002666 case MVT::i1:
Owen Anderson9f944592009-08-11 20:47:22 +00002667 case MVT::i32:
Craig Topperabadc662012-04-20 06:31:50 +00002668 RC = &PPC::GPRCRegClass;
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002669 break;
Owen Anderson9f944592009-08-11 20:47:22 +00002670 case MVT::f32:
Craig Topperabadc662012-04-20 06:31:50 +00002671 RC = &PPC::F4RCRegClass;
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002672 break;
Owen Anderson9f944592009-08-11 20:47:22 +00002673 case MVT::f64:
Eric Christopherb1aaebe2014-06-12 22:38:18 +00002674 if (Subtarget.hasVSX())
Hal Finkel19be5062014-03-29 05:29:01 +00002675 RC = &PPC::VSFRCRegClass;
2676 else
2677 RC = &PPC::F8RCRegClass;
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002678 break;
Owen Anderson9f944592009-08-11 20:47:22 +00002679 case MVT::v16i8:
2680 case MVT::v8i16:
2681 case MVT::v4i32:
Hal Finkel7811c612014-03-28 19:58:11 +00002682 RC = &PPC::VRRCRegClass;
2683 break;
Hal Finkelc93a9a22015-02-25 01:06:45 +00002684 case MVT::v4f32:
2685 RC = Subtarget.hasQPX() ? &PPC::QSRCRegClass : &PPC::VRRCRegClass;
2686 break;
Hal Finkel27774d92014-03-13 07:58:58 +00002687 case MVT::v2f64:
Hal Finkela6c8b512014-03-26 16:12:58 +00002688 case MVT::v2i64:
Hal Finkel7811c612014-03-28 19:58:11 +00002689 RC = &PPC::VSHRCRegClass;
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002690 break;
Hal Finkelc93a9a22015-02-25 01:06:45 +00002691 case MVT::v4f64:
2692 RC = &PPC::QFRCRegClass;
2693 break;
2694 case MVT::v4i1:
2695 RC = &PPC::QBRCRegClass;
2696 break;
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002697 }
Wesley Peck527da1b2010-11-23 03:31:01 +00002698
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002699 // Transform the arguments stored in physical registers into virtual ones.
Devang Patelf3292b22011-02-21 23:21:26 +00002700 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
Hal Finkel940ab932014-02-28 00:27:01 +00002701 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, Reg,
2702 ValVT == MVT::i1 ? MVT::i32 : ValVT);
2703
2704 if (ValVT == MVT::i1)
2705 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgValue);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002706
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002707 InVals.push_back(ArgValue);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002708 } else {
2709 // Argument stored in memory.
2710 assert(VA.isMemLoc());
2711
Hal Finkel940ab932014-02-28 00:27:01 +00002712 unsigned ArgSize = VA.getLocVT().getStoreSize();
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002713 int FI = MFI->CreateFixedObject(ArgSize, VA.getLocMemOffset(),
Evan Cheng0664a672010-07-03 00:40:23 +00002714 isImmutable);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002715
2716 // Create load nodes to retrieve arguments from the stack.
2717 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
Chris Lattner7727d052010-09-21 06:44:06 +00002718 InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
2719 MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00002720 false, false, false, 0));
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002721 }
2722 }
2723
2724 // Assign locations to all of the incoming aggregate by value arguments.
2725 // Aggregates passed by value are stored in the local variable space of the
2726 // caller's stack frame, right above the parameter list area.
2727 SmallVector<CCValAssign, 16> ByValArgLocs;
Eric Christopher0713a9d2011-06-08 23:55:35 +00002728 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(),
Eric Christopherb5217502014-08-06 18:45:26 +00002729 ByValArgLocs, *DAG.getContext());
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002730
2731 // Reserve stack space for the allocations in CCInfo.
2732 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize);
2733
Bill Schmidtef17c142013-02-06 17:33:58 +00002734 CCByValInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4_ByVal);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002735
2736 // Area that is at least reserved in the caller of this function.
2737 unsigned MinReservedArea = CCByValInfo.getNextStackOffset();
Ulrich Weigand8ca988f2014-06-23 14:15:53 +00002738 MinReservedArea = std::max(MinReservedArea, LinkageSize);
Wesley Peck527da1b2010-11-23 03:31:01 +00002739
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002740 // Set the size that is at least reserved in caller of this function. Tail
2741 // call optimized function's reserved stack space needs to be aligned so that
2742 // taking the difference between two stack areas will result in an aligned
2743 // stack.
Eric Christophercccae792015-01-30 22:02:31 +00002744 MinReservedArea =
2745 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea);
Ulrich Weigand2bffb952014-06-23 13:08:27 +00002746 FuncInfo->setMinReservedArea(MinReservedArea);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002747
2748 SmallVector<SDValue, 8> MemOps;
Wesley Peck527da1b2010-11-23 03:31:01 +00002749
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002750 // If the function takes variable number of arguments, make a frame index for
2751 // the start of the first vararg value... for expansion of llvm.va_start.
2752 if (isVarArg) {
Craig Topper840beec2014-04-04 05:16:06 +00002753 static const MCPhysReg GPArgRegs[] = {
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002754 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
2755 PPC::R7, PPC::R8, PPC::R9, PPC::R10,
2756 };
2757 const unsigned NumGPArgRegs = array_lengthof(GPArgRegs);
2758
Craig Topper840beec2014-04-04 05:16:06 +00002759 static const MCPhysReg FPArgRegs[] = {
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002760 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7,
2761 PPC::F8
2762 };
Joerg Sonnenbergereb8655a2014-08-08 16:46:10 +00002763 unsigned NumFPArgRegs = array_lengthof(FPArgRegs);
2764 if (DisablePPCFloatInVariadic)
2765 NumFPArgRegs = 0;
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002766
Tim Northover3b6b7ca2015-02-21 02:11:17 +00002767 FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(GPArgRegs));
2768 FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(FPArgRegs));
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002769
2770 // Make room for NumGPArgRegs and NumFPArgRegs.
2771 int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 +
Craig Topper7ff15922014-09-10 04:51:36 +00002772 NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8;
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002773
Dan Gohman31ae5862010-04-17 14:41:14 +00002774 FuncInfo->setVarArgsStackOffset(
2775 MFI->CreateFixedObject(PtrVT.getSizeInBits()/8,
Evan Cheng0664a672010-07-03 00:40:23 +00002776 CCInfo.getNextStackOffset(), true));
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002777
Dan Gohman31ae5862010-04-17 14:41:14 +00002778 FuncInfo->setVarArgsFrameIndex(MFI->CreateStackObject(Depth, 8, false));
2779 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002780
Jakob Stoklund Olesen6c4353e2010-10-11 20:43:09 +00002781 // The fixed integer arguments of a variadic function are stored to the
2782 // VarArgsFrameIndex on the stack so that they may be loaded by deferencing
2783 // the result of va_next.
2784 for (unsigned GPRIndex = 0; GPRIndex != NumGPArgRegs; ++GPRIndex) {
2785 // Get an existing live-in vreg, or add a new one.
2786 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(GPArgRegs[GPRIndex]);
2787 if (!VReg)
Devang Patelf3292b22011-02-21 23:21:26 +00002788 VReg = MF.addLiveIn(GPArgRegs[GPRIndex], &PPC::GPRCRegClass);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002789
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002790 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
Chris Lattner676c61d2010-09-21 18:41:36 +00002791 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
2792 MachinePointerInfo(), false, false, 0);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002793 MemOps.push_back(Store);
2794 // Increment the address by four for the next argument to store
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002795 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002796 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
2797 }
2798
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00002799 // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6
2800 // is set.
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002801 // The double arguments are stored to the VarArgsFrameIndex
2802 // on the stack.
Jakob Stoklund Olesen6c4353e2010-10-11 20:43:09 +00002803 for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) {
2804 // Get an existing live-in vreg, or add a new one.
2805 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(FPArgRegs[FPRIndex]);
2806 if (!VReg)
Devang Patelf3292b22011-02-21 23:21:26 +00002807 VReg = MF.addLiveIn(FPArgRegs[FPRIndex], &PPC::F8RCRegClass);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002808
Owen Anderson9f944592009-08-11 20:47:22 +00002809 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::f64);
Chris Lattner676c61d2010-09-21 18:41:36 +00002810 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
2811 MachinePointerInfo(), false, false, 0);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002812 MemOps.push_back(Store);
2813 // Increment the address by eight for the next argument to store
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002814 SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8, dl,
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002815 PtrVT);
2816 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
2817 }
2818 }
2819
2820 if (!MemOps.empty())
Craig Topper48d114b2014-04-26 18:35:24 +00002821 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002822
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002823 return Chain;
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002824}
2825
Bill Schmidt57d6de52012-10-23 15:51:16 +00002826// PPC64 passes i8, i16, and i32 values in i64 registers. Promote
2827// value to MVT::i64 and then truncate to the correct register size.
2828SDValue
2829PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags, EVT ObjectVT,
2830 SelectionDAG &DAG, SDValue ArgVal,
Andrew Trickef9de2a2013-05-25 02:42:55 +00002831 SDLoc dl) const {
Bill Schmidt57d6de52012-10-23 15:51:16 +00002832 if (Flags.isSExt())
2833 ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal,
2834 DAG.getValueType(ObjectVT));
2835 else if (Flags.isZExt())
2836 ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal,
2837 DAG.getValueType(ObjectVT));
Matt Arsenault758659232013-05-18 00:21:46 +00002838
Hal Finkel940ab932014-02-28 00:27:01 +00002839 return DAG.getNode(ISD::TRUNCATE, dl, ObjectVT, ArgVal);
Bill Schmidt57d6de52012-10-23 15:51:16 +00002840}
2841
Tilmann Schellerb93960d2009-07-03 06:45:56 +00002842SDValue
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002843PPCTargetLowering::LowerFormalArguments_64SVR4(
2844 SDValue Chain,
2845 CallingConv::ID CallConv, bool isVarArg,
2846 const SmallVectorImpl<ISD::InputArg>
2847 &Ins,
Andrew Trickef9de2a2013-05-25 02:42:55 +00002848 SDLoc dl, SelectionDAG &DAG,
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002849 SmallVectorImpl<SDValue> &InVals) const {
2850 // TODO: add description of PPC stack frame format, or at least some docs.
2851 //
Ulrich Weigand8658f172014-07-20 23:43:15 +00002852 bool isELFv2ABI = Subtarget.isELFv2ABI();
Ulrich Weigand59c6ab22014-06-20 16:34:05 +00002853 bool isLittleEndian = Subtarget.isLittleEndian();
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002854 MachineFunction &MF = DAG.getMachineFunction();
2855 MachineFrameInfo *MFI = MF.getFrameInfo();
2856 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2857
Hal Finkelf81b6dd2015-01-18 12:08:47 +00002858 assert(!(CallConv == CallingConv::Fast && isVarArg) &&
2859 "fastcc not supported on varargs functions");
2860
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002861 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
2862 // Potential tail calls could cause overwriting of argument stack slots.
2863 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
2864 (CallConv == CallingConv::Fast));
2865 unsigned PtrByteSize = 8;
Eric Christophera4ae2132015-02-13 22:22:57 +00002866 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002867
Craig Topper840beec2014-04-04 05:16:06 +00002868 static const MCPhysReg GPR[] = {
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002869 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
2870 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
2871 };
Craig Topper840beec2014-04-04 05:16:06 +00002872 static const MCPhysReg VR[] = {
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002873 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
2874 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
2875 };
Craig Topper840beec2014-04-04 05:16:06 +00002876 static const MCPhysReg VSRH[] = {
Hal Finkel7811c612014-03-28 19:58:11 +00002877 PPC::VSH2, PPC::VSH3, PPC::VSH4, PPC::VSH5, PPC::VSH6, PPC::VSH7, PPC::VSH8,
2878 PPC::VSH9, PPC::VSH10, PPC::VSH11, PPC::VSH12, PPC::VSH13
2879 };
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002880
2881 const unsigned Num_GPR_Regs = array_lengthof(GPR);
2882 const unsigned Num_FPR_Regs = 13;
2883 const unsigned Num_VR_Regs = array_lengthof(VR);
Hal Finkelc93a9a22015-02-25 01:06:45 +00002884 const unsigned Num_QFPR_Regs = Num_FPR_Regs;
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002885
Ulrich Weigand8658f172014-07-20 23:43:15 +00002886 // Do a first pass over the arguments to determine whether the ABI
2887 // guarantees that our caller has allocated the parameter save area
2888 // on its stack frame. In the ELFv1 ABI, this is always the case;
2889 // in the ELFv2 ABI, it is true if this is a vararg function or if
2890 // any parameter is located in a stack slot.
2891
2892 bool HasParameterArea = !isELFv2ABI || isVarArg;
2893 unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize;
2894 unsigned NumBytes = LinkageSize;
2895 unsigned AvailableFPRs = Num_FPR_Regs;
2896 unsigned AvailableVRs = Num_VR_Regs;
2897 for (unsigned i = 0, e = Ins.size(); i != e; ++i)
Ulrich Weigand85d5df22014-07-21 00:13:26 +00002898 if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags,
Ulrich Weigand8658f172014-07-20 23:43:15 +00002899 PtrByteSize, LinkageSize, ParamAreaSize,
Hal Finkelc93a9a22015-02-25 01:06:45 +00002900 NumBytes, AvailableFPRs, AvailableVRs,
2901 Subtarget.hasQPX()))
Ulrich Weigand8658f172014-07-20 23:43:15 +00002902 HasParameterArea = true;
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002903
2904 // Add DAG nodes to load the arguments or copy them out of registers. On
2905 // entry to a function on PPC, the arguments start after the linkage area,
2906 // although the first ones are often in registers.
2907
Ulrich Weigand8658f172014-07-20 23:43:15 +00002908 unsigned ArgOffset = LinkageSize;
Hal Finkelf81b6dd2015-01-18 12:08:47 +00002909 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
Hal Finkelc93a9a22015-02-25 01:06:45 +00002910 unsigned &QFPR_idx = FPR_idx;
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002911 SmallVector<SDValue, 8> MemOps;
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002912 Function::const_arg_iterator FuncArg = MF.getFunction()->arg_begin();
Bill Schmidt6631e942013-02-20 17:31:41 +00002913 unsigned CurArgIdx = 0;
2914 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) {
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002915 SDValue ArgVal;
2916 bool needsLoad = false;
2917 EVT ObjectVT = Ins[ArgNo].VT;
Ulrich Weigand85d5df22014-07-21 00:13:26 +00002918 EVT OrigVT = Ins[ArgNo].ArgVT;
Hal Finkel940ab932014-02-28 00:27:01 +00002919 unsigned ObjSize = ObjectVT.getStoreSize();
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002920 unsigned ArgSize = ObjSize;
2921 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags;
Andrew Trick05938a52015-02-16 18:10:47 +00002922 if (Ins[ArgNo].isOrigArg()) {
2923 std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx);
2924 CurArgIdx = Ins[ArgNo].getOrigArgIndex();
2925 }
Hal Finkelf81b6dd2015-01-18 12:08:47 +00002926 // We re-align the argument offset for each argument, except when using the
2927 // fast calling convention, when we need to make sure we do that only when
2928 // we'll actually use a stack slot.
2929 unsigned CurArgOffset, Align;
2930 auto ComputeArgOffset = [&]() {
2931 /* Respect alignment of argument on the stack. */
2932 Align = CalculateStackSlotAlignment(ObjectVT, OrigVT, Flags, PtrByteSize);
2933 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align;
2934 CurArgOffset = ArgOffset;
2935 };
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002936
Hal Finkelf81b6dd2015-01-18 12:08:47 +00002937 if (CallConv != CallingConv::Fast) {
2938 ComputeArgOffset();
2939
2940 /* Compute GPR index associated with argument offset. */
2941 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
2942 GPR_idx = std::min(GPR_idx, Num_GPR_Regs);
2943 }
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002944
2945 // FIXME the codegen can be much improved in some cases.
2946 // We do not have to keep everything in memory.
2947 if (Flags.isByVal()) {
Andrew Trick05938a52015-02-16 18:10:47 +00002948 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit");
2949
Hal Finkelf81b6dd2015-01-18 12:08:47 +00002950 if (CallConv == CallingConv::Fast)
2951 ComputeArgOffset();
2952
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002953 // ObjSize is the true size, ArgSize rounded up to multiple of registers.
2954 ObjSize = Flags.getByValSize();
2955 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
Bill Schmidt9953cf22012-10-31 01:15:05 +00002956 // Empty aggregate parameters do not take up registers. Examples:
2957 // struct { } a;
2958 // union { } b;
2959 // int c[0];
2960 // etc. However, we have to provide a place-holder in InVals, so
2961 // pretend we have an 8-byte item at the current address for that
2962 // purpose.
2963 if (!ObjSize) {
2964 int FI = MFI->CreateFixedObject(PtrByteSize, ArgOffset, true);
2965 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
2966 InVals.push_back(FIN);
2967 continue;
2968 }
Hal Finkel262a2242013-09-12 23:20:06 +00002969
Ulrich Weigand24195972014-07-20 22:36:52 +00002970 // Create a stack object covering all stack doublewords occupied
Ulrich Weigand8658f172014-07-20 23:43:15 +00002971 // by the argument. If the argument is (fully or partially) on
2972 // the stack, or if the argument is fully in registers but the
2973 // caller has allocated the parameter save anyway, we can refer
2974 // directly to the caller's stack frame. Otherwise, create a
2975 // local copy in our own frame.
2976 int FI;
2977 if (HasParameterArea ||
2978 ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize)
Hal Finkel41a55ad2014-08-16 00:17:05 +00002979 FI = MFI->CreateFixedObject(ArgSize, ArgOffset, false, true);
Ulrich Weigand8658f172014-07-20 23:43:15 +00002980 else
2981 FI = MFI->CreateStackObject(ArgSize, Align, false);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002982 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
Bill Schmidt6ed3b992012-10-25 13:38:09 +00002983
Ulrich Weigand24195972014-07-20 22:36:52 +00002984 // Handle aggregates smaller than 8 bytes.
2985 if (ObjSize < PtrByteSize) {
2986 // The value of the object is its address, which differs from the
2987 // address of the enclosing doubleword on big-endian systems.
2988 SDValue Arg = FIN;
2989 if (!isLittleEndian) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00002990 SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, dl, PtrVT);
Ulrich Weigand24195972014-07-20 22:36:52 +00002991 Arg = DAG.getNode(ISD::ADD, dl, ArgOff.getValueType(), Arg, ArgOff);
2992 }
2993 InVals.push_back(Arg);
2994
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002995 if (GPR_idx != Num_GPR_Regs) {
Hal Finkelf81b6dd2015-01-18 12:08:47 +00002996 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00002997 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
Bill Schmidt6ed3b992012-10-25 13:38:09 +00002998 SDValue Store;
2999
3000 if (ObjSize==1 || ObjSize==2 || ObjSize==4) {
3001 EVT ObjType = (ObjSize == 1 ? MVT::i8 :
3002 (ObjSize == 2 ? MVT::i16 : MVT::i32));
Ulrich Weigand24195972014-07-20 22:36:52 +00003003 Store = DAG.getTruncStore(Val.getValue(1), dl, Val, Arg,
Hal Finkel3e4a34c2014-01-21 20:15:58 +00003004 MachinePointerInfo(FuncArg),
Bill Schmidt6ed3b992012-10-25 13:38:09 +00003005 ObjType, false, false, 0);
3006 } else {
3007 // For sizes that don't fit a truncating store (3, 5, 6, 7),
3008 // store the whole register as-is to the parameter save area
Ulrich Weigand24195972014-07-20 22:36:52 +00003009 // slot.
Bill Schmidt6ed3b992012-10-25 13:38:09 +00003010 Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
Hal Finkel3e4a34c2014-01-21 20:15:58 +00003011 MachinePointerInfo(FuncArg),
Bill Schmidt6ed3b992012-10-25 13:38:09 +00003012 false, false, 0);
3013 }
3014
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003015 MemOps.push_back(Store);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003016 }
Bill Schmidt6ed3b992012-10-25 13:38:09 +00003017 // Whether we copied from a register or not, advance the offset
3018 // into the parameter save area by a full doubleword.
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003019 ArgOffset += PtrByteSize;
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003020 continue;
3021 }
Bill Schmidt6ed3b992012-10-25 13:38:09 +00003022
Ulrich Weigand24195972014-07-20 22:36:52 +00003023 // The value of the object is its address, which is the address of
3024 // its first stack doubleword.
3025 InVals.push_back(FIN);
3026
3027 // Store whatever pieces of the object are in registers to memory.
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003028 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) {
Ulrich Weigand24195972014-07-20 22:36:52 +00003029 if (GPR_idx == Num_GPR_Regs)
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003030 break;
Ulrich Weigand24195972014-07-20 22:36:52 +00003031
3032 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
3033 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
3034 SDValue Addr = FIN;
3035 if (j) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00003036 SDValue Off = DAG.getConstant(j, dl, PtrVT);
Ulrich Weigand24195972014-07-20 22:36:52 +00003037 Addr = DAG.getNode(ISD::ADD, dl, Off.getValueType(), Addr, Off);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003038 }
Ulrich Weigand24195972014-07-20 22:36:52 +00003039 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, Addr,
3040 MachinePointerInfo(FuncArg, j),
3041 false, false, 0);
3042 MemOps.push_back(Store);
3043 ++GPR_idx;
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003044 }
Ulrich Weigand24195972014-07-20 22:36:52 +00003045 ArgOffset += ArgSize;
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003046 continue;
3047 }
3048
3049 switch (ObjectVT.getSimpleVT().SimpleTy) {
3050 default: llvm_unreachable("Unhandled argument type!");
Hal Finkel940ab932014-02-28 00:27:01 +00003051 case MVT::i1:
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003052 case MVT::i32:
3053 case MVT::i64:
Ulrich Weigand85d5df22014-07-21 00:13:26 +00003054 // These can be scalar arguments or elements of an integer array type
3055 // passed directly. Clang may use those instead of "byval" aggregate
3056 // types to avoid forcing arguments to memory unnecessarily.
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003057 if (GPR_idx != Num_GPR_Regs) {
Hal Finkelf81b6dd2015-01-18 12:08:47 +00003058 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003059 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
3060
Hal Finkel940ab932014-02-28 00:27:01 +00003061 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003062 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote
3063 // value to MVT::i64 and then truncate to the correct register size.
Bill Schmidt57d6de52012-10-23 15:51:16 +00003064 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003065 } else {
Hal Finkelf81b6dd2015-01-18 12:08:47 +00003066 if (CallConv == CallingConv::Fast)
3067 ComputeArgOffset();
3068
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003069 needsLoad = true;
3070 ArgSize = PtrByteSize;
3071 }
Hal Finkelf81b6dd2015-01-18 12:08:47 +00003072 if (CallConv != CallingConv::Fast || needsLoad)
3073 ArgOffset += 8;
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003074 break;
3075
3076 case MVT::f32:
3077 case MVT::f64:
Ulrich Weigand85d5df22014-07-21 00:13:26 +00003078 // These can be scalar arguments or elements of a float array type
3079 // passed directly. The latter are used to implement ELFv2 homogenous
3080 // float aggregates.
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003081 if (FPR_idx != Num_FPR_Regs) {
3082 unsigned VReg;
3083
3084 if (ObjectVT == MVT::f32)
3085 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass);
3086 else
Eric Christophercccae792015-01-30 22:02:31 +00003087 VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasVSX()
3088 ? &PPC::VSFRCRegClass
3089 : &PPC::F8RCRegClass);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003090
3091 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
3092 ++FPR_idx;
Hal Finkelf81b6dd2015-01-18 12:08:47 +00003093 } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) {
Hal Finkel8ea446b2015-01-18 14:31:10 +00003094 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
3095 // once we support fp <-> gpr moves.
3096
Ulrich Weigand85d5df22014-07-21 00:13:26 +00003097 // This can only ever happen in the presence of f32 array types,
3098 // since otherwise we never run out of FPRs before running out
3099 // of GPRs.
Hal Finkelf81b6dd2015-01-18 12:08:47 +00003100 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass);
Ulrich Weigand85d5df22014-07-21 00:13:26 +00003101 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
3102
3103 if (ObjectVT == MVT::f32) {
3104 if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0))
3105 ArgVal = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgVal,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00003106 DAG.getConstant(32, dl, MVT::i32));
Ulrich Weigand85d5df22014-07-21 00:13:26 +00003107 ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal);
3108 }
3109
3110 ArgVal = DAG.getNode(ISD::BITCAST, dl, ObjectVT, ArgVal);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003111 } else {
Hal Finkelf81b6dd2015-01-18 12:08:47 +00003112 if (CallConv == CallingConv::Fast)
3113 ComputeArgOffset();
3114
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003115 needsLoad = true;
3116 }
3117
Ulrich Weigand85d5df22014-07-21 00:13:26 +00003118 // When passing an array of floats, the array occupies consecutive
3119 // space in the argument area; only round up to the next doubleword
3120 // at the end of the array. Otherwise, each float takes 8 bytes.
Hal Finkelf81b6dd2015-01-18 12:08:47 +00003121 if (CallConv != CallingConv::Fast || needsLoad) {
3122 ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize;
3123 ArgOffset += ArgSize;
3124 if (Flags.isInConsecutiveRegsLast())
3125 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
3126 }
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003127 break;
3128 case MVT::v4f32:
3129 case MVT::v4i32:
3130 case MVT::v8i16:
3131 case MVT::v16i8:
Hal Finkel27774d92014-03-13 07:58:58 +00003132 case MVT::v2f64:
Hal Finkela6c8b512014-03-26 16:12:58 +00003133 case MVT::v2i64:
Hal Finkelc93a9a22015-02-25 01:06:45 +00003134 if (!Subtarget.hasQPX()) {
Ulrich Weigand85d5df22014-07-21 00:13:26 +00003135 // These can be scalar arguments or elements of a vector array type
3136 // passed directly. The latter are used to implement ELFv2 homogenous
3137 // vector aggregates.
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003138 if (VR_idx != Num_VR_Regs) {
Hal Finkel7811c612014-03-28 19:58:11 +00003139 unsigned VReg = (ObjectVT == MVT::v2f64 || ObjectVT == MVT::v2i64) ?
3140 MF.addLiveIn(VSRH[VR_idx], &PPC::VSHRCRegClass) :
3141 MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003142 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003143 ++VR_idx;
3144 } else {
Hal Finkelf81b6dd2015-01-18 12:08:47 +00003145 if (CallConv == CallingConv::Fast)
3146 ComputeArgOffset();
3147
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003148 needsLoad = true;
3149 }
Hal Finkelf81b6dd2015-01-18 12:08:47 +00003150 if (CallConv != CallingConv::Fast || needsLoad)
3151 ArgOffset += 16;
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003152 break;
Hal Finkelc93a9a22015-02-25 01:06:45 +00003153 } // not QPX
3154
3155 assert(ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 &&
3156 "Invalid QPX parameter type");
3157 /* fall through */
3158
3159 case MVT::v4f64:
3160 case MVT::v4i1:
3161 // QPX vectors are treated like their scalar floating-point subregisters
3162 // (except that they're larger).
3163 unsigned Sz = ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 ? 16 : 32;
3164 if (QFPR_idx != Num_QFPR_Regs) {
3165 const TargetRegisterClass *RC;
3166 switch (ObjectVT.getSimpleVT().SimpleTy) {
3167 case MVT::v4f64: RC = &PPC::QFRCRegClass; break;
3168 case MVT::v4f32: RC = &PPC::QSRCRegClass; break;
3169 default: RC = &PPC::QBRCRegClass; break;
3170 }
3171
3172 unsigned VReg = MF.addLiveIn(QFPR[QFPR_idx], RC);
3173 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
3174 ++QFPR_idx;
3175 } else {
3176 if (CallConv == CallingConv::Fast)
3177 ComputeArgOffset();
3178 needsLoad = true;
3179 }
3180 if (CallConv != CallingConv::Fast || needsLoad)
3181 ArgOffset += Sz;
3182 break;
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003183 }
3184
3185 // We need to load the argument to a virtual register if we determined
3186 // above that we ran out of physical registers of the appropriate type.
3187 if (needsLoad) {
Ulrich Weigand59c6ab22014-06-20 16:34:05 +00003188 if (ObjSize < ArgSize && !isLittleEndian)
3189 CurArgOffset += ArgSize - ObjSize;
3190 int FI = MFI->CreateFixedObject(ObjSize, CurArgOffset, isImmutable);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003191 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
3192 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo(),
3193 false, false, false, 0);
3194 }
3195
3196 InVals.push_back(ArgVal);
3197 }
3198
Ulrich Weigand2bffb952014-06-23 13:08:27 +00003199 // Area that is at least reserved in the caller of this function.
Ulrich Weigandec2bf932014-07-07 19:26:41 +00003200 unsigned MinReservedArea;
Ulrich Weigand8658f172014-07-20 23:43:15 +00003201 if (HasParameterArea)
3202 MinReservedArea = std::max(ArgOffset, LinkageSize + 8 * PtrByteSize);
3203 else
3204 MinReservedArea = LinkageSize;
Ulrich Weigand2bffb952014-06-23 13:08:27 +00003205
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003206 // Set the size that is at least reserved in caller of this function. Tail
Bill Schmidt57d6de52012-10-23 15:51:16 +00003207 // call optimized functions' reserved stack space needs to be aligned so that
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003208 // taking the difference between two stack areas will result in an aligned
3209 // stack.
Eric Christophercccae792015-01-30 22:02:31 +00003210 MinReservedArea =
3211 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea);
Ulrich Weigand2bffb952014-06-23 13:08:27 +00003212 FuncInfo->setMinReservedArea(MinReservedArea);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003213
3214 // If the function takes variable number of arguments, make a frame index for
3215 // the start of the first vararg value... for expansion of llvm.va_start.
3216 if (isVarArg) {
3217 int Depth = ArgOffset;
3218
3219 FuncInfo->setVarArgsFrameIndex(
Bill Schmidt57d6de52012-10-23 15:51:16 +00003220 MFI->CreateFixedObject(PtrByteSize, Depth, true));
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003221 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
3222
3223 // If this function is vararg, store any remaining integer argument regs
3224 // to their spots on the stack so that they may be loaded by deferencing the
3225 // result of va_next.
Ulrich Weigandec2bf932014-07-07 19:26:41 +00003226 for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
3227 GPR_idx < Num_GPR_Regs; ++GPR_idx) {
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003228 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
3229 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
3230 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
3231 MachinePointerInfo(), false, false, 0);
3232 MemOps.push_back(Store);
3233 // Increment the address by four for the next argument to store
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00003234 SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003235 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
3236 }
3237 }
3238
3239 if (!MemOps.empty())
Craig Topper48d114b2014-04-26 18:35:24 +00003240 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003241
3242 return Chain;
3243}
3244
3245SDValue
3246PPCTargetLowering::LowerFormalArguments_Darwin(
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003247 SDValue Chain,
Sandeep Patel68c5f472009-09-02 08:44:58 +00003248 CallingConv::ID CallConv, bool isVarArg,
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003249 const SmallVectorImpl<ISD::InputArg>
3250 &Ins,
Andrew Trickef9de2a2013-05-25 02:42:55 +00003251 SDLoc dl, SelectionDAG &DAG,
Dan Gohman21cea8a2010-04-17 15:26:15 +00003252 SmallVectorImpl<SDValue> &InVals) const {
Chris Lattner4302e8f2006-05-16 18:18:50 +00003253 // TODO: add description of PPC stack frame format, or at least some docs.
3254 //
3255 MachineFunction &MF = DAG.getMachineFunction();
3256 MachineFrameInfo *MFI = MF.getFrameInfo();
Dan Gohman31ae5862010-04-17 14:41:14 +00003257 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
Scott Michelcf0da6c2009-02-17 22:15:04 +00003258
Owen Anderson53aa7a92009-08-10 22:56:29 +00003259 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Owen Anderson9f944592009-08-11 20:47:22 +00003260 bool isPPC64 = PtrVT == MVT::i64;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003261 // Potential tail calls could cause overwriting of argument stack slots.
Nick Lewycky50f02cb2011-12-02 22:16:29 +00003262 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
3263 (CallConv == CallingConv::Fast));
Jim Laskeyf4e2e002006-11-28 14:53:52 +00003264 unsigned PtrByteSize = isPPC64 ? 8 : 4;
Eric Christophera4ae2132015-02-13 22:22:57 +00003265 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
Ulrich Weigand8ca988f2014-06-23 14:15:53 +00003266 unsigned ArgOffset = LinkageSize;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003267 // Area that is at least reserved in caller of this function.
3268 unsigned MinReservedArea = ArgOffset;
3269
Craig Topper840beec2014-04-04 05:16:06 +00003270 static const MCPhysReg GPR_32[] = { // 32-bit registers.
Chris Lattner4302e8f2006-05-16 18:18:50 +00003271 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
3272 PPC::R7, PPC::R8, PPC::R9, PPC::R10,
3273 };
Craig Topper840beec2014-04-04 05:16:06 +00003274 static const MCPhysReg GPR_64[] = { // 64-bit registers.
Chris Lattnerec78cad2006-06-26 22:48:35 +00003275 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
3276 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
3277 };
Craig Topper840beec2014-04-04 05:16:06 +00003278 static const MCPhysReg VR[] = {
Chris Lattner4302e8f2006-05-16 18:18:50 +00003279 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
3280 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
3281 };
Chris Lattnerec78cad2006-06-26 22:48:35 +00003282
Owen Andersone2f23a32007-09-07 04:06:50 +00003283 const unsigned Num_GPR_Regs = array_lengthof(GPR_32);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003284 const unsigned Num_FPR_Regs = 13;
Owen Andersone2f23a32007-09-07 04:06:50 +00003285 const unsigned Num_VR_Regs = array_lengthof( VR);
Jim Laskey48850c12006-11-16 22:43:37 +00003286
3287 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
Scott Michelcf0da6c2009-02-17 22:15:04 +00003288
Craig Topper840beec2014-04-04 05:16:06 +00003289 const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32;
Scott Michelcf0da6c2009-02-17 22:15:04 +00003290
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00003291 // In 32-bit non-varargs functions, the stack space for vectors is after the
3292 // stack space for non-vectors. We do not use this space unless we have
3293 // too many vectors to fit in registers, something that only occurs in
Scott Michelcf0da6c2009-02-17 22:15:04 +00003294 // constructed examples:), but we have to walk the arglist to figure
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00003295 // that out...for the pathological case, compute VecArgOffset as the
3296 // start of the vector parameter area. Computing VecArgOffset is the
3297 // entire point of the following loop.
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00003298 unsigned VecArgOffset = ArgOffset;
3299 if (!isVarArg && !isPPC64) {
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003300 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e;
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00003301 ++ArgNo) {
Owen Anderson53aa7a92009-08-10 22:56:29 +00003302 EVT ObjectVT = Ins[ArgNo].VT;
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003303 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags;
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00003304
Duncan Sandsd97eea32008-03-21 09:14:45 +00003305 if (Flags.isByVal()) {
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00003306 // ObjSize is the true size, ArgSize rounded up to multiple of regs.
Benjamin Kramer084b9f42012-01-20 14:42:32 +00003307 unsigned ObjSize = Flags.getByValSize();
Scott Michelcf0da6c2009-02-17 22:15:04 +00003308 unsigned ArgSize =
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00003309 ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
3310 VecArgOffset += ArgSize;
3311 continue;
3312 }
3313
Owen Anderson9f944592009-08-11 20:47:22 +00003314 switch(ObjectVT.getSimpleVT().SimpleTy) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00003315 default: llvm_unreachable("Unhandled argument type!");
Hal Finkel5cae2162014-02-28 01:17:25 +00003316 case MVT::i1:
Owen Anderson9f944592009-08-11 20:47:22 +00003317 case MVT::i32:
3318 case MVT::f32:
Bill Schmidt019cc6f2012-09-19 15:42:13 +00003319 VecArgOffset += 4;
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00003320 break;
Owen Anderson9f944592009-08-11 20:47:22 +00003321 case MVT::i64: // PPC64
3322 case MVT::f64:
Bill Schmidt019cc6f2012-09-19 15:42:13 +00003323 // FIXME: We are guaranteed to be !isPPC64 at this point.
3324 // Does MVT::i64 apply?
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00003325 VecArgOffset += 8;
3326 break;
Owen Anderson9f944592009-08-11 20:47:22 +00003327 case MVT::v4f32:
3328 case MVT::v4i32:
3329 case MVT::v8i16:
3330 case MVT::v16i8:
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00003331 // Nothing to do, we're only looking at Nonvector args here.
3332 break;
3333 }
3334 }
3335 }
3336 // We've found where the vector parameter area in memory is. Skip the
3337 // first 12 parameters; these don't use that memory.
3338 VecArgOffset = ((VecArgOffset+15)/16)*16;
3339 VecArgOffset += 12*16;
3340
Chris Lattner4302e8f2006-05-16 18:18:50 +00003341 // Add DAG nodes to load the arguments or copy them out of registers. On
Jim Laskey48850c12006-11-16 22:43:37 +00003342 // entry to a function on PPC, the arguments start after the linkage area,
3343 // although the first ones are often in registers.
Nicolas Geoffray7aad9282007-03-13 15:02:46 +00003344
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003345 SmallVector<SDValue, 8> MemOps;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003346 unsigned nAltivecParamsAtEnd = 0;
Roman Divackyca103892012-09-24 20:47:19 +00003347 Function::const_arg_iterator FuncArg = MF.getFunction()->arg_begin();
Bill Schmidt38b6cb52013-05-08 17:22:33 +00003348 unsigned CurArgIdx = 0;
3349 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003350 SDValue ArgVal;
Chris Lattner4302e8f2006-05-16 18:18:50 +00003351 bool needsLoad = false;
Owen Anderson53aa7a92009-08-10 22:56:29 +00003352 EVT ObjectVT = Ins[ArgNo].VT;
Duncan Sands13237ac2008-06-06 12:08:01 +00003353 unsigned ObjSize = ObjectVT.getSizeInBits()/8;
Jim Laskey152671f2006-11-29 13:37:09 +00003354 unsigned ArgSize = ObjSize;
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003355 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags;
Andrew Trick05938a52015-02-16 18:10:47 +00003356 if (Ins[ArgNo].isOrigArg()) {
3357 std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx);
3358 CurArgIdx = Ins[ArgNo].getOrigArgIndex();
3359 }
Chris Lattner318f0d22006-05-16 18:51:52 +00003360 unsigned CurArgOffset = ArgOffset;
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003361
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003362 // Varargs or 64 bit Altivec parameters are padded to a 16 byte boundary.
Owen Anderson9f944592009-08-11 20:47:22 +00003363 if (ObjectVT==MVT::v4f32 || ObjectVT==MVT::v4i32 ||
3364 ObjectVT==MVT::v8i16 || ObjectVT==MVT::v16i8) {
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003365 if (isVarArg || isPPC64) {
3366 MinReservedArea = ((MinReservedArea+15)/16)*16;
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003367 MinReservedArea += CalculateStackSlotSize(ObjectVT,
Dan Gohmand3fe1742008-09-13 01:54:27 +00003368 Flags,
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003369 PtrByteSize);
3370 } else nAltivecParamsAtEnd++;
3371 } else
3372 // Calculate min reserved area.
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003373 MinReservedArea += CalculateStackSlotSize(Ins[ArgNo].VT,
Dan Gohmand3fe1742008-09-13 01:54:27 +00003374 Flags,
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003375 PtrByteSize);
3376
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003377 // FIXME the codegen can be much improved in some cases.
3378 // We do not have to keep everything in memory.
Duncan Sandsd97eea32008-03-21 09:14:45 +00003379 if (Flags.isByVal()) {
Andrew Trick05938a52015-02-16 18:10:47 +00003380 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit");
3381
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003382 // ObjSize is the true size, ArgSize rounded up to multiple of registers.
Duncan Sandsd97eea32008-03-21 09:14:45 +00003383 ObjSize = Flags.getByValSize();
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003384 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003385 // Objects of size 1 and 2 are right justified, everything else is
3386 // left justified. This means the memory address is adjusted forwards.
Dale Johannesen21a8f142008-03-08 01:41:42 +00003387 if (ObjSize==1 || ObjSize==2) {
3388 CurArgOffset = CurArgOffset + (4 - ObjSize);
3389 }
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003390 // The value of the object is its address.
Hal Finkel41a55ad2014-08-16 00:17:05 +00003391 int FI = MFI->CreateFixedObject(ObjSize, CurArgOffset, false, true);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003392 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003393 InVals.push_back(FIN);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003394 if (ObjSize==1 || ObjSize==2) {
Dale Johannesen21a8f142008-03-08 01:41:42 +00003395 if (GPR_idx != Num_GPR_Regs) {
Roman Divackyd0419622011-06-17 15:21:10 +00003396 unsigned VReg;
3397 if (isPPC64)
3398 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
3399 else
3400 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003401 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
Bill Schmidt57d6de52012-10-23 15:51:16 +00003402 EVT ObjType = ObjSize == 1 ? MVT::i8 : MVT::i16;
Scott Michelcf0da6c2009-02-17 22:15:04 +00003403 SDValue Store = DAG.getTruncStore(Val.getValue(1), dl, Val, FIN,
Hal Finkel3e4a34c2014-01-21 20:15:58 +00003404 MachinePointerInfo(FuncArg),
Bill Schmidt019cc6f2012-09-19 15:42:13 +00003405 ObjType, false, false, 0);
Dale Johannesen21a8f142008-03-08 01:41:42 +00003406 MemOps.push_back(Store);
3407 ++GPR_idx;
Dale Johannesen21a8f142008-03-08 01:41:42 +00003408 }
Wesley Peck527da1b2010-11-23 03:31:01 +00003409
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003410 ArgOffset += PtrByteSize;
Wesley Peck527da1b2010-11-23 03:31:01 +00003411
Dale Johannesen21a8f142008-03-08 01:41:42 +00003412 continue;
3413 }
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003414 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) {
3415 // Store whatever pieces of the object are in registers
Bill Schmidt019cc6f2012-09-19 15:42:13 +00003416 // to memory. ArgOffset will be the address of the beginning
3417 // of the object.
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003418 if (GPR_idx != Num_GPR_Regs) {
Roman Divackyd0419622011-06-17 15:21:10 +00003419 unsigned VReg;
3420 if (isPPC64)
3421 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
3422 else
3423 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass);
Evan Cheng0664a672010-07-03 00:40:23 +00003424 int FI = MFI->CreateFixedObject(PtrByteSize, ArgOffset, true);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003425 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003426 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
Bill Schmidtd1fa36f2012-10-05 21:27:08 +00003427 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
Hal Finkel3e4a34c2014-01-21 20:15:58 +00003428 MachinePointerInfo(FuncArg, j),
David Greene87a5abe2010-02-15 16:56:53 +00003429 false, false, 0);
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003430 MemOps.push_back(Store);
3431 ++GPR_idx;
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003432 ArgOffset += PtrByteSize;
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003433 } else {
3434 ArgOffset += ArgSize - (ArgOffset-CurArgOffset);
3435 break;
3436 }
3437 }
3438 continue;
3439 }
3440
Owen Anderson9f944592009-08-11 20:47:22 +00003441 switch (ObjectVT.getSimpleVT().SimpleTy) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00003442 default: llvm_unreachable("Unhandled argument type!");
Hal Finkel5cae2162014-02-28 01:17:25 +00003443 case MVT::i1:
Owen Anderson9f944592009-08-11 20:47:22 +00003444 case MVT::i32:
Bill Wendling968f32c2008-03-07 20:49:02 +00003445 if (!isPPC64) {
Bill Wendling968f32c2008-03-07 20:49:02 +00003446 if (GPR_idx != Num_GPR_Regs) {
Devang Patelf3292b22011-02-21 23:21:26 +00003447 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass);
Owen Anderson9f944592009-08-11 20:47:22 +00003448 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
Hal Finkel7f908e82014-03-06 00:45:19 +00003449
3450 if (ObjectVT == MVT::i1)
3451 ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgVal);
3452
Bill Wendling968f32c2008-03-07 20:49:02 +00003453 ++GPR_idx;
3454 } else {
3455 needsLoad = true;
3456 ArgSize = PtrByteSize;
3457 }
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003458 // All int arguments reserve stack space in the Darwin ABI.
3459 ArgOffset += PtrByteSize;
Bill Wendling968f32c2008-03-07 20:49:02 +00003460 break;
Chris Lattner4302e8f2006-05-16 18:18:50 +00003461 }
Bill Wendling968f32c2008-03-07 20:49:02 +00003462 // FALLTHROUGH
Owen Anderson9f944592009-08-11 20:47:22 +00003463 case MVT::i64: // PPC64
Chris Lattnerec78cad2006-06-26 22:48:35 +00003464 if (GPR_idx != Num_GPR_Regs) {
Devang Patelf3292b22011-02-21 23:21:26 +00003465 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
Owen Anderson9f944592009-08-11 20:47:22 +00003466 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
Bill Wendling968f32c2008-03-07 20:49:02 +00003467
Hal Finkel940ab932014-02-28 00:27:01 +00003468 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
Bill Wendling968f32c2008-03-07 20:49:02 +00003469 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote
Owen Anderson9f944592009-08-11 20:47:22 +00003470 // value to MVT::i64 and then truncate to the correct register size.
Bill Schmidt57d6de52012-10-23 15:51:16 +00003471 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
Bill Wendling968f32c2008-03-07 20:49:02 +00003472
Chris Lattnerec78cad2006-06-26 22:48:35 +00003473 ++GPR_idx;
3474 } else {
3475 needsLoad = true;
Evan Cheng0f0aee22008-07-24 08:17:07 +00003476 ArgSize = PtrByteSize;
Chris Lattnerec78cad2006-06-26 22:48:35 +00003477 }
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003478 // All int arguments reserve stack space in the Darwin ABI.
3479 ArgOffset += 8;
Chris Lattnerec78cad2006-06-26 22:48:35 +00003480 break;
Scott Michelcf0da6c2009-02-17 22:15:04 +00003481
Owen Anderson9f944592009-08-11 20:47:22 +00003482 case MVT::f32:
3483 case MVT::f64:
Chris Lattner318f0d22006-05-16 18:51:52 +00003484 // Every 4 bytes of argument space consumes one of the GPRs available for
3485 // argument passing.
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003486 if (GPR_idx != Num_GPR_Regs) {
Chris Lattner26e2fcd2006-05-16 18:58:15 +00003487 ++GPR_idx;
Chris Lattner2cca3852006-11-18 01:57:19 +00003488 if (ObjSize == 8 && GPR_idx != Num_GPR_Regs && !isPPC64)
Chris Lattner26e2fcd2006-05-16 18:58:15 +00003489 ++GPR_idx;
Chris Lattner318f0d22006-05-16 18:51:52 +00003490 }
Chris Lattner26e2fcd2006-05-16 18:58:15 +00003491 if (FPR_idx != Num_FPR_Regs) {
Chris Lattner4302e8f2006-05-16 18:18:50 +00003492 unsigned VReg;
Tilmann Scheller98bdaaa2009-07-03 06:43:35 +00003493
Owen Anderson9f944592009-08-11 20:47:22 +00003494 if (ObjectVT == MVT::f32)
Devang Patelf3292b22011-02-21 23:21:26 +00003495 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass);
Chris Lattner4302e8f2006-05-16 18:18:50 +00003496 else
Devang Patelf3292b22011-02-21 23:21:26 +00003497 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F8RCRegClass);
Tilmann Scheller98bdaaa2009-07-03 06:43:35 +00003498
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003499 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
Chris Lattner4302e8f2006-05-16 18:18:50 +00003500 ++FPR_idx;
3501 } else {
3502 needsLoad = true;
3503 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00003504
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003505 // All FP arguments reserve stack space in the Darwin ABI.
3506 ArgOffset += isPPC64 ? 8 : ObjSize;
Chris Lattner4302e8f2006-05-16 18:18:50 +00003507 break;
Owen Anderson9f944592009-08-11 20:47:22 +00003508 case MVT::v4f32:
3509 case MVT::v4i32:
3510 case MVT::v8i16:
3511 case MVT::v16i8:
Dale Johannesenb28456e2008-03-12 00:22:17 +00003512 // Note that vector arguments in registers don't reserve stack space,
3513 // except in varargs functions.
Chris Lattner26e2fcd2006-05-16 18:58:15 +00003514 if (VR_idx != Num_VR_Regs) {
Devang Patelf3292b22011-02-21 23:21:26 +00003515 unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003516 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
Dale Johannesenb28456e2008-03-12 00:22:17 +00003517 if (isVarArg) {
3518 while ((ArgOffset % 16) != 0) {
3519 ArgOffset += PtrByteSize;
3520 if (GPR_idx != Num_GPR_Regs)
3521 GPR_idx++;
3522 }
3523 ArgOffset += 16;
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00003524 GPR_idx = std::min(GPR_idx+4, Num_GPR_Regs); // FIXME correct for ppc64?
Dale Johannesenb28456e2008-03-12 00:22:17 +00003525 }
Chris Lattner4302e8f2006-05-16 18:18:50 +00003526 ++VR_idx;
3527 } else {
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00003528 if (!isVarArg && !isPPC64) {
3529 // Vectors go after all the nonvectors.
3530 CurArgOffset = VecArgOffset;
3531 VecArgOffset += 16;
3532 } else {
3533 // Vectors are aligned.
3534 ArgOffset = ((ArgOffset+15)/16)*16;
3535 CurArgOffset = ArgOffset;
3536 ArgOffset += 16;
Dale Johannesen0d982562008-03-12 00:49:20 +00003537 }
Chris Lattner4302e8f2006-05-16 18:18:50 +00003538 needsLoad = true;
3539 }
3540 break;
3541 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00003542
Chris Lattner4302e8f2006-05-16 18:18:50 +00003543 // We need to load the argument to a virtual register if we determined above
Chris Lattnerf6518cf2008-02-13 07:35:30 +00003544 // that we ran out of physical registers of the appropriate type.
Chris Lattner4302e8f2006-05-16 18:18:50 +00003545 if (needsLoad) {
Chris Lattnerf6518cf2008-02-13 07:35:30 +00003546 int FI = MFI->CreateFixedObject(ObjSize,
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003547 CurArgOffset + (ArgSize - ObjSize),
Evan Cheng0664a672010-07-03 00:40:23 +00003548 isImmutable);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003549 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
Chris Lattner7727d052010-09-21 06:44:06 +00003550 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00003551 false, false, false, 0);
Chris Lattner4302e8f2006-05-16 18:18:50 +00003552 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00003553
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003554 InVals.push_back(ArgVal);
Chris Lattner4302e8f2006-05-16 18:18:50 +00003555 }
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003556
Ulrich Weigand2bffb952014-06-23 13:08:27 +00003557 // Allow for Altivec parameters at the end, if needed.
3558 if (nAltivecParamsAtEnd) {
3559 MinReservedArea = ((MinReservedArea+15)/16)*16;
3560 MinReservedArea += 16*nAltivecParamsAtEnd;
3561 }
3562
3563 // Area that is at least reserved in the caller of this function.
Ulrich Weigand8ca988f2014-06-23 14:15:53 +00003564 MinReservedArea = std::max(MinReservedArea, LinkageSize + 8 * PtrByteSize);
Ulrich Weigand2bffb952014-06-23 13:08:27 +00003565
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003566 // Set the size that is at least reserved in caller of this function. Tail
Bill Schmidt57d6de52012-10-23 15:51:16 +00003567 // call optimized functions' reserved stack space needs to be aligned so that
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003568 // taking the difference between two stack areas will result in an aligned
3569 // stack.
Eric Christophercccae792015-01-30 22:02:31 +00003570 MinReservedArea =
3571 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea);
Ulrich Weigand2bffb952014-06-23 13:08:27 +00003572 FuncInfo->setMinReservedArea(MinReservedArea);
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003573
Chris Lattner4302e8f2006-05-16 18:18:50 +00003574 // If the function takes variable number of arguments, make a frame index for
3575 // the start of the first vararg value... for expansion of llvm.va_start.
Chris Lattner4302e8f2006-05-16 18:18:50 +00003576 if (isVarArg) {
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003577 int Depth = ArgOffset;
Scott Michelcf0da6c2009-02-17 22:15:04 +00003578
Dan Gohman31ae5862010-04-17 14:41:14 +00003579 FuncInfo->setVarArgsFrameIndex(
3580 MFI->CreateFixedObject(PtrVT.getSizeInBits()/8,
Evan Cheng0664a672010-07-03 00:40:23 +00003581 Depth, true));
Dan Gohman31ae5862010-04-17 14:41:14 +00003582 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
Scott Michelcf0da6c2009-02-17 22:15:04 +00003583
Chris Lattner4302e8f2006-05-16 18:18:50 +00003584 // If this function is vararg, store any remaining integer argument regs
3585 // to their spots on the stack so that they may be loaded by deferencing the
3586 // result of va_next.
Chris Lattner26e2fcd2006-05-16 18:58:15 +00003587 for (; GPR_idx != Num_GPR_Regs; ++GPR_idx) {
Chris Lattner2cca3852006-11-18 01:57:19 +00003588 unsigned VReg;
Wesley Peck527da1b2010-11-23 03:31:01 +00003589
Chris Lattner2cca3852006-11-18 01:57:19 +00003590 if (isPPC64)
Devang Patelf3292b22011-02-21 23:21:26 +00003591 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
Chris Lattner2cca3852006-11-18 01:57:19 +00003592 else
Devang Patelf3292b22011-02-21 23:21:26 +00003593 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass);
Chris Lattner2cca3852006-11-18 01:57:19 +00003594
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003595 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
Chris Lattner676c61d2010-09-21 18:41:36 +00003596 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
3597 MachinePointerInfo(), false, false, 0);
Chris Lattner4302e8f2006-05-16 18:18:50 +00003598 MemOps.push_back(Store);
3599 // Increment the address by four for the next argument to store
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00003600 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT);
Dale Johannesen679073b2009-02-04 02:34:38 +00003601 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
Chris Lattner4302e8f2006-05-16 18:18:50 +00003602 }
Chris Lattner4302e8f2006-05-16 18:18:50 +00003603 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00003604
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003605 if (!MemOps.empty())
Craig Topper48d114b2014-04-26 18:35:24 +00003606 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
Dale Johannesenbfa252d2008-03-07 20:27:40 +00003607
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003608 return Chain;
Chris Lattner4302e8f2006-05-16 18:18:50 +00003609}
3610
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003611/// CalculateTailCallSPDiff - Get the amount the stack pointer has to be
Chris Lattner0ab5e2c2011-04-15 05:18:47 +00003612/// adjusted to accommodate the arguments for the tailcall.
Dale Johannesen86dcae12009-11-24 01:09:07 +00003613static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall,
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003614 unsigned ParamSize) {
3615
Dale Johannesen86dcae12009-11-24 01:09:07 +00003616 if (!isTailCall) return 0;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003617
3618 PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
3619 unsigned CallerMinReservedArea = FI->getMinReservedArea();
3620 int SPDiff = (int)CallerMinReservedArea - (int)ParamSize;
3621 // Remember only if the new adjustement is bigger.
3622 if (SPDiff < FI->getTailCallSPDelta())
3623 FI->setTailCallSPDelta(SPDiff);
3624
3625 return SPDiff;
3626}
3627
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003628/// IsEligibleForTailCallOptimization - Check whether the call is eligible
3629/// for tail call optimization. Targets which want to do tail call
3630/// optimization should implement this function.
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003631bool
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003632PPCTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
Sandeep Patel68c5f472009-09-02 08:44:58 +00003633 CallingConv::ID CalleeCC,
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003634 bool isVarArg,
3635 const SmallVectorImpl<ISD::InputArg> &Ins,
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003636 SelectionDAG& DAG) const {
Nick Lewycky50f02cb2011-12-02 22:16:29 +00003637 if (!getTargetMachine().Options.GuaranteedTailCallOpt)
Evan Cheng25217ff2010-01-29 23:05:56 +00003638 return false;
3639
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003640 // Variable argument functions are not supported.
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003641 if (isVarArg)
Dan Gohmaneffb8942008-09-12 16:56:44 +00003642 return false;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003643
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003644 MachineFunction &MF = DAG.getMachineFunction();
Sandeep Patel68c5f472009-09-02 08:44:58 +00003645 CallingConv::ID CallerCC = MF.getFunction()->getCallingConv();
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003646 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) {
3647 // Functions containing by val parameters are not supported.
3648 for (unsigned i = 0; i != Ins.size(); i++) {
3649 ISD::ArgFlagsTy Flags = Ins[i].Flags;
3650 if (Flags.isByVal()) return false;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003651 }
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003652
Alp Tokerf907b892013-12-05 05:44:44 +00003653 // Non-PIC/GOT tail calls are supported.
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003654 if (getTargetMachine().getRelocationModel() != Reloc::PIC_)
3655 return true;
3656
3657 // At the moment we can only do local tail calls (in same module, hidden
3658 // or protected) if we are generating PIC.
3659 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
3660 return G->getGlobal()->hasHiddenVisibility()
3661 || G->getGlobal()->hasProtectedVisibility();
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003662 }
3663
3664 return false;
3665}
3666
Chris Lattnereb755fc2006-05-17 19:00:46 +00003667/// isCallCompatibleAddress - Return the immediate to use if the specified
3668/// 32-bit value is representable in the immediate field of a BxA instruction.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003669static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) {
Chris Lattnereb755fc2006-05-17 19:00:46 +00003670 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
Craig Topper062a2ba2014-04-25 05:30:21 +00003671 if (!C) return nullptr;
Scott Michelcf0da6c2009-02-17 22:15:04 +00003672
Dan Gohmaneffb8942008-09-12 16:56:44 +00003673 int Addr = C->getZExtValue();
Chris Lattnereb755fc2006-05-17 19:00:46 +00003674 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero.
Richard Smith228e6d42012-08-24 23:29:28 +00003675 SignExtend32<26>(Addr) != Addr)
Craig Topper062a2ba2014-04-25 05:30:21 +00003676 return nullptr; // Top 6 bits have to be sext of immediate.
Scott Michelcf0da6c2009-02-17 22:15:04 +00003677
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00003678 return DAG.getConstant((int)C->getZExtValue() >> 2, SDLoc(Op),
Gabor Greiff304a7a2008-08-28 21:40:38 +00003679 DAG.getTargetLoweringInfo().getPointerTy()).getNode();
Chris Lattnereb755fc2006-05-17 19:00:46 +00003680}
3681
Dan Gohmand78c4002008-05-13 00:00:25 +00003682namespace {
3683
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003684struct TailCallArgumentInfo {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003685 SDValue Arg;
3686 SDValue FrameIdxOp;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003687 int FrameIdx;
3688
3689 TailCallArgumentInfo() : FrameIdx(0) {}
3690};
3691
Dan Gohmand78c4002008-05-13 00:00:25 +00003692}
3693
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003694/// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot.
3695static void
3696StoreTailCallArgumentsToStackSlot(SelectionDAG &DAG,
Evan Cheng0e9d9ca2009-10-18 18:16:27 +00003697 SDValue Chain,
Craig Topperb94011f2013-07-14 04:42:23 +00003698 const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs,
3699 SmallVectorImpl<SDValue> &MemOpChains,
Andrew Trickef9de2a2013-05-25 02:42:55 +00003700 SDLoc dl) {
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003701 for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003702 SDValue Arg = TailCallArgs[i].Arg;
3703 SDValue FIN = TailCallArgs[i].FrameIdxOp;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003704 int FI = TailCallArgs[i].FrameIdx;
3705 // Store relative to framepointer.
Dale Johannesen021052a2009-02-04 20:06:27 +00003706 MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, FIN,
Chris Lattner7727d052010-09-21 06:44:06 +00003707 MachinePointerInfo::getFixedStack(FI),
3708 false, false, 0));
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003709 }
3710}
3711
3712/// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to
3713/// the appropriate stack slot for the tail call optimized function call.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003714static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG,
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003715 MachineFunction &MF,
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003716 SDValue Chain,
3717 SDValue OldRetAddr,
3718 SDValue OldFP,
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003719 int SPDiff,
3720 bool isPPC64,
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003721 bool isDarwinABI,
Andrew Trickef9de2a2013-05-25 02:42:55 +00003722 SDLoc dl) {
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003723 if (SPDiff) {
3724 // Calculate the new stack slot for the return address.
3725 int SlotSize = isPPC64 ? 8 : 4;
Eric Christopherdc3a8a42015-02-13 00:39:38 +00003726 const PPCFrameLowering *FL =
3727 MF.getSubtarget<PPCSubtarget>().getFrameLowering();
3728 int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset();
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003729 int NewRetAddr = MF.getFrameInfo()->CreateFixedObject(SlotSize,
Evan Cheng0664a672010-07-03 00:40:23 +00003730 NewRetAddrLoc, true);
Owen Anderson9f944592009-08-11 20:47:22 +00003731 EVT VT = isPPC64 ? MVT::i64 : MVT::i32;
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003732 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewRetAddr, VT);
Dale Johannesen021052a2009-02-04 20:06:27 +00003733 Chain = DAG.getStore(Chain, dl, OldRetAddr, NewRetAddrFrIdx,
Chris Lattner7727d052010-09-21 06:44:06 +00003734 MachinePointerInfo::getFixedStack(NewRetAddr),
David Greene87a5abe2010-02-15 16:56:53 +00003735 false, false, 0);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00003736
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00003737 // When using the 32/64-bit SVR4 ABI there is no need to move the FP stack
3738 // slot as the FP is never overwritten.
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003739 if (isDarwinABI) {
Eric Christopherdc3a8a42015-02-13 00:39:38 +00003740 int NewFPLoc = SPDiff + FL->getFramePointerSaveOffset();
David Greene1fbe0542009-11-12 20:49:22 +00003741 int NewFPIdx = MF.getFrameInfo()->CreateFixedObject(SlotSize, NewFPLoc,
Evan Cheng0664a672010-07-03 00:40:23 +00003742 true);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00003743 SDValue NewFramePtrIdx = DAG.getFrameIndex(NewFPIdx, VT);
3744 Chain = DAG.getStore(Chain, dl, OldFP, NewFramePtrIdx,
Chris Lattner7727d052010-09-21 06:44:06 +00003745 MachinePointerInfo::getFixedStack(NewFPIdx),
David Greene87a5abe2010-02-15 16:56:53 +00003746 false, false, 0);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00003747 }
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003748 }
3749 return Chain;
3750}
3751
3752/// CalculateTailCallArgDest - Remember Argument for later processing. Calculate
3753/// the position of the argument.
3754static void
3755CalculateTailCallArgDest(SelectionDAG &DAG, MachineFunction &MF, bool isPPC64,
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003756 SDValue Arg, int SPDiff, unsigned ArgOffset,
Craig Topperb94011f2013-07-14 04:42:23 +00003757 SmallVectorImpl<TailCallArgumentInfo>& TailCallArguments) {
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003758 int Offset = ArgOffset + SPDiff;
Duncan Sands13237ac2008-06-06 12:08:01 +00003759 uint32_t OpSize = (Arg.getValueType().getSizeInBits()+7)/8;
Evan Cheng0664a672010-07-03 00:40:23 +00003760 int FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true);
Owen Anderson9f944592009-08-11 20:47:22 +00003761 EVT VT = isPPC64 ? MVT::i64 : MVT::i32;
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003762 SDValue FIN = DAG.getFrameIndex(FI, VT);
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003763 TailCallArgumentInfo Info;
3764 Info.Arg = Arg;
3765 Info.FrameIdxOp = FIN;
3766 Info.FrameIdx = FI;
3767 TailCallArguments.push_back(Info);
3768}
3769
3770/// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address
3771/// stack slot. Returns the chain as result and the loaded frame pointers in
3772/// LROpOut/FPOpout. Used when tail calling.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003773SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr(SelectionDAG & DAG,
Dale Johannesen021052a2009-02-04 20:06:27 +00003774 int SPDiff,
3775 SDValue Chain,
3776 SDValue &LROpOut,
3777 SDValue &FPOpOut,
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003778 bool isDarwinABI,
Andrew Trickef9de2a2013-05-25 02:42:55 +00003779 SDLoc dl) const {
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003780 if (SPDiff) {
3781 // Load the LR and FP stack slot for later adjusting.
Eric Christopherb1aaebe2014-06-12 22:38:18 +00003782 EVT VT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003783 LROpOut = getReturnAddrFrameIndex(DAG);
Chris Lattner7727d052010-09-21 06:44:06 +00003784 LROpOut = DAG.getLoad(VT, dl, Chain, LROpOut, MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00003785 false, false, false, 0);
Gabor Greiff304a7a2008-08-28 21:40:38 +00003786 Chain = SDValue(LROpOut.getNode(), 1);
Wesley Peck527da1b2010-11-23 03:31:01 +00003787
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00003788 // When using the 32/64-bit SVR4 ABI there is no need to load the FP stack
3789 // slot as the FP is never overwritten.
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003790 if (isDarwinABI) {
Tilmann Schellerb93960d2009-07-03 06:45:56 +00003791 FPOpOut = getFramePointerFrameIndex(DAG);
Chris Lattner7727d052010-09-21 06:44:06 +00003792 FPOpOut = DAG.getLoad(VT, dl, Chain, FPOpOut, MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00003793 false, false, false, 0);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00003794 Chain = SDValue(FPOpOut.getNode(), 1);
3795 }
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003796 }
3797 return Chain;
3798}
3799
Dale Johannesen85d41a12008-03-04 23:17:14 +00003800/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
Scott Michelcf0da6c2009-02-17 22:15:04 +00003801/// by "Src" to address "Dst" of size "Size". Alignment information is
Dale Johannesen85d41a12008-03-04 23:17:14 +00003802/// specified by the specific parameter attribute. The copy will be passed as
3803/// a byval function parameter.
3804/// Sometimes what we are copying is the end of a larger object, the part that
3805/// does not fit in registers.
Scott Michelcf0da6c2009-02-17 22:15:04 +00003806static SDValue
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003807CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain,
Duncan Sandsd97eea32008-03-21 09:14:45 +00003808 ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
Andrew Trickef9de2a2013-05-25 02:42:55 +00003809 SDLoc dl) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00003810 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
Dale Johannesen85263882009-02-04 01:17:06 +00003811 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
Krzysztof Parzyszeka46c36b2015-04-13 17:16:45 +00003812 false, false, false, MachinePointerInfo(),
Nick Lewyckyaad475b2014-04-15 07:22:52 +00003813 MachinePointerInfo());
Dale Johannesen85d41a12008-03-04 23:17:14 +00003814}
Chris Lattner43df5b32007-02-25 05:34:32 +00003815
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003816/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
3817/// tail calls.
3818static void
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003819LowerMemOpCallTo(SelectionDAG &DAG, MachineFunction &MF, SDValue Chain,
3820 SDValue Arg, SDValue PtrOff, int SPDiff,
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003821 unsigned ArgOffset, bool isPPC64, bool isTailCall,
Craig Topperb94011f2013-07-14 04:42:23 +00003822 bool isVector, SmallVectorImpl<SDValue> &MemOpChains,
3823 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments,
Andrew Trickef9de2a2013-05-25 02:42:55 +00003824 SDLoc dl) {
Owen Anderson53aa7a92009-08-10 22:56:29 +00003825 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003826 if (!isTailCall) {
3827 if (isVector) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003828 SDValue StackPtr;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003829 if (isPPC64)
Owen Anderson9f944592009-08-11 20:47:22 +00003830 StackPtr = DAG.getRegister(PPC::X1, MVT::i64);
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003831 else
Owen Anderson9f944592009-08-11 20:47:22 +00003832 StackPtr = DAG.getRegister(PPC::R1, MVT::i32);
Dale Johannesen021052a2009-02-04 20:06:27 +00003833 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00003834 DAG.getConstant(ArgOffset, dl, PtrVT));
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003835 }
Chris Lattner676c61d2010-09-21 18:41:36 +00003836 MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff,
3837 MachinePointerInfo(), false, false, 0));
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00003838 // Calculate and remember argument location.
3839 } else CalculateTailCallArgDest(DAG, MF, isPPC64, Arg, SPDiff, ArgOffset,
3840 TailCallArguments);
3841}
3842
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003843static
3844void PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain,
Andrew Trickef9de2a2013-05-25 02:42:55 +00003845 SDLoc dl, bool isPPC64, int SPDiff, unsigned NumBytes,
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003846 SDValue LROp, SDValue FPOp, bool isDarwinABI,
Craig Topperb94011f2013-07-14 04:42:23 +00003847 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003848 MachineFunction &MF = DAG.getMachineFunction();
3849
3850 // Emit a sequence of copyto/copyfrom virtual registers for arguments that
3851 // might overwrite each other in case of tail call optimization.
3852 SmallVector<SDValue, 8> MemOpChains2;
Chris Lattner0ab5e2c2011-04-15 05:18:47 +00003853 // Do not flag preceding copytoreg stuff together with the following stuff.
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003854 InFlag = SDValue();
3855 StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArguments,
3856 MemOpChains2, dl);
3857 if (!MemOpChains2.empty())
Craig Topper48d114b2014-04-26 18:35:24 +00003858 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003859
3860 // Store the return address to the appropriate stack slot.
3861 Chain = EmitTailCallStoreFPAndRetAddr(DAG, MF, Chain, LROp, FPOp, SPDiff,
3862 isPPC64, isDarwinABI, dl);
3863
3864 // Emit callseq_end just before tailcall node.
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00003865 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
3866 DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003867 InFlag = Chain.getValue(1);
3868}
3869
Hal Finkel87deb0b2015-01-12 04:34:47 +00003870// Is this global address that of a function that can be called by name? (as
3871// opposed to something that must hold a descriptor for an indirect call).
3872static bool isFunctionGlobalAddress(SDValue Callee) {
3873 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
3874 if (Callee.getOpcode() == ISD::GlobalTLSAddress ||
3875 Callee.getOpcode() == ISD::TargetGlobalTLSAddress)
3876 return false;
3877
3878 return G->getGlobal()->getType()->getElementType()->isFunctionTy();
3879 }
3880
3881 return false;
3882}
3883
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003884static
3885unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
Hal Finkele2ab0f12015-01-15 21:17:34 +00003886 SDValue &Chain, SDValue CallSeqStart, SDLoc dl, int SPDiff,
3887 bool isTailCall, bool IsPatchPoint,
Craig Topperb94011f2013-07-14 04:42:23 +00003888 SmallVectorImpl<std::pair<unsigned, SDValue> > &RegsToPass,
3889 SmallVectorImpl<SDValue> &Ops, std::vector<EVT> &NodeTys,
Hal Finkele2ab0f12015-01-15 21:17:34 +00003890 ImmutableCallSite *CS, const PPCSubtarget &Subtarget) {
Wesley Peck527da1b2010-11-23 03:31:01 +00003891
Eric Christopherb1aaebe2014-06-12 22:38:18 +00003892 bool isPPC64 = Subtarget.isPPC64();
3893 bool isSVR4ABI = Subtarget.isSVR4ABI();
Ulrich Weigandaa0ac4f2014-07-20 23:31:44 +00003894 bool isELFv2ABI = Subtarget.isELFv2ABI();
Chris Lattnerdf8e17d2010-11-14 23:42:06 +00003895
Owen Anderson53aa7a92009-08-10 22:56:29 +00003896 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Owen Anderson9f944592009-08-11 20:47:22 +00003897 NodeTys.push_back(MVT::Other); // Returns a chain
Chris Lattner3e5fbd72010-12-21 02:38:05 +00003898 NodeTys.push_back(MVT::Glue); // Returns a flag for retval copy to use.
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003899
Ulrich Weigandf62e83f2013-03-22 15:24:13 +00003900 unsigned CallOpc = PPCISD::CALL;
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003901
Torok Edwin31e90d22010-08-04 20:47:44 +00003902 bool needIndirectCall = true;
Ulrich Weigand9aa09ef2014-06-18 16:14:04 +00003903 if (!isSVR4ABI || !isPPC64)
3904 if (SDNode *Dest = isBLACompatibleAddress(Callee, DAG)) {
3905 // If this is an absolute destination address, use the munged value.
3906 Callee = SDValue(Dest, 0);
3907 needIndirectCall = false;
3908 }
Wesley Peck527da1b2010-11-23 03:31:01 +00003909
Hal Finkel87deb0b2015-01-12 04:34:47 +00003910 if (isFunctionGlobalAddress(Callee)) {
3911 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Callee);
3912 // A call to a TLS address is actually an indirect call to a
3913 // thread-specific pointer.
Eric Christopher79cc1e32014-09-02 22:28:02 +00003914 unsigned OpFlags = 0;
3915 if ((DAG.getTarget().getRelocationModel() != Reloc::Static &&
3916 (Subtarget.getTargetTriple().isMacOSX() &&
3917 Subtarget.getTargetTriple().isMacOSXVersionLT(10, 5)) &&
3918 (G->getGlobal()->isDeclaration() ||
3919 G->getGlobal()->isWeakForLinker())) ||
3920 (Subtarget.isTargetELF() && !isPPC64 &&
3921 !G->getGlobal()->hasLocalLinkage() &&
3922 DAG.getTarget().getRelocationModel() == Reloc::PIC_)) {
3923 // PC-relative references to external symbols should go through $stub,
3924 // unless we're building with the leopard linker or later, which
3925 // automatically synthesizes these stubs.
3926 OpFlags = PPCII::MO_PLT_OR_STUB;
Eric Christopherb9fd9ed2014-08-07 22:02:54 +00003927 }
Eric Christopher79cc1e32014-09-02 22:28:02 +00003928
3929 // If the callee is a GlobalAddress/ExternalSymbol node (quite common,
3930 // every direct call is) turn it into a TargetGlobalAddress /
3931 // TargetExternalSymbol node so that legalize doesn't hack it.
3932 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl,
3933 Callee.getValueType(), 0, OpFlags);
3934 needIndirectCall = false;
Torok Edwin31e90d22010-08-04 20:47:44 +00003935 }
Wesley Peck527da1b2010-11-23 03:31:01 +00003936
Torok Edwin31e90d22010-08-04 20:47:44 +00003937 if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
Chris Lattnerdf8e17d2010-11-14 23:42:06 +00003938 unsigned char OpFlags = 0;
Wesley Peck527da1b2010-11-23 03:31:01 +00003939
Hal Finkel3ee2af72014-07-18 23:29:49 +00003940 if ((DAG.getTarget().getRelocationModel() != Reloc::Static &&
3941 (Subtarget.getTargetTriple().isMacOSX() &&
3942 Subtarget.getTargetTriple().isMacOSXVersionLT(10, 5))) ||
3943 (Subtarget.isTargetELF() && !isPPC64 &&
Justin Hibbits17744c12015-01-10 07:50:31 +00003944 DAG.getTarget().getRelocationModel() == Reloc::PIC_)) {
Chris Lattnerdf8e17d2010-11-14 23:42:06 +00003945 // PC-relative references to external symbols should go through $stub,
3946 // unless we're building with the leopard linker or later, which
3947 // automatically synthesizes these stubs.
Hal Finkel3ee2af72014-07-18 23:29:49 +00003948 OpFlags = PPCII::MO_PLT_OR_STUB;
Chris Lattnerdf8e17d2010-11-14 23:42:06 +00003949 }
Wesley Peck527da1b2010-11-23 03:31:01 +00003950
Chris Lattnerdf8e17d2010-11-14 23:42:06 +00003951 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), Callee.getValueType(),
3952 OpFlags);
3953 needIndirectCall = false;
Torok Edwin31e90d22010-08-04 20:47:44 +00003954 }
Wesley Peck527da1b2010-11-23 03:31:01 +00003955
Hal Finkel934361a2015-01-14 01:07:51 +00003956 if (IsPatchPoint) {
3957 // We'll form an invalid direct call when lowering a patchpoint; the full
3958 // sequence for an indirect call is complicated, and many of the
3959 // instructions introduced might have side effects (and, thus, can't be
3960 // removed later). The call itself will be removed as soon as the
3961 // argument/return lowering is complete, so the fact that it has the wrong
3962 // kind of operands should not really matter.
3963 needIndirectCall = false;
3964 }
3965
Torok Edwin31e90d22010-08-04 20:47:44 +00003966 if (needIndirectCall) {
Tilmann Scheller773f14c2009-07-03 06:47:08 +00003967 // Otherwise, this is an indirect call. We have to use a MTCTR/BCTRL pair
3968 // to do the call, we can't use PPCISD::CALL.
3969 SDValue MTCTROps[] = {Chain, Callee, InFlag};
Tilmann Scheller79fef932009-12-18 13:00:15 +00003970
Hal Finkel63fb9282015-01-13 18:25:05 +00003971 if (isSVR4ABI && isPPC64 && !isELFv2ABI) {
Tilmann Scheller79fef932009-12-18 13:00:15 +00003972 // Function pointers in the 64-bit SVR4 ABI do not point to the function
3973 // entry point, but to the function descriptor (the function entry point
3974 // address is part of the function descriptor though).
3975 // The function descriptor is a three doubleword structure with the
3976 // following fields: function entry point, TOC base address and
3977 // environment pointer.
3978 // Thus for a call through a function pointer, the following actions need
3979 // to be performed:
3980 // 1. Save the TOC of the caller in the TOC save area of its stack
Bill Schmidt57d6de52012-10-23 15:51:16 +00003981 // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()).
Tilmann Scheller79fef932009-12-18 13:00:15 +00003982 // 2. Load the address of the function entry point from the function
3983 // descriptor.
3984 // 3. Load the TOC of the callee from the function descriptor into r2.
3985 // 4. Load the environment pointer from the function descriptor into
3986 // r11.
3987 // 5. Branch to the function entry point address.
3988 // 6. On return of the callee, the TOC of the caller needs to be
3989 // restored (this is done in FinishCall()).
3990 //
Hal Finkele2ab0f12015-01-15 21:17:34 +00003991 // The loads are scheduled at the beginning of the call sequence, and the
3992 // register copies are flagged together to ensure that no other
Tilmann Scheller79fef932009-12-18 13:00:15 +00003993 // operations can be scheduled in between. E.g. without flagging the
Hal Finkele2ab0f12015-01-15 21:17:34 +00003994 // copies together, a TOC access in the caller could be scheduled between
3995 // the assignment of the callee TOC and the branch to the callee, which
Tilmann Scheller79fef932009-12-18 13:00:15 +00003996 // results in the TOC access going through the TOC of the callee instead
3997 // of going through the TOC of the caller, which leads to incorrect code.
3998
3999 // Load the address of the function entry point from the function
4000 // descriptor.
Hal Finkele2ab0f12015-01-15 21:17:34 +00004001 SDValue LDChain = CallSeqStart.getValue(CallSeqStart->getNumValues()-1);
4002 if (LDChain.getValueType() == MVT::Glue)
4003 LDChain = CallSeqStart.getValue(CallSeqStart->getNumValues()-2);
4004
4005 bool LoadsInv = Subtarget.hasInvariantFunctionDescriptors();
4006
4007 MachinePointerInfo MPI(CS ? CS->getCalledValue() : nullptr);
4008 SDValue LoadFuncPtr = DAG.getLoad(MVT::i64, dl, LDChain, Callee, MPI,
4009 false, false, LoadsInv, 8);
Tilmann Scheller79fef932009-12-18 13:00:15 +00004010
4011 // Load environment pointer into r11.
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004012 SDValue PtrOff = DAG.getIntPtrConstant(16, dl);
Tilmann Scheller79fef932009-12-18 13:00:15 +00004013 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, PtrOff);
Hal Finkele2ab0f12015-01-15 21:17:34 +00004014 SDValue LoadEnvPtr = DAG.getLoad(MVT::i64, dl, LDChain, AddPtr,
4015 MPI.getWithOffset(16), false, false,
4016 LoadsInv, 8);
4017
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004018 SDValue TOCOff = DAG.getIntPtrConstant(8, dl);
Hal Finkele2ab0f12015-01-15 21:17:34 +00004019 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, TOCOff);
4020 SDValue TOCPtr = DAG.getLoad(MVT::i64, dl, LDChain, AddTOC,
4021 MPI.getWithOffset(8), false, false,
4022 LoadsInv, 8);
4023
Hal Finkele6698d52015-02-01 15:03:28 +00004024 setUsesTOCBasePtr(DAG);
Hal Finkele2ab0f12015-01-15 21:17:34 +00004025 SDValue TOCVal = DAG.getCopyToReg(Chain, dl, PPC::X2, TOCPtr,
4026 InFlag);
4027 Chain = TOCVal.getValue(0);
4028 InFlag = TOCVal.getValue(1);
Tilmann Scheller79fef932009-12-18 13:00:15 +00004029
4030 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, PPC::X11, LoadEnvPtr,
4031 InFlag);
Hal Finkele2ab0f12015-01-15 21:17:34 +00004032
Tilmann Scheller79fef932009-12-18 13:00:15 +00004033 Chain = EnvVal.getValue(0);
4034 InFlag = EnvVal.getValue(1);
4035
Tilmann Scheller79fef932009-12-18 13:00:15 +00004036 MTCTROps[0] = Chain;
4037 MTCTROps[1] = LoadFuncPtr;
4038 MTCTROps[2] = InFlag;
4039 }
4040
Hal Finkel63fb9282015-01-13 18:25:05 +00004041 Chain = DAG.getNode(PPCISD::MTCTR, dl, NodeTys,
4042 makeArrayRef(MTCTROps, InFlag.getNode() ? 3 : 2));
4043 InFlag = Chain.getValue(1);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004044
4045 NodeTys.clear();
Owen Anderson9f944592009-08-11 20:47:22 +00004046 NodeTys.push_back(MVT::Other);
Chris Lattner3e5fbd72010-12-21 02:38:05 +00004047 NodeTys.push_back(MVT::Glue);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004048 Ops.push_back(Chain);
Ulrich Weigandf62e83f2013-03-22 15:24:13 +00004049 CallOpc = PPCISD::BCTRL;
Craig Topper062a2ba2014-04-25 05:30:21 +00004050 Callee.setNode(nullptr);
Ulrich Weigandf62e83f2013-03-22 15:24:13 +00004051 // Add use of X11 (holding environment pointer)
Hal Finkel63fb9282015-01-13 18:25:05 +00004052 if (isSVR4ABI && isPPC64 && !isELFv2ABI)
Ulrich Weigandf62e83f2013-03-22 15:24:13 +00004053 Ops.push_back(DAG.getRegister(PPC::X11, PtrVT));
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004054 // Add CTR register as callee so a bctr can be emitted later.
4055 if (isTailCall)
Roman Divackya4a59ae2011-06-03 15:47:49 +00004056 Ops.push_back(DAG.getRegister(isPPC64 ? PPC::CTR8 : PPC::CTR, PtrVT));
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004057 }
4058
4059 // If this is a direct call, pass the chain and the callee.
4060 if (Callee.getNode()) {
4061 Ops.push_back(Chain);
4062 Ops.push_back(Callee);
4063 }
4064 // If this is a tail call add stack pointer delta.
4065 if (isTailCall)
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004066 Ops.push_back(DAG.getConstant(SPDiff, dl, MVT::i32));
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004067
4068 // Add argument registers to the end of the list so that they are known live
4069 // into the call.
4070 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
4071 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
4072 RegsToPass[i].second.getValueType()));
4073
Hal Finkelaf519932015-01-19 07:20:27 +00004074 // All calls, in both the ELF V1 and V2 ABIs, need the TOC register live
4075 // into the call.
Hal Finkele6698d52015-02-01 15:03:28 +00004076 if (isSVR4ABI && isPPC64 && !IsPatchPoint) {
4077 setUsesTOCBasePtr(DAG);
Ulrich Weigandaa0ac4f2014-07-20 23:31:44 +00004078 Ops.push_back(DAG.getRegister(PPC::X2, PtrVT));
Hal Finkele6698d52015-02-01 15:03:28 +00004079 }
Ulrich Weigandaa0ac4f2014-07-20 23:31:44 +00004080
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004081 return CallOpc;
4082}
4083
Roman Divacky76293062012-09-18 16:47:58 +00004084static
4085bool isLocalCall(const SDValue &Callee)
4086{
4087 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
Roman Divacky09adf3d2012-09-18 18:27:49 +00004088 return !G->getGlobal()->isDeclaration() &&
4089 !G->getGlobal()->isWeakForLinker();
Roman Divacky76293062012-09-18 16:47:58 +00004090 return false;
4091}
4092
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004093SDValue
4094PPCTargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
Sandeep Patel68c5f472009-09-02 08:44:58 +00004095 CallingConv::ID CallConv, bool isVarArg,
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004096 const SmallVectorImpl<ISD::InputArg> &Ins,
Andrew Trickef9de2a2013-05-25 02:42:55 +00004097 SDLoc dl, SelectionDAG &DAG,
Dan Gohman21cea8a2010-04-17 15:26:15 +00004098 SmallVectorImpl<SDValue> &InVals) const {
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004099
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004100 SmallVector<CCValAssign, 16> RVLocs;
Eric Christopherb5217502014-08-06 18:45:26 +00004101 CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
4102 *DAG.getContext());
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004103 CCRetInfo.AnalyzeCallResult(Ins, RetCC_PPC);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004104
4105 // Copy all of the result registers out of their specified physreg.
4106 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
4107 CCValAssign &VA = RVLocs[i];
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004108 assert(VA.isRegLoc() && "Can only return in registers!");
Ulrich Weigand339d0592012-11-05 19:39:45 +00004109
4110 SDValue Val = DAG.getCopyFromReg(Chain, dl,
4111 VA.getLocReg(), VA.getLocVT(), InFlag);
4112 Chain = Val.getValue(1);
4113 InFlag = Val.getValue(2);
4114
4115 switch (VA.getLocInfo()) {
4116 default: llvm_unreachable("Unknown loc info!");
4117 case CCValAssign::Full: break;
4118 case CCValAssign::AExt:
4119 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
4120 break;
4121 case CCValAssign::ZExt:
4122 Val = DAG.getNode(ISD::AssertZext, dl, VA.getLocVT(), Val,
4123 DAG.getValueType(VA.getValVT()));
4124 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
4125 break;
4126 case CCValAssign::SExt:
4127 Val = DAG.getNode(ISD::AssertSext, dl, VA.getLocVT(), Val,
4128 DAG.getValueType(VA.getValVT()));
4129 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
4130 break;
4131 }
4132
4133 InVals.push_back(Val);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004134 }
4135
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004136 return Chain;
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004137}
4138
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004139SDValue
Andrew Trickef9de2a2013-05-25 02:42:55 +00004140PPCTargetLowering::FinishCall(CallingConv::ID CallConv, SDLoc dl,
Hal Finkel934361a2015-01-14 01:07:51 +00004141 bool isTailCall, bool isVarArg, bool IsPatchPoint,
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004142 SelectionDAG &DAG,
4143 SmallVector<std::pair<unsigned, SDValue>, 8>
4144 &RegsToPass,
4145 SDValue InFlag, SDValue Chain,
Hal Finkele2ab0f12015-01-15 21:17:34 +00004146 SDValue CallSeqStart, SDValue &Callee,
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004147 int SPDiff, unsigned NumBytes,
4148 const SmallVectorImpl<ISD::InputArg> &Ins,
Hal Finkele2ab0f12015-01-15 21:17:34 +00004149 SmallVectorImpl<SDValue> &InVals,
4150 ImmutableCallSite *CS) const {
Ulrich Weigand8658f172014-07-20 23:43:15 +00004151
Owen Anderson53aa7a92009-08-10 22:56:29 +00004152 std::vector<EVT> NodeTys;
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004153 SmallVector<SDValue, 8> Ops;
Hal Finkele2ab0f12015-01-15 21:17:34 +00004154 unsigned CallOpc = PrepareCall(DAG, Callee, InFlag, Chain, CallSeqStart, dl,
4155 SPDiff, isTailCall, IsPatchPoint, RegsToPass,
4156 Ops, NodeTys, CS, Subtarget);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004157
Hal Finkel5ab37802012-08-28 02:10:27 +00004158 // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls
Eric Christopherb1aaebe2014-06-12 22:38:18 +00004159 if (isVarArg && Subtarget.isSVR4ABI() && !Subtarget.isPPC64())
Hal Finkel5ab37802012-08-28 02:10:27 +00004160 Ops.push_back(DAG.getRegister(PPC::CR1EQ, MVT::i32));
4161
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004162 // When performing tail call optimization the callee pops its arguments off
4163 // the stack. Account for this here so these bytes can be pushed back on in
Eli Bendersky8da87162013-02-21 20:05:00 +00004164 // PPCFrameLowering::eliminateCallFramePseudoInstr.
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004165 int BytesCalleePops =
Nick Lewycky50f02cb2011-12-02 22:16:29 +00004166 (CallConv == CallingConv::Fast &&
4167 getTargetMachine().Options.GuaranteedTailCallOpt) ? NumBytes : 0;
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004168
Roman Divackyef21be22012-03-06 16:41:49 +00004169 // Add a register mask operand representing the call-preserved registers.
Eric Christophercccae792015-01-30 22:02:31 +00004170 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
Eric Christopher9deb75d2015-03-11 22:42:13 +00004171 const uint32_t *Mask =
4172 TRI->getCallPreservedMask(DAG.getMachineFunction(), CallConv);
Roman Divackyef21be22012-03-06 16:41:49 +00004173 assert(Mask && "Missing call preserved mask for calling convention");
4174 Ops.push_back(DAG.getRegisterMask(Mask));
4175
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004176 if (InFlag.getNode())
4177 Ops.push_back(InFlag);
4178
4179 // Emit tail call.
4180 if (isTailCall) {
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004181 assert(((Callee.getOpcode() == ISD::Register &&
4182 cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) ||
4183 Callee.getOpcode() == ISD::TargetExternalSymbol ||
4184 Callee.getOpcode() == ISD::TargetGlobalAddress ||
4185 isa<ConstantSDNode>(Callee)) &&
4186 "Expecting an global address, external symbol, absolute value or register");
4187
Craig Topper48d114b2014-04-26 18:35:24 +00004188 return DAG.getNode(PPCISD::TC_RETURN, dl, MVT::Other, Ops);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004189 }
4190
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00004191 // Add a NOP immediately after the branch instruction when using the 64-bit
4192 // SVR4 ABI. At link time, if caller and callee are in a different module and
4193 // thus have a different TOC, the call will be replaced with a call to a stub
4194 // function which saves the current TOC, loads the TOC of the callee and
4195 // branches to the callee. The NOP will be replaced with a load instruction
4196 // which restores the TOC of the caller from the TOC save slot of the current
4197 // stack frame. If caller and callee belong to the same module (and have the
4198 // same TOC), the NOP will remain unchanged.
Hal Finkel51861b42012-03-31 14:45:15 +00004199
Hal Finkel934361a2015-01-14 01:07:51 +00004200 if (!isTailCall && Subtarget.isSVR4ABI()&& Subtarget.isPPC64() &&
4201 !IsPatchPoint) {
Ulrich Weigandf62e83f2013-03-22 15:24:13 +00004202 if (CallOpc == PPCISD::BCTRL) {
Tilmann Scheller79fef932009-12-18 13:00:15 +00004203 // This is a call through a function pointer.
4204 // Restore the caller TOC from the save area into R2.
4205 // See PrepareCall() for more information about calls through function
4206 // pointers in the 64-bit SVR4 ABI.
4207 // We are using a target-specific load with r2 hard coded, because the
4208 // result of a target-independent load would never go directly into r2,
4209 // since r2 is a reserved register (which prevents the register allocator
4210 // from allocating it), resulting in an additional register being
4211 // allocated and an unnecessary move instruction being generated.
Hal Finkelfc096c92014-12-23 22:29:40 +00004212 CallOpc = PPCISD::BCTRL_LOAD_TOC;
4213
4214 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
4215 SDValue StackPtr = DAG.getRegister(PPC::X1, PtrVT);
Eric Christopher736d39e2015-02-13 00:39:36 +00004216 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004217 SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
Hal Finkelfc096c92014-12-23 22:29:40 +00004218 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, StackPtr, TOCOff);
4219
4220 // The address needs to go after the chain input but before the flag (or
4221 // any other variadic arguments).
4222 Ops.insert(std::next(Ops.begin()), AddTOC);
Bill Schmidtcea15962013-09-26 17:09:28 +00004223 } else if ((CallOpc == PPCISD::CALL) &&
4224 (!isLocalCall(Callee) ||
Bill Schmidt82f1c772015-02-10 19:09:05 +00004225 DAG.getTarget().getRelocationModel() == Reloc::PIC_))
Roman Divacky76293062012-09-18 16:47:58 +00004226 // Otherwise insert NOP for non-local calls.
Ulrich Weigandf62e83f2013-03-22 15:24:13 +00004227 CallOpc = PPCISD::CALL_NOP;
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00004228 }
4229
Craig Topper48d114b2014-04-26 18:35:24 +00004230 Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops);
Hal Finkel51861b42012-03-31 14:45:15 +00004231 InFlag = Chain.getValue(1);
4232
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004233 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
4234 DAG.getIntPtrConstant(BytesCalleePops, dl, true),
Andrew Trickad6d08a2013-05-29 22:03:55 +00004235 InFlag, dl);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004236 if (!Ins.empty())
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004237 InFlag = Chain.getValue(1);
4238
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004239 return LowerCallResult(Chain, InFlag, CallConv, isVarArg,
4240 Ins, dl, DAG, InVals);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004241}
4242
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004243SDValue
Justin Holewinskiaa583972012-05-25 16:35:28 +00004244PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
Dan Gohman21cea8a2010-04-17 15:26:15 +00004245 SmallVectorImpl<SDValue> &InVals) const {
Justin Holewinskiaa583972012-05-25 16:35:28 +00004246 SelectionDAG &DAG = CLI.DAG;
Craig Topperb94011f2013-07-14 04:42:23 +00004247 SDLoc &dl = CLI.DL;
4248 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
4249 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
4250 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
Justin Holewinskiaa583972012-05-25 16:35:28 +00004251 SDValue Chain = CLI.Chain;
4252 SDValue Callee = CLI.Callee;
4253 bool &isTailCall = CLI.IsTailCall;
4254 CallingConv::ID CallConv = CLI.CallConv;
4255 bool isVarArg = CLI.IsVarArg;
Hal Finkel934361a2015-01-14 01:07:51 +00004256 bool IsPatchPoint = CLI.IsPatchPoint;
Hal Finkele2ab0f12015-01-15 21:17:34 +00004257 ImmutableCallSite *CS = CLI.CS;
Justin Holewinskiaa583972012-05-25 16:35:28 +00004258
Evan Cheng67a69dd2010-01-27 00:07:07 +00004259 if (isTailCall)
4260 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, isVarArg,
4261 Ins, DAG);
4262
Hal Finkele2ab0f12015-01-15 21:17:34 +00004263 if (!isTailCall && CS && CS->isMustTailCall())
Reid Kleckner5772b772014-04-24 20:14:34 +00004264 report_fatal_error("failed to perform tail call elimination on a call "
4265 "site marked musttail");
4266
Eric Christopherb1aaebe2014-06-12 22:38:18 +00004267 if (Subtarget.isSVR4ABI()) {
4268 if (Subtarget.isPPC64())
Bill Schmidt57d6de52012-10-23 15:51:16 +00004269 return LowerCall_64SVR4(Chain, Callee, CallConv, isVarArg,
Hal Finkel934361a2015-01-14 01:07:51 +00004270 isTailCall, IsPatchPoint, Outs, OutVals, Ins,
Hal Finkele2ab0f12015-01-15 21:17:34 +00004271 dl, DAG, InVals, CS);
Bill Schmidt57d6de52012-10-23 15:51:16 +00004272 else
4273 return LowerCall_32SVR4(Chain, Callee, CallConv, isVarArg,
Hal Finkel934361a2015-01-14 01:07:51 +00004274 isTailCall, IsPatchPoint, Outs, OutVals, Ins,
Hal Finkele2ab0f12015-01-15 21:17:34 +00004275 dl, DAG, InVals, CS);
Bill Schmidt57d6de52012-10-23 15:51:16 +00004276 }
Chris Lattnerdf8e17d2010-11-14 23:42:06 +00004277
Bill Schmidt57d6de52012-10-23 15:51:16 +00004278 return LowerCall_Darwin(Chain, Callee, CallConv, isVarArg,
Hal Finkel934361a2015-01-14 01:07:51 +00004279 isTailCall, IsPatchPoint, Outs, OutVals, Ins,
Hal Finkele2ab0f12015-01-15 21:17:34 +00004280 dl, DAG, InVals, CS);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004281}
4282
4283SDValue
Bill Schmidt019cc6f2012-09-19 15:42:13 +00004284PPCTargetLowering::LowerCall_32SVR4(SDValue Chain, SDValue Callee,
4285 CallingConv::ID CallConv, bool isVarArg,
Hal Finkel934361a2015-01-14 01:07:51 +00004286 bool isTailCall, bool IsPatchPoint,
Bill Schmidt019cc6f2012-09-19 15:42:13 +00004287 const SmallVectorImpl<ISD::OutputArg> &Outs,
4288 const SmallVectorImpl<SDValue> &OutVals,
4289 const SmallVectorImpl<ISD::InputArg> &Ins,
Andrew Trickef9de2a2013-05-25 02:42:55 +00004290 SDLoc dl, SelectionDAG &DAG,
Hal Finkele2ab0f12015-01-15 21:17:34 +00004291 SmallVectorImpl<SDValue> &InVals,
4292 ImmutableCallSite *CS) const {
Bill Schmidt019cc6f2012-09-19 15:42:13 +00004293 // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description
Tilmann Schellerd1aaa322009-08-15 11:54:46 +00004294 // of the 32-bit SVR4 ABI stack frame layout.
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004295
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004296 assert((CallConv == CallingConv::C ||
4297 CallConv == CallingConv::Fast) && "Unknown calling convention!");
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004298
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004299 unsigned PtrByteSize = 4;
4300
4301 MachineFunction &MF = DAG.getMachineFunction();
4302
4303 // Mark this function as potentially containing a function that contains a
4304 // tail call. As a consequence the frame pointer will be used for dynamicalloc
4305 // and restoring the callers stack pointer in this functions epilog. This is
4306 // done because by tail calling the called function might overwrite the value
4307 // in this function's (MF) stack pointer stack slot 0(SP).
Nick Lewycky50f02cb2011-12-02 22:16:29 +00004308 if (getTargetMachine().Options.GuaranteedTailCallOpt &&
4309 CallConv == CallingConv::Fast)
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004310 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
Wesley Peck527da1b2010-11-23 03:31:01 +00004311
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004312 // Count how many bytes are to be pushed on the stack, including the linkage
4313 // area, parameter list area and the part of the local variable space which
4314 // contains copies of aggregates which are passed by value.
4315
4316 // Assign locations to all of the outgoing arguments.
4317 SmallVector<CCValAssign, 16> ArgLocs;
Eric Christopherb5217502014-08-06 18:45:26 +00004318 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4319 *DAG.getContext());
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004320
4321 // Reserve space for the linkage area on the stack.
Eric Christophera4ae2132015-02-13 22:22:57 +00004322 CCInfo.AllocateStack(Subtarget.getFrameLowering()->getLinkageSize(),
Ulrich Weigand8658f172014-07-20 23:43:15 +00004323 PtrByteSize);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004324
4325 if (isVarArg) {
4326 // Handle fixed and variable vector arguments differently.
4327 // Fixed vector arguments go into registers as long as registers are
4328 // available. Variable vector arguments always go into memory.
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004329 unsigned NumArgs = Outs.size();
Wesley Peck527da1b2010-11-23 03:31:01 +00004330
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004331 for (unsigned i = 0; i != NumArgs; ++i) {
Duncan Sandsf5dda012010-11-03 11:35:31 +00004332 MVT ArgVT = Outs[i].VT;
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004333 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004334 bool Result;
Wesley Peck527da1b2010-11-23 03:31:01 +00004335
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004336 if (Outs[i].IsFixed) {
Bill Schmidtef17c142013-02-06 17:33:58 +00004337 Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags,
4338 CCInfo);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004339 } else {
Bill Schmidtef17c142013-02-06 17:33:58 +00004340 Result = CC_PPC32_SVR4_VarArg(i, ArgVT, ArgVT, CCValAssign::Full,
4341 ArgFlags, CCInfo);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004342 }
Wesley Peck527da1b2010-11-23 03:31:01 +00004343
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004344 if (Result) {
Torok Edwinfb8d6d52009-07-08 20:53:28 +00004345#ifndef NDEBUG
Chris Lattner13626022009-08-23 06:03:38 +00004346 errs() << "Call operand #" << i << " has unhandled type "
Duncan Sandsf5dda012010-11-03 11:35:31 +00004347 << EVT(ArgVT).getEVTString() << "\n";
Torok Edwinfb8d6d52009-07-08 20:53:28 +00004348#endif
Craig Toppere73658d2014-04-28 04:05:08 +00004349 llvm_unreachable(nullptr);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004350 }
4351 }
4352 } else {
4353 // All arguments are treated the same.
Bill Schmidtef17c142013-02-06 17:33:58 +00004354 CCInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004355 }
Wesley Peck527da1b2010-11-23 03:31:01 +00004356
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004357 // Assign locations to all of the outgoing aggregate by value arguments.
4358 SmallVector<CCValAssign, 16> ByValArgLocs;
Eric Christopher0713a9d2011-06-08 23:55:35 +00004359 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(),
Eric Christopherb5217502014-08-06 18:45:26 +00004360 ByValArgLocs, *DAG.getContext());
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004361
4362 // Reserve stack space for the allocations in CCInfo.
4363 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize);
4364
Bill Schmidtef17c142013-02-06 17:33:58 +00004365 CCByValInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4_ByVal);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004366
4367 // Size of the linkage area, parameter list area and the part of the local
4368 // space variable where copies of aggregates which are passed by value are
4369 // stored.
4370 unsigned NumBytes = CCByValInfo.getNextStackOffset();
Wesley Peck527da1b2010-11-23 03:31:01 +00004371
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004372 // Calculate by how many bytes the stack has to be adjusted in case of tail
4373 // call optimization.
4374 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes);
4375
4376 // Adjust the stack pointer for the new arguments...
4377 // These operations are automatically eliminated by the prolog/epilog pass
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004378 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
Andrew Trickad6d08a2013-05-29 22:03:55 +00004379 dl);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004380 SDValue CallSeqStart = Chain;
4381
4382 // Load the return address and frame pointer so it can be moved somewhere else
4383 // later.
4384 SDValue LROp, FPOp;
4385 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, false,
4386 dl);
4387
4388 // Set up a copy of the stack pointer for use loading and storing any
4389 // arguments that may not fit in the registers available for argument
4390 // passing.
Owen Anderson9f944592009-08-11 20:47:22 +00004391 SDValue StackPtr = DAG.getRegister(PPC::R1, MVT::i32);
Wesley Peck527da1b2010-11-23 03:31:01 +00004392
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004393 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
4394 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
4395 SmallVector<SDValue, 8> MemOpChains;
4396
Roman Divacky71038e72011-08-30 17:04:16 +00004397 bool seenFloatArg = false;
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004398 // Walk the register/memloc assignments, inserting copies/loads.
4399 for (unsigned i = 0, j = 0, e = ArgLocs.size();
4400 i != e;
4401 ++i) {
4402 CCValAssign &VA = ArgLocs[i];
Dan Gohmanfe7532a2010-07-07 15:54:55 +00004403 SDValue Arg = OutVals[i];
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004404 ISD::ArgFlagsTy Flags = Outs[i].Flags;
Wesley Peck527da1b2010-11-23 03:31:01 +00004405
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004406 if (Flags.isByVal()) {
4407 // Argument is an aggregate which is passed by value, thus we need to
4408 // create a copy of it in the local variable space of the current stack
4409 // frame (which is the stack frame of the caller) and pass the address of
4410 // this copy to the callee.
4411 assert((j < ByValArgLocs.size()) && "Index out of bounds!");
4412 CCValAssign &ByValVA = ByValArgLocs[j++];
4413 assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!");
Wesley Peck527da1b2010-11-23 03:31:01 +00004414
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004415 // Memory reserved in the local variable space of the callers stack frame.
4416 unsigned LocMemOffset = ByValVA.getLocMemOffset();
Wesley Peck527da1b2010-11-23 03:31:01 +00004417
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004418 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004419 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
Wesley Peck527da1b2010-11-23 03:31:01 +00004420
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004421 // Create a copy of the argument in the local area of the current
4422 // stack frame.
4423 SDValue MemcpyCall =
4424 CreateCopyOfByValArgument(Arg, PtrOff,
4425 CallSeqStart.getNode()->getOperand(0),
4426 Flags, DAG, dl);
Wesley Peck527da1b2010-11-23 03:31:01 +00004427
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004428 // This must go outside the CALLSEQ_START..END.
4429 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall,
Andrew Trickad6d08a2013-05-29 22:03:55 +00004430 CallSeqStart.getNode()->getOperand(1),
4431 SDLoc(MemcpyCall));
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004432 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(),
4433 NewCallSeqStart.getNode());
4434 Chain = CallSeqStart = NewCallSeqStart;
Wesley Peck527da1b2010-11-23 03:31:01 +00004435
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004436 // Pass the address of the aggregate copy on the stack either in a
4437 // physical register or in the parameter list area of the current stack
4438 // frame to the callee.
4439 Arg = PtrOff;
4440 }
Wesley Peck527da1b2010-11-23 03:31:01 +00004441
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004442 if (VA.isRegLoc()) {
Hal Finkel2a9d3182014-03-06 00:23:33 +00004443 if (Arg.getValueType() == MVT::i1)
4444 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Arg);
4445
Roman Divacky71038e72011-08-30 17:04:16 +00004446 seenFloatArg |= VA.getLocVT().isFloatingPoint();
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004447 // Put argument in a physical register.
4448 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
4449 } else {
4450 // Put argument in the parameter list area of the current stack frame.
4451 assert(VA.isMemLoc());
4452 unsigned LocMemOffset = VA.getLocMemOffset();
4453
4454 if (!isTailCall) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004455 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004456 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
4457
4458 MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff,
Chris Lattner676c61d2010-09-21 18:41:36 +00004459 MachinePointerInfo(),
David Greene87a5abe2010-02-15 16:56:53 +00004460 false, false, 0));
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004461 } else {
4462 // Calculate and remember argument location.
4463 CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset,
4464 TailCallArguments);
4465 }
4466 }
4467 }
Wesley Peck527da1b2010-11-23 03:31:01 +00004468
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004469 if (!MemOpChains.empty())
Craig Topper48d114b2014-04-26 18:35:24 +00004470 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
Wesley Peck527da1b2010-11-23 03:31:01 +00004471
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004472 // Build a sequence of copy-to-reg nodes chained together with token chain
4473 // and flag operands which copy the outgoing args into the appropriate regs.
4474 SDValue InFlag;
4475 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
4476 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
4477 RegsToPass[i].second, InFlag);
4478 InFlag = Chain.getValue(1);
4479 }
Wesley Peck527da1b2010-11-23 03:31:01 +00004480
Hal Finkel5ab37802012-08-28 02:10:27 +00004481 // Set CR bit 6 to true if this is a vararg call with floating args passed in
4482 // registers.
4483 if (isVarArg) {
NAKAMURA Takumiac490292012-08-30 15:52:29 +00004484 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
4485 SDValue Ops[] = { Chain, InFlag };
4486
Hal Finkel5ab37802012-08-28 02:10:27 +00004487 Chain = DAG.getNode(seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET,
Craig Topper2d2aa0c2014-04-30 07:17:30 +00004488 dl, VTs, makeArrayRef(Ops, InFlag.getNode() ? 2 : 1));
NAKAMURA Takumiac490292012-08-30 15:52:29 +00004489
Hal Finkel5ab37802012-08-28 02:10:27 +00004490 InFlag = Chain.getValue(1);
4491 }
4492
Chris Lattnerdf8e17d2010-11-14 23:42:06 +00004493 if (isTailCall)
Tilmann Scheller773f14c2009-07-03 06:47:08 +00004494 PrepareTailCall(DAG, InFlag, Chain, dl, false, SPDiff, NumBytes, LROp, FPOp,
4495 false, TailCallArguments);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004496
Hal Finkel934361a2015-01-14 01:07:51 +00004497 return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint, DAG,
Hal Finkele2ab0f12015-01-15 21:17:34 +00004498 RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff,
4499 NumBytes, Ins, InVals, CS);
Tilmann Schellerb93960d2009-07-03 06:45:56 +00004500}
4501
Bill Schmidt57d6de52012-10-23 15:51:16 +00004502// Copy an argument into memory, being careful to do this outside the
4503// call sequence for the call to which the argument belongs.
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004504SDValue
Bill Schmidt57d6de52012-10-23 15:51:16 +00004505PPCTargetLowering::createMemcpyOutsideCallSeq(SDValue Arg, SDValue PtrOff,
4506 SDValue CallSeqStart,
4507 ISD::ArgFlagsTy Flags,
4508 SelectionDAG &DAG,
Andrew Trickef9de2a2013-05-25 02:42:55 +00004509 SDLoc dl) const {
Bill Schmidt57d6de52012-10-23 15:51:16 +00004510 SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff,
4511 CallSeqStart.getNode()->getOperand(0),
4512 Flags, DAG, dl);
4513 // The MEMCPY must go outside the CALLSEQ_START..END.
4514 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall,
Andrew Trickad6d08a2013-05-29 22:03:55 +00004515 CallSeqStart.getNode()->getOperand(1),
4516 SDLoc(MemcpyCall));
Bill Schmidt57d6de52012-10-23 15:51:16 +00004517 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(),
4518 NewCallSeqStart.getNode());
4519 return NewCallSeqStart;
4520}
4521
4522SDValue
4523PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
Sandeep Patel68c5f472009-09-02 08:44:58 +00004524 CallingConv::ID CallConv, bool isVarArg,
Hal Finkel934361a2015-01-14 01:07:51 +00004525 bool isTailCall, bool IsPatchPoint,
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004526 const SmallVectorImpl<ISD::OutputArg> &Outs,
Dan Gohmanfe7532a2010-07-07 15:54:55 +00004527 const SmallVectorImpl<SDValue> &OutVals,
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004528 const SmallVectorImpl<ISD::InputArg> &Ins,
Andrew Trickef9de2a2013-05-25 02:42:55 +00004529 SDLoc dl, SelectionDAG &DAG,
Hal Finkele2ab0f12015-01-15 21:17:34 +00004530 SmallVectorImpl<SDValue> &InVals,
4531 ImmutableCallSite *CS) const {
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00004532
Ulrich Weigandaa0ac4f2014-07-20 23:31:44 +00004533 bool isELFv2ABI = Subtarget.isELFv2ABI();
Ulrich Weigand59c6ab22014-06-20 16:34:05 +00004534 bool isLittleEndian = Subtarget.isLittleEndian();
Bill Schmidt57d6de52012-10-23 15:51:16 +00004535 unsigned NumOps = Outs.size();
Bill Schmidt019cc6f2012-09-19 15:42:13 +00004536
Bill Schmidt57d6de52012-10-23 15:51:16 +00004537 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
4538 unsigned PtrByteSize = 8;
4539
4540 MachineFunction &MF = DAG.getMachineFunction();
4541
4542 // Mark this function as potentially containing a function that contains a
4543 // tail call. As a consequence the frame pointer will be used for dynamicalloc
4544 // and restoring the callers stack pointer in this functions epilog. This is
4545 // done because by tail calling the called function might overwrite the value
4546 // in this function's (MF) stack pointer stack slot 0(SP).
4547 if (getTargetMachine().Options.GuaranteedTailCallOpt &&
4548 CallConv == CallingConv::Fast)
4549 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
4550
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004551 assert(!(CallConv == CallingConv::Fast && isVarArg) &&
4552 "fastcc not supported on varargs functions");
4553
Bill Schmidt57d6de52012-10-23 15:51:16 +00004554 // Count how many bytes are to be pushed on the stack, including the linkage
Ulrich Weigand8658f172014-07-20 23:43:15 +00004555 // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes
4556 // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage
4557 // area is 32 bytes reserved space for [SP][CR][LR][TOC].
Eric Christophera4ae2132015-02-13 22:22:57 +00004558 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
Ulrich Weigand8ca988f2014-06-23 14:15:53 +00004559 unsigned NumBytes = LinkageSize;
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004560 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
Hal Finkelc93a9a22015-02-25 01:06:45 +00004561 unsigned &QFPR_idx = FPR_idx;
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004562
4563 static const MCPhysReg GPR[] = {
4564 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
4565 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
4566 };
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004567 static const MCPhysReg VR[] = {
4568 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
4569 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
4570 };
4571 static const MCPhysReg VSRH[] = {
4572 PPC::VSH2, PPC::VSH3, PPC::VSH4, PPC::VSH5, PPC::VSH6, PPC::VSH7, PPC::VSH8,
4573 PPC::VSH9, PPC::VSH10, PPC::VSH11, PPC::VSH12, PPC::VSH13
4574 };
4575
4576 const unsigned NumGPRs = array_lengthof(GPR);
4577 const unsigned NumFPRs = 13;
4578 const unsigned NumVRs = array_lengthof(VR);
Hal Finkelc93a9a22015-02-25 01:06:45 +00004579 const unsigned NumQFPRs = NumFPRs;
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004580
4581 // When using the fast calling convention, we don't provide backing for
4582 // arguments that will be in registers.
4583 unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0;
Ulrich Weigand2bffb952014-06-23 13:08:27 +00004584
4585 // Add up all the space actually used.
4586 for (unsigned i = 0; i != NumOps; ++i) {
4587 ISD::ArgFlagsTy Flags = Outs[i].Flags;
4588 EVT ArgVT = Outs[i].VT;
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004589 EVT OrigVT = Outs[i].ArgVT;
Ulrich Weigand2bffb952014-06-23 13:08:27 +00004590
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004591 if (CallConv == CallingConv::Fast) {
4592 if (Flags.isByVal())
4593 NumGPRsUsed += (Flags.getByValSize()+7)/8;
4594 else
4595 switch (ArgVT.getSimpleVT().SimpleTy) {
4596 default: llvm_unreachable("Unexpected ValueType for argument!");
4597 case MVT::i1:
4598 case MVT::i32:
4599 case MVT::i64:
4600 if (++NumGPRsUsed <= NumGPRs)
4601 continue;
4602 break;
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004603 case MVT::v4i32:
4604 case MVT::v8i16:
4605 case MVT::v16i8:
4606 case MVT::v2f64:
4607 case MVT::v2i64:
4608 if (++NumVRsUsed <= NumVRs)
4609 continue;
4610 break;
Hal Finkelc93a9a22015-02-25 01:06:45 +00004611 case MVT::v4f32:
4612 // When using QPX, this is handled like a FP register, otherwise, it
4613 // is an Altivec register.
4614 if (Subtarget.hasQPX()) {
4615 if (++NumFPRsUsed <= NumFPRs)
4616 continue;
4617 } else {
4618 if (++NumVRsUsed <= NumVRs)
4619 continue;
4620 }
4621 break;
4622 case MVT::f32:
4623 case MVT::f64:
4624 case MVT::v4f64: // QPX
4625 case MVT::v4i1: // QPX
4626 if (++NumFPRsUsed <= NumFPRs)
4627 continue;
4628 break;
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004629 }
4630 }
4631
Ulrich Weigandec2bf932014-07-07 19:26:41 +00004632 /* Respect alignment of argument on the stack. */
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004633 unsigned Align =
4634 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
Ulrich Weigandec2bf932014-07-07 19:26:41 +00004635 NumBytes = ((NumBytes + Align - 1) / Align) * Align;
Ulrich Weigand2bffb952014-06-23 13:08:27 +00004636
4637 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004638 if (Flags.isInConsecutiveRegsLast())
4639 NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
Ulrich Weigand2bffb952014-06-23 13:08:27 +00004640 }
4641
Ulrich Weigandec2bf932014-07-07 19:26:41 +00004642 unsigned NumBytesActuallyUsed = NumBytes;
4643
Ulrich Weigand2bffb952014-06-23 13:08:27 +00004644 // The prolog code of the callee may store up to 8 GPR argument registers to
4645 // the stack, allowing va_start to index over them in memory if its varargs.
4646 // Because we cannot tell if this is needed on the caller side, we have to
4647 // conservatively assume that it is needed. As such, make sure we have at
4648 // least enough stack space for the caller to store the 8 GPRs.
Ulrich Weigand8658f172014-07-20 23:43:15 +00004649 // FIXME: On ELFv2, it may be unnecessary to allocate the parameter area.
Ulrich Weigand8ca988f2014-06-23 14:15:53 +00004650 NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize);
Ulrich Weigand2bffb952014-06-23 13:08:27 +00004651
4652 // Tail call needs the stack to be aligned.
4653 if (getTargetMachine().Options.GuaranteedTailCallOpt &&
4654 CallConv == CallingConv::Fast)
Eric Christophercccae792015-01-30 22:02:31 +00004655 NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes);
Bill Schmidt57d6de52012-10-23 15:51:16 +00004656
4657 // Calculate by how many bytes the stack has to be adjusted in case of tail
4658 // call optimization.
4659 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes);
4660
4661 // To protect arguments on the stack from being clobbered in a tail call,
4662 // force all the loads to happen before doing any other lowering.
4663 if (isTailCall)
4664 Chain = DAG.getStackArgumentTokenFactor(Chain);
4665
4666 // Adjust the stack pointer for the new arguments...
4667 // These operations are automatically eliminated by the prolog/epilog pass
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004668 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
Andrew Trickad6d08a2013-05-29 22:03:55 +00004669 dl);
Bill Schmidt57d6de52012-10-23 15:51:16 +00004670 SDValue CallSeqStart = Chain;
4671
4672 // Load the return address and frame pointer so it can be move somewhere else
4673 // later.
4674 SDValue LROp, FPOp;
4675 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, true,
4676 dl);
4677
4678 // Set up a copy of the stack pointer for use loading and storing any
4679 // arguments that may not fit in the registers available for argument
4680 // passing.
4681 SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64);
4682
4683 // Figure out which arguments are going to go in registers, and which in
4684 // memory. Also, if this is a vararg function, floating point operations
4685 // must be stored to our stack, and loaded into integer regs as well, if
4686 // any integer regs are available for argument passing.
Ulrich Weigand8ca988f2014-06-23 14:15:53 +00004687 unsigned ArgOffset = LinkageSize;
Bill Schmidt57d6de52012-10-23 15:51:16 +00004688
4689 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
4690 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
4691
4692 SmallVector<SDValue, 8> MemOpChains;
4693 for (unsigned i = 0; i != NumOps; ++i) {
4694 SDValue Arg = OutVals[i];
4695 ISD::ArgFlagsTy Flags = Outs[i].Flags;
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004696 EVT ArgVT = Outs[i].VT;
4697 EVT OrigVT = Outs[i].ArgVT;
Bill Schmidt57d6de52012-10-23 15:51:16 +00004698
4699 // PtrOff will be used to store the current argument to the stack if a
4700 // register cannot be found for it.
4701 SDValue PtrOff;
4702
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004703 // We re-align the argument offset for each argument, except when using the
4704 // fast calling convention, when we need to make sure we do that only when
4705 // we'll actually use a stack slot.
4706 auto ComputePtrOff = [&]() {
4707 /* Respect alignment of argument on the stack. */
4708 unsigned Align =
4709 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
4710 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align;
Bill Schmidt57d6de52012-10-23 15:51:16 +00004711
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004712 PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType());
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004713
4714 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
4715 };
4716
4717 if (CallConv != CallingConv::Fast) {
4718 ComputePtrOff();
4719
4720 /* Compute GPR index associated with argument offset. */
4721 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
4722 GPR_idx = std::min(GPR_idx, NumGPRs);
4723 }
Bill Schmidt57d6de52012-10-23 15:51:16 +00004724
4725 // Promote integers to 64-bit values.
Hal Finkel940ab932014-02-28 00:27:01 +00004726 if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) {
Bill Schmidt57d6de52012-10-23 15:51:16 +00004727 // FIXME: Should this use ANY_EXTEND if neither sext nor zext?
4728 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4729 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg);
4730 }
4731
4732 // FIXME memcpy is used way more than necessary. Correctness first.
4733 // Note: "by value" is code for passing a structure by value, not
4734 // basic types.
4735 if (Flags.isByVal()) {
4736 // Note: Size includes alignment padding, so
4737 // struct x { short a; char b; }
4738 // will have Size = 4. With #pragma pack(1), it will have Size = 3.
4739 // These are the proper values we need for right-justifying the
4740 // aggregate in a parameter register.
4741 unsigned Size = Flags.getByValSize();
Bill Schmidt9953cf22012-10-31 01:15:05 +00004742
4743 // An empty aggregate parameter takes up no storage and no
4744 // registers.
4745 if (Size == 0)
4746 continue;
4747
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004748 if (CallConv == CallingConv::Fast)
4749 ComputePtrOff();
4750
Bill Schmidt57d6de52012-10-23 15:51:16 +00004751 // All aggregates smaller than 8 bytes must be passed right-justified.
4752 if (Size==1 || Size==2 || Size==4) {
4753 EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32);
4754 if (GPR_idx != NumGPRs) {
4755 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg,
4756 MachinePointerInfo(), VT,
Louis Gerbarg67474e32014-07-31 21:45:05 +00004757 false, false, false, 0);
Bill Schmidt57d6de52012-10-23 15:51:16 +00004758 MemOpChains.push_back(Load.getValue(1));
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004759 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
Bill Schmidt57d6de52012-10-23 15:51:16 +00004760
4761 ArgOffset += PtrByteSize;
4762 continue;
4763 }
4764 }
4765
4766 if (GPR_idx == NumGPRs && Size < 8) {
Ulrich Weigand59c6ab22014-06-20 16:34:05 +00004767 SDValue AddPtr = PtrOff;
4768 if (!isLittleEndian) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004769 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl,
Ulrich Weigand59c6ab22014-06-20 16:34:05 +00004770 PtrOff.getValueType());
4771 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
4772 }
Bill Schmidt57d6de52012-10-23 15:51:16 +00004773 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
4774 CallSeqStart,
4775 Flags, DAG, dl);
4776 ArgOffset += PtrByteSize;
4777 continue;
4778 }
4779 // Copy entire object into memory. There are cases where gcc-generated
4780 // code assumes it is there, even if it could be put entirely into
4781 // registers. (This is not what the doc says.)
4782
4783 // FIXME: The above statement is likely due to a misunderstanding of the
4784 // documents. All arguments must be copied into the parameter area BY
4785 // THE CALLEE in the event that the callee takes the address of any
4786 // formal argument. That has not yet been implemented. However, it is
4787 // reasonable to use the stack area as a staging area for the register
4788 // load.
4789
4790 // Skip this for small aggregates, as we will use the same slot for a
4791 // right-justified copy, below.
4792 if (Size >= 8)
4793 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff,
4794 CallSeqStart,
4795 Flags, DAG, dl);
4796
4797 // When a register is available, pass a small aggregate right-justified.
4798 if (Size < 8 && GPR_idx != NumGPRs) {
4799 // The easiest way to get this right-justified in a register
4800 // is to copy the structure into the rightmost portion of a
4801 // local variable slot, then load the whole slot into the
4802 // register.
4803 // FIXME: The memcpy seems to produce pretty awful code for
4804 // small aggregates, particularly for packed ones.
Matt Arsenault758659232013-05-18 00:21:46 +00004805 // FIXME: It would be preferable to use the slot in the
Bill Schmidt57d6de52012-10-23 15:51:16 +00004806 // parameter save area instead of a new local variable.
Ulrich Weigand59c6ab22014-06-20 16:34:05 +00004807 SDValue AddPtr = PtrOff;
4808 if (!isLittleEndian) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004809 SDValue Const = DAG.getConstant(8 - Size, dl, PtrOff.getValueType());
Ulrich Weigand59c6ab22014-06-20 16:34:05 +00004810 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
4811 }
Bill Schmidt57d6de52012-10-23 15:51:16 +00004812 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
4813 CallSeqStart,
4814 Flags, DAG, dl);
4815
4816 // Load the slot into the register.
4817 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, PtrOff,
4818 MachinePointerInfo(),
4819 false, false, false, 0);
4820 MemOpChains.push_back(Load.getValue(1));
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004821 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
Bill Schmidt57d6de52012-10-23 15:51:16 +00004822
4823 // Done with this argument.
4824 ArgOffset += PtrByteSize;
4825 continue;
4826 }
4827
4828 // For aggregates larger than PtrByteSize, copy the pieces of the
4829 // object that fit into registers from the parameter save area.
4830 for (unsigned j=0; j<Size; j+=PtrByteSize) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004831 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType());
Bill Schmidt57d6de52012-10-23 15:51:16 +00004832 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
4833 if (GPR_idx != NumGPRs) {
4834 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg,
4835 MachinePointerInfo(),
4836 false, false, false, 0);
4837 MemOpChains.push_back(Load.getValue(1));
4838 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
4839 ArgOffset += PtrByteSize;
4840 } else {
4841 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize;
4842 break;
4843 }
4844 }
4845 continue;
4846 }
4847
Craig Topper56710102013-08-15 02:33:50 +00004848 switch (Arg.getSimpleValueType().SimpleTy) {
Bill Schmidt57d6de52012-10-23 15:51:16 +00004849 default: llvm_unreachable("Unexpected ValueType for argument!");
Hal Finkel940ab932014-02-28 00:27:01 +00004850 case MVT::i1:
Bill Schmidt57d6de52012-10-23 15:51:16 +00004851 case MVT::i32:
4852 case MVT::i64:
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004853 // These can be scalar arguments or elements of an integer array type
4854 // passed directly. Clang may use those instead of "byval" aggregate
4855 // types to avoid forcing arguments to memory unnecessarily.
Bill Schmidt57d6de52012-10-23 15:51:16 +00004856 if (GPR_idx != NumGPRs) {
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004857 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg));
Bill Schmidt57d6de52012-10-23 15:51:16 +00004858 } else {
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004859 if (CallConv == CallingConv::Fast)
4860 ComputePtrOff();
4861
Bill Schmidt57d6de52012-10-23 15:51:16 +00004862 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
4863 true, isTailCall, false, MemOpChains,
4864 TailCallArguments, dl);
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004865 if (CallConv == CallingConv::Fast)
4866 ArgOffset += PtrByteSize;
Bill Schmidt57d6de52012-10-23 15:51:16 +00004867 }
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004868 if (CallConv != CallingConv::Fast)
4869 ArgOffset += PtrByteSize;
Bill Schmidt57d6de52012-10-23 15:51:16 +00004870 break;
4871 case MVT::f32:
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004872 case MVT::f64: {
4873 // These can be scalar arguments or elements of a float array type
4874 // passed directly. The latter are used to implement ELFv2 homogenous
4875 // float aggregates.
4876
4877 // Named arguments go into FPRs first, and once they overflow, the
4878 // remaining arguments go into GPRs and then the parameter save area.
4879 // Unnamed arguments for vararg functions always go to GPRs and
4880 // then the parameter save area. For now, put all arguments to vararg
4881 // routines always in both locations (FPR *and* GPR or stack slot).
4882 bool NeedGPROrStack = isVarArg || FPR_idx == NumFPRs;
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004883 bool NeededLoad = false;
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004884
4885 // First load the argument into the next available FPR.
4886 if (FPR_idx != NumFPRs)
Bill Schmidt57d6de52012-10-23 15:51:16 +00004887 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg));
4888
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004889 // Next, load the argument into GPR or stack slot if needed.
4890 if (!NeedGPROrStack)
4891 ;
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004892 else if (GPR_idx != NumGPRs && CallConv != CallingConv::Fast) {
Hal Finkel8ea446b2015-01-18 14:31:10 +00004893 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
4894 // once we support fp <-> gpr moves.
4895
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004896 // In the non-vararg case, this can only ever happen in the
4897 // presence of f32 array types, since otherwise we never run
4898 // out of FPRs before running out of GPRs.
4899 SDValue ArgVal;
Bill Schmidtbd4ac262012-10-29 21:18:16 +00004900
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004901 // Double values are always passed in a single GPR.
4902 if (Arg.getValueType() != MVT::f32) {
4903 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg);
Bill Schmidt57d6de52012-10-23 15:51:16 +00004904
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004905 // Non-array float values are extended and passed in a GPR.
4906 } else if (!Flags.isInConsecutiveRegs()) {
4907 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg);
4908 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal);
4909
4910 // If we have an array of floats, we collect every odd element
4911 // together with its predecessor into one GPR.
4912 } else if (ArgOffset % PtrByteSize != 0) {
4913 SDValue Lo, Hi;
4914 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::i32, OutVals[i - 1]);
4915 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg);
4916 if (!isLittleEndian)
4917 std::swap(Lo, Hi);
4918 ArgVal = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
4919
4920 // The final element, if even, goes into the first half of a GPR.
4921 } else if (Flags.isInConsecutiveRegsLast()) {
4922 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg);
4923 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal);
4924 if (!isLittleEndian)
4925 ArgVal = DAG.getNode(ISD::SHL, dl, MVT::i64, ArgVal,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004926 DAG.getConstant(32, dl, MVT::i32));
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004927
4928 // Non-final even elements are skipped; they will be handled
4929 // together the with subsequent argument on the next go-around.
4930 } else
4931 ArgVal = SDValue();
4932
4933 if (ArgVal.getNode())
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004934 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], ArgVal));
Bill Schmidt57d6de52012-10-23 15:51:16 +00004935 } else {
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004936 if (CallConv == CallingConv::Fast)
4937 ComputePtrOff();
4938
Bill Schmidt57d6de52012-10-23 15:51:16 +00004939 // Single-precision floating-point values are mapped to the
4940 // second (rightmost) word of the stack doubleword.
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004941 if (Arg.getValueType() == MVT::f32 &&
4942 !isLittleEndian && !Flags.isInConsecutiveRegs()) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00004943 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType());
Bill Schmidt57d6de52012-10-23 15:51:16 +00004944 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour);
4945 }
4946
4947 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
4948 true, isTailCall, false, MemOpChains,
4949 TailCallArguments, dl);
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004950
4951 NeededLoad = true;
Bill Schmidt57d6de52012-10-23 15:51:16 +00004952 }
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004953 // When passing an array of floats, the array occupies consecutive
4954 // space in the argument area; only round up to the next doubleword
4955 // at the end of the array. Otherwise, each float takes 8 bytes.
Hal Finkelf81b6dd2015-01-18 12:08:47 +00004956 if (CallConv != CallingConv::Fast || NeededLoad) {
4957 ArgOffset += (Arg.getValueType() == MVT::f32 &&
4958 Flags.isInConsecutiveRegs()) ? 4 : 8;
4959 if (Flags.isInConsecutiveRegsLast())
4960 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4961 }
Bill Schmidt57d6de52012-10-23 15:51:16 +00004962 break;
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004963 }
Bill Schmidt57d6de52012-10-23 15:51:16 +00004964 case MVT::v4f32:
4965 case MVT::v4i32:
4966 case MVT::v8i16:
4967 case MVT::v16i8:
Hal Finkel27774d92014-03-13 07:58:58 +00004968 case MVT::v2f64:
Hal Finkela6c8b512014-03-26 16:12:58 +00004969 case MVT::v2i64:
Hal Finkelc93a9a22015-02-25 01:06:45 +00004970 if (!Subtarget.hasQPX()) {
Ulrich Weigand85d5df22014-07-21 00:13:26 +00004971 // These can be scalar arguments or elements of a vector array type
4972 // passed directly. The latter are used to implement ELFv2 homogenous
4973 // vector aggregates.
4974
Ulrich Weigand9ba552d2014-06-23 12:36:34 +00004975 // For a varargs call, named arguments go into VRs or on the stack as
4976 // usual; unnamed arguments always go to the stack or the corresponding
4977 // GPRs when within range. For now, we always put the value in both
4978 // locations (or even all three).
Bill Schmidt57d6de52012-10-23 15:51:16 +00004979 if (isVarArg) {
Bill Schmidt57d6de52012-10-23 15:51:16 +00004980 // We could elide this store in the case where the object fits
4981 // entirely in R registers. Maybe later.
Bill Schmidt57d6de52012-10-23 15:51:16 +00004982 SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff,
4983 MachinePointerInfo(), false, false, 0);
4984 MemOpChains.push_back(Store);
4985 if (VR_idx != NumVRs) {
4986 SDValue Load = DAG.getLoad(MVT::v4f32, dl, Store, PtrOff,
4987 MachinePointerInfo(),
4988 false, false, false, 0);
4989 MemOpChains.push_back(Load.getValue(1));
Hal Finkel7811c612014-03-28 19:58:11 +00004990
4991 unsigned VReg = (Arg.getSimpleValueType() == MVT::v2f64 ||
4992 Arg.getSimpleValueType() == MVT::v2i64) ?
4993 VSRH[VR_idx] : VR[VR_idx];
4994 ++VR_idx;
4995
4996 RegsToPass.push_back(std::make_pair(VReg, Load));
Bill Schmidt57d6de52012-10-23 15:51:16 +00004997 }
4998 ArgOffset += 16;
4999 for (unsigned i=0; i<16; i+=PtrByteSize) {
5000 if (GPR_idx == NumGPRs)
5001 break;
5002 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005003 DAG.getConstant(i, dl, PtrVT));
Bill Schmidt57d6de52012-10-23 15:51:16 +00005004 SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(),
5005 false, false, false, 0);
5006 MemOpChains.push_back(Load.getValue(1));
5007 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
5008 }
5009 break;
5010 }
5011
Ulrich Weigand9ba552d2014-06-23 12:36:34 +00005012 // Non-varargs Altivec params go into VRs or on the stack.
Bill Schmidt57d6de52012-10-23 15:51:16 +00005013 if (VR_idx != NumVRs) {
Hal Finkel7811c612014-03-28 19:58:11 +00005014 unsigned VReg = (Arg.getSimpleValueType() == MVT::v2f64 ||
5015 Arg.getSimpleValueType() == MVT::v2i64) ?
5016 VSRH[VR_idx] : VR[VR_idx];
5017 ++VR_idx;
5018
5019 RegsToPass.push_back(std::make_pair(VReg, Arg));
Bill Schmidt57d6de52012-10-23 15:51:16 +00005020 } else {
Hal Finkelf81b6dd2015-01-18 12:08:47 +00005021 if (CallConv == CallingConv::Fast)
5022 ComputePtrOff();
5023
Bill Schmidt57d6de52012-10-23 15:51:16 +00005024 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
5025 true, isTailCall, true, MemOpChains,
5026 TailCallArguments, dl);
Hal Finkelf81b6dd2015-01-18 12:08:47 +00005027 if (CallConv == CallingConv::Fast)
5028 ArgOffset += 16;
Bill Schmidt57d6de52012-10-23 15:51:16 +00005029 }
Hal Finkelf81b6dd2015-01-18 12:08:47 +00005030
5031 if (CallConv != CallingConv::Fast)
5032 ArgOffset += 16;
Bill Schmidt57d6de52012-10-23 15:51:16 +00005033 break;
Hal Finkelc93a9a22015-02-25 01:06:45 +00005034 } // not QPX
5035
5036 assert(Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32 &&
5037 "Invalid QPX parameter type");
5038
5039 /* fall through */
5040 case MVT::v4f64:
5041 case MVT::v4i1: {
5042 bool IsF32 = Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32;
5043 if (isVarArg) {
5044 // We could elide this store in the case where the object fits
5045 // entirely in R registers. Maybe later.
5046 SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff,
5047 MachinePointerInfo(), false, false, 0);
5048 MemOpChains.push_back(Store);
5049 if (QFPR_idx != NumQFPRs) {
5050 SDValue Load = DAG.getLoad(IsF32 ? MVT::v4f32 : MVT::v4f64, dl,
5051 Store, PtrOff, MachinePointerInfo(),
5052 false, false, false, 0);
5053 MemOpChains.push_back(Load.getValue(1));
5054 RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Load));
5055 }
5056 ArgOffset += (IsF32 ? 16 : 32);
Aaron Ballman70c27de2015-02-25 13:02:23 +00005057 for (unsigned i = 0; i < (IsF32 ? 16U : 32U); i += PtrByteSize) {
Hal Finkelc93a9a22015-02-25 01:06:45 +00005058 if (GPR_idx == NumGPRs)
5059 break;
5060 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005061 DAG.getConstant(i, dl, PtrVT));
Hal Finkelc93a9a22015-02-25 01:06:45 +00005062 SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(),
5063 false, false, false, 0);
5064 MemOpChains.push_back(Load.getValue(1));
5065 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
5066 }
5067 break;
5068 }
5069
5070 // Non-varargs QPX params go into registers or on the stack.
5071 if (QFPR_idx != NumQFPRs) {
5072 RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Arg));
5073 } else {
5074 if (CallConv == CallingConv::Fast)
5075 ComputePtrOff();
5076
5077 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
5078 true, isTailCall, true, MemOpChains,
5079 TailCallArguments, dl);
5080 if (CallConv == CallingConv::Fast)
5081 ArgOffset += (IsF32 ? 16 : 32);
5082 }
5083
5084 if (CallConv != CallingConv::Fast)
5085 ArgOffset += (IsF32 ? 16 : 32);
5086 break;
5087 }
Bill Schmidt57d6de52012-10-23 15:51:16 +00005088 }
5089 }
5090
Ulrich Weigandec2bf932014-07-07 19:26:41 +00005091 assert(NumBytesActuallyUsed == ArgOffset);
Ulrich Weigandde8641b2014-07-07 19:39:44 +00005092 (void)NumBytesActuallyUsed;
Ulrich Weigandec2bf932014-07-07 19:26:41 +00005093
Bill Schmidt57d6de52012-10-23 15:51:16 +00005094 if (!MemOpChains.empty())
Craig Topper48d114b2014-04-26 18:35:24 +00005095 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
Bill Schmidt57d6de52012-10-23 15:51:16 +00005096
5097 // Check if this is an indirect call (MTCTR/BCTRL).
5098 // See PrepareCall() for more information about calls through function
5099 // pointers in the 64-bit SVR4 ABI.
Hal Finkel934361a2015-01-14 01:07:51 +00005100 if (!isTailCall && !IsPatchPoint &&
Hal Finkel87deb0b2015-01-12 04:34:47 +00005101 !isFunctionGlobalAddress(Callee) &&
5102 !isa<ExternalSymbolSDNode>(Callee)) {
Bill Schmidt57d6de52012-10-23 15:51:16 +00005103 // Load r2 into a virtual register and store it to the TOC save area.
Hal Finkele6698d52015-02-01 15:03:28 +00005104 setUsesTOCBasePtr(DAG);
Bill Schmidt57d6de52012-10-23 15:51:16 +00005105 SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64);
5106 // TOC save area offset.
Eric Christopher736d39e2015-02-13 00:39:36 +00005107 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005108 SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
Bill Schmidt57d6de52012-10-23 15:51:16 +00005109 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
Hal Finkele2ab0f12015-01-15 21:17:34 +00005110 Chain = DAG.getStore(Val.getValue(1), dl, Val, AddPtr,
5111 MachinePointerInfo::getStack(TOCSaveOffset),
Bill Schmidt57d6de52012-10-23 15:51:16 +00005112 false, false, 0);
Ulrich Weigandaa0ac4f2014-07-20 23:31:44 +00005113 // In the ELFv2 ABI, R12 must contain the address of an indirect callee.
5114 // This does not mean the MTCTR instruction must use R12; it's easier
5115 // to model this as an extra parameter, so do that.
Hal Finkel934361a2015-01-14 01:07:51 +00005116 if (isELFv2ABI && !IsPatchPoint)
Ulrich Weigandaa0ac4f2014-07-20 23:31:44 +00005117 RegsToPass.push_back(std::make_pair((unsigned)PPC::X12, Callee));
Bill Schmidt57d6de52012-10-23 15:51:16 +00005118 }
5119
5120 // Build a sequence of copy-to-reg nodes chained together with token chain
5121 // and flag operands which copy the outgoing args into the appropriate regs.
5122 SDValue InFlag;
5123 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
5124 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
5125 RegsToPass[i].second, InFlag);
5126 InFlag = Chain.getValue(1);
5127 }
5128
5129 if (isTailCall)
5130 PrepareTailCall(DAG, InFlag, Chain, dl, true, SPDiff, NumBytes, LROp,
5131 FPOp, true, TailCallArguments);
5132
Hal Finkel934361a2015-01-14 01:07:51 +00005133 return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint, DAG,
Hal Finkele2ab0f12015-01-15 21:17:34 +00005134 RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff,
5135 NumBytes, Ins, InVals, CS);
Bill Schmidt57d6de52012-10-23 15:51:16 +00005136}
5137
5138SDValue
5139PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
5140 CallingConv::ID CallConv, bool isVarArg,
Hal Finkel934361a2015-01-14 01:07:51 +00005141 bool isTailCall, bool IsPatchPoint,
Bill Schmidt57d6de52012-10-23 15:51:16 +00005142 const SmallVectorImpl<ISD::OutputArg> &Outs,
5143 const SmallVectorImpl<SDValue> &OutVals,
5144 const SmallVectorImpl<ISD::InputArg> &Ins,
Andrew Trickef9de2a2013-05-25 02:42:55 +00005145 SDLoc dl, SelectionDAG &DAG,
Hal Finkele2ab0f12015-01-15 21:17:34 +00005146 SmallVectorImpl<SDValue> &InVals,
5147 ImmutableCallSite *CS) const {
Bill Schmidt57d6de52012-10-23 15:51:16 +00005148
5149 unsigned NumOps = Outs.size();
Scott Michelcf0da6c2009-02-17 22:15:04 +00005150
Owen Anderson53aa7a92009-08-10 22:56:29 +00005151 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Owen Anderson9f944592009-08-11 20:47:22 +00005152 bool isPPC64 = PtrVT == MVT::i64;
Chris Lattnerec78cad2006-06-26 22:48:35 +00005153 unsigned PtrByteSize = isPPC64 ? 8 : 4;
Scott Michelcf0da6c2009-02-17 22:15:04 +00005154
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005155 MachineFunction &MF = DAG.getMachineFunction();
5156
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005157 // Mark this function as potentially containing a function that contains a
5158 // tail call. As a consequence the frame pointer will be used for dynamicalloc
5159 // and restoring the callers stack pointer in this functions epilog. This is
5160 // done because by tail calling the called function might overwrite the value
5161 // in this function's (MF) stack pointer stack slot 0(SP).
Nick Lewycky50f02cb2011-12-02 22:16:29 +00005162 if (getTargetMachine().Options.GuaranteedTailCallOpt &&
5163 CallConv == CallingConv::Fast)
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005164 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
5165
Chris Lattneraa40ec12006-05-16 22:56:08 +00005166 // Count how many bytes are to be pushed on the stack, including the linkage
Chris Lattnerec78cad2006-06-26 22:48:35 +00005167 // area, and parameter passing area. We start with 24/48 bytes, which is
Chris Lattnerb7552a82006-05-17 00:15:40 +00005168 // prereserved space for [SP][CR][LR][3 x unused].
Eric Christophera4ae2132015-02-13 22:22:57 +00005169 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
Ulrich Weigand8ca988f2014-06-23 14:15:53 +00005170 unsigned NumBytes = LinkageSize;
Ulrich Weigand2bffb952014-06-23 13:08:27 +00005171
5172 // Add up all the space actually used.
5173 // In 32-bit non-varargs calls, Altivec parameters all go at the end; usually
5174 // they all go in registers, but we must reserve stack space for them for
5175 // possible use by the caller. In varargs or 64-bit calls, parameters are
5176 // assigned stack space in order, with padding so Altivec parameters are
5177 // 16-byte aligned.
5178 unsigned nAltivecParamsAtEnd = 0;
5179 for (unsigned i = 0; i != NumOps; ++i) {
5180 ISD::ArgFlagsTy Flags = Outs[i].Flags;
5181 EVT ArgVT = Outs[i].VT;
5182 // Varargs Altivec parameters are padded to a 16 byte boundary.
5183 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
5184 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
5185 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64) {
5186 if (!isVarArg && !isPPC64) {
5187 // Non-varargs Altivec parameters go after all the non-Altivec
5188 // parameters; handle those later so we know how much padding we need.
5189 nAltivecParamsAtEnd++;
5190 continue;
5191 }
5192 // Varargs and 64-bit Altivec parameters are padded to 16 byte boundary.
5193 NumBytes = ((NumBytes+15)/16)*16;
5194 }
5195 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
5196 }
5197
5198 // Allow for Altivec parameters at the end, if needed.
5199 if (nAltivecParamsAtEnd) {
5200 NumBytes = ((NumBytes+15)/16)*16;
5201 NumBytes += 16*nAltivecParamsAtEnd;
5202 }
5203
5204 // The prolog code of the callee may store up to 8 GPR argument registers to
5205 // the stack, allowing va_start to index over them in memory if its varargs.
5206 // Because we cannot tell if this is needed on the caller side, we have to
5207 // conservatively assume that it is needed. As such, make sure we have at
5208 // least enough stack space for the caller to store the 8 GPRs.
Ulrich Weigand8ca988f2014-06-23 14:15:53 +00005209 NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize);
Ulrich Weigand2bffb952014-06-23 13:08:27 +00005210
5211 // Tail call needs the stack to be aligned.
5212 if (getTargetMachine().Options.GuaranteedTailCallOpt &&
5213 CallConv == CallingConv::Fast)
Eric Christophercccae792015-01-30 22:02:31 +00005214 NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes);
Dale Johannesenb28456e2008-03-12 00:22:17 +00005215
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005216 // Calculate by how many bytes the stack has to be adjusted in case of tail
5217 // call optimization.
5218 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005219
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00005220 // To protect arguments on the stack from being clobbered in a tail call,
5221 // force all the loads to happen before doing any other lowering.
5222 if (isTailCall)
5223 Chain = DAG.getStackArgumentTokenFactor(Chain);
5224
Chris Lattnerb7552a82006-05-17 00:15:40 +00005225 // Adjust the stack pointer for the new arguments...
5226 // These operations are automatically eliminated by the prolog/epilog pass
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005227 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
Andrew Trickad6d08a2013-05-29 22:03:55 +00005228 dl);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005229 SDValue CallSeqStart = Chain;
Scott Michelcf0da6c2009-02-17 22:15:04 +00005230
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005231 // Load the return address and frame pointer so it can be move somewhere else
5232 // later.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005233 SDValue LROp, FPOp;
Tilmann Schellerb93960d2009-07-03 06:45:56 +00005234 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, true,
5235 dl);
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005236
Chris Lattnerb7552a82006-05-17 00:15:40 +00005237 // Set up a copy of the stack pointer for use loading and storing any
5238 // arguments that may not fit in the registers available for argument
5239 // passing.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005240 SDValue StackPtr;
Chris Lattnerec78cad2006-06-26 22:48:35 +00005241 if (isPPC64)
Owen Anderson9f944592009-08-11 20:47:22 +00005242 StackPtr = DAG.getRegister(PPC::X1, MVT::i64);
Chris Lattnerec78cad2006-06-26 22:48:35 +00005243 else
Owen Anderson9f944592009-08-11 20:47:22 +00005244 StackPtr = DAG.getRegister(PPC::R1, MVT::i32);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005245
Chris Lattnerb7552a82006-05-17 00:15:40 +00005246 // Figure out which arguments are going to go in registers, and which in
5247 // memory. Also, if this is a vararg function, floating point operations
5248 // must be stored to our stack, and loaded into integer regs as well, if
5249 // any integer regs are available for argument passing.
Ulrich Weigand8ca988f2014-06-23 14:15:53 +00005250 unsigned ArgOffset = LinkageSize;
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005251 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
Scott Michelcf0da6c2009-02-17 22:15:04 +00005252
Craig Topper840beec2014-04-04 05:16:06 +00005253 static const MCPhysReg GPR_32[] = { // 32-bit registers.
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005254 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
5255 PPC::R7, PPC::R8, PPC::R9, PPC::R10,
5256 };
Craig Topper840beec2014-04-04 05:16:06 +00005257 static const MCPhysReg GPR_64[] = { // 64-bit registers.
Chris Lattnerec78cad2006-06-26 22:48:35 +00005258 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
5259 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
5260 };
Craig Topper840beec2014-04-04 05:16:06 +00005261 static const MCPhysReg VR[] = {
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005262 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
5263 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
5264 };
Owen Andersone2f23a32007-09-07 04:06:50 +00005265 const unsigned NumGPRs = array_lengthof(GPR_32);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00005266 const unsigned NumFPRs = 13;
Tilmann Scheller98bdaaa2009-07-03 06:43:35 +00005267 const unsigned NumVRs = array_lengthof(VR);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005268
Craig Topper840beec2014-04-04 05:16:06 +00005269 const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32;
Chris Lattnerec78cad2006-06-26 22:48:35 +00005270
Tilmann Scheller773f14c2009-07-03 06:47:08 +00005271 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005272 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
5273
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005274 SmallVector<SDValue, 8> MemOpChains;
Evan Chengc2cd4732006-05-25 00:57:32 +00005275 for (unsigned i = 0; i != NumOps; ++i) {
Dan Gohmanfe7532a2010-07-07 15:54:55 +00005276 SDValue Arg = OutVals[i];
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00005277 ISD::ArgFlagsTy Flags = Outs[i].Flags;
Nicolas Geoffray7aad9282007-03-13 15:02:46 +00005278
Chris Lattnerb7552a82006-05-17 00:15:40 +00005279 // PtrOff will be used to store the current argument to the stack if a
5280 // register cannot be found for it.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005281 SDValue PtrOff;
Scott Michelcf0da6c2009-02-17 22:15:04 +00005282
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005283 PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType());
Nicolas Geoffray7aad9282007-03-13 15:02:46 +00005284
Dale Johannesen679073b2009-02-04 02:34:38 +00005285 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
Chris Lattnerec78cad2006-06-26 22:48:35 +00005286
5287 // On PPC64, promote integers to 64-bit values.
Owen Anderson9f944592009-08-11 20:47:22 +00005288 if (isPPC64 && Arg.getValueType() == MVT::i32) {
Duncan Sandsd97eea32008-03-21 09:14:45 +00005289 // FIXME: Should this use ANY_EXTEND if neither sext nor zext?
5290 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
Owen Anderson9f944592009-08-11 20:47:22 +00005291 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg);
Chris Lattnerec78cad2006-06-26 22:48:35 +00005292 }
Dale Johannesen85d41a12008-03-04 23:17:14 +00005293
Dale Johannesenbfa252d2008-03-07 20:27:40 +00005294 // FIXME memcpy is used way more than necessary. Correctness first.
Bill Schmidt019cc6f2012-09-19 15:42:13 +00005295 // Note: "by value" is code for passing a structure by value, not
5296 // basic types.
Duncan Sandsd97eea32008-03-21 09:14:45 +00005297 if (Flags.isByVal()) {
5298 unsigned Size = Flags.getByValSize();
Bill Schmidt57d6de52012-10-23 15:51:16 +00005299 // Very small objects are passed right-justified. Everything else is
5300 // passed left-justified.
5301 if (Size==1 || Size==2) {
5302 EVT VT = (Size==1) ? MVT::i8 : MVT::i16;
Dale Johannesenbfa252d2008-03-07 20:27:40 +00005303 if (GPR_idx != NumGPRs) {
Stuart Hastings81c43062011-02-16 16:23:55 +00005304 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg,
Chris Lattner3d178ed2010-09-21 17:04:51 +00005305 MachinePointerInfo(), VT,
Louis Gerbarg67474e32014-07-31 21:45:05 +00005306 false, false, false, 0);
Dale Johannesenbfa252d2008-03-07 20:27:40 +00005307 MemOpChains.push_back(Load.getValue(1));
5308 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
Tilmann Scheller773f14c2009-07-03 06:47:08 +00005309
5310 ArgOffset += PtrByteSize;
Dale Johannesenbfa252d2008-03-07 20:27:40 +00005311 } else {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005312 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl,
Bill Schmidt48081ca2012-10-16 13:30:53 +00005313 PtrOff.getValueType());
Dale Johannesen679073b2009-02-04 02:34:38 +00005314 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
Bill Schmidt57d6de52012-10-23 15:51:16 +00005315 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
5316 CallSeqStart,
5317 Flags, DAG, dl);
Dale Johannesenbfa252d2008-03-07 20:27:40 +00005318 ArgOffset += PtrByteSize;
5319 }
5320 continue;
5321 }
Dale Johannesen92dcf1e2008-03-17 02:13:43 +00005322 // Copy entire object into memory. There are cases where gcc-generated
5323 // code assumes it is there, even if it could be put entirely into
5324 // registers. (This is not what the doc says.)
Bill Schmidt57d6de52012-10-23 15:51:16 +00005325 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff,
5326 CallSeqStart,
5327 Flags, DAG, dl);
Bill Schmidt019cc6f2012-09-19 15:42:13 +00005328
5329 // For small aggregates (Darwin only) and aggregates >= PtrByteSize,
5330 // copy the pieces of the object that fit into registers from the
5331 // parameter save area.
Dale Johannesen85d41a12008-03-04 23:17:14 +00005332 for (unsigned j=0; j<Size; j+=PtrByteSize) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005333 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType());
Dale Johannesen679073b2009-02-04 02:34:38 +00005334 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
Dale Johannesen85d41a12008-03-04 23:17:14 +00005335 if (GPR_idx != NumGPRs) {
Chris Lattner7727d052010-09-21 06:44:06 +00005336 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg,
5337 MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00005338 false, false, false, 0);
Dale Johannesen0d235052008-03-05 23:31:27 +00005339 MemOpChains.push_back(Load.getValue(1));
Dale Johannesen85d41a12008-03-04 23:17:14 +00005340 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
Tilmann Scheller773f14c2009-07-03 06:47:08 +00005341 ArgOffset += PtrByteSize;
Dale Johannesen85d41a12008-03-04 23:17:14 +00005342 } else {
Dale Johannesen92dcf1e2008-03-17 02:13:43 +00005343 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize;
Dale Johannesenbfa252d2008-03-07 20:27:40 +00005344 break;
Dale Johannesen85d41a12008-03-04 23:17:14 +00005345 }
5346 }
5347 continue;
5348 }
5349
Craig Topper56710102013-08-15 02:33:50 +00005350 switch (Arg.getSimpleValueType().SimpleTy) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00005351 default: llvm_unreachable("Unexpected ValueType for argument!");
Hal Finkel5cae2162014-02-28 01:17:25 +00005352 case MVT::i1:
Owen Anderson9f944592009-08-11 20:47:22 +00005353 case MVT::i32:
5354 case MVT::i64:
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005355 if (GPR_idx != NumGPRs) {
Hal Finkel7f908e82014-03-06 00:45:19 +00005356 if (Arg.getValueType() == MVT::i1)
5357 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, PtrVT, Arg);
5358
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005359 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg));
Chris Lattnerb7552a82006-05-17 00:15:40 +00005360 } else {
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005361 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
5362 isPPC64, isTailCall, false, MemOpChains,
Dale Johannesen021052a2009-02-04 20:06:27 +00005363 TailCallArguments, dl);
Chris Lattnerb7552a82006-05-17 00:15:40 +00005364 }
Tilmann Scheller773f14c2009-07-03 06:47:08 +00005365 ArgOffset += PtrByteSize;
Chris Lattnerb7552a82006-05-17 00:15:40 +00005366 break;
Owen Anderson9f944592009-08-11 20:47:22 +00005367 case MVT::f32:
5368 case MVT::f64:
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005369 if (FPR_idx != NumFPRs) {
5370 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg));
5371
Chris Lattnerb7552a82006-05-17 00:15:40 +00005372 if (isVarArg) {
Chris Lattner676c61d2010-09-21 18:41:36 +00005373 SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff,
5374 MachinePointerInfo(), false, false, 0);
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005375 MemOpChains.push_back(Store);
5376
Chris Lattnerb7552a82006-05-17 00:15:40 +00005377 // Float varargs are always shadowed in available integer registers
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005378 if (GPR_idx != NumGPRs) {
Chris Lattner7727d052010-09-21 06:44:06 +00005379 SDValue Load = DAG.getLoad(PtrVT, dl, Store, PtrOff,
Pete Cooper82cd9e82011-11-08 18:42:53 +00005380 MachinePointerInfo(), false, false,
5381 false, 0);
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005382 MemOpChains.push_back(Load.getValue(1));
Tilmann Scheller773f14c2009-07-03 06:47:08 +00005383 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
Chris Lattnerb7552a82006-05-17 00:15:40 +00005384 }
Owen Anderson9f944592009-08-11 20:47:22 +00005385 if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && !isPPC64){
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005386 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType());
Dale Johannesen679073b2009-02-04 02:34:38 +00005387 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour);
Chris Lattner7727d052010-09-21 06:44:06 +00005388 SDValue Load = DAG.getLoad(PtrVT, dl, Store, PtrOff,
5389 MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00005390 false, false, false, 0);
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005391 MemOpChains.push_back(Load.getValue(1));
Tilmann Scheller773f14c2009-07-03 06:47:08 +00005392 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
Chris Lattneraa40ec12006-05-16 22:56:08 +00005393 }
5394 } else {
Chris Lattnerb7552a82006-05-17 00:15:40 +00005395 // If we have any FPRs remaining, we may also have GPRs remaining.
5396 // Args passed in FPRs consume either 1 (f32) or 2 (f64) available
5397 // GPRs.
Tilmann Scheller773f14c2009-07-03 06:47:08 +00005398 if (GPR_idx != NumGPRs)
5399 ++GPR_idx;
Owen Anderson9f944592009-08-11 20:47:22 +00005400 if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 &&
Tilmann Scheller773f14c2009-07-03 06:47:08 +00005401 !isPPC64) // PPC64 has 64-bit GPR's obviously :)
5402 ++GPR_idx;
Chris Lattneraa40ec12006-05-16 22:56:08 +00005403 }
Bill Schmidt57d6de52012-10-23 15:51:16 +00005404 } else
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005405 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
5406 isPPC64, isTailCall, false, MemOpChains,
Dale Johannesen021052a2009-02-04 20:06:27 +00005407 TailCallArguments, dl);
Tilmann Scheller773f14c2009-07-03 06:47:08 +00005408 if (isPPC64)
5409 ArgOffset += 8;
5410 else
Owen Anderson9f944592009-08-11 20:47:22 +00005411 ArgOffset += Arg.getValueType() == MVT::f32 ? 4 : 8;
Chris Lattnerb7552a82006-05-17 00:15:40 +00005412 break;
Owen Anderson9f944592009-08-11 20:47:22 +00005413 case MVT::v4f32:
5414 case MVT::v4i32:
5415 case MVT::v8i16:
5416 case MVT::v16i8:
Dale Johannesenb28456e2008-03-12 00:22:17 +00005417 if (isVarArg) {
5418 // These go aligned on the stack, or in the corresponding R registers
Scott Michelcf0da6c2009-02-17 22:15:04 +00005419 // when within range. The Darwin PPC ABI doc claims they also go in
Dale Johannesenb28456e2008-03-12 00:22:17 +00005420 // V registers; in fact gcc does this only for arguments that are
5421 // prototyped, not for those that match the ... We do it for all
5422 // arguments, seems to work.
5423 while (ArgOffset % 16 !=0) {
5424 ArgOffset += PtrByteSize;
5425 if (GPR_idx != NumGPRs)
5426 GPR_idx++;
5427 }
5428 // We could elide this store in the case where the object fits
5429 // entirely in R registers. Maybe later.
Scott Michelcf0da6c2009-02-17 22:15:04 +00005430 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005431 DAG.getConstant(ArgOffset, dl, PtrVT));
Chris Lattner676c61d2010-09-21 18:41:36 +00005432 SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff,
5433 MachinePointerInfo(), false, false, 0);
Dale Johannesenb28456e2008-03-12 00:22:17 +00005434 MemOpChains.push_back(Store);
5435 if (VR_idx != NumVRs) {
Wesley Peck527da1b2010-11-23 03:31:01 +00005436 SDValue Load = DAG.getLoad(MVT::v4f32, dl, Store, PtrOff,
Chris Lattner7727d052010-09-21 06:44:06 +00005437 MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00005438 false, false, false, 0);
Dale Johannesenb28456e2008-03-12 00:22:17 +00005439 MemOpChains.push_back(Load.getValue(1));
5440 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load));
5441 }
5442 ArgOffset += 16;
5443 for (unsigned i=0; i<16; i+=PtrByteSize) {
5444 if (GPR_idx == NumGPRs)
5445 break;
Dale Johannesen679073b2009-02-04 02:34:38 +00005446 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005447 DAG.getConstant(i, dl, PtrVT));
Chris Lattner7727d052010-09-21 06:44:06 +00005448 SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00005449 false, false, false, 0);
Dale Johannesenb28456e2008-03-12 00:22:17 +00005450 MemOpChains.push_back(Load.getValue(1));
5451 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
5452 }
5453 break;
5454 }
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005455
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00005456 // Non-varargs Altivec params generally go in registers, but have
5457 // stack space allocated at the end.
5458 if (VR_idx != NumVRs) {
5459 // Doesn't have GPR space allocated.
5460 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg));
5461 } else if (nAltivecParamsAtEnd==0) {
5462 // We are emitting Altivec params in order.
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005463 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
5464 isPPC64, isTailCall, true, MemOpChains,
Dale Johannesen021052a2009-02-04 20:06:27 +00005465 TailCallArguments, dl);
Dale Johannesenb28456e2008-03-12 00:22:17 +00005466 ArgOffset += 16;
Dale Johannesenb28456e2008-03-12 00:22:17 +00005467 }
Chris Lattnerb7552a82006-05-17 00:15:40 +00005468 break;
Chris Lattneraa40ec12006-05-16 22:56:08 +00005469 }
Chris Lattneraa40ec12006-05-16 22:56:08 +00005470 }
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00005471 // If all Altivec parameters fit in registers, as they usually do,
5472 // they get stack space following the non-Altivec parameters. We
5473 // don't track this here because nobody below needs it.
5474 // If there are more Altivec parameters than fit in registers emit
5475 // the stores here.
5476 if (!isVarArg && nAltivecParamsAtEnd > NumVRs) {
5477 unsigned j = 0;
5478 // Offset is aligned; skip 1st 12 params which go in V registers.
5479 ArgOffset = ((ArgOffset+15)/16)*16;
5480 ArgOffset += 12*16;
5481 for (unsigned i = 0; i != NumOps; ++i) {
Dan Gohmanfe7532a2010-07-07 15:54:55 +00005482 SDValue Arg = OutVals[i];
5483 EVT ArgType = Outs[i].VT;
Owen Anderson9f944592009-08-11 20:47:22 +00005484 if (ArgType==MVT::v4f32 || ArgType==MVT::v4i32 ||
5485 ArgType==MVT::v8i16 || ArgType==MVT::v16i8) {
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00005486 if (++j > NumVRs) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005487 SDValue PtrOff;
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005488 // We are emitting Altivec params in order.
5489 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
5490 isPPC64, isTailCall, true, MemOpChains,
Dale Johannesen021052a2009-02-04 20:06:27 +00005491 TailCallArguments, dl);
Dale Johannesen0dfd3f32008-03-14 17:41:26 +00005492 ArgOffset += 16;
5493 }
5494 }
5495 }
5496 }
5497
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005498 if (!MemOpChains.empty())
Craig Topper48d114b2014-04-26 18:35:24 +00005499 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005500
Dale Johannesen90eab672010-03-09 20:15:42 +00005501 // On Darwin, R12 must contain the address of an indirect callee. This does
5502 // not mean the MTCTR instruction must use R12; it's easier to model this as
5503 // an extra parameter, so do that.
Wesley Peck527da1b2010-11-23 03:31:01 +00005504 if (!isTailCall &&
Hal Finkel87deb0b2015-01-12 04:34:47 +00005505 !isFunctionGlobalAddress(Callee) &&
5506 !isa<ExternalSymbolSDNode>(Callee) &&
Dale Johannesen90eab672010-03-09 20:15:42 +00005507 !isBLACompatibleAddress(Callee, DAG))
5508 RegsToPass.push_back(std::make_pair((unsigned)(isPPC64 ? PPC::X12 :
5509 PPC::R12), Callee));
5510
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005511 // Build a sequence of copy-to-reg nodes chained together with token chain
5512 // and flag operands which copy the outgoing args into the appropriate regs.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005513 SDValue InFlag;
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005514 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
Scott Michelcf0da6c2009-02-17 22:15:04 +00005515 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
Dale Johannesen679073b2009-02-04 02:34:38 +00005516 RegsToPass[i].second, InFlag);
Chris Lattnerb1e9e372006-05-17 06:01:33 +00005517 InFlag = Chain.getValue(1);
5518 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00005519
Chris Lattnerdf8e17d2010-11-14 23:42:06 +00005520 if (isTailCall)
Tilmann Scheller773f14c2009-07-03 06:47:08 +00005521 PrepareTailCall(DAG, InFlag, Chain, dl, isPPC64, SPDiff, NumBytes, LROp,
5522 FPOp, true, TailCallArguments);
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005523
Hal Finkel934361a2015-01-14 01:07:51 +00005524 return FinishCall(CallConv, dl, isTailCall, isVarArg, IsPatchPoint, DAG,
Hal Finkele2ab0f12015-01-15 21:17:34 +00005525 RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff,
5526 NumBytes, Ins, InVals, CS);
Chris Lattneraa40ec12006-05-16 22:56:08 +00005527}
5528
Hal Finkel450128a2011-10-14 19:51:36 +00005529bool
5530PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
5531 MachineFunction &MF, bool isVarArg,
5532 const SmallVectorImpl<ISD::OutputArg> &Outs,
5533 LLVMContext &Context) const {
5534 SmallVector<CCValAssign, 16> RVLocs;
Eric Christopherb5217502014-08-06 18:45:26 +00005535 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
Hal Finkel450128a2011-10-14 19:51:36 +00005536 return CCInfo.CheckReturn(Outs, RetCC_PPC);
5537}
5538
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00005539SDValue
5540PPCTargetLowering::LowerReturn(SDValue Chain,
Sandeep Patel68c5f472009-09-02 08:44:58 +00005541 CallingConv::ID CallConv, bool isVarArg,
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00005542 const SmallVectorImpl<ISD::OutputArg> &Outs,
Dan Gohmanfe7532a2010-07-07 15:54:55 +00005543 const SmallVectorImpl<SDValue> &OutVals,
Andrew Trickef9de2a2013-05-25 02:42:55 +00005544 SDLoc dl, SelectionDAG &DAG) const {
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00005545
Chris Lattner4f2e4e02007-03-06 00:59:59 +00005546 SmallVector<CCValAssign, 16> RVLocs;
Eric Christopherb5217502014-08-06 18:45:26 +00005547 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
5548 *DAG.getContext());
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00005549 CCInfo.AnalyzeReturn(Outs, RetCC_PPC);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005550
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005551 SDValue Flag;
Jakob Stoklund Olesen8660a8c2013-02-05 18:12:00 +00005552 SmallVector<SDValue, 4> RetOps(1, Chain);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005553
Chris Lattner4f2e4e02007-03-06 00:59:59 +00005554 // Copy the result values into the output registers.
5555 for (unsigned i = 0; i != RVLocs.size(); ++i) {
5556 CCValAssign &VA = RVLocs[i];
5557 assert(VA.isRegLoc() && "Can only return in registers!");
Ulrich Weigand339d0592012-11-05 19:39:45 +00005558
5559 SDValue Arg = OutVals[i];
5560
5561 switch (VA.getLocInfo()) {
5562 default: llvm_unreachable("Unknown loc info!");
5563 case CCValAssign::Full: break;
5564 case CCValAssign::AExt:
5565 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
5566 break;
5567 case CCValAssign::ZExt:
5568 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
5569 break;
5570 case CCValAssign::SExt:
5571 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
5572 break;
5573 }
5574
5575 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag);
Chris Lattner4f2e4e02007-03-06 00:59:59 +00005576 Flag = Chain.getValue(1);
Jakob Stoklund Olesen8660a8c2013-02-05 18:12:00 +00005577 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
Chris Lattner4f2e4e02007-03-06 00:59:59 +00005578 }
5579
Jakob Stoklund Olesen8660a8c2013-02-05 18:12:00 +00005580 RetOps[0] = Chain; // Update chain.
5581
5582 // Add the flag if we have it.
Gabor Greiff304a7a2008-08-28 21:40:38 +00005583 if (Flag.getNode())
Jakob Stoklund Olesen8660a8c2013-02-05 18:12:00 +00005584 RetOps.push_back(Flag);
5585
Craig Topper48d114b2014-04-26 18:35:24 +00005586 return DAG.getNode(PPCISD::RET_FLAG, dl, MVT::Other, RetOps);
Chris Lattner4211ca92006-04-14 06:01:58 +00005587}
5588
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005589SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG,
Dan Gohman21cea8a2010-04-17 15:26:15 +00005590 const PPCSubtarget &Subtarget) const {
Jim Laskeye4f4d042006-12-04 22:04:42 +00005591 // When we pop the dynamic allocation we need to restore the SP link.
Andrew Trickef9de2a2013-05-25 02:42:55 +00005592 SDLoc dl(Op);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005593
Jim Laskeye4f4d042006-12-04 22:04:42 +00005594 // Get the corect type for pointers.
Owen Anderson53aa7a92009-08-10 22:56:29 +00005595 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Jim Laskeye4f4d042006-12-04 22:04:42 +00005596
5597 // Construct the stack pointer operand.
Dale Johannesen86dcae12009-11-24 01:09:07 +00005598 bool isPPC64 = Subtarget.isPPC64();
5599 unsigned SP = isPPC64 ? PPC::X1 : PPC::R1;
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005600 SDValue StackPtr = DAG.getRegister(SP, PtrVT);
Jim Laskeye4f4d042006-12-04 22:04:42 +00005601
5602 // Get the operands for the STACKRESTORE.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005603 SDValue Chain = Op.getOperand(0);
5604 SDValue SaveSP = Op.getOperand(1);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005605
Jim Laskeye4f4d042006-12-04 22:04:42 +00005606 // Load the old link SP.
Chris Lattner7727d052010-09-21 06:44:06 +00005607 SDValue LoadLinkSP = DAG.getLoad(PtrVT, dl, Chain, StackPtr,
5608 MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00005609 false, false, false, 0);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005610
Jim Laskeye4f4d042006-12-04 22:04:42 +00005611 // Restore the stack pointer.
Dale Johannesen021052a2009-02-04 20:06:27 +00005612 Chain = DAG.getCopyToReg(LoadLinkSP.getValue(1), dl, SP, SaveSP);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005613
Jim Laskeye4f4d042006-12-04 22:04:42 +00005614 // Store the old link SP.
Chris Lattner676c61d2010-09-21 18:41:36 +00005615 return DAG.getStore(Chain, dl, LoadLinkSP, StackPtr, MachinePointerInfo(),
David Greene87a5abe2010-02-15 16:56:53 +00005616 false, false, 0);
Jim Laskeye4f4d042006-12-04 22:04:42 +00005617}
5618
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005619
5620
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005621SDValue
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005622PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG & DAG) const {
Jim Laskey48850c12006-11-16 22:43:37 +00005623 MachineFunction &MF = DAG.getMachineFunction();
Eric Christopherb1aaebe2014-06-12 22:38:18 +00005624 bool isPPC64 = Subtarget.isPPC64();
Owen Anderson53aa7a92009-08-10 22:56:29 +00005625 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005626
5627 // Get current frame pointer save index. The users of this index will be
5628 // primarily DYNALLOC instructions.
5629 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
5630 int RASI = FI->getReturnAddrSaveIndex();
5631
5632 // If the frame pointer save index hasn't been defined yet.
5633 if (!RASI) {
5634 // Find out what the fix offset of the frame pointer save area.
Eric Christopherf71609b2015-02-13 00:39:27 +00005635 int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset();
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005636 // Allocate the frame index for frame pointer save area.
Hal Finkel6e27c6d2014-12-23 09:45:06 +00005637 RASI = MF.getFrameInfo()->CreateFixedObject(isPPC64? 8 : 4, LROffset, false);
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005638 // Save the result.
5639 FI->setReturnAddrSaveIndex(RASI);
5640 }
5641 return DAG.getFrameIndex(RASI, PtrVT);
5642}
5643
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005644SDValue
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005645PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const {
5646 MachineFunction &MF = DAG.getMachineFunction();
Eric Christopherb1aaebe2014-06-12 22:38:18 +00005647 bool isPPC64 = Subtarget.isPPC64();
Owen Anderson53aa7a92009-08-10 22:56:29 +00005648 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Jim Laskey48850c12006-11-16 22:43:37 +00005649
5650 // Get current frame pointer save index. The users of this index will be
5651 // primarily DYNALLOC instructions.
5652 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
5653 int FPSI = FI->getFramePointerSaveIndex();
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005654
Jim Laskey48850c12006-11-16 22:43:37 +00005655 // If the frame pointer save index hasn't been defined yet.
5656 if (!FPSI) {
5657 // Find out what the fix offset of the frame pointer save area.
Eric Christopherdc3a8a42015-02-13 00:39:38 +00005658 int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset();
Jim Laskey48850c12006-11-16 22:43:37 +00005659 // Allocate the frame index for frame pointer save area.
Evan Cheng0664a672010-07-03 00:40:23 +00005660 FPSI = MF.getFrameInfo()->CreateFixedObject(isPPC64? 8 : 4, FPOffset, true);
Jim Laskey48850c12006-11-16 22:43:37 +00005661 // Save the result.
Scott Michelcf0da6c2009-02-17 22:15:04 +00005662 FI->setFramePointerSaveIndex(FPSI);
Jim Laskey48850c12006-11-16 22:43:37 +00005663 }
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005664 return DAG.getFrameIndex(FPSI, PtrVT);
5665}
Jim Laskey48850c12006-11-16 22:43:37 +00005666
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005667SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
Arnold Schwaighoferbe0de342008-04-30 09:16:33 +00005668 SelectionDAG &DAG,
Dan Gohman21cea8a2010-04-17 15:26:15 +00005669 const PPCSubtarget &Subtarget) const {
Jim Laskey48850c12006-11-16 22:43:37 +00005670 // Get the inputs.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005671 SDValue Chain = Op.getOperand(0);
5672 SDValue Size = Op.getOperand(1);
Andrew Trickef9de2a2013-05-25 02:42:55 +00005673 SDLoc dl(Op);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005674
Jim Laskey48850c12006-11-16 22:43:37 +00005675 // Get the corect type for pointers.
Owen Anderson53aa7a92009-08-10 22:56:29 +00005676 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Jim Laskey48850c12006-11-16 22:43:37 +00005677 // Negate the size.
Dale Johannesen400dc2e2009-02-06 21:50:26 +00005678 SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005679 DAG.getConstant(0, dl, PtrVT), Size);
Jim Laskey48850c12006-11-16 22:43:37 +00005680 // Construct a node for the frame pointer save index.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005681 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
Jim Laskey48850c12006-11-16 22:43:37 +00005682 // Build a DYNALLOC node.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005683 SDValue Ops[3] = { Chain, NegSize, FPSIdx };
Owen Anderson9f944592009-08-11 20:47:22 +00005684 SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other);
Craig Topper48d114b2014-04-26 18:35:24 +00005685 return DAG.getNode(PPCISD::DYNALLOC, dl, VTs, Ops);
Jim Laskey48850c12006-11-16 22:43:37 +00005686}
5687
Hal Finkel756810f2013-03-21 21:37:52 +00005688SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
5689 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00005690 SDLoc DL(Op);
Hal Finkel756810f2013-03-21 21:37:52 +00005691 return DAG.getNode(PPCISD::EH_SJLJ_SETJMP, DL,
5692 DAG.getVTList(MVT::i32, MVT::Other),
5693 Op.getOperand(0), Op.getOperand(1));
5694}
5695
5696SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
5697 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00005698 SDLoc DL(Op);
Hal Finkel756810f2013-03-21 21:37:52 +00005699 return DAG.getNode(PPCISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
5700 Op.getOperand(0), Op.getOperand(1));
5701}
5702
Hal Finkel940ab932014-02-28 00:27:01 +00005703SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
Hal Finkelc93a9a22015-02-25 01:06:45 +00005704 if (Op.getValueType().isVector())
5705 return LowerVectorLoad(Op, DAG);
5706
Hal Finkel940ab932014-02-28 00:27:01 +00005707 assert(Op.getValueType() == MVT::i1 &&
5708 "Custom lowering only for i1 loads");
5709
5710 // First, load 8 bits into 32 bits, then truncate to 1 bit.
5711
5712 SDLoc dl(Op);
5713 LoadSDNode *LD = cast<LoadSDNode>(Op);
5714
5715 SDValue Chain = LD->getChain();
5716 SDValue BasePtr = LD->getBasePtr();
5717 MachineMemOperand *MMO = LD->getMemOperand();
5718
5719 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(), Chain,
5720 BasePtr, MVT::i8, MMO);
5721 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD);
5722
5723 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) };
Craig Topper64941d92014-04-27 19:20:57 +00005724 return DAG.getMergeValues(Ops, dl);
Hal Finkel940ab932014-02-28 00:27:01 +00005725}
5726
5727SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
Hal Finkelc93a9a22015-02-25 01:06:45 +00005728 if (Op.getOperand(1).getValueType().isVector())
5729 return LowerVectorStore(Op, DAG);
5730
Hal Finkel940ab932014-02-28 00:27:01 +00005731 assert(Op.getOperand(1).getValueType() == MVT::i1 &&
5732 "Custom lowering only for i1 stores");
5733
5734 // First, zero extend to 32 bits, then use a truncating store to 8 bits.
5735
5736 SDLoc dl(Op);
5737 StoreSDNode *ST = cast<StoreSDNode>(Op);
5738
5739 SDValue Chain = ST->getChain();
5740 SDValue BasePtr = ST->getBasePtr();
5741 SDValue Value = ST->getValue();
5742 MachineMemOperand *MMO = ST->getMemOperand();
5743
5744 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(), Value);
5745 return DAG.getTruncStore(Chain, dl, Value, BasePtr, MVT::i8, MMO);
5746}
5747
5748// FIXME: Remove this once the ANDI glue bug is fixed:
5749SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
5750 assert(Op.getValueType() == MVT::i1 &&
5751 "Custom lowering only for i1 results");
5752
5753 SDLoc DL(Op);
5754 return DAG.getNode(PPCISD::ANDIo_1_GT_BIT, DL, MVT::i1,
5755 Op.getOperand(0));
5756}
5757
Chris Lattner4211ca92006-04-14 06:01:58 +00005758/// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when
5759/// possible.
Dan Gohman21cea8a2010-04-17 15:26:15 +00005760SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
Chris Lattner4211ca92006-04-14 06:01:58 +00005761 // Not FP? Not a fsel.
Duncan Sands13237ac2008-06-06 12:08:01 +00005762 if (!Op.getOperand(0).getValueType().isFloatingPoint() ||
5763 !Op.getOperand(2).getValueType().isFloatingPoint())
Eli Friedman5806e182009-05-28 04:31:08 +00005764 return Op;
Scott Michelcf0da6c2009-02-17 22:15:04 +00005765
Hal Finkel81f87992013-04-07 22:11:09 +00005766 // We might be able to do better than this under some circumstances, but in
5767 // general, fsel-based lowering of select is a finite-math-only optimization.
5768 // For more information, see section F.3 of the 2.06 ISA specification.
5769 if (!DAG.getTarget().Options.NoInfsFPMath ||
5770 !DAG.getTarget().Options.NoNaNsFPMath)
5771 return Op;
Scott Michelcf0da6c2009-02-17 22:15:04 +00005772
Hal Finkel81f87992013-04-07 22:11:09 +00005773 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
Scott Michelcf0da6c2009-02-17 22:15:04 +00005774
Owen Anderson53aa7a92009-08-10 22:56:29 +00005775 EVT ResVT = Op.getValueType();
5776 EVT CmpVT = Op.getOperand(0).getValueType();
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005777 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
5778 SDValue TV = Op.getOperand(2), FV = Op.getOperand(3);
Andrew Trickef9de2a2013-05-25 02:42:55 +00005779 SDLoc dl(Op);
Scott Michelcf0da6c2009-02-17 22:15:04 +00005780
Chris Lattner4211ca92006-04-14 06:01:58 +00005781 // If the RHS of the comparison is a 0.0, we don't need to do the
5782 // subtraction at all.
Hal Finkel81f87992013-04-07 22:11:09 +00005783 SDValue Sel1;
Chris Lattner4211ca92006-04-14 06:01:58 +00005784 if (isFloatingPointZero(RHS))
5785 switch (CC) {
5786 default: break; // SETUO etc aren't handled by fsel.
Hal Finkel81f87992013-04-07 22:11:09 +00005787 case ISD::SETNE:
5788 std::swap(TV, FV);
5789 case ISD::SETEQ:
5790 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
5791 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS);
5792 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV);
5793 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits
5794 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1);
5795 return DAG.getNode(PPCISD::FSEL, dl, ResVT,
5796 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), Sel1, FV);
Chris Lattner4211ca92006-04-14 06:01:58 +00005797 case ISD::SETULT:
5798 case ISD::SETLT:
5799 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt
Chris Lattnerb56d22c2006-05-24 00:06:44 +00005800 case ISD::SETOGE:
Chris Lattner4211ca92006-04-14 06:01:58 +00005801 case ISD::SETGE:
Owen Anderson9f944592009-08-11 20:47:22 +00005802 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
5803 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS);
Dale Johannesen400dc2e2009-02-06 21:50:26 +00005804 return DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV);
Chris Lattner4211ca92006-04-14 06:01:58 +00005805 case ISD::SETUGT:
5806 case ISD::SETGT:
5807 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt
Chris Lattnerb56d22c2006-05-24 00:06:44 +00005808 case ISD::SETOLE:
Chris Lattner4211ca92006-04-14 06:01:58 +00005809 case ISD::SETLE:
Owen Anderson9f944592009-08-11 20:47:22 +00005810 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
5811 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS);
Dale Johannesen400dc2e2009-02-06 21:50:26 +00005812 return DAG.getNode(PPCISD::FSEL, dl, ResVT,
Owen Anderson9f944592009-08-11 20:47:22 +00005813 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), TV, FV);
Chris Lattner4211ca92006-04-14 06:01:58 +00005814 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00005815
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005816 SDValue Cmp;
Chris Lattner4211ca92006-04-14 06:01:58 +00005817 switch (CC) {
5818 default: break; // SETUO etc aren't handled by fsel.
Hal Finkel81f87992013-04-07 22:11:09 +00005819 case ISD::SETNE:
5820 std::swap(TV, FV);
5821 case ISD::SETEQ:
5822 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS);
5823 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
5824 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
5825 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV);
5826 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits
5827 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1);
5828 return DAG.getNode(PPCISD::FSEL, dl, ResVT,
5829 DAG.getNode(ISD::FNEG, dl, MVT::f64, Cmp), Sel1, FV);
Chris Lattner4211ca92006-04-14 06:01:58 +00005830 case ISD::SETULT:
5831 case ISD::SETLT:
Dale Johannesen400dc2e2009-02-06 21:50:26 +00005832 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS);
Owen Anderson9f944592009-08-11 20:47:22 +00005833 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
5834 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
Hal Finkel81f87992013-04-07 22:11:09 +00005835 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV);
Chris Lattnerb56d22c2006-05-24 00:06:44 +00005836 case ISD::SETOGE:
Chris Lattner4211ca92006-04-14 06:01:58 +00005837 case ISD::SETGE:
Dale Johannesen400dc2e2009-02-06 21:50:26 +00005838 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS);
Owen Anderson9f944592009-08-11 20:47:22 +00005839 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
5840 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
Hal Finkel81f87992013-04-07 22:11:09 +00005841 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV);
Chris Lattner4211ca92006-04-14 06:01:58 +00005842 case ISD::SETUGT:
5843 case ISD::SETGT:
Dale Johannesen400dc2e2009-02-06 21:50:26 +00005844 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS);
Owen Anderson9f944592009-08-11 20:47:22 +00005845 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
5846 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
Hal Finkel81f87992013-04-07 22:11:09 +00005847 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV);
Chris Lattnerb56d22c2006-05-24 00:06:44 +00005848 case ISD::SETOLE:
Chris Lattner4211ca92006-04-14 06:01:58 +00005849 case ISD::SETLE:
Dale Johannesen400dc2e2009-02-06 21:50:26 +00005850 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS);
Owen Anderson9f944592009-08-11 20:47:22 +00005851 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
5852 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
Hal Finkel81f87992013-04-07 22:11:09 +00005853 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV);
Chris Lattner4211ca92006-04-14 06:01:58 +00005854 }
Eli Friedman5806e182009-05-28 04:31:08 +00005855 return Op;
Chris Lattner4211ca92006-04-14 06:01:58 +00005856}
5857
Hal Finkeled844c42015-01-06 22:31:02 +00005858void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
5859 SelectionDAG &DAG,
5860 SDLoc dl) const {
Duncan Sands13237ac2008-06-06 12:08:01 +00005861 assert(Op.getOperand(0).getValueType().isFloatingPoint());
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005862 SDValue Src = Op.getOperand(0);
Owen Anderson9f944592009-08-11 20:47:22 +00005863 if (Src.getValueType() == MVT::f32)
5864 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src);
Duncan Sands2a287912008-07-19 16:26:02 +00005865
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00005866 SDValue Tmp;
Craig Topper56710102013-08-15 02:33:50 +00005867 switch (Op.getSimpleValueType().SimpleTy) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00005868 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!");
Owen Anderson9f944592009-08-11 20:47:22 +00005869 case MVT::i32:
Eric Christophercccae792015-01-30 22:02:31 +00005870 Tmp = DAG.getNode(
5871 Op.getOpcode() == ISD::FP_TO_SINT
5872 ? PPCISD::FCTIWZ
5873 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ),
5874 dl, MVT::f64, Src);
Chris Lattner4211ca92006-04-14 06:01:58 +00005875 break;
Owen Anderson9f944592009-08-11 20:47:22 +00005876 case MVT::i64:
Eric Christopherb1aaebe2014-06-12 22:38:18 +00005877 assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) &&
Hal Finkel3f88d082013-04-01 18:42:58 +00005878 "i64 FP_TO_UINT is supported only with FPCVT");
Hal Finkelf6d45f22013-04-01 17:52:07 +00005879 Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ :
5880 PPCISD::FCTIDUZ,
5881 dl, MVT::f64, Src);
Chris Lattner4211ca92006-04-14 06:01:58 +00005882 break;
5883 }
Duncan Sands2a287912008-07-19 16:26:02 +00005884
Chris Lattner4211ca92006-04-14 06:01:58 +00005885 // Convert the FP value to an int value through memory.
Eric Christopherb1aaebe2014-06-12 22:38:18 +00005886 bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() &&
5887 (Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT());
Hal Finkelf6d45f22013-04-01 17:52:07 +00005888 SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64);
5889 int FI = cast<FrameIndexSDNode>(FIPtr)->getIndex();
5890 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(FI);
Duncan Sands2a287912008-07-19 16:26:02 +00005891
Chris Lattner06a49542007-10-15 20:14:52 +00005892 // Emit a store to the stack slot.
Hal Finkelf6d45f22013-04-01 17:52:07 +00005893 SDValue Chain;
5894 if (i32Stack) {
5895 MachineFunction &MF = DAG.getMachineFunction();
5896 MachineMemOperand *MMO =
5897 MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, 4);
5898 SDValue Ops[] = { DAG.getEntryNode(), Tmp, FIPtr };
5899 Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl,
Craig Topper206fcd42014-04-26 19:29:41 +00005900 DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO);
Hal Finkelf6d45f22013-04-01 17:52:07 +00005901 } else
5902 Chain = DAG.getStore(DAG.getEntryNode(), dl, Tmp, FIPtr,
5903 MPI, false, false, 0);
Chris Lattner06a49542007-10-15 20:14:52 +00005904
5905 // Result is a load from the stack slot. If loading 4 bytes, make sure to
5906 // add in a bias.
Hal Finkelf6d45f22013-04-01 17:52:07 +00005907 if (Op.getValueType() == MVT::i32 && !i32Stack) {
Dale Johannesen021052a2009-02-04 20:06:27 +00005908 FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00005909 DAG.getConstant(4, dl, FIPtr.getValueType()));
Hal Finkeled844c42015-01-06 22:31:02 +00005910 MPI = MPI.getWithOffset(4);
Hal Finkelf6d45f22013-04-01 17:52:07 +00005911 }
5912
Hal Finkeled844c42015-01-06 22:31:02 +00005913 RLI.Chain = Chain;
5914 RLI.Ptr = FIPtr;
5915 RLI.MPI = MPI;
5916}
5917
Nemanja Ivanovicc38b5312015-04-11 10:40:42 +00005918/// \brief Custom lowers floating point to integer conversions to use
5919/// the direct move instructions available in ISA 2.07 to avoid the
5920/// need for load/store combinations.
5921SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op,
5922 SelectionDAG &DAG,
5923 SDLoc dl) const {
5924 assert(Op.getOperand(0).getValueType().isFloatingPoint());
5925 SDValue Src = Op.getOperand(0);
5926
5927 if (Src.getValueType() == MVT::f32)
5928 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src);
5929
5930 SDValue Tmp;
5931 switch (Op.getSimpleValueType().SimpleTy) {
5932 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!");
5933 case MVT::i32:
5934 Tmp = DAG.getNode(
5935 Op.getOpcode() == ISD::FP_TO_SINT
5936 ? PPCISD::FCTIWZ
5937 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ),
5938 dl, MVT::f64, Src);
5939 Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i32, Tmp);
5940 break;
5941 case MVT::i64:
5942 assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) &&
5943 "i64 FP_TO_UINT is supported only with FPCVT");
5944 Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ :
5945 PPCISD::FCTIDUZ,
5946 dl, MVT::f64, Src);
5947 Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i64, Tmp);
5948 break;
5949 }
5950 return Tmp;
5951}
5952
Hal Finkeled844c42015-01-06 22:31:02 +00005953SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
5954 SDLoc dl) const {
Nemanja Ivanovicc38b5312015-04-11 10:40:42 +00005955 if (Subtarget.hasDirectMove() && Subtarget.isPPC64())
5956 return LowerFP_TO_INTDirectMove(Op, DAG, dl);
5957
Hal Finkeled844c42015-01-06 22:31:02 +00005958 ReuseLoadInfo RLI;
5959 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
5960
5961 return DAG.getLoad(Op.getValueType(), dl, RLI.Chain, RLI.Ptr, RLI.MPI, false,
5962 false, RLI.IsInvariant, RLI.Alignment, RLI.AAInfo,
5963 RLI.Ranges);
5964}
5965
5966// We're trying to insert a regular store, S, and then a load, L. If the
5967// incoming value, O, is a load, we might just be able to have our load use the
5968// address used by O. However, we don't know if anything else will store to
5969// that address before we can load from it. To prevent this situation, we need
5970// to insert our load, L, into the chain as a peer of O. To do this, we give L
5971// the same chain operand as O, we create a token factor from the chain results
5972// of O and L, and we replace all uses of O's chain result with that token
5973// factor (see spliceIntoChain below for this last part).
5974bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT,
5975 ReuseLoadInfo &RLI,
Hal Finkel6c392692015-01-09 01:34:30 +00005976 SelectionDAG &DAG,
5977 ISD::LoadExtType ET) const {
Hal Finkeled844c42015-01-06 22:31:02 +00005978 SDLoc dl(Op);
Hal Finkel6c392692015-01-09 01:34:30 +00005979 if (ET == ISD::NON_EXTLOAD &&
5980 (Op.getOpcode() == ISD::FP_TO_UINT ||
Hal Finkeled844c42015-01-06 22:31:02 +00005981 Op.getOpcode() == ISD::FP_TO_SINT) &&
5982 isOperationLegalOrCustom(Op.getOpcode(),
5983 Op.getOperand(0).getValueType())) {
5984
5985 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
5986 return true;
5987 }
5988
5989 LoadSDNode *LD = dyn_cast<LoadSDNode>(Op);
Hal Finkel6c392692015-01-09 01:34:30 +00005990 if (!LD || LD->getExtensionType() != ET || LD->isVolatile() ||
5991 LD->isNonTemporal())
Hal Finkeled844c42015-01-06 22:31:02 +00005992 return false;
5993 if (LD->getMemoryVT() != MemVT)
5994 return false;
5995
5996 RLI.Ptr = LD->getBasePtr();
5997 if (LD->isIndexed() && LD->getOffset().getOpcode() != ISD::UNDEF) {
5998 assert(LD->getAddressingMode() == ISD::PRE_INC &&
5999 "Non-pre-inc AM on PPC?");
6000 RLI.Ptr = DAG.getNode(ISD::ADD, dl, RLI.Ptr.getValueType(), RLI.Ptr,
6001 LD->getOffset());
6002 }
6003
6004 RLI.Chain = LD->getChain();
6005 RLI.MPI = LD->getPointerInfo();
6006 RLI.IsInvariant = LD->isInvariant();
6007 RLI.Alignment = LD->getAlignment();
6008 RLI.AAInfo = LD->getAAInfo();
6009 RLI.Ranges = LD->getRanges();
6010
6011 RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1);
6012 return true;
6013}
6014
6015// Given the head of the old chain, ResChain, insert a token factor containing
6016// it and NewResChain, and make users of ResChain now be users of that token
6017// factor.
6018void PPCTargetLowering::spliceIntoChain(SDValue ResChain,
6019 SDValue NewResChain,
6020 SelectionDAG &DAG) const {
6021 if (!ResChain)
6022 return;
6023
6024 SDLoc dl(NewResChain);
6025
6026 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
6027 NewResChain, DAG.getUNDEF(MVT::Other));
6028 assert(TF.getNode() != NewResChain.getNode() &&
6029 "A new TF really is required here");
6030
6031 DAG.ReplaceAllUsesOfValueWith(ResChain, TF);
6032 DAG.UpdateNodeOperands(TF.getNode(), ResChain, NewResChain);
Chris Lattner4211ca92006-04-14 06:01:58 +00006033}
6034
Nemanja Ivanovicc38b5312015-04-11 10:40:42 +00006035/// \brief Custom lowers integer to floating point conversions to use
6036/// the direct move instructions available in ISA 2.07 to avoid the
6037/// need for load/store combinations.
6038SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op,
6039 SelectionDAG &DAG,
6040 SDLoc dl) const {
6041 assert((Op.getValueType() == MVT::f32 ||
6042 Op.getValueType() == MVT::f64) &&
6043 "Invalid floating point type as target of conversion");
6044 assert(Subtarget.hasFPCVT() &&
6045 "Int to FP conversions with direct moves require FPCVT");
6046 SDValue FP;
6047 SDValue Src = Op.getOperand(0);
6048 bool SinglePrec = Op.getValueType() == MVT::f32;
6049 bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32;
6050 bool Signed = Op.getOpcode() == ISD::SINT_TO_FP;
6051 unsigned ConvOp = Signed ? (SinglePrec ? PPCISD::FCFIDS : PPCISD::FCFID) :
6052 (SinglePrec ? PPCISD::FCFIDUS : PPCISD::FCFIDU);
6053
6054 if (WordInt) {
6055 FP = DAG.getNode(Signed ? PPCISD::MTVSRA : PPCISD::MTVSRZ,
6056 dl, MVT::f64, Src);
6057 FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP);
6058 }
6059 else {
6060 FP = DAG.getNode(PPCISD::MTVSRA, dl, MVT::f64, Src);
6061 FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP);
6062 }
6063
6064 return FP;
6065}
6066
Hal Finkelf6d45f22013-04-01 17:52:07 +00006067SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
Hal Finkeled844c42015-01-06 22:31:02 +00006068 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00006069 SDLoc dl(Op);
Hal Finkelc93a9a22015-02-25 01:06:45 +00006070
6071 if (Subtarget.hasQPX() && Op.getOperand(0).getValueType() == MVT::v4i1) {
6072 if (Op.getValueType() != MVT::v4f32 && Op.getValueType() != MVT::v4f64)
6073 return SDValue();
6074
6075 SDValue Value = Op.getOperand(0);
6076 // The values are now known to be -1 (false) or 1 (true). To convert this
6077 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
6078 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
6079 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
6080
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006081 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::f64);
Hal Finkelc93a9a22015-02-25 01:06:45 +00006082 FPHalfs = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64,
6083 FPHalfs, FPHalfs, FPHalfs, FPHalfs);
6084
6085 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
6086
6087 if (Op.getValueType() != MVT::v4f64)
6088 Value = DAG.getNode(ISD::FP_ROUND, dl,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006089 Op.getValueType(), Value,
6090 DAG.getIntPtrConstant(1, dl));
Hal Finkelc93a9a22015-02-25 01:06:45 +00006091 return Value;
6092 }
6093
Dan Gohmand6819da2008-03-11 01:59:03 +00006094 // Don't handle ppc_fp128 here; let it be lowered to a libcall.
Owen Anderson9f944592009-08-11 20:47:22 +00006095 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006096 return SDValue();
Dan Gohmand6819da2008-03-11 01:59:03 +00006097
Hal Finkel6a56b212014-03-05 22:14:00 +00006098 if (Op.getOperand(0).getValueType() == MVT::i1)
6099 return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Op.getOperand(0),
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006100 DAG.getConstantFP(1.0, dl, Op.getValueType()),
6101 DAG.getConstantFP(0.0, dl, Op.getValueType()));
Hal Finkel6a56b212014-03-05 22:14:00 +00006102
Nemanja Ivanovicc38b5312015-04-11 10:40:42 +00006103 // If we have direct moves, we can do all the conversion, skip the store/load
6104 // however, without FPCVT we can't do most conversions.
6105 if (Subtarget.hasDirectMove() && Subtarget.isPPC64() && Subtarget.hasFPCVT())
6106 return LowerINT_TO_FPDirectMove(Op, DAG, dl);
6107
Eric Christopherb1aaebe2014-06-12 22:38:18 +00006108 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) &&
Hal Finkelf6d45f22013-04-01 17:52:07 +00006109 "UINT_TO_FP is supported only with FPCVT");
6110
6111 // If we have FCFIDS, then use it when converting to single-precision.
Hal Finkel93d75ea2013-04-02 03:29:51 +00006112 // Otherwise, convert to double-precision and then round.
Eric Christophercccae792015-01-30 22:02:31 +00006113 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
6114 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS
6115 : PPCISD::FCFIDS)
6116 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU
6117 : PPCISD::FCFID);
6118 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
6119 ? MVT::f32
6120 : MVT::f64;
Hal Finkelf6d45f22013-04-01 17:52:07 +00006121
Owen Anderson9f944592009-08-11 20:47:22 +00006122 if (Op.getOperand(0).getValueType() == MVT::i64) {
Ulrich Weigandd34b5bd2012-10-18 13:16:11 +00006123 SDValue SINT = Op.getOperand(0);
6124 // When converting to single-precision, we actually need to convert
6125 // to double-precision first and then round to single-precision.
6126 // To avoid double-rounding effects during that operation, we have
6127 // to prepare the input operand. Bits that might be truncated when
6128 // converting to double-precision are replaced by a bit that won't
6129 // be lost at this stage, but is below the single-precision rounding
6130 // position.
6131 //
6132 // However, if -enable-unsafe-fp-math is in effect, accept double
6133 // rounding to avoid the extra overhead.
6134 if (Op.getValueType() == MVT::f32 &&
Eric Christopherb1aaebe2014-06-12 22:38:18 +00006135 !Subtarget.hasFPCVT() &&
Ulrich Weigandd34b5bd2012-10-18 13:16:11 +00006136 !DAG.getTarget().Options.UnsafeFPMath) {
6137
6138 // Twiddle input to make sure the low 11 bits are zero. (If this
6139 // is the case, we are guaranteed the value will fit into the 53 bit
6140 // mantissa of an IEEE double-precision value without rounding.)
6141 // If any of those low 11 bits were not zero originally, make sure
6142 // bit 12 (value 2048) is set instead, so that the final rounding
6143 // to single-precision gets the correct result.
6144 SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006145 SINT, DAG.getConstant(2047, dl, MVT::i64));
Ulrich Weigandd34b5bd2012-10-18 13:16:11 +00006146 Round = DAG.getNode(ISD::ADD, dl, MVT::i64,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006147 Round, DAG.getConstant(2047, dl, MVT::i64));
Ulrich Weigandd34b5bd2012-10-18 13:16:11 +00006148 Round = DAG.getNode(ISD::OR, dl, MVT::i64, Round, SINT);
6149 Round = DAG.getNode(ISD::AND, dl, MVT::i64,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006150 Round, DAG.getConstant(-2048, dl, MVT::i64));
Ulrich Weigandd34b5bd2012-10-18 13:16:11 +00006151
6152 // However, we cannot use that value unconditionally: if the magnitude
6153 // of the input value is small, the bit-twiddling we did above might
6154 // end up visibly changing the output. Fortunately, in that case, we
6155 // don't need to twiddle bits since the original input will convert
6156 // exactly to double-precision floating-point already. Therefore,
6157 // construct a conditional to use the original value if the top 11
6158 // bits are all sign-bit copies, and use the rounded value computed
6159 // above otherwise.
6160 SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006161 SINT, DAG.getConstant(53, dl, MVT::i32));
Ulrich Weigandd34b5bd2012-10-18 13:16:11 +00006162 Cond = DAG.getNode(ISD::ADD, dl, MVT::i64,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006163 Cond, DAG.getConstant(1, dl, MVT::i64));
Ulrich Weigandd34b5bd2012-10-18 13:16:11 +00006164 Cond = DAG.getSetCC(dl, MVT::i32,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006165 Cond, DAG.getConstant(1, dl, MVT::i64), ISD::SETUGT);
Ulrich Weigandd34b5bd2012-10-18 13:16:11 +00006166
6167 SINT = DAG.getNode(ISD::SELECT, dl, MVT::i64, Cond, Round, SINT);
6168 }
Hal Finkelf6d45f22013-04-01 17:52:07 +00006169
Hal Finkeled844c42015-01-06 22:31:02 +00006170 ReuseLoadInfo RLI;
6171 SDValue Bits;
6172
Hal Finkel6c392692015-01-09 01:34:30 +00006173 MachineFunction &MF = DAG.getMachineFunction();
Hal Finkeled844c42015-01-06 22:31:02 +00006174 if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) {
6175 Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, false,
6176 false, RLI.IsInvariant, RLI.Alignment, RLI.AAInfo,
6177 RLI.Ranges);
6178 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG);
Hal Finkel6c392692015-01-09 01:34:30 +00006179 } else if (Subtarget.hasLFIWAX() &&
6180 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) {
6181 MachineMemOperand *MMO =
6182 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
6183 RLI.Alignment, RLI.AAInfo, RLI.Ranges);
6184 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
6185 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl,
6186 DAG.getVTList(MVT::f64, MVT::Other),
6187 Ops, MVT::i32, MMO);
6188 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG);
6189 } else if (Subtarget.hasFPCVT() &&
6190 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) {
6191 MachineMemOperand *MMO =
6192 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
6193 RLI.Alignment, RLI.AAInfo, RLI.Ranges);
6194 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
6195 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl,
6196 DAG.getVTList(MVT::f64, MVT::Other),
6197 Ops, MVT::i32, MMO);
6198 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG);
6199 } else if (((Subtarget.hasLFIWAX() &&
6200 SINT.getOpcode() == ISD::SIGN_EXTEND) ||
6201 (Subtarget.hasFPCVT() &&
6202 SINT.getOpcode() == ISD::ZERO_EXTEND)) &&
6203 SINT.getOperand(0).getValueType() == MVT::i32) {
6204 MachineFrameInfo *FrameInfo = MF.getFrameInfo();
6205 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
6206
6207 int FrameIdx = FrameInfo->CreateStackObject(4, 4, false);
6208 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
6209
6210 SDValue Store =
6211 DAG.getStore(DAG.getEntryNode(), dl, SINT.getOperand(0), FIdx,
6212 MachinePointerInfo::getFixedStack(FrameIdx),
6213 false, false, 0);
6214
6215 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&
6216 "Expected an i32 store");
6217
6218 RLI.Ptr = FIdx;
6219 RLI.Chain = Store;
6220 RLI.MPI = MachinePointerInfo::getFixedStack(FrameIdx);
6221 RLI.Alignment = 4;
6222
6223 MachineMemOperand *MMO =
6224 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
6225 RLI.Alignment, RLI.AAInfo, RLI.Ranges);
6226 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
6227 Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ?
6228 PPCISD::LFIWZX : PPCISD::LFIWAX,
6229 dl, DAG.getVTList(MVT::f64, MVT::Other),
6230 Ops, MVT::i32, MMO);
Hal Finkeled844c42015-01-06 22:31:02 +00006231 } else
6232 Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT);
6233
Hal Finkelf6d45f22013-04-01 17:52:07 +00006234 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Bits);
6235
Eric Christopherb1aaebe2014-06-12 22:38:18 +00006236 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT())
Scott Michelcf0da6c2009-02-17 22:15:04 +00006237 FP = DAG.getNode(ISD::FP_ROUND, dl,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006238 MVT::f32, FP, DAG.getIntPtrConstant(0, dl));
Chris Lattner4211ca92006-04-14 06:01:58 +00006239 return FP;
6240 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00006241
Owen Anderson9f944592009-08-11 20:47:22 +00006242 assert(Op.getOperand(0).getValueType() == MVT::i32 &&
Hal Finkelf6d45f22013-04-01 17:52:07 +00006243 "Unhandled INT_TO_FP type in custom expander!");
Chris Lattner4211ca92006-04-14 06:01:58 +00006244 // Since we only generate this in 64-bit mode, we can take advantage of
6245 // 64-bit registers. In particular, sign extend the input value into the
6246 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack
6247 // then lfd it and fcfid it.
Dan Gohman48b185d2009-09-25 20:36:54 +00006248 MachineFunction &MF = DAG.getMachineFunction();
6249 MachineFrameInfo *FrameInfo = MF.getFrameInfo();
Owen Anderson53aa7a92009-08-10 22:56:29 +00006250 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Scott Michelcf0da6c2009-02-17 22:15:04 +00006251
Hal Finkelbeb296b2013-03-31 10:12:51 +00006252 SDValue Ld;
Eric Christopherb1aaebe2014-06-12 22:38:18 +00006253 if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) {
Hal Finkeled844c42015-01-06 22:31:02 +00006254 ReuseLoadInfo RLI;
6255 bool ReusingLoad;
6256 if (!(ReusingLoad = canReuseLoadAddress(Op.getOperand(0), MVT::i32, RLI,
6257 DAG))) {
6258 int FrameIdx = FrameInfo->CreateStackObject(4, 4, false);
6259 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
Scott Michelcf0da6c2009-02-17 22:15:04 +00006260
Hal Finkeled844c42015-01-06 22:31:02 +00006261 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx,
6262 MachinePointerInfo::getFixedStack(FrameIdx),
6263 false, false, 0);
Hal Finkele53429a2013-03-31 01:58:02 +00006264
Hal Finkeled844c42015-01-06 22:31:02 +00006265 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&
6266 "Expected an i32 store");
6267
6268 RLI.Ptr = FIdx;
6269 RLI.Chain = Store;
6270 RLI.MPI = MachinePointerInfo::getFixedStack(FrameIdx);
6271 RLI.Alignment = 4;
6272 }
6273
Hal Finkelbeb296b2013-03-31 10:12:51 +00006274 MachineMemOperand *MMO =
Hal Finkeled844c42015-01-06 22:31:02 +00006275 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
6276 RLI.Alignment, RLI.AAInfo, RLI.Ranges);
6277 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Hal Finkelf6d45f22013-04-01 17:52:07 +00006278 Ld = DAG.getMemIntrinsicNode(Op.getOpcode() == ISD::UINT_TO_FP ?
6279 PPCISD::LFIWZX : PPCISD::LFIWAX,
6280 dl, DAG.getVTList(MVT::f64, MVT::Other),
Craig Topper206fcd42014-04-26 19:29:41 +00006281 Ops, MVT::i32, MMO);
Hal Finkeled844c42015-01-06 22:31:02 +00006282 if (ReusingLoad)
6283 spliceIntoChain(RLI.ResChain, Ld.getValue(1), DAG);
Hal Finkelbeb296b2013-03-31 10:12:51 +00006284 } else {
Eric Christopherb1aaebe2014-06-12 22:38:18 +00006285 assert(Subtarget.isPPC64() &&
Hal Finkelf6d45f22013-04-01 17:52:07 +00006286 "i32->FP without LFIWAX supported only on PPC64");
6287
Hal Finkelbeb296b2013-03-31 10:12:51 +00006288 int FrameIdx = FrameInfo->CreateStackObject(8, 8, false);
6289 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
6290
6291 SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64,
6292 Op.getOperand(0));
6293
6294 // STD the extended value into the stack slot.
6295 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Ext64, FIdx,
6296 MachinePointerInfo::getFixedStack(FrameIdx),
6297 false, false, 0);
6298
6299 // Load the value as a double.
6300 Ld = DAG.getLoad(MVT::f64, dl, Store, FIdx,
6301 MachinePointerInfo::getFixedStack(FrameIdx),
6302 false, false, false, 0);
6303 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00006304
Chris Lattner4211ca92006-04-14 06:01:58 +00006305 // FCFID it and return it.
Hal Finkelf6d45f22013-04-01 17:52:07 +00006306 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Ld);
Eric Christopherb1aaebe2014-06-12 22:38:18 +00006307 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT())
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006308 FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP,
6309 DAG.getIntPtrConstant(0, dl));
Chris Lattner4211ca92006-04-14 06:01:58 +00006310 return FP;
6311}
6312
Dan Gohman21cea8a2010-04-17 15:26:15 +00006313SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
6314 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00006315 SDLoc dl(Op);
Dale Johannesen5c94cb32008-01-18 19:55:37 +00006316 /*
6317 The rounding mode is in bits 30:31 of FPSR, and has the following
6318 settings:
6319 00 Round to nearest
6320 01 Round to 0
6321 10 Round to +inf
6322 11 Round to -inf
6323
6324 FLT_ROUNDS, on the other hand, expects the following:
6325 -1 Undefined
6326 0 Round to 0
6327 1 Round to nearest
6328 2 Round to +inf
6329 3 Round to -inf
6330
6331 To perform the conversion, we do:
6332 ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1))
6333 */
6334
6335 MachineFunction &MF = DAG.getMachineFunction();
Owen Anderson53aa7a92009-08-10 22:56:29 +00006336 EVT VT = Op.getValueType();
6337 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Dale Johannesen5c94cb32008-01-18 19:55:37 +00006338
6339 // Save FP Control Word to register
Benjamin Kramerfdf362b2013-03-07 20:33:29 +00006340 EVT NodeTys[] = {
6341 MVT::f64, // return register
6342 MVT::Glue // unused in this context
6343 };
Craig Topper2d2aa0c2014-04-30 07:17:30 +00006344 SDValue Chain = DAG.getNode(PPCISD::MFFS, dl, NodeTys, None);
Dale Johannesen5c94cb32008-01-18 19:55:37 +00006345
6346 // Save FP register to stack slot
David Greene1fbe0542009-11-12 20:49:22 +00006347 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8, false);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006348 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
Dale Johannesen021052a2009-02-04 20:06:27 +00006349 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Chain,
Chris Lattner676c61d2010-09-21 18:41:36 +00006350 StackSlot, MachinePointerInfo(), false, false,0);
Dale Johannesen5c94cb32008-01-18 19:55:37 +00006351
6352 // Load FP Control Word from low 32 bits of stack slot.
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006353 SDValue Four = DAG.getConstant(4, dl, PtrVT);
Dale Johannesen021052a2009-02-04 20:06:27 +00006354 SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four);
Chris Lattner7727d052010-09-21 06:44:06 +00006355 SDValue CWD = DAG.getLoad(MVT::i32, dl, Store, Addr, MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00006356 false, false, false, 0);
Dale Johannesen5c94cb32008-01-18 19:55:37 +00006357
6358 // Transform as necessary
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006359 SDValue CWD1 =
Owen Anderson9f944592009-08-11 20:47:22 +00006360 DAG.getNode(ISD::AND, dl, MVT::i32,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006361 CWD, DAG.getConstant(3, dl, MVT::i32));
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006362 SDValue CWD2 =
Owen Anderson9f944592009-08-11 20:47:22 +00006363 DAG.getNode(ISD::SRL, dl, MVT::i32,
6364 DAG.getNode(ISD::AND, dl, MVT::i32,
6365 DAG.getNode(ISD::XOR, dl, MVT::i32,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006366 CWD, DAG.getConstant(3, dl, MVT::i32)),
6367 DAG.getConstant(3, dl, MVT::i32)),
6368 DAG.getConstant(1, dl, MVT::i32));
Dale Johannesen5c94cb32008-01-18 19:55:37 +00006369
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006370 SDValue RetVal =
Owen Anderson9f944592009-08-11 20:47:22 +00006371 DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1, CWD2);
Dale Johannesen5c94cb32008-01-18 19:55:37 +00006372
Duncan Sands13237ac2008-06-06 12:08:01 +00006373 return DAG.getNode((VT.getSizeInBits() < 16 ?
Dale Johannesen021052a2009-02-04 20:06:27 +00006374 ISD::TRUNCATE : ISD::ZERO_EXTEND), dl, VT, RetVal);
Dale Johannesen5c94cb32008-01-18 19:55:37 +00006375}
6376
Dan Gohman21cea8a2010-04-17 15:26:15 +00006377SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const {
Owen Anderson53aa7a92009-08-10 22:56:29 +00006378 EVT VT = Op.getValueType();
Duncan Sands13237ac2008-06-06 12:08:01 +00006379 unsigned BitWidth = VT.getSizeInBits();
Andrew Trickef9de2a2013-05-25 02:42:55 +00006380 SDLoc dl(Op);
Dan Gohman8d2ead22008-03-07 20:36:53 +00006381 assert(Op.getNumOperands() == 3 &&
6382 VT == Op.getOperand(1).getValueType() &&
6383 "Unexpected SHL!");
Scott Michelcf0da6c2009-02-17 22:15:04 +00006384
Chris Lattner601b8652006-09-20 03:47:40 +00006385 // Expand into a bunch of logical ops. Note that these ops
Chris Lattner4211ca92006-04-14 06:01:58 +00006386 // depend on the PPC behavior for oversized shift amounts.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006387 SDValue Lo = Op.getOperand(0);
6388 SDValue Hi = Op.getOperand(1);
6389 SDValue Amt = Op.getOperand(2);
Owen Anderson53aa7a92009-08-10 22:56:29 +00006390 EVT AmtVT = Amt.getValueType();
Scott Michelcf0da6c2009-02-17 22:15:04 +00006391
Dale Johannesen7ae8c8b2009-02-05 00:20:09 +00006392 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006393 DAG.getConstant(BitWidth, dl, AmtVT), Amt);
Dale Johannesen7ae8c8b2009-02-05 00:20:09 +00006394 SDValue Tmp2 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Amt);
6395 SDValue Tmp3 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Tmp1);
6396 SDValue Tmp4 = DAG.getNode(ISD::OR , dl, VT, Tmp2, Tmp3);
6397 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006398 DAG.getConstant(-BitWidth, dl, AmtVT));
Dale Johannesen7ae8c8b2009-02-05 00:20:09 +00006399 SDValue Tmp6 = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Tmp5);
6400 SDValue OutHi = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6);
6401 SDValue OutLo = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Amt);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006402 SDValue OutOps[] = { OutLo, OutHi };
Craig Topper64941d92014-04-27 19:20:57 +00006403 return DAG.getMergeValues(OutOps, dl);
Chris Lattner4211ca92006-04-14 06:01:58 +00006404}
6405
Dan Gohman21cea8a2010-04-17 15:26:15 +00006406SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const {
Owen Anderson53aa7a92009-08-10 22:56:29 +00006407 EVT VT = Op.getValueType();
Andrew Trickef9de2a2013-05-25 02:42:55 +00006408 SDLoc dl(Op);
Duncan Sands13237ac2008-06-06 12:08:01 +00006409 unsigned BitWidth = VT.getSizeInBits();
Dan Gohman8d2ead22008-03-07 20:36:53 +00006410 assert(Op.getNumOperands() == 3 &&
6411 VT == Op.getOperand(1).getValueType() &&
6412 "Unexpected SRL!");
Scott Michelcf0da6c2009-02-17 22:15:04 +00006413
Dan Gohman8d2ead22008-03-07 20:36:53 +00006414 // Expand into a bunch of logical ops. Note that these ops
Chris Lattner4211ca92006-04-14 06:01:58 +00006415 // depend on the PPC behavior for oversized shift amounts.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006416 SDValue Lo = Op.getOperand(0);
6417 SDValue Hi = Op.getOperand(1);
6418 SDValue Amt = Op.getOperand(2);
Owen Anderson53aa7a92009-08-10 22:56:29 +00006419 EVT AmtVT = Amt.getValueType();
Scott Michelcf0da6c2009-02-17 22:15:04 +00006420
Dale Johannesen7ae8c8b2009-02-05 00:20:09 +00006421 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006422 DAG.getConstant(BitWidth, dl, AmtVT), Amt);
Dale Johannesen7ae8c8b2009-02-05 00:20:09 +00006423 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt);
6424 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1);
6425 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
6426 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006427 DAG.getConstant(-BitWidth, dl, AmtVT));
Dale Johannesen7ae8c8b2009-02-05 00:20:09 +00006428 SDValue Tmp6 = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Tmp5);
6429 SDValue OutLo = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6);
6430 SDValue OutHi = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Amt);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006431 SDValue OutOps[] = { OutLo, OutHi };
Craig Topper64941d92014-04-27 19:20:57 +00006432 return DAG.getMergeValues(OutOps, dl);
Chris Lattner4211ca92006-04-14 06:01:58 +00006433}
6434
Dan Gohman21cea8a2010-04-17 15:26:15 +00006435SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00006436 SDLoc dl(Op);
Owen Anderson53aa7a92009-08-10 22:56:29 +00006437 EVT VT = Op.getValueType();
Duncan Sands13237ac2008-06-06 12:08:01 +00006438 unsigned BitWidth = VT.getSizeInBits();
Dan Gohman8d2ead22008-03-07 20:36:53 +00006439 assert(Op.getNumOperands() == 3 &&
6440 VT == Op.getOperand(1).getValueType() &&
6441 "Unexpected SRA!");
Scott Michelcf0da6c2009-02-17 22:15:04 +00006442
Dan Gohman8d2ead22008-03-07 20:36:53 +00006443 // Expand into a bunch of logical ops, followed by a select_cc.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006444 SDValue Lo = Op.getOperand(0);
6445 SDValue Hi = Op.getOperand(1);
6446 SDValue Amt = Op.getOperand(2);
Owen Anderson53aa7a92009-08-10 22:56:29 +00006447 EVT AmtVT = Amt.getValueType();
Scott Michelcf0da6c2009-02-17 22:15:04 +00006448
Dale Johannesenf2bb6f02009-02-04 01:48:28 +00006449 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006450 DAG.getConstant(BitWidth, dl, AmtVT), Amt);
Dale Johannesenf2bb6f02009-02-04 01:48:28 +00006451 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt);
6452 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1);
6453 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
6454 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006455 DAG.getConstant(-BitWidth, dl, AmtVT));
Dale Johannesenf2bb6f02009-02-04 01:48:28 +00006456 SDValue Tmp6 = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Tmp5);
6457 SDValue OutHi = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Amt);
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006458 SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, dl, AmtVT),
Duncan Sands13105742008-10-30 19:28:32 +00006459 Tmp4, Tmp6, ISD::SETLE);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006460 SDValue OutOps[] = { OutLo, OutHi };
Craig Topper64941d92014-04-27 19:20:57 +00006461 return DAG.getMergeValues(OutOps, dl);
Chris Lattner4211ca92006-04-14 06:01:58 +00006462}
6463
6464//===----------------------------------------------------------------------===//
6465// Vector related lowering.
6466//
6467
Chris Lattner2a099c02006-04-17 06:00:21 +00006468/// BuildSplatI - Build a canonical splati of Val with an element size of
6469/// SplatSize. Cast the result to VT.
Owen Anderson53aa7a92009-08-10 22:56:29 +00006470static SDValue BuildSplatI(int Val, unsigned SplatSize, EVT VT,
Andrew Trickef9de2a2013-05-25 02:42:55 +00006471 SelectionDAG &DAG, SDLoc dl) {
Chris Lattner2a099c02006-04-17 06:00:21 +00006472 assert(Val >= -16 && Val <= 15 && "vsplti is out of range!");
Chris Lattner09ed0ff2006-12-01 01:45:39 +00006473
Benjamin Kramer7149aab2015-03-01 18:09:56 +00006474 static const MVT VTys[] = { // canonical VT to use for each size.
Owen Anderson9f944592009-08-11 20:47:22 +00006475 MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32
Chris Lattner2a099c02006-04-17 06:00:21 +00006476 };
Chris Lattner09ed0ff2006-12-01 01:45:39 +00006477
Owen Anderson9f944592009-08-11 20:47:22 +00006478 EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1];
Scott Michelcf0da6c2009-02-17 22:15:04 +00006479
Chris Lattner09ed0ff2006-12-01 01:45:39 +00006480 // Force vspltis[hw] -1 to vspltisb -1 to canonicalize.
6481 if (Val == -1)
6482 SplatSize = 1;
Scott Michelcf0da6c2009-02-17 22:15:04 +00006483
Owen Anderson53aa7a92009-08-10 22:56:29 +00006484 EVT CanonicalVT = VTys[SplatSize-1];
Scott Michelcf0da6c2009-02-17 22:15:04 +00006485
Chris Lattner2a099c02006-04-17 06:00:21 +00006486 // Build a canonical splat for this value.
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006487 SDValue Elt = DAG.getConstant(Val, dl, MVT::i32);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006488 SmallVector<SDValue, 8> Ops;
Duncan Sands13237ac2008-06-06 12:08:01 +00006489 Ops.assign(CanonicalVT.getVectorNumElements(), Elt);
Craig Topper48d114b2014-04-26 18:35:24 +00006490 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, dl, CanonicalVT, Ops);
Wesley Peck527da1b2010-11-23 03:31:01 +00006491 return DAG.getNode(ISD::BITCAST, dl, ReqVT, Res);
Chris Lattner2a099c02006-04-17 06:00:21 +00006492}
6493
Hal Finkelcf2e9082013-05-24 23:00:14 +00006494/// BuildIntrinsicOp - Return a unary operator intrinsic node with the
6495/// specified intrinsic ID.
6496static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op,
Andrew Trickef9de2a2013-05-25 02:42:55 +00006497 SelectionDAG &DAG, SDLoc dl,
Hal Finkelcf2e9082013-05-24 23:00:14 +00006498 EVT DestVT = MVT::Other) {
6499 if (DestVT == MVT::Other) DestVT = Op.getValueType();
6500 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006501 DAG.getConstant(IID, dl, MVT::i32), Op);
Hal Finkelcf2e9082013-05-24 23:00:14 +00006502}
6503
Chris Lattnera2cae1b2006-04-18 03:24:30 +00006504/// BuildIntrinsicOp - Return a binary operator intrinsic node with the
Chris Lattner1b3806a2006-04-17 06:58:41 +00006505/// specified intrinsic ID.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006506static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS,
Andrew Trickef9de2a2013-05-25 02:42:55 +00006507 SelectionDAG &DAG, SDLoc dl,
Owen Anderson9f944592009-08-11 20:47:22 +00006508 EVT DestVT = MVT::Other) {
6509 if (DestVT == MVT::Other) DestVT = LHS.getValueType();
Dale Johannesen9f3f72f2009-02-06 01:31:28 +00006510 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006511 DAG.getConstant(IID, dl, MVT::i32), LHS, RHS);
Chris Lattner1b3806a2006-04-17 06:58:41 +00006512}
6513
Chris Lattnera2cae1b2006-04-18 03:24:30 +00006514/// BuildIntrinsicOp - Return a ternary operator intrinsic node with the
6515/// specified intrinsic ID.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006516static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1,
Dale Johannesen9f3f72f2009-02-06 01:31:28 +00006517 SDValue Op2, SelectionDAG &DAG,
Andrew Trickef9de2a2013-05-25 02:42:55 +00006518 SDLoc dl, EVT DestVT = MVT::Other) {
Owen Anderson9f944592009-08-11 20:47:22 +00006519 if (DestVT == MVT::Other) DestVT = Op0.getValueType();
Dale Johannesen9f3f72f2009-02-06 01:31:28 +00006520 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006521 DAG.getConstant(IID, dl, MVT::i32), Op0, Op1, Op2);
Chris Lattnera2cae1b2006-04-18 03:24:30 +00006522}
6523
6524
Chris Lattner264c9082006-04-17 17:55:10 +00006525/// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified
6526/// amount. The result has the specified value type.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006527static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt,
Andrew Trickef9de2a2013-05-25 02:42:55 +00006528 EVT VT, SelectionDAG &DAG, SDLoc dl) {
Chris Lattner264c9082006-04-17 17:55:10 +00006529 // Force LHS/RHS to be the right type.
Wesley Peck527da1b2010-11-23 03:31:01 +00006530 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, LHS);
6531 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, RHS);
Duncan Sandsb0e39382008-07-21 10:20:31 +00006532
Nate Begeman8d6d4b92009-04-27 18:41:29 +00006533 int Ops[16];
Chris Lattner264c9082006-04-17 17:55:10 +00006534 for (unsigned i = 0; i != 16; ++i)
Nate Begeman8d6d4b92009-04-27 18:41:29 +00006535 Ops[i] = i + Amt;
Owen Anderson9f944592009-08-11 20:47:22 +00006536 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, LHS, RHS, Ops);
Wesley Peck527da1b2010-11-23 03:31:01 +00006537 return DAG.getNode(ISD::BITCAST, dl, VT, T);
Chris Lattner264c9082006-04-17 17:55:10 +00006538}
6539
Chris Lattner19e90552006-04-14 05:19:18 +00006540// If this is a case we can't handle, return null and let the default
6541// expansion code take care of it. If we CAN select this case, and if it
6542// selects to a single instruction, return Op. Otherwise, if we can codegen
6543// this case more efficiently than a constant pool load, lower it to the
6544// sequence of ops that should be used.
Dan Gohman21cea8a2010-04-17 15:26:15 +00006545SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
6546 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00006547 SDLoc dl(Op);
Bob Wilsond8ea0e12009-03-01 01:13:55 +00006548 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
Craig Toppere73658d2014-04-28 04:05:08 +00006549 assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR");
Scott Michelbb878282009-02-25 03:12:50 +00006550
Hal Finkelc93a9a22015-02-25 01:06:45 +00006551 if (Subtarget.hasQPX() && Op.getValueType() == MVT::v4i1) {
6552 // We first build an i32 vector, load it into a QPX register,
6553 // then convert it to a floating-point vector and compare it
6554 // to a zero vector to get the boolean result.
6555 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
6556 int FrameIdx = FrameInfo->CreateStackObject(16, 16, false);
6557 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FrameIdx);
6558 EVT PtrVT = getPointerTy();
6559 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
6560
6561 assert(BVN->getNumOperands() == 4 &&
6562 "BUILD_VECTOR for v4i1 does not have 4 operands");
6563
6564 bool IsConst = true;
6565 for (unsigned i = 0; i < 4; ++i) {
6566 if (BVN->getOperand(i).getOpcode() == ISD::UNDEF) continue;
6567 if (!isa<ConstantSDNode>(BVN->getOperand(i))) {
6568 IsConst = false;
6569 break;
6570 }
6571 }
6572
6573 if (IsConst) {
6574 Constant *One =
6575 ConstantFP::get(Type::getFloatTy(*DAG.getContext()), 1.0);
6576 Constant *NegOne =
6577 ConstantFP::get(Type::getFloatTy(*DAG.getContext()), -1.0);
6578
6579 SmallVector<Constant*, 4> CV(4, NegOne);
6580 for (unsigned i = 0; i < 4; ++i) {
6581 if (BVN->getOperand(i).getOpcode() == ISD::UNDEF)
6582 CV[i] = UndefValue::get(Type::getFloatTy(*DAG.getContext()));
6583 else if (cast<ConstantSDNode>(BVN->getOperand(i))->
6584 getConstantIntValue()->isZero())
6585 continue;
6586 else
6587 CV[i] = One;
6588 }
6589
6590 Constant *CP = ConstantVector::get(CV);
6591 SDValue CPIdx = DAG.getConstantPool(CP, getPointerTy(),
6592 16 /* alignment */);
6593
6594 SmallVector<SDValue, 2> Ops;
6595 Ops.push_back(DAG.getEntryNode());
6596 Ops.push_back(CPIdx);
6597
6598 SmallVector<EVT, 2> ValueVTs;
6599 ValueVTs.push_back(MVT::v4i1);
6600 ValueVTs.push_back(MVT::Other); // chain
6601 SDVTList VTs = DAG.getVTList(ValueVTs);
6602
6603 return DAG.getMemIntrinsicNode(PPCISD::QVLFSb,
6604 dl, VTs, Ops, MVT::v4f32,
6605 MachinePointerInfo::getConstantPool());
6606 }
6607
6608 SmallVector<SDValue, 4> Stores;
6609 for (unsigned i = 0; i < 4; ++i) {
6610 if (BVN->getOperand(i).getOpcode() == ISD::UNDEF) continue;
6611
6612 unsigned Offset = 4*i;
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006613 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
Hal Finkelc93a9a22015-02-25 01:06:45 +00006614 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
6615
6616 unsigned StoreSize = BVN->getOperand(i).getValueType().getStoreSize();
6617 if (StoreSize > 4) {
6618 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
6619 BVN->getOperand(i), Idx,
6620 PtrInfo.getWithOffset(Offset),
6621 MVT::i32, false, false, 0));
6622 } else {
6623 SDValue StoreValue = BVN->getOperand(i);
6624 if (StoreSize < 4)
6625 StoreValue = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, StoreValue);
6626
6627 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl,
6628 StoreValue, Idx,
6629 PtrInfo.getWithOffset(Offset),
6630 false, false, 0));
6631 }
6632 }
6633
6634 SDValue StoreChain;
6635 if (!Stores.empty())
6636 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
6637 else
6638 StoreChain = DAG.getEntryNode();
6639
6640 // Now load from v4i32 into the QPX register; this will extend it to
6641 // v4i64 but not yet convert it to a floating point. Nevertheless, this
6642 // is typed as v4f64 because the QPX register integer states are not
6643 // explicitly represented.
6644
6645 SmallVector<SDValue, 2> Ops;
6646 Ops.push_back(StoreChain);
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006647 Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, dl, MVT::i32));
Hal Finkelc93a9a22015-02-25 01:06:45 +00006648 Ops.push_back(FIdx);
6649
6650 SmallVector<EVT, 2> ValueVTs;
6651 ValueVTs.push_back(MVT::v4f64);
6652 ValueVTs.push_back(MVT::Other); // chain
6653 SDVTList VTs = DAG.getVTList(ValueVTs);
6654
6655 SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN,
6656 dl, VTs, Ops, MVT::v4i32, PtrInfo);
6657 LoadedVect = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006658 DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, dl, MVT::i32),
Hal Finkelc93a9a22015-02-25 01:06:45 +00006659 LoadedVect);
6660
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006661 SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::f64);
Hal Finkelc93a9a22015-02-25 01:06:45 +00006662 FPZeros = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64,
6663 FPZeros, FPZeros, FPZeros, FPZeros);
6664
6665 return DAG.getSetCC(dl, MVT::v4i1, LoadedVect, FPZeros, ISD::SETEQ);
6666 }
6667
6668 // All other QPX vectors are handled by generic code.
6669 if (Subtarget.hasQPX())
6670 return SDValue();
6671
Bob Wilson85cefe82009-03-02 23:24:16 +00006672 // Check if this is a splat of a constant value.
6673 APInt APSplatBits, APSplatUndef;
6674 unsigned SplatBitSize;
Bob Wilsond8ea0e12009-03-01 01:13:55 +00006675 bool HasAnyUndefs;
Bob Wilson530e0382009-03-03 19:26:27 +00006676 if (! BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize,
Bill Schmidt91dd7652015-04-03 13:48:24 +00006677 HasAnyUndefs, 0, !Subtarget.isLittleEndian()) ||
6678 SplatBitSize > 32)
Bob Wilson530e0382009-03-03 19:26:27 +00006679 return SDValue();
Evan Chenga49de9d2009-02-25 22:49:59 +00006680
Bob Wilson530e0382009-03-03 19:26:27 +00006681 unsigned SplatBits = APSplatBits.getZExtValue();
6682 unsigned SplatUndef = APSplatUndef.getZExtValue();
6683 unsigned SplatSize = SplatBitSize / 8;
Scott Michelcf0da6c2009-02-17 22:15:04 +00006684
Bob Wilson530e0382009-03-03 19:26:27 +00006685 // First, handle single instruction cases.
6686
6687 // All zeros?
6688 if (SplatBits == 0) {
6689 // Canonicalize all zero vectors to be v4i32.
Owen Anderson9f944592009-08-11 20:47:22 +00006690 if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006691 SDValue Z = DAG.getConstant(0, dl, MVT::i32);
Owen Anderson9f944592009-08-11 20:47:22 +00006692 Z = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Z, Z, Z, Z);
Wesley Peck527da1b2010-11-23 03:31:01 +00006693 Op = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Z);
Chris Lattner19e90552006-04-14 05:19:18 +00006694 }
Bob Wilson530e0382009-03-03 19:26:27 +00006695 return Op;
6696 }
Chris Lattnerfa5aa392006-04-16 01:01:29 +00006697
Bob Wilson530e0382009-03-03 19:26:27 +00006698 // If the sign extended value is in the range [-16,15], use VSPLTI[bhw].
6699 int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >>
6700 (32-SplatBitSize));
6701 if (SextVal >= -16 && SextVal <= 15)
6702 return BuildSplatI(SextVal, SplatSize, Op.getValueType(), DAG, dl);
Scott Michelcf0da6c2009-02-17 22:15:04 +00006703
6704
Bob Wilson530e0382009-03-03 19:26:27 +00006705 // Two instruction sequences.
Scott Michelcf0da6c2009-02-17 22:15:04 +00006706
Bob Wilson530e0382009-03-03 19:26:27 +00006707 // If this value is in the range [-32,30] and is even, use:
Bill Schmidtc6cbecc2013-02-20 20:41:42 +00006708 // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2)
6709 // If this value is in the range [17,31] and is odd, use:
6710 // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16)
6711 // If this value is in the range [-31,-17] and is odd, use:
6712 // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16)
6713 // Note the last two are three-instruction sequences.
6714 if (SextVal >= -32 && SextVal <= 31) {
6715 // To avoid having these optimizations undone by constant folding,
6716 // we convert to a pseudo that will be expanded later into one of
6717 // the above forms.
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006718 SDValue Elt = DAG.getConstant(SextVal, dl, MVT::i32);
Bill Schmidt71dddd52014-05-27 15:57:51 +00006719 EVT VT = (SplatSize == 1 ? MVT::v16i8 :
6720 (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32));
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006721 SDValue EltSize = DAG.getConstant(SplatSize, dl, MVT::i32);
Bill Schmidt71dddd52014-05-27 15:57:51 +00006722 SDValue RetVal = DAG.getNode(PPCISD::VADD_SPLAT, dl, VT, Elt, EltSize);
6723 if (VT == Op.getValueType())
6724 return RetVal;
6725 else
6726 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), RetVal);
Bob Wilson530e0382009-03-03 19:26:27 +00006727 }
6728
6729 // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is
6730 // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important
6731 // for fneg/fabs.
6732 if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) {
6733 // Make -1 and vspltisw -1:
Owen Anderson9f944592009-08-11 20:47:22 +00006734 SDValue OnesV = BuildSplatI(-1, 4, MVT::v4i32, DAG, dl);
Bob Wilson530e0382009-03-03 19:26:27 +00006735
6736 // Make the VSLW intrinsic, computing 0x8000_0000.
6737 SDValue Res = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, OnesV,
6738 OnesV, DAG, dl);
6739
6740 // xor by OnesV to invert it.
Owen Anderson9f944592009-08-11 20:47:22 +00006741 Res = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Res, OnesV);
Wesley Peck527da1b2010-11-23 03:31:01 +00006742 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
Bob Wilson530e0382009-03-03 19:26:27 +00006743 }
6744
6745 // Check to see if this is a wide variety of vsplti*, binop self cases.
6746 static const signed char SplatCsts[] = {
6747 -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7,
6748 -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16
6749 };
6750
6751 for (unsigned idx = 0; idx < array_lengthof(SplatCsts); ++idx) {
6752 // Indirect through the SplatCsts array so that we favor 'vsplti -1' for
6753 // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1'
6754 int i = SplatCsts[idx];
6755
6756 // Figure out what shift amount will be used by altivec if shifted by i in
6757 // this splat size.
6758 unsigned TypeShiftAmt = i & (SplatBitSize-1);
6759
6760 // vsplti + shl self.
Richard Smith228e6d42012-08-24 23:29:28 +00006761 if (SextVal == (int)((unsigned)i << TypeShiftAmt)) {
Owen Anderson9f944592009-08-11 20:47:22 +00006762 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
Bob Wilson530e0382009-03-03 19:26:27 +00006763 static const unsigned IIDs[] = { // Intrinsic to use for each size.
6764 Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0,
6765 Intrinsic::ppc_altivec_vslw
6766 };
6767 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl);
Wesley Peck527da1b2010-11-23 03:31:01 +00006768 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
Chris Lattner2a099c02006-04-17 06:00:21 +00006769 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00006770
Bob Wilson530e0382009-03-03 19:26:27 +00006771 // vsplti + srl self.
6772 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) {
Owen Anderson9f944592009-08-11 20:47:22 +00006773 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
Bob Wilson530e0382009-03-03 19:26:27 +00006774 static const unsigned IIDs[] = { // Intrinsic to use for each size.
6775 Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0,
6776 Intrinsic::ppc_altivec_vsrw
6777 };
6778 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl);
Wesley Peck527da1b2010-11-23 03:31:01 +00006779 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
Chris Lattner1b3806a2006-04-17 06:58:41 +00006780 }
6781
Bob Wilson530e0382009-03-03 19:26:27 +00006782 // vsplti + sra self.
6783 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) {
Owen Anderson9f944592009-08-11 20:47:22 +00006784 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
Bob Wilson530e0382009-03-03 19:26:27 +00006785 static const unsigned IIDs[] = { // Intrinsic to use for each size.
6786 Intrinsic::ppc_altivec_vsrab, Intrinsic::ppc_altivec_vsrah, 0,
6787 Intrinsic::ppc_altivec_vsraw
6788 };
6789 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl);
Wesley Peck527da1b2010-11-23 03:31:01 +00006790 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
Chris Lattner1b3806a2006-04-17 06:58:41 +00006791 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00006792
Bob Wilson530e0382009-03-03 19:26:27 +00006793 // vsplti + rol self.
6794 if (SextVal == (int)(((unsigned)i << TypeShiftAmt) |
6795 ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) {
Owen Anderson9f944592009-08-11 20:47:22 +00006796 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
Bob Wilson530e0382009-03-03 19:26:27 +00006797 static const unsigned IIDs[] = { // Intrinsic to use for each size.
6798 Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0,
6799 Intrinsic::ppc_altivec_vrlw
6800 };
6801 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl);
Wesley Peck527da1b2010-11-23 03:31:01 +00006802 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
Bob Wilson530e0382009-03-03 19:26:27 +00006803 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00006804
Bob Wilson530e0382009-03-03 19:26:27 +00006805 // t = vsplti c, result = vsldoi t, t, 1
Richard Smith228e6d42012-08-24 23:29:28 +00006806 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) {
Owen Anderson9f944592009-08-11 20:47:22 +00006807 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
Bob Wilson530e0382009-03-03 19:26:27 +00006808 return BuildVSLDOI(T, T, 1, Op.getValueType(), DAG, dl);
Chris Lattnere54133c2006-04-17 18:09:22 +00006809 }
Bob Wilson530e0382009-03-03 19:26:27 +00006810 // t = vsplti c, result = vsldoi t, t, 2
Richard Smith228e6d42012-08-24 23:29:28 +00006811 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) {
Owen Anderson9f944592009-08-11 20:47:22 +00006812 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
Bob Wilson530e0382009-03-03 19:26:27 +00006813 return BuildVSLDOI(T, T, 2, Op.getValueType(), DAG, dl);
Chris Lattner19e90552006-04-14 05:19:18 +00006814 }
Bob Wilson530e0382009-03-03 19:26:27 +00006815 // t = vsplti c, result = vsldoi t, t, 3
Richard Smith228e6d42012-08-24 23:29:28 +00006816 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) {
Owen Anderson9f944592009-08-11 20:47:22 +00006817 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
Bob Wilson530e0382009-03-03 19:26:27 +00006818 return BuildVSLDOI(T, T, 3, Op.getValueType(), DAG, dl);
6819 }
6820 }
6821
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006822 return SDValue();
Chris Lattner19e90552006-04-14 05:19:18 +00006823}
6824
Chris Lattner071ad012006-04-17 05:28:54 +00006825/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
6826/// the specified operations to build the shuffle.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006827static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
Scott Michelcf0da6c2009-02-17 22:15:04 +00006828 SDValue RHS, SelectionDAG &DAG,
Andrew Trickef9de2a2013-05-25 02:42:55 +00006829 SDLoc dl) {
Chris Lattner071ad012006-04-17 05:28:54 +00006830 unsigned OpNum = (PFEntry >> 26) & 0x0F;
Bill Wendling95e1af22008-09-17 00:30:57 +00006831 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
Chris Lattner071ad012006-04-17 05:28:54 +00006832 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
Scott Michelcf0da6c2009-02-17 22:15:04 +00006833
Chris Lattner071ad012006-04-17 05:28:54 +00006834 enum {
Chris Lattnerd2ca9ab2006-05-16 04:20:24 +00006835 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
Chris Lattner071ad012006-04-17 05:28:54 +00006836 OP_VMRGHW,
6837 OP_VMRGLW,
6838 OP_VSPLTISW0,
6839 OP_VSPLTISW1,
6840 OP_VSPLTISW2,
6841 OP_VSPLTISW3,
6842 OP_VSLDOI4,
6843 OP_VSLDOI8,
Chris Lattneraa2372562006-05-24 17:04:05 +00006844 OP_VSLDOI12
Chris Lattner071ad012006-04-17 05:28:54 +00006845 };
Scott Michelcf0da6c2009-02-17 22:15:04 +00006846
Chris Lattner071ad012006-04-17 05:28:54 +00006847 if (OpNum == OP_COPY) {
6848 if (LHSID == (1*9+2)*9+3) return LHS;
6849 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
6850 return RHS;
6851 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00006852
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006853 SDValue OpLHS, OpRHS;
Dale Johannesen9f3f72f2009-02-06 01:31:28 +00006854 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
6855 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
Scott Michelcf0da6c2009-02-17 22:15:04 +00006856
Nate Begeman8d6d4b92009-04-27 18:41:29 +00006857 int ShufIdxs[16];
Chris Lattner071ad012006-04-17 05:28:54 +00006858 switch (OpNum) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00006859 default: llvm_unreachable("Unknown i32 permute!");
Chris Lattner071ad012006-04-17 05:28:54 +00006860 case OP_VMRGHW:
6861 ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3;
6862 ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19;
6863 ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7;
6864 ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23;
6865 break;
6866 case OP_VMRGLW:
6867 ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11;
6868 ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27;
6869 ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15;
6870 ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31;
6871 break;
6872 case OP_VSPLTISW0:
6873 for (unsigned i = 0; i != 16; ++i)
6874 ShufIdxs[i] = (i&3)+0;
6875 break;
6876 case OP_VSPLTISW1:
6877 for (unsigned i = 0; i != 16; ++i)
6878 ShufIdxs[i] = (i&3)+4;
6879 break;
6880 case OP_VSPLTISW2:
6881 for (unsigned i = 0; i != 16; ++i)
6882 ShufIdxs[i] = (i&3)+8;
6883 break;
6884 case OP_VSPLTISW3:
6885 for (unsigned i = 0; i != 16; ++i)
6886 ShufIdxs[i] = (i&3)+12;
6887 break;
6888 case OP_VSLDOI4:
Dale Johannesen9f3f72f2009-02-06 01:31:28 +00006889 return BuildVSLDOI(OpLHS, OpRHS, 4, OpLHS.getValueType(), DAG, dl);
Chris Lattner071ad012006-04-17 05:28:54 +00006890 case OP_VSLDOI8:
Dale Johannesen9f3f72f2009-02-06 01:31:28 +00006891 return BuildVSLDOI(OpLHS, OpRHS, 8, OpLHS.getValueType(), DAG, dl);
Chris Lattner071ad012006-04-17 05:28:54 +00006892 case OP_VSLDOI12:
Dale Johannesen9f3f72f2009-02-06 01:31:28 +00006893 return BuildVSLDOI(OpLHS, OpRHS, 12, OpLHS.getValueType(), DAG, dl);
Chris Lattner071ad012006-04-17 05:28:54 +00006894 }
Owen Anderson53aa7a92009-08-10 22:56:29 +00006895 EVT VT = OpLHS.getValueType();
Wesley Peck527da1b2010-11-23 03:31:01 +00006896 OpLHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpLHS);
6897 OpRHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpRHS);
Owen Anderson9f944592009-08-11 20:47:22 +00006898 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, OpLHS, OpRHS, ShufIdxs);
Wesley Peck527da1b2010-11-23 03:31:01 +00006899 return DAG.getNode(ISD::BITCAST, dl, VT, T);
Chris Lattner071ad012006-04-17 05:28:54 +00006900}
6901
Chris Lattner19e90552006-04-14 05:19:18 +00006902/// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this
6903/// is a shuffle we can handle in a single instruction, return it. Otherwise,
6904/// return the code it can be lowered into. Worst case, it can always be
6905/// lowered into a vperm.
Scott Michelcf0da6c2009-02-17 22:15:04 +00006906SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
Dan Gohman21cea8a2010-04-17 15:26:15 +00006907 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00006908 SDLoc dl(Op);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00006909 SDValue V1 = Op.getOperand(0);
6910 SDValue V2 = Op.getOperand(1);
Nate Begeman8d6d4b92009-04-27 18:41:29 +00006911 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
Owen Anderson53aa7a92009-08-10 22:56:29 +00006912 EVT VT = Op.getValueType();
Eric Christopherb1aaebe2014-06-12 22:38:18 +00006913 bool isLittleEndian = Subtarget.isLittleEndian();
Scott Michelcf0da6c2009-02-17 22:15:04 +00006914
Hal Finkelc93a9a22015-02-25 01:06:45 +00006915 if (Subtarget.hasQPX()) {
6916 if (VT.getVectorNumElements() != 4)
6917 return SDValue();
6918
6919 if (V2.getOpcode() == ISD::UNDEF) V2 = V1;
6920
6921 int AlignIdx = PPC::isQVALIGNIShuffleMask(SVOp);
6922 if (AlignIdx != -1) {
6923 return DAG.getNode(PPCISD::QVALIGNI, dl, VT, V1, V2,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006924 DAG.getConstant(AlignIdx, dl, MVT::i32));
Hal Finkelc93a9a22015-02-25 01:06:45 +00006925 } else if (SVOp->isSplat()) {
6926 int SplatIdx = SVOp->getSplatIndex();
6927 if (SplatIdx >= 4) {
6928 std::swap(V1, V2);
6929 SplatIdx -= 4;
6930 }
6931
6932 // FIXME: If SplatIdx == 0 and the input came from a load, then there is
6933 // nothing to do.
6934
6935 return DAG.getNode(PPCISD::QVESPLATI, dl, VT, V1,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006936 DAG.getConstant(SplatIdx, dl, MVT::i32));
Hal Finkelc93a9a22015-02-25 01:06:45 +00006937 }
6938
6939 // Lower this into a qvgpci/qvfperm pair.
6940
6941 // Compute the qvgpci literal
6942 unsigned idx = 0;
6943 for (unsigned i = 0; i < 4; ++i) {
6944 int m = SVOp->getMaskElt(i);
6945 unsigned mm = m >= 0 ? (unsigned) m : i;
6946 idx |= mm << (3-i)*3;
6947 }
6948
6949 SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00006950 DAG.getConstant(idx, dl, MVT::i32));
Hal Finkelc93a9a22015-02-25 01:06:45 +00006951 return DAG.getNode(PPCISD::QVFPERM, dl, VT, V1, V2, V3);
6952 }
6953
Chris Lattner19e90552006-04-14 05:19:18 +00006954 // Cases that are handled by instructions that take permute immediates
6955 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be
6956 // selected by the instruction selector.
6957 if (V2.getOpcode() == ISD::UNDEF) {
Nate Begeman8d6d4b92009-04-27 18:41:29 +00006958 if (PPC::isSplatShuffleMask(SVOp, 1) ||
6959 PPC::isSplatShuffleMask(SVOp, 2) ||
6960 PPC::isSplatShuffleMask(SVOp, 4) ||
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00006961 PPC::isVPKUWUMShuffleMask(SVOp, 1, DAG) ||
6962 PPC::isVPKUHUMShuffleMask(SVOp, 1, DAG) ||
Bill Schmidt42a69362014-08-05 20:47:25 +00006963 PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) != -1 ||
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00006964 PPC::isVMRGLShuffleMask(SVOp, 1, 1, DAG) ||
6965 PPC::isVMRGLShuffleMask(SVOp, 2, 1, DAG) ||
6966 PPC::isVMRGLShuffleMask(SVOp, 4, 1, DAG) ||
6967 PPC::isVMRGHShuffleMask(SVOp, 1, 1, DAG) ||
6968 PPC::isVMRGHShuffleMask(SVOp, 2, 1, DAG) ||
6969 PPC::isVMRGHShuffleMask(SVOp, 4, 1, DAG)) {
Chris Lattner19e90552006-04-14 05:19:18 +00006970 return Op;
6971 }
6972 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00006973
Chris Lattner19e90552006-04-14 05:19:18 +00006974 // Altivec has a variety of "shuffle immediates" that take two vector inputs
6975 // and produce a fixed permutation. If any of these match, do not lower to
6976 // VPERM.
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00006977 unsigned int ShuffleKind = isLittleEndian ? 2 : 0;
Ulrich Weigandcc9909b2014-08-04 13:53:40 +00006978 if (PPC::isVPKUWUMShuffleMask(SVOp, ShuffleKind, DAG) ||
6979 PPC::isVPKUHUMShuffleMask(SVOp, ShuffleKind, DAG) ||
Bill Schmidt42a69362014-08-05 20:47:25 +00006980 PPC::isVSLDOIShuffleMask(SVOp, ShuffleKind, DAG) != -1 ||
Bill Schmidtc9fa5dd2014-07-25 01:55:55 +00006981 PPC::isVMRGLShuffleMask(SVOp, 1, ShuffleKind, DAG) ||
6982 PPC::isVMRGLShuffleMask(SVOp, 2, ShuffleKind, DAG) ||
6983 PPC::isVMRGLShuffleMask(SVOp, 4, ShuffleKind, DAG) ||
6984 PPC::isVMRGHShuffleMask(SVOp, 1, ShuffleKind, DAG) ||
6985 PPC::isVMRGHShuffleMask(SVOp, 2, ShuffleKind, DAG) ||
6986 PPC::isVMRGHShuffleMask(SVOp, 4, ShuffleKind, DAG))
Chris Lattner19e90552006-04-14 05:19:18 +00006987 return Op;
Scott Michelcf0da6c2009-02-17 22:15:04 +00006988
Chris Lattner071ad012006-04-17 05:28:54 +00006989 // Check to see if this is a shuffle of 4-byte values. If so, we can use our
6990 // perfect shuffle table to emit an optimal matching sequence.
Benjamin Kramer339ced42012-01-15 13:16:05 +00006991 ArrayRef<int> PermMask = SVOp->getMask();
Wesley Peck527da1b2010-11-23 03:31:01 +00006992
Chris Lattner071ad012006-04-17 05:28:54 +00006993 unsigned PFIndexes[4];
6994 bool isFourElementShuffle = true;
6995 for (unsigned i = 0; i != 4 && isFourElementShuffle; ++i) { // Element number
6996 unsigned EltNo = 8; // Start out undef.
6997 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte.
Nate Begeman8d6d4b92009-04-27 18:41:29 +00006998 if (PermMask[i*4+j] < 0)
Chris Lattner071ad012006-04-17 05:28:54 +00006999 continue; // Undef, ignore it.
Scott Michelcf0da6c2009-02-17 22:15:04 +00007000
Nate Begeman8d6d4b92009-04-27 18:41:29 +00007001 unsigned ByteSource = PermMask[i*4+j];
Chris Lattner071ad012006-04-17 05:28:54 +00007002 if ((ByteSource & 3) != j) {
7003 isFourElementShuffle = false;
7004 break;
7005 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00007006
Chris Lattner071ad012006-04-17 05:28:54 +00007007 if (EltNo == 8) {
7008 EltNo = ByteSource/4;
7009 } else if (EltNo != ByteSource/4) {
7010 isFourElementShuffle = false;
7011 break;
7012 }
7013 }
7014 PFIndexes[i] = EltNo;
7015 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00007016
7017 // If this shuffle can be expressed as a shuffle of 4-byte elements, use the
Chris Lattner071ad012006-04-17 05:28:54 +00007018 // perfect shuffle vector to determine if it is cost effective to do this as
7019 // discrete instructions, or whether we should use a vperm.
Bill Schmidtf910a062014-06-10 14:35:01 +00007020 // For now, we skip this for little endian until such time as we have a
7021 // little-endian perfect shuffle table.
7022 if (isFourElementShuffle && !isLittleEndian) {
Chris Lattner071ad012006-04-17 05:28:54 +00007023 // Compute the index in the perfect shuffle table.
Scott Michelcf0da6c2009-02-17 22:15:04 +00007024 unsigned PFTableIndex =
Chris Lattner071ad012006-04-17 05:28:54 +00007025 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
Scott Michelcf0da6c2009-02-17 22:15:04 +00007026
Chris Lattner071ad012006-04-17 05:28:54 +00007027 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
7028 unsigned Cost = (PFEntry >> 30);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007029
Chris Lattner071ad012006-04-17 05:28:54 +00007030 // Determining when to avoid vperm is tricky. Many things affect the cost
7031 // of vperm, particularly how many times the perm mask needs to be computed.
7032 // For example, if the perm mask can be hoisted out of a loop or is already
7033 // used (perhaps because there are multiple permutes with the same shuffle
7034 // mask?) the vperm has a cost of 1. OTOH, hoisting the permute mask out of
7035 // the loop requires an extra register.
7036 //
7037 // As a compromise, we only emit discrete instructions if the shuffle can be
Scott Michelcf0da6c2009-02-17 22:15:04 +00007038 // generated in 3 or fewer operations. When we have loop information
Chris Lattner071ad012006-04-17 05:28:54 +00007039 // available, if this block is within a loop, we should avoid using vperm
7040 // for 3-operation perms and use a constant pool load instead.
Scott Michelcf0da6c2009-02-17 22:15:04 +00007041 if (Cost < 3)
Dale Johannesen9f3f72f2009-02-06 01:31:28 +00007042 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
Chris Lattner071ad012006-04-17 05:28:54 +00007043 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00007044
Chris Lattner19e90552006-04-14 05:19:18 +00007045 // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant
7046 // vector that will get spilled to the constant pool.
7047 if (V2.getOpcode() == ISD::UNDEF) V2 = V1;
Scott Michelcf0da6c2009-02-17 22:15:04 +00007048
Chris Lattner19e90552006-04-14 05:19:18 +00007049 // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except
7050 // that it is in input element units, not in bytes. Convert now.
Bill Schmidt4aedff82014-06-06 14:06:26 +00007051
7052 // For little endian, the order of the input vectors is reversed, and
7053 // the permutation mask is complemented with respect to 31. This is
7054 // necessary to produce proper semantics with the big-endian-biased vperm
7055 // instruction.
Owen Anderson53aa7a92009-08-10 22:56:29 +00007056 EVT EltVT = V1.getValueType().getVectorElementType();
Duncan Sands13237ac2008-06-06 12:08:01 +00007057 unsigned BytesPerElement = EltVT.getSizeInBits()/8;
Scott Michelcf0da6c2009-02-17 22:15:04 +00007058
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007059 SmallVector<SDValue, 16> ResultMask;
Nate Begeman8d6d4b92009-04-27 18:41:29 +00007060 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
7061 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i];
Scott Michelcf0da6c2009-02-17 22:15:04 +00007062
Chris Lattner19e90552006-04-14 05:19:18 +00007063 for (unsigned j = 0; j != BytesPerElement; ++j)
Bill Schmidt4aedff82014-06-06 14:06:26 +00007064 if (isLittleEndian)
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007065 ResultMask.push_back(DAG.getConstant(31 - (SrcElt*BytesPerElement + j),
7066 dl, MVT::i32));
Bill Schmidt4aedff82014-06-06 14:06:26 +00007067 else
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007068 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement + j, dl,
Bill Schmidt4aedff82014-06-06 14:06:26 +00007069 MVT::i32));
Chris Lattner19e90552006-04-14 05:19:18 +00007070 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00007071
Owen Anderson9f944592009-08-11 20:47:22 +00007072 SDValue VPermMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i8,
Craig Topper48d114b2014-04-26 18:35:24 +00007073 ResultMask);
Bill Schmidt4aedff82014-06-06 14:06:26 +00007074 if (isLittleEndian)
7075 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(),
7076 V2, V1, VPermMask);
7077 else
7078 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(),
7079 V1, V2, VPermMask);
Chris Lattner19e90552006-04-14 05:19:18 +00007080}
7081
Chris Lattner9754d142006-04-18 17:59:36 +00007082/// getAltivecCompareInfo - Given an intrinsic, return false if it is not an
7083/// altivec comparison. If it is, return true and fill in Opc/isDot with
7084/// information about the intrinsic.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007085static bool getAltivecCompareInfo(SDValue Intrin, int &CompareOpc,
Kit Barton0cfa7b72015-03-03 19:55:45 +00007086 bool &isDot, const PPCSubtarget &Subtarget) {
Dan Gohmaneffb8942008-09-12 16:56:44 +00007087 unsigned IntrinsicID =
7088 cast<ConstantSDNode>(Intrin.getOperand(0))->getZExtValue();
Chris Lattner9754d142006-04-18 17:59:36 +00007089 CompareOpc = -1;
7090 isDot = false;
7091 switch (IntrinsicID) {
7092 default: return false;
7093 // Comparison predicates.
Chris Lattner4211ca92006-04-14 06:01:58 +00007094 case Intrinsic::ppc_altivec_vcmpbfp_p: CompareOpc = 966; isDot = 1; break;
7095 case Intrinsic::ppc_altivec_vcmpeqfp_p: CompareOpc = 198; isDot = 1; break;
7096 case Intrinsic::ppc_altivec_vcmpequb_p: CompareOpc = 6; isDot = 1; break;
7097 case Intrinsic::ppc_altivec_vcmpequh_p: CompareOpc = 70; isDot = 1; break;
7098 case Intrinsic::ppc_altivec_vcmpequw_p: CompareOpc = 134; isDot = 1; break;
Kit Barton0cfa7b72015-03-03 19:55:45 +00007099 case Intrinsic::ppc_altivec_vcmpequd_p:
7100 if (Subtarget.hasP8Altivec()) {
7101 CompareOpc = 199;
7102 isDot = 1;
7103 }
7104 else
7105 return false;
7106
7107 break;
Chris Lattner4211ca92006-04-14 06:01:58 +00007108 case Intrinsic::ppc_altivec_vcmpgefp_p: CompareOpc = 454; isDot = 1; break;
7109 case Intrinsic::ppc_altivec_vcmpgtfp_p: CompareOpc = 710; isDot = 1; break;
7110 case Intrinsic::ppc_altivec_vcmpgtsb_p: CompareOpc = 774; isDot = 1; break;
7111 case Intrinsic::ppc_altivec_vcmpgtsh_p: CompareOpc = 838; isDot = 1; break;
7112 case Intrinsic::ppc_altivec_vcmpgtsw_p: CompareOpc = 902; isDot = 1; break;
Kit Barton0cfa7b72015-03-03 19:55:45 +00007113 case Intrinsic::ppc_altivec_vcmpgtsd_p:
7114 if (Subtarget.hasP8Altivec()) {
7115 CompareOpc = 967;
7116 isDot = 1;
7117 }
7118 else
7119 return false;
7120
7121 break;
Chris Lattner4211ca92006-04-14 06:01:58 +00007122 case Intrinsic::ppc_altivec_vcmpgtub_p: CompareOpc = 518; isDot = 1; break;
7123 case Intrinsic::ppc_altivec_vcmpgtuh_p: CompareOpc = 582; isDot = 1; break;
7124 case Intrinsic::ppc_altivec_vcmpgtuw_p: CompareOpc = 646; isDot = 1; break;
Kit Barton0cfa7b72015-03-03 19:55:45 +00007125 case Intrinsic::ppc_altivec_vcmpgtud_p:
7126 if (Subtarget.hasP8Altivec()) {
7127 CompareOpc = 711;
7128 isDot = 1;
7129 }
7130 else
7131 return false;
Scott Michelcf0da6c2009-02-17 22:15:04 +00007132
Kit Barton0cfa7b72015-03-03 19:55:45 +00007133 break;
7134
Chris Lattner4211ca92006-04-14 06:01:58 +00007135 // Normal Comparisons.
7136 case Intrinsic::ppc_altivec_vcmpbfp: CompareOpc = 966; isDot = 0; break;
7137 case Intrinsic::ppc_altivec_vcmpeqfp: CompareOpc = 198; isDot = 0; break;
7138 case Intrinsic::ppc_altivec_vcmpequb: CompareOpc = 6; isDot = 0; break;
7139 case Intrinsic::ppc_altivec_vcmpequh: CompareOpc = 70; isDot = 0; break;
7140 case Intrinsic::ppc_altivec_vcmpequw: CompareOpc = 134; isDot = 0; break;
Kit Barton0cfa7b72015-03-03 19:55:45 +00007141 case Intrinsic::ppc_altivec_vcmpequd:
7142 if (Subtarget.hasP8Altivec()) {
7143 CompareOpc = 199;
7144 isDot = 0;
7145 }
7146 else
7147 return false;
7148
7149 break;
Chris Lattner4211ca92006-04-14 06:01:58 +00007150 case Intrinsic::ppc_altivec_vcmpgefp: CompareOpc = 454; isDot = 0; break;
7151 case Intrinsic::ppc_altivec_vcmpgtfp: CompareOpc = 710; isDot = 0; break;
7152 case Intrinsic::ppc_altivec_vcmpgtsb: CompareOpc = 774; isDot = 0; break;
7153 case Intrinsic::ppc_altivec_vcmpgtsh: CompareOpc = 838; isDot = 0; break;
7154 case Intrinsic::ppc_altivec_vcmpgtsw: CompareOpc = 902; isDot = 0; break;
Kit Barton0cfa7b72015-03-03 19:55:45 +00007155 case Intrinsic::ppc_altivec_vcmpgtsd:
7156 if (Subtarget.hasP8Altivec()) {
7157 CompareOpc = 967;
7158 isDot = 0;
7159 }
7160 else
7161 return false;
7162
7163 break;
Chris Lattner4211ca92006-04-14 06:01:58 +00007164 case Intrinsic::ppc_altivec_vcmpgtub: CompareOpc = 518; isDot = 0; break;
7165 case Intrinsic::ppc_altivec_vcmpgtuh: CompareOpc = 582; isDot = 0; break;
7166 case Intrinsic::ppc_altivec_vcmpgtuw: CompareOpc = 646; isDot = 0; break;
Kit Barton0cfa7b72015-03-03 19:55:45 +00007167 case Intrinsic::ppc_altivec_vcmpgtud:
7168 if (Subtarget.hasP8Altivec()) {
7169 CompareOpc = 711;
7170 isDot = 0;
7171 }
7172 else
7173 return false;
7174
7175 break;
Chris Lattner4211ca92006-04-14 06:01:58 +00007176 }
Chris Lattner9754d142006-04-18 17:59:36 +00007177 return true;
7178}
7179
7180/// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom
7181/// lower, do it, otherwise return null.
Scott Michelcf0da6c2009-02-17 22:15:04 +00007182SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
Dan Gohman21cea8a2010-04-17 15:26:15 +00007183 SelectionDAG &DAG) const {
Chris Lattner9754d142006-04-18 17:59:36 +00007184 // If this is a lowered altivec predicate compare, CompareOpc is set to the
7185 // opcode number of the comparison.
Andrew Trickef9de2a2013-05-25 02:42:55 +00007186 SDLoc dl(Op);
Chris Lattner9754d142006-04-18 17:59:36 +00007187 int CompareOpc;
7188 bool isDot;
Kit Barton0cfa7b72015-03-03 19:55:45 +00007189 if (!getAltivecCompareInfo(Op, CompareOpc, isDot, Subtarget))
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007190 return SDValue(); // Don't custom lower most intrinsics.
Scott Michelcf0da6c2009-02-17 22:15:04 +00007191
Chris Lattner9754d142006-04-18 17:59:36 +00007192 // If this is a non-dot comparison, make the VCMP node and we are done.
Chris Lattner4211ca92006-04-14 06:01:58 +00007193 if (!isDot) {
Dale Johannesenf80493b2009-02-05 22:07:54 +00007194 SDValue Tmp = DAG.getNode(PPCISD::VCMP, dl, Op.getOperand(2).getValueType(),
Chris Lattner9fa851b2010-03-14 22:44:11 +00007195 Op.getOperand(1), Op.getOperand(2),
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007196 DAG.getConstant(CompareOpc, dl, MVT::i32));
Wesley Peck527da1b2010-11-23 03:31:01 +00007197 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Tmp);
Chris Lattner4211ca92006-04-14 06:01:58 +00007198 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00007199
Chris Lattner4211ca92006-04-14 06:01:58 +00007200 // Create the PPCISD altivec 'dot' comparison node.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007201 SDValue Ops[] = {
Chris Lattnerd66f14e2006-08-11 17:18:05 +00007202 Op.getOperand(2), // LHS
7203 Op.getOperand(3), // RHS
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007204 DAG.getConstant(CompareOpc, dl, MVT::i32)
Chris Lattnerd66f14e2006-08-11 17:18:05 +00007205 };
Benjamin Kramerfdf362b2013-03-07 20:33:29 +00007206 EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue };
Craig Topper48d114b2014-04-26 18:35:24 +00007207 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007208
Chris Lattner4211ca92006-04-14 06:01:58 +00007209 // Now that we have the comparison, emit a copy from the CR to a GPR.
7210 // This is flagged to the above dot comparison.
Ulrich Weigandd5ebc622013-07-03 17:05:42 +00007211 SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32,
Owen Anderson9f944592009-08-11 20:47:22 +00007212 DAG.getRegister(PPC::CR6, MVT::i32),
Scott Michelcf0da6c2009-02-17 22:15:04 +00007213 CompNode.getValue(1));
7214
Chris Lattner4211ca92006-04-14 06:01:58 +00007215 // Unpack the result based on how the target uses it.
7216 unsigned BitNo; // Bit # of CR6.
7217 bool InvertBit; // Invert result?
Dan Gohmaneffb8942008-09-12 16:56:44 +00007218 switch (cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue()) {
Chris Lattner4211ca92006-04-14 06:01:58 +00007219 default: // Can't happen, don't crash on invalid number though.
7220 case 0: // Return the value of the EQ bit of CR6.
7221 BitNo = 0; InvertBit = false;
7222 break;
7223 case 1: // Return the inverted value of the EQ bit of CR6.
7224 BitNo = 0; InvertBit = true;
7225 break;
7226 case 2: // Return the value of the LT bit of CR6.
7227 BitNo = 2; InvertBit = false;
7228 break;
7229 case 3: // Return the inverted value of the LT bit of CR6.
7230 BitNo = 2; InvertBit = true;
7231 break;
7232 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00007233
Chris Lattner4211ca92006-04-14 06:01:58 +00007234 // Shift the bit into the low position.
Owen Anderson9f944592009-08-11 20:47:22 +00007235 Flags = DAG.getNode(ISD::SRL, dl, MVT::i32, Flags,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007236 DAG.getConstant(8 - (3 - BitNo), dl, MVT::i32));
Chris Lattner4211ca92006-04-14 06:01:58 +00007237 // Isolate the bit.
Owen Anderson9f944592009-08-11 20:47:22 +00007238 Flags = DAG.getNode(ISD::AND, dl, MVT::i32, Flags,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007239 DAG.getConstant(1, dl, MVT::i32));
Scott Michelcf0da6c2009-02-17 22:15:04 +00007240
Chris Lattner4211ca92006-04-14 06:01:58 +00007241 // If we are supposed to, toggle the bit.
7242 if (InvertBit)
Owen Anderson9f944592009-08-11 20:47:22 +00007243 Flags = DAG.getNode(ISD::XOR, dl, MVT::i32, Flags,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007244 DAG.getConstant(1, dl, MVT::i32));
Chris Lattner4211ca92006-04-14 06:01:58 +00007245 return Flags;
7246}
7247
Hal Finkel5c0d1452014-03-30 13:22:59 +00007248SDValue PPCTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
7249 SelectionDAG &DAG) const {
7250 SDLoc dl(Op);
7251 // For v2i64 (VSX), we can pattern patch the v2i32 case (using fp <-> int
7252 // instructions), but for smaller types, we need to first extend up to v2i32
7253 // before doing going farther.
7254 if (Op.getValueType() == MVT::v2i64) {
7255 EVT ExtVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
7256 if (ExtVT != MVT::v2i32) {
7257 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(0));
7258 Op = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, Op,
7259 DAG.getValueType(EVT::getVectorVT(*DAG.getContext(),
7260 ExtVT.getVectorElementType(), 4)));
7261 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Op);
7262 Op = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v2i64, Op,
7263 DAG.getValueType(MVT::v2i32));
7264 }
7265
7266 return Op;
7267 }
7268
7269 return SDValue();
7270}
7271
Scott Michelcf0da6c2009-02-17 22:15:04 +00007272SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
Dan Gohman21cea8a2010-04-17 15:26:15 +00007273 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00007274 SDLoc dl(Op);
Chris Lattner4211ca92006-04-14 06:01:58 +00007275 // Create a stack slot that is 16-byte aligned.
7276 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
David Greene1fbe0542009-11-12 20:49:22 +00007277 int FrameIdx = FrameInfo->CreateStackObject(16, 16, false);
Dale Johannesen81bfca72010-05-03 22:59:34 +00007278 EVT PtrVT = getPointerTy();
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007279 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007280
Chris Lattner4211ca92006-04-14 06:01:58 +00007281 // Store the input value into Value#0 of the stack slot.
Dale Johannesen021052a2009-02-04 20:06:27 +00007282 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl,
Chris Lattner676c61d2010-09-21 18:41:36 +00007283 Op.getOperand(0), FIdx, MachinePointerInfo(),
David Greene87a5abe2010-02-15 16:56:53 +00007284 false, false, 0);
Chris Lattner4211ca92006-04-14 06:01:58 +00007285 // Load it out.
Chris Lattner7727d052010-09-21 06:44:06 +00007286 return DAG.getLoad(Op.getValueType(), dl, Store, FIdx, MachinePointerInfo(),
Pete Cooper82cd9e82011-11-08 18:42:53 +00007287 false, false, false, 0);
Chris Lattner4211ca92006-04-14 06:01:58 +00007288}
7289
Hal Finkelc93a9a22015-02-25 01:06:45 +00007290SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
7291 SelectionDAG &DAG) const {
7292 SDLoc dl(Op);
7293 SDNode *N = Op.getNode();
7294
7295 assert(N->getOperand(0).getValueType() == MVT::v4i1 &&
7296 "Unknown extract_vector_elt type");
7297
7298 SDValue Value = N->getOperand(0);
7299
7300 // The first part of this is like the store lowering except that we don't
7301 // need to track the chain.
7302
7303 // The values are now known to be -1 (false) or 1 (true). To convert this
7304 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
7305 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
7306 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
7307
7308 // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to
7309 // understand how to form the extending load.
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007310 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::f64);
Hal Finkelc93a9a22015-02-25 01:06:45 +00007311 FPHalfs = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64,
7312 FPHalfs, FPHalfs, FPHalfs, FPHalfs);
7313
7314 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
7315
7316 // Now convert to an integer and store.
7317 Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007318 DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32),
Hal Finkelc93a9a22015-02-25 01:06:45 +00007319 Value);
7320
7321 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
7322 int FrameIdx = FrameInfo->CreateStackObject(16, 16, false);
7323 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FrameIdx);
7324 EVT PtrVT = getPointerTy();
7325 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
7326
7327 SDValue StoreChain = DAG.getEntryNode();
7328 SmallVector<SDValue, 2> Ops;
7329 Ops.push_back(StoreChain);
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007330 Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32));
Hal Finkelc93a9a22015-02-25 01:06:45 +00007331 Ops.push_back(Value);
7332 Ops.push_back(FIdx);
7333
7334 SmallVector<EVT, 2> ValueVTs;
7335 ValueVTs.push_back(MVT::Other); // chain
7336 SDVTList VTs = DAG.getVTList(ValueVTs);
7337
7338 StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID,
7339 dl, VTs, Ops, MVT::v4i32, PtrInfo);
7340
7341 // Extract the value requested.
7342 unsigned Offset = 4*cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007343 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
Hal Finkelc93a9a22015-02-25 01:06:45 +00007344 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
7345
7346 SDValue IntVal = DAG.getLoad(MVT::i32, dl, StoreChain, Idx,
7347 PtrInfo.getWithOffset(Offset),
7348 false, false, false, 0);
7349
7350 if (!Subtarget.useCRBits())
7351 return IntVal;
7352
7353 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, IntVal);
7354}
7355
7356/// Lowering for QPX v4i1 loads
7357SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
7358 SelectionDAG &DAG) const {
7359 SDLoc dl(Op);
7360 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
7361 SDValue LoadChain = LN->getChain();
7362 SDValue BasePtr = LN->getBasePtr();
7363
7364 if (Op.getValueType() == MVT::v4f64 ||
7365 Op.getValueType() == MVT::v4f32) {
7366 EVT MemVT = LN->getMemoryVT();
7367 unsigned Alignment = LN->getAlignment();
7368
7369 // If this load is properly aligned, then it is legal.
7370 if (Alignment >= MemVT.getStoreSize())
7371 return Op;
7372
7373 EVT ScalarVT = Op.getValueType().getScalarType(),
7374 ScalarMemVT = MemVT.getScalarType();
7375 unsigned Stride = ScalarMemVT.getStoreSize();
7376
7377 SmallVector<SDValue, 8> Vals, LoadChains;
7378 for (unsigned Idx = 0; Idx < 4; ++Idx) {
7379 SDValue Load;
7380 if (ScalarVT != ScalarMemVT)
7381 Load =
7382 DAG.getExtLoad(LN->getExtensionType(), dl, ScalarVT, LoadChain,
7383 BasePtr,
7384 LN->getPointerInfo().getWithOffset(Idx*Stride),
7385 ScalarMemVT, LN->isVolatile(), LN->isNonTemporal(),
7386 LN->isInvariant(), MinAlign(Alignment, Idx*Stride),
7387 LN->getAAInfo());
7388 else
7389 Load =
7390 DAG.getLoad(ScalarVT, dl, LoadChain, BasePtr,
7391 LN->getPointerInfo().getWithOffset(Idx*Stride),
7392 LN->isVolatile(), LN->isNonTemporal(),
7393 LN->isInvariant(), MinAlign(Alignment, Idx*Stride),
7394 LN->getAAInfo());
7395
7396 if (Idx == 0 && LN->isIndexed()) {
7397 assert(LN->getAddressingMode() == ISD::PRE_INC &&
7398 "Unknown addressing mode on vector load");
7399 Load = DAG.getIndexedLoad(Load, dl, BasePtr, LN->getOffset(),
7400 LN->getAddressingMode());
7401 }
7402
7403 Vals.push_back(Load);
7404 LoadChains.push_back(Load.getValue(1));
7405
7406 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007407 DAG.getConstant(Stride, dl,
7408 BasePtr.getValueType()));
Hal Finkelc93a9a22015-02-25 01:06:45 +00007409 }
7410
7411 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
7412 SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007413 Op.getValueType(), Vals);
Hal Finkelc93a9a22015-02-25 01:06:45 +00007414
7415 if (LN->isIndexed()) {
7416 SDValue RetOps[] = { Value, Vals[0].getValue(1), TF };
7417 return DAG.getMergeValues(RetOps, dl);
7418 }
7419
7420 SDValue RetOps[] = { Value, TF };
7421 return DAG.getMergeValues(RetOps, dl);
7422 }
7423
7424 assert(Op.getValueType() == MVT::v4i1 && "Unknown load to lower");
7425 assert(LN->isUnindexed() && "Indexed v4i1 loads are not supported");
7426
7427 // To lower v4i1 from a byte array, we load the byte elements of the
7428 // vector and then reuse the BUILD_VECTOR logic.
7429
7430 SmallVector<SDValue, 4> VectElmts, VectElmtChains;
7431 for (unsigned i = 0; i < 4; ++i) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007432 SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
Hal Finkelc93a9a22015-02-25 01:06:45 +00007433 Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx);
7434
7435 VectElmts.push_back(DAG.getExtLoad(ISD::EXTLOAD,
7436 dl, MVT::i32, LoadChain, Idx,
7437 LN->getPointerInfo().getWithOffset(i),
7438 MVT::i8 /* memory type */,
7439 LN->isVolatile(), LN->isNonTemporal(),
7440 LN->isInvariant(),
7441 1 /* alignment */, LN->getAAInfo()));
7442 VectElmtChains.push_back(VectElmts[i].getValue(1));
7443 }
7444
7445 LoadChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, VectElmtChains);
7446 SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i1, VectElmts);
7447
7448 SDValue RVals[] = { Value, LoadChain };
7449 return DAG.getMergeValues(RVals, dl);
7450}
7451
7452/// Lowering for QPX v4i1 stores
7453SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
7454 SelectionDAG &DAG) const {
7455 SDLoc dl(Op);
7456 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
7457 SDValue StoreChain = SN->getChain();
7458 SDValue BasePtr = SN->getBasePtr();
7459 SDValue Value = SN->getValue();
7460
7461 if (Value.getValueType() == MVT::v4f64 ||
7462 Value.getValueType() == MVT::v4f32) {
7463 EVT MemVT = SN->getMemoryVT();
7464 unsigned Alignment = SN->getAlignment();
7465
7466 // If this store is properly aligned, then it is legal.
7467 if (Alignment >= MemVT.getStoreSize())
7468 return Op;
7469
7470 EVT ScalarVT = Value.getValueType().getScalarType(),
7471 ScalarMemVT = MemVT.getScalarType();
7472 unsigned Stride = ScalarMemVT.getStoreSize();
7473
7474 SmallVector<SDValue, 8> Stores;
7475 for (unsigned Idx = 0; Idx < 4; ++Idx) {
7476 SDValue Ex =
7477 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007478 DAG.getConstant(Idx, dl, getVectorIdxTy()));
Hal Finkelc93a9a22015-02-25 01:06:45 +00007479 SDValue Store;
7480 if (ScalarVT != ScalarMemVT)
7481 Store =
7482 DAG.getTruncStore(StoreChain, dl, Ex, BasePtr,
7483 SN->getPointerInfo().getWithOffset(Idx*Stride),
7484 ScalarMemVT, SN->isVolatile(), SN->isNonTemporal(),
7485 MinAlign(Alignment, Idx*Stride), SN->getAAInfo());
7486 else
7487 Store =
7488 DAG.getStore(StoreChain, dl, Ex, BasePtr,
7489 SN->getPointerInfo().getWithOffset(Idx*Stride),
7490 SN->isVolatile(), SN->isNonTemporal(),
7491 MinAlign(Alignment, Idx*Stride), SN->getAAInfo());
7492
7493 if (Idx == 0 && SN->isIndexed()) {
7494 assert(SN->getAddressingMode() == ISD::PRE_INC &&
7495 "Unknown addressing mode on vector store");
7496 Store = DAG.getIndexedStore(Store, dl, BasePtr, SN->getOffset(),
7497 SN->getAddressingMode());
7498 }
7499
7500 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007501 DAG.getConstant(Stride, dl,
7502 BasePtr.getValueType()));
Hal Finkelc93a9a22015-02-25 01:06:45 +00007503 Stores.push_back(Store);
7504 }
7505
7506 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
7507
7508 if (SN->isIndexed()) {
7509 SDValue RetOps[] = { TF, Stores[0].getValue(1) };
7510 return DAG.getMergeValues(RetOps, dl);
7511 }
7512
7513 return TF;
7514 }
7515
7516 assert(SN->isUnindexed() && "Indexed v4i1 stores are not supported");
7517 assert(Value.getValueType() == MVT::v4i1 && "Unknown store to lower");
7518
7519 // The values are now known to be -1 (false) or 1 (true). To convert this
7520 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5).
7521 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5
7522 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value);
7523
7524 // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to
7525 // understand how to form the extending load.
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007526 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::f64);
Hal Finkelc93a9a22015-02-25 01:06:45 +00007527 FPHalfs = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f64,
7528 FPHalfs, FPHalfs, FPHalfs, FPHalfs);
7529
7530 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs);
7531
7532 // Now convert to an integer and store.
7533 Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007534 DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32),
Hal Finkelc93a9a22015-02-25 01:06:45 +00007535 Value);
7536
7537 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
7538 int FrameIdx = FrameInfo->CreateStackObject(16, 16, false);
7539 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FrameIdx);
7540 EVT PtrVT = getPointerTy();
7541 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
7542
7543 SmallVector<SDValue, 2> Ops;
7544 Ops.push_back(StoreChain);
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007545 Ops.push_back(DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32));
Hal Finkelc93a9a22015-02-25 01:06:45 +00007546 Ops.push_back(Value);
7547 Ops.push_back(FIdx);
7548
7549 SmallVector<EVT, 2> ValueVTs;
7550 ValueVTs.push_back(MVT::Other); // chain
7551 SDVTList VTs = DAG.getVTList(ValueVTs);
7552
7553 StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID,
7554 dl, VTs, Ops, MVT::v4i32, PtrInfo);
7555
7556 // Move data into the byte array.
7557 SmallVector<SDValue, 4> Loads, LoadChains;
7558 for (unsigned i = 0; i < 4; ++i) {
7559 unsigned Offset = 4*i;
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007560 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType());
Hal Finkelc93a9a22015-02-25 01:06:45 +00007561 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx);
7562
7563 Loads.push_back(DAG.getLoad(MVT::i32, dl, StoreChain, Idx,
7564 PtrInfo.getWithOffset(Offset),
7565 false, false, false, 0));
7566 LoadChains.push_back(Loads[i].getValue(1));
7567 }
7568
7569 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
7570
7571 SmallVector<SDValue, 4> Stores;
7572 for (unsigned i = 0; i < 4; ++i) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007573 SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType());
Hal Finkelc93a9a22015-02-25 01:06:45 +00007574 Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx);
7575
7576 Stores.push_back(DAG.getTruncStore(StoreChain, dl, Loads[i], Idx,
7577 SN->getPointerInfo().getWithOffset(i),
7578 MVT::i8 /* memory type */,
7579 SN->isNonTemporal(), SN->isVolatile(),
7580 1 /* alignment */, SN->getAAInfo()));
7581 }
7582
7583 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
7584
7585 return StoreChain;
7586}
7587
Dan Gohman21cea8a2010-04-17 15:26:15 +00007588SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00007589 SDLoc dl(Op);
Owen Anderson9f944592009-08-11 20:47:22 +00007590 if (Op.getValueType() == MVT::v4i32) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007591 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007592
Owen Anderson9f944592009-08-11 20:47:22 +00007593 SDValue Zero = BuildSplatI( 0, 1, MVT::v4i32, DAG, dl);
7594 SDValue Neg16 = BuildSplatI(-16, 4, MVT::v4i32, DAG, dl);//+16 as shift amt.
Scott Michelcf0da6c2009-02-17 22:15:04 +00007595
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007596 SDValue RHSSwap = // = vrlw RHS, 16
Dale Johannesen9f3f72f2009-02-06 01:31:28 +00007597 BuildIntrinsicOp(Intrinsic::ppc_altivec_vrlw, RHS, Neg16, DAG, dl);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007598
Chris Lattner7e4398742006-04-18 03:43:48 +00007599 // Shrinkify inputs to v8i16.
Wesley Peck527da1b2010-11-23 03:31:01 +00007600 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, LHS);
7601 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHS);
7602 RHSSwap = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHSSwap);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007603
Chris Lattner7e4398742006-04-18 03:43:48 +00007604 // Low parts multiplied together, generating 32-bit results (we ignore the
7605 // top parts).
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007606 SDValue LoProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmulouh,
Owen Anderson9f944592009-08-11 20:47:22 +00007607 LHS, RHS, DAG, dl, MVT::v4i32);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007608
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007609 SDValue HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmsumuhm,
Owen Anderson9f944592009-08-11 20:47:22 +00007610 LHS, RHSSwap, Zero, DAG, dl, MVT::v4i32);
Chris Lattner7e4398742006-04-18 03:43:48 +00007611 // Shift the high parts up 16 bits.
Scott Michelcf0da6c2009-02-17 22:15:04 +00007612 HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, HiProd,
Dale Johannesen9f3f72f2009-02-06 01:31:28 +00007613 Neg16, DAG, dl);
Owen Anderson9f944592009-08-11 20:47:22 +00007614 return DAG.getNode(ISD::ADD, dl, MVT::v4i32, LoProd, HiProd);
7615 } else if (Op.getValueType() == MVT::v8i16) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007616 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007617
Owen Anderson9f944592009-08-11 20:47:22 +00007618 SDValue Zero = BuildSplatI(0, 1, MVT::v8i16, DAG, dl);
Chris Lattner7e4398742006-04-18 03:43:48 +00007619
Chris Lattner96d50482006-04-18 04:28:57 +00007620 return BuildIntrinsicOp(Intrinsic::ppc_altivec_vmladduhm,
Dale Johannesen9f3f72f2009-02-06 01:31:28 +00007621 LHS, RHS, Zero, DAG, dl);
Owen Anderson9f944592009-08-11 20:47:22 +00007622 } else if (Op.getValueType() == MVT::v16i8) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007623 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
Eric Christopherb1aaebe2014-06-12 22:38:18 +00007624 bool isLittleEndian = Subtarget.isLittleEndian();
Scott Michelcf0da6c2009-02-17 22:15:04 +00007625
Chris Lattnerd6d82aa2006-04-18 03:57:35 +00007626 // Multiply the even 8-bit parts, producing 16-bit sums.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007627 SDValue EvenParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuleub,
Owen Anderson9f944592009-08-11 20:47:22 +00007628 LHS, RHS, DAG, dl, MVT::v8i16);
Wesley Peck527da1b2010-11-23 03:31:01 +00007629 EvenParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, EvenParts);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007630
Chris Lattnerd6d82aa2006-04-18 03:57:35 +00007631 // Multiply the odd 8-bit parts, producing 16-bit sums.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00007632 SDValue OddParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuloub,
Owen Anderson9f944592009-08-11 20:47:22 +00007633 LHS, RHS, DAG, dl, MVT::v8i16);
Wesley Peck527da1b2010-11-23 03:31:01 +00007634 OddParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OddParts);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007635
Bill Schmidt42995e82014-06-09 16:06:29 +00007636 // Merge the results together. Because vmuleub and vmuloub are
7637 // instructions with a big-endian bias, we must reverse the
7638 // element numbering and reverse the meaning of "odd" and "even"
7639 // when generating little endian code.
Nate Begeman8d6d4b92009-04-27 18:41:29 +00007640 int Ops[16];
Chris Lattnerd6d82aa2006-04-18 03:57:35 +00007641 for (unsigned i = 0; i != 8; ++i) {
Bill Schmidt42995e82014-06-09 16:06:29 +00007642 if (isLittleEndian) {
7643 Ops[i*2 ] = 2*i;
7644 Ops[i*2+1] = 2*i+16;
7645 } else {
7646 Ops[i*2 ] = 2*i+1;
7647 Ops[i*2+1] = 2*i+1+16;
7648 }
Chris Lattnerd6d82aa2006-04-18 03:57:35 +00007649 }
Bill Schmidt42995e82014-06-09 16:06:29 +00007650 if (isLittleEndian)
7651 return DAG.getVectorShuffle(MVT::v16i8, dl, OddParts, EvenParts, Ops);
7652 else
7653 return DAG.getVectorShuffle(MVT::v16i8, dl, EvenParts, OddParts, Ops);
Chris Lattner7e4398742006-04-18 03:43:48 +00007654 } else {
Torok Edwinfbcc6632009-07-14 16:55:14 +00007655 llvm_unreachable("Unknown mul to lower!");
Chris Lattner7e4398742006-04-18 03:43:48 +00007656 }
Chris Lattnera2cae1b2006-04-18 03:24:30 +00007657}
7658
Chris Lattnerf3d06c62005-08-26 00:52:45 +00007659/// LowerOperation - Provide custom lowering hooks for some operations.
7660///
Dan Gohman21cea8a2010-04-17 15:26:15 +00007661SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
Chris Lattnerf3d06c62005-08-26 00:52:45 +00007662 switch (Op.getOpcode()) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00007663 default: llvm_unreachable("Wasn't expecting to be able to lower this!");
Chris Lattner4211ca92006-04-14 06:01:58 +00007664 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
Bob Wilsonf84f7102009-11-04 21:31:18 +00007665 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
Chris Lattner4211ca92006-04-14 06:01:58 +00007666 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
Roman Divackye3f15c982012-06-04 17:36:38 +00007667 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
Nate Begeman4ca2ea52006-04-22 18:53:45 +00007668 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
Chris Lattner4211ca92006-04-14 06:01:58 +00007669 case ISD::SETCC: return LowerSETCC(Op, DAG);
Duncan Sandsa0984362011-09-06 13:37:06 +00007670 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
7671 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007672 case ISD::VASTART:
Eric Christopherb1aaebe2014-06-12 22:38:18 +00007673 return LowerVASTART(Op, DAG, Subtarget);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007674
7675 case ISD::VAARG:
Eric Christopherb1aaebe2014-06-12 22:38:18 +00007676 return LowerVAARG(Op, DAG, Subtarget);
Nicolas Geoffray23710a72007-04-03 13:59:52 +00007677
Roman Divackyc3825df2013-07-25 21:36:47 +00007678 case ISD::VACOPY:
Eric Christopherb1aaebe2014-06-12 22:38:18 +00007679 return LowerVACOPY(Op, DAG, Subtarget);
Roman Divackyc3825df2013-07-25 21:36:47 +00007680
Eric Christopherb1aaebe2014-06-12 22:38:18 +00007681 case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG, Subtarget);
Chris Lattner43df5b32007-02-25 05:34:32 +00007682 case ISD::DYNAMIC_STACKALLOC:
Eric Christopherb1aaebe2014-06-12 22:38:18 +00007683 return LowerDYNAMIC_STACKALLOC(Op, DAG, Subtarget);
Evan Cheng51096af2008-04-19 01:30:48 +00007684
Hal Finkel756810f2013-03-21 21:37:52 +00007685 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
7686 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
7687
Hal Finkel940ab932014-02-28 00:27:01 +00007688 case ISD::LOAD: return LowerLOAD(Op, DAG);
7689 case ISD::STORE: return LowerSTORE(Op, DAG);
7690 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
Chris Lattner4211ca92006-04-14 06:01:58 +00007691 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
Dale Johannesen37bc85f2009-06-04 20:53:52 +00007692 case ISD::FP_TO_UINT:
7693 case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG,
Hal Finkeled844c42015-01-06 22:31:02 +00007694 SDLoc(Op));
Hal Finkelf6d45f22013-04-01 17:52:07 +00007695 case ISD::UINT_TO_FP:
7696 case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
Dan Gohman9ba4d762008-01-31 00:41:03 +00007697 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG);
Chris Lattner4a66d692006-03-22 05:30:33 +00007698
Chris Lattner4211ca92006-04-14 06:01:58 +00007699 // Lower 64-bit shifts.
Chris Lattner601b8652006-09-20 03:47:40 +00007700 case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG);
7701 case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG);
7702 case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG);
Chris Lattner4a66d692006-03-22 05:30:33 +00007703
Chris Lattner4211ca92006-04-14 06:01:58 +00007704 // Vector-related lowering.
7705 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
7706 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
7707 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
7708 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
Hal Finkel5c0d1452014-03-30 13:22:59 +00007709 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
Hal Finkelc93a9a22015-02-25 01:06:45 +00007710 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
Chris Lattnera2cae1b2006-04-18 03:24:30 +00007711 case ISD::MUL: return LowerMUL(Op, DAG);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007712
Hal Finkel25c19922013-05-15 21:37:41 +00007713 // For counter-based loop handling.
7714 case ISD::INTRINSIC_W_CHAIN: return SDValue();
7715
Chris Lattnerf6a81562007-12-08 06:59:59 +00007716 // Frame & Return address.
7717 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
Nicolas Geoffray75ab9792007-03-01 13:11:38 +00007718 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
Chris Lattnere675a082005-08-31 20:23:54 +00007719 }
Chris Lattnerf3d06c62005-08-26 00:52:45 +00007720}
7721
Duncan Sands6ed40142008-12-01 11:39:25 +00007722void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
7723 SmallVectorImpl<SDValue>&Results,
Dan Gohman21cea8a2010-04-17 15:26:15 +00007724 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00007725 SDLoc dl(N);
Chris Lattner57ee7c62007-11-28 18:44:47 +00007726 switch (N->getOpcode()) {
Duncan Sands4068a7f2008-10-28 15:00:32 +00007727 default:
Craig Toppere55c5562012-02-07 02:50:20 +00007728 llvm_unreachable("Do not know how to custom type legalize this operation!");
Hal Finkelbbdee932014-12-02 22:01:00 +00007729 case ISD::READCYCLECOUNTER: {
7730 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
7731 SDValue RTB = DAG.getNode(PPCISD::READ_TIME_BASE, dl, VTs, N->getOperand(0));
7732
7733 Results.push_back(RTB);
7734 Results.push_back(RTB.getValue(1));
7735 Results.push_back(RTB.getValue(2));
7736 break;
7737 }
Hal Finkel25c19922013-05-15 21:37:41 +00007738 case ISD::INTRINSIC_W_CHAIN: {
7739 if (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() !=
7740 Intrinsic::ppc_is_decremented_ctr_nonzero)
7741 break;
7742
7743 assert(N->getValueType(0) == MVT::i1 &&
7744 "Unexpected result type for CTR decrement intrinsic");
Matt Arsenault758659232013-05-18 00:21:46 +00007745 EVT SVT = getSetCCResultType(*DAG.getContext(), N->getValueType(0));
Hal Finkel25c19922013-05-15 21:37:41 +00007746 SDVTList VTs = DAG.getVTList(SVT, MVT::Other);
7747 SDValue NewInt = DAG.getNode(N->getOpcode(), dl, VTs, N->getOperand(0),
7748 N->getOperand(1));
7749
7750 Results.push_back(NewInt);
7751 Results.push_back(NewInt.getValue(1));
7752 break;
7753 }
Roman Divacky4394e682011-06-28 15:30:42 +00007754 case ISD::VAARG: {
Eric Christophercccae792015-01-30 22:02:31 +00007755 if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64())
Roman Divacky4394e682011-06-28 15:30:42 +00007756 return;
7757
7758 EVT VT = N->getValueType(0);
7759
7760 if (VT == MVT::i64) {
Eric Christopherb1aaebe2014-06-12 22:38:18 +00007761 SDValue NewNode = LowerVAARG(SDValue(N, 1), DAG, Subtarget);
Roman Divacky4394e682011-06-28 15:30:42 +00007762
7763 Results.push_back(NewNode);
7764 Results.push_back(NewNode.getValue(1));
7765 }
7766 return;
7767 }
Duncan Sands6ed40142008-12-01 11:39:25 +00007768 case ISD::FP_ROUND_INREG: {
Owen Anderson9f944592009-08-11 20:47:22 +00007769 assert(N->getValueType(0) == MVT::ppcf128);
7770 assert(N->getOperand(0).getValueType() == MVT::ppcf128);
Scott Michelcf0da6c2009-02-17 22:15:04 +00007771 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl,
Owen Anderson9f944592009-08-11 20:47:22 +00007772 MVT::f64, N->getOperand(0),
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007773 DAG.getIntPtrConstant(0, dl));
Dale Johannesenf80493b2009-02-05 22:07:54 +00007774 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl,
Owen Anderson9f944592009-08-11 20:47:22 +00007775 MVT::f64, N->getOperand(0),
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00007776 DAG.getIntPtrConstant(1, dl));
Duncan Sands6ed40142008-12-01 11:39:25 +00007777
Ulrich Weigand874fc622013-03-26 10:56:22 +00007778 // Add the two halves of the long double in round-to-zero mode.
7779 SDValue FPreg = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi);
Duncan Sands6ed40142008-12-01 11:39:25 +00007780
7781 // We know the low half is about to be thrown away, so just use something
7782 // convenient.
Owen Anderson9f944592009-08-11 20:47:22 +00007783 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::ppcf128,
Dale Johannesenf80493b2009-02-05 22:07:54 +00007784 FPreg, FPreg));
Duncan Sands6ed40142008-12-01 11:39:25 +00007785 return;
Duncan Sands2a287912008-07-19 16:26:02 +00007786 }
Duncan Sands6ed40142008-12-01 11:39:25 +00007787 case ISD::FP_TO_SINT:
Hal Finkel93138502015-04-10 03:39:00 +00007788 case ISD::FP_TO_UINT:
Bill Schmidt41221692013-07-09 18:50:20 +00007789 // LowerFP_TO_INT() can only handle f32 and f64.
7790 if (N->getOperand(0).getValueType() == MVT::ppcf128)
7791 return;
Dale Johannesen37bc85f2009-06-04 20:53:52 +00007792 Results.push_back(LowerFP_TO_INT(SDValue(N, 0), DAG, dl));
Duncan Sands6ed40142008-12-01 11:39:25 +00007793 return;
Chris Lattner57ee7c62007-11-28 18:44:47 +00007794 }
7795}
7796
7797
Chris Lattner4211ca92006-04-14 06:01:58 +00007798//===----------------------------------------------------------------------===//
7799// Other Lowering Code
7800//===----------------------------------------------------------------------===//
7801
Robin Morisset22129962014-09-23 20:46:49 +00007802static Instruction* callIntrinsic(IRBuilder<> &Builder, Intrinsic::ID Id) {
7803 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
7804 Function *Func = Intrinsic::getDeclaration(M, Id);
7805 return Builder.CreateCall(Func);
7806}
7807
7808// The mappings for emitLeading/TrailingFence is taken from
7809// http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
7810Instruction* PPCTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
7811 AtomicOrdering Ord, bool IsStore,
7812 bool IsLoad) const {
7813 if (Ord == SequentiallyConsistent)
7814 return callIntrinsic(Builder, Intrinsic::ppc_sync);
7815 else if (isAtLeastRelease(Ord))
7816 return callIntrinsic(Builder, Intrinsic::ppc_lwsync);
7817 else
7818 return nullptr;
7819}
7820
7821Instruction* PPCTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
7822 AtomicOrdering Ord, bool IsStore,
7823 bool IsLoad) const {
7824 if (IsLoad && isAtLeastAcquire(Ord))
7825 return callIntrinsic(Builder, Intrinsic::ppc_lwsync);
7826 // FIXME: this is too conservative, a dependent branch + isync is enough.
7827 // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and
7828 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
7829 // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification.
7830 else
7831 return nullptr;
7832}
7833
Chris Lattner9b577f12005-08-26 21:23:58 +00007834MachineBasicBlock *
Dale Johannesend4eb0522008-08-25 22:34:37 +00007835PPCTargetLowering::EmitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB,
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00007836 unsigned AtomicSize,
7837 unsigned BinOpcode) const {
Dale Johannesenf0a88d62008-08-29 18:29:46 +00007838 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0.
Eric Christophercccae792015-01-30 22:02:31 +00007839 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
Dale Johannesend4eb0522008-08-25 22:34:37 +00007840
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00007841 auto LoadMnemonic = PPC::LDARX;
7842 auto StoreMnemonic = PPC::STDCX;
7843 switch (AtomicSize) {
7844 default:
7845 llvm_unreachable("Unexpected size of atomic entity");
7846 case 1:
7847 LoadMnemonic = PPC::LBARX;
7848 StoreMnemonic = PPC::STBCX;
7849 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4");
7850 break;
7851 case 2:
7852 LoadMnemonic = PPC::LHARX;
7853 StoreMnemonic = PPC::STHCX;
7854 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4");
7855 break;
7856 case 4:
7857 LoadMnemonic = PPC::LWARX;
7858 StoreMnemonic = PPC::STWCX;
7859 break;
7860 case 8:
7861 LoadMnemonic = PPC::LDARX;
7862 StoreMnemonic = PPC::STDCX;
7863 break;
7864 }
7865
Dale Johannesend4eb0522008-08-25 22:34:37 +00007866 const BasicBlock *LLVM_BB = BB->getBasicBlock();
7867 MachineFunction *F = BB->getParent();
7868 MachineFunction::iterator It = BB;
7869 ++It;
7870
7871 unsigned dest = MI->getOperand(0).getReg();
7872 unsigned ptrA = MI->getOperand(1).getReg();
7873 unsigned ptrB = MI->getOperand(2).getReg();
7874 unsigned incr = MI->getOperand(3).getReg();
Dale Johannesene9f623e2009-02-13 02:27:39 +00007875 DebugLoc dl = MI->getDebugLoc();
Dale Johannesend4eb0522008-08-25 22:34:37 +00007876
7877 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB);
7878 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB);
7879 F->insert(It, loopMBB);
7880 F->insert(It, exitMBB);
Dan Gohman34396292010-07-06 20:24:04 +00007881 exitMBB->splice(exitMBB->begin(), BB,
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00007882 std::next(MachineBasicBlock::iterator(MI)), BB->end());
Dan Gohman34396292010-07-06 20:24:04 +00007883 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
Dale Johannesend4eb0522008-08-25 22:34:37 +00007884
7885 MachineRegisterInfo &RegInfo = F->getRegInfo();
Dale Johannesenf0a88d62008-08-29 18:29:46 +00007886 unsigned TmpReg = (!BinOpcode) ? incr :
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00007887 RegInfo.createVirtualRegister( AtomicSize == 8 ? &PPC::G8RCRegClass
Craig Topper61e88f42014-11-21 05:58:21 +00007888 : &PPC::GPRCRegClass);
Dale Johannesend4eb0522008-08-25 22:34:37 +00007889
7890 // thisMBB:
7891 // ...
7892 // fallthrough --> loopMBB
7893 BB->addSuccessor(loopMBB);
7894
7895 // loopMBB:
7896 // l[wd]arx dest, ptr
7897 // add r0, dest, incr
7898 // st[wd]cx. r0, ptr
7899 // bne- loopMBB
7900 // fallthrough --> exitMBB
7901 BB = loopMBB;
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00007902 BuildMI(BB, dl, TII->get(LoadMnemonic), dest)
Dale Johannesend4eb0522008-08-25 22:34:37 +00007903 .addReg(ptrA).addReg(ptrB);
Dale Johannesenf0a88d62008-08-29 18:29:46 +00007904 if (BinOpcode)
Dale Johannesene9f623e2009-02-13 02:27:39 +00007905 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg).addReg(incr).addReg(dest);
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00007906 BuildMI(BB, dl, TII->get(StoreMnemonic))
Dale Johannesend4eb0522008-08-25 22:34:37 +00007907 .addReg(TmpReg).addReg(ptrA).addReg(ptrB);
Dale Johannesene9f623e2009-02-13 02:27:39 +00007908 BuildMI(BB, dl, TII->get(PPC::BCC))
Scott Michelcf0da6c2009-02-17 22:15:04 +00007909 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB);
Dale Johannesend4eb0522008-08-25 22:34:37 +00007910 BB->addSuccessor(loopMBB);
7911 BB->addSuccessor(exitMBB);
7912
7913 // exitMBB:
7914 // ...
7915 BB = exitMBB;
7916 return BB;
7917}
7918
7919MachineBasicBlock *
Scott Michelcf0da6c2009-02-17 22:15:04 +00007920PPCTargetLowering::EmitPartwordAtomicBinary(MachineInstr *MI,
Dale Johannesena32affb2008-08-28 17:53:09 +00007921 MachineBasicBlock *BB,
7922 bool is8bit, // operation
Dan Gohman747e55b2009-02-07 16:15:20 +00007923 unsigned BinOpcode) const {
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00007924 // If we support part-word atomic mnemonics, just use them
7925 if (Subtarget.hasPartwordAtomics())
7926 return EmitAtomicBinary(MI, BB, is8bit ? 1 : 2, BinOpcode);
7927
Dale Johannesenf0a88d62008-08-29 18:29:46 +00007928 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0.
Eric Christophercccae792015-01-30 22:02:31 +00007929 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
Dale Johannesena32affb2008-08-28 17:53:09 +00007930 // In 64 bit mode we have to use 64 bits for addresses, even though the
7931 // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address
7932 // registers without caring whether they're 32 or 64, but here we're
7933 // doing actual arithmetic on the addresses.
Eric Christopherb1aaebe2014-06-12 22:38:18 +00007934 bool is64bit = Subtarget.isPPC64();
Hal Finkelf70c41e2013-03-21 23:45:03 +00007935 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
Dale Johannesena32affb2008-08-28 17:53:09 +00007936
7937 const BasicBlock *LLVM_BB = BB->getBasicBlock();
7938 MachineFunction *F = BB->getParent();
7939 MachineFunction::iterator It = BB;
7940 ++It;
7941
7942 unsigned dest = MI->getOperand(0).getReg();
7943 unsigned ptrA = MI->getOperand(1).getReg();
7944 unsigned ptrB = MI->getOperand(2).getReg();
7945 unsigned incr = MI->getOperand(3).getReg();
Dale Johannesene9f623e2009-02-13 02:27:39 +00007946 DebugLoc dl = MI->getDebugLoc();
Dale Johannesena32affb2008-08-28 17:53:09 +00007947
7948 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB);
7949 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB);
7950 F->insert(It, loopMBB);
7951 F->insert(It, exitMBB);
Dan Gohman34396292010-07-06 20:24:04 +00007952 exitMBB->splice(exitMBB->begin(), BB,
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00007953 std::next(MachineBasicBlock::iterator(MI)), BB->end());
Dan Gohman34396292010-07-06 20:24:04 +00007954 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
Dale Johannesena32affb2008-08-28 17:53:09 +00007955
7956 MachineRegisterInfo &RegInfo = F->getRegInfo();
Craig Topper61e88f42014-11-21 05:58:21 +00007957 const TargetRegisterClass *RC = is64bit ? &PPC::G8RCRegClass
7958 : &PPC::GPRCRegClass;
Dale Johannesena32affb2008-08-28 17:53:09 +00007959 unsigned PtrReg = RegInfo.createVirtualRegister(RC);
7960 unsigned Shift1Reg = RegInfo.createVirtualRegister(RC);
7961 unsigned ShiftReg = RegInfo.createVirtualRegister(RC);
7962 unsigned Incr2Reg = RegInfo.createVirtualRegister(RC);
7963 unsigned MaskReg = RegInfo.createVirtualRegister(RC);
7964 unsigned Mask2Reg = RegInfo.createVirtualRegister(RC);
7965 unsigned Mask3Reg = RegInfo.createVirtualRegister(RC);
7966 unsigned Tmp2Reg = RegInfo.createVirtualRegister(RC);
7967 unsigned Tmp3Reg = RegInfo.createVirtualRegister(RC);
7968 unsigned Tmp4Reg = RegInfo.createVirtualRegister(RC);
Dale Johannesenf0a88d62008-08-29 18:29:46 +00007969 unsigned TmpDestReg = RegInfo.createVirtualRegister(RC);
Dale Johannesena32affb2008-08-28 17:53:09 +00007970 unsigned Ptr1Reg;
Dale Johannesenf0a88d62008-08-29 18:29:46 +00007971 unsigned TmpReg = (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(RC);
Dale Johannesena32affb2008-08-28 17:53:09 +00007972
7973 // thisMBB:
7974 // ...
7975 // fallthrough --> loopMBB
7976 BB->addSuccessor(loopMBB);
7977
7978 // The 4-byte load must be aligned, while a char or short may be
7979 // anywhere in the word. Hence all this nasty bookkeeping code.
7980 // add ptr1, ptrA, ptrB [copy if ptrA==0]
7981 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27]
Dale Johannesenbc698292008-09-02 20:30:23 +00007982 // xori shift, shift1, 24 [16]
Dale Johannesena32affb2008-08-28 17:53:09 +00007983 // rlwinm ptr, ptr1, 0, 0, 29
7984 // slw incr2, incr, shift
7985 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535]
7986 // slw mask, mask2, shift
7987 // loopMBB:
Dale Johannesen340d2642008-08-30 00:08:53 +00007988 // lwarx tmpDest, ptr
Dale Johannesenf0a88d62008-08-29 18:29:46 +00007989 // add tmp, tmpDest, incr2
7990 // andc tmp2, tmpDest, mask
Dale Johannesena32affb2008-08-28 17:53:09 +00007991 // and tmp3, tmp, mask
7992 // or tmp4, tmp3, tmp2
Dale Johannesen340d2642008-08-30 00:08:53 +00007993 // stwcx. tmp4, ptr
Dale Johannesena32affb2008-08-28 17:53:09 +00007994 // bne- loopMBB
7995 // fallthrough --> exitMBB
Dale Johannesenf0a88d62008-08-29 18:29:46 +00007996 // srw dest, tmpDest, shift
Jakob Stoklund Olesen7067bff2011-04-04 17:07:06 +00007997 if (ptrA != ZeroReg) {
Dale Johannesena32affb2008-08-28 17:53:09 +00007998 Ptr1Reg = RegInfo.createVirtualRegister(RC);
Dale Johannesene9f623e2009-02-13 02:27:39 +00007999 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg)
Dale Johannesena32affb2008-08-28 17:53:09 +00008000 .addReg(ptrA).addReg(ptrB);
8001 } else {
8002 Ptr1Reg = ptrB;
8003 }
Dale Johannesene9f623e2009-02-13 02:27:39 +00008004 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg).addReg(Ptr1Reg)
Dale Johannesena32affb2008-08-28 17:53:09 +00008005 .addImm(3).addImm(27).addImm(is8bit ? 28 : 27);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008006 BuildMI(BB, dl, TII->get(is64bit ? PPC::XORI8 : PPC::XORI), ShiftReg)
Dale Johannesena32affb2008-08-28 17:53:09 +00008007 .addReg(Shift1Reg).addImm(is8bit ? 24 : 16);
8008 if (is64bit)
Dale Johannesene9f623e2009-02-13 02:27:39 +00008009 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg)
Dale Johannesena32affb2008-08-28 17:53:09 +00008010 .addReg(Ptr1Reg).addImm(0).addImm(61);
8011 else
Dale Johannesene9f623e2009-02-13 02:27:39 +00008012 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg)
Dale Johannesena32affb2008-08-28 17:53:09 +00008013 .addReg(Ptr1Reg).addImm(0).addImm(0).addImm(29);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008014 BuildMI(BB, dl, TII->get(PPC::SLW), Incr2Reg)
Dale Johannesena32affb2008-08-28 17:53:09 +00008015 .addReg(incr).addReg(ShiftReg);
8016 if (is8bit)
Dale Johannesene9f623e2009-02-13 02:27:39 +00008017 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255);
Dale Johannesena32affb2008-08-28 17:53:09 +00008018 else {
Dale Johannesene9f623e2009-02-13 02:27:39 +00008019 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0);
8020 BuildMI(BB, dl, TII->get(PPC::ORI),Mask2Reg).addReg(Mask3Reg).addImm(65535);
Dale Johannesena32affb2008-08-28 17:53:09 +00008021 }
Dale Johannesene9f623e2009-02-13 02:27:39 +00008022 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg)
Dale Johannesena32affb2008-08-28 17:53:09 +00008023 .addReg(Mask2Reg).addReg(ShiftReg);
8024
8025 BB = loopMBB;
Dale Johannesene9f623e2009-02-13 02:27:39 +00008026 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg)
Jakob Stoklund Olesen7067bff2011-04-04 17:07:06 +00008027 .addReg(ZeroReg).addReg(PtrReg);
Dale Johannesenf0a88d62008-08-29 18:29:46 +00008028 if (BinOpcode)
Dale Johannesene9f623e2009-02-13 02:27:39 +00008029 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg)
Dale Johannesenf0a88d62008-08-29 18:29:46 +00008030 .addReg(Incr2Reg).addReg(TmpDestReg);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008031 BuildMI(BB, dl, TII->get(is64bit ? PPC::ANDC8 : PPC::ANDC), Tmp2Reg)
Dale Johannesenf0a88d62008-08-29 18:29:46 +00008032 .addReg(TmpDestReg).addReg(MaskReg);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008033 BuildMI(BB, dl, TII->get(is64bit ? PPC::AND8 : PPC::AND), Tmp3Reg)
Dale Johannesena32affb2008-08-28 17:53:09 +00008034 .addReg(TmpReg).addReg(MaskReg);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008035 BuildMI(BB, dl, TII->get(is64bit ? PPC::OR8 : PPC::OR), Tmp4Reg)
Dale Johannesena32affb2008-08-28 17:53:09 +00008036 .addReg(Tmp3Reg).addReg(Tmp2Reg);
Bill Schmidt3581cd42013-04-02 18:37:08 +00008037 BuildMI(BB, dl, TII->get(PPC::STWCX))
Jakob Stoklund Olesen7067bff2011-04-04 17:07:06 +00008038 .addReg(Tmp4Reg).addReg(ZeroReg).addReg(PtrReg);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008039 BuildMI(BB, dl, TII->get(PPC::BCC))
Scott Michelcf0da6c2009-02-17 22:15:04 +00008040 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB);
Dale Johannesena32affb2008-08-28 17:53:09 +00008041 BB->addSuccessor(loopMBB);
8042 BB->addSuccessor(exitMBB);
8043
8044 // exitMBB:
8045 // ...
8046 BB = exitMBB;
Jakob Stoklund Olesen13ce2362011-04-04 17:57:29 +00008047 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest).addReg(TmpDestReg)
8048 .addReg(ShiftReg);
Dale Johannesena32affb2008-08-28 17:53:09 +00008049 return BB;
8050}
8051
Hal Finkel756810f2013-03-21 21:37:52 +00008052llvm::MachineBasicBlock*
8053PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr *MI,
8054 MachineBasicBlock *MBB) const {
8055 DebugLoc DL = MI->getDebugLoc();
Eric Christophercccae792015-01-30 22:02:31 +00008056 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
Hal Finkel756810f2013-03-21 21:37:52 +00008057
8058 MachineFunction *MF = MBB->getParent();
8059 MachineRegisterInfo &MRI = MF->getRegInfo();
8060
8061 const BasicBlock *BB = MBB->getBasicBlock();
8062 MachineFunction::iterator I = MBB;
8063 ++I;
8064
8065 // Memory Reference
8066 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
8067 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
8068
8069 unsigned DstReg = MI->getOperand(0).getReg();
8070 const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
8071 assert(RC->hasType(MVT::i32) && "Invalid destination!");
8072 unsigned mainDstReg = MRI.createVirtualRegister(RC);
8073 unsigned restoreDstReg = MRI.createVirtualRegister(RC);
8074
8075 MVT PVT = getPointerTy();
8076 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
8077 "Invalid Pointer Size!");
8078 // For v = setjmp(buf), we generate
8079 //
8080 // thisMBB:
8081 // SjLjSetup mainMBB
8082 // bl mainMBB
8083 // v_restore = 1
8084 // b sinkMBB
8085 //
8086 // mainMBB:
8087 // buf[LabelOffset] = LR
8088 // v_main = 0
8089 //
8090 // sinkMBB:
8091 // v = phi(main, restore)
8092 //
8093
8094 MachineBasicBlock *thisMBB = MBB;
8095 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
8096 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
8097 MF->insert(I, mainMBB);
8098 MF->insert(I, sinkMBB);
8099
8100 MachineInstrBuilder MIB;
8101
8102 // Transfer the remainder of BB and its successor edges to sinkMBB.
8103 sinkMBB->splice(sinkMBB->begin(), MBB,
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00008104 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
Hal Finkel756810f2013-03-21 21:37:52 +00008105 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
8106
8107 // Note that the structure of the jmp_buf used here is not compatible
8108 // with that used by libc, and is not designed to be. Specifically, it
8109 // stores only those 'reserved' registers that LLVM does not otherwise
8110 // understand how to spill. Also, by convention, by the time this
8111 // intrinsic is called, Clang has already stored the frame address in the
8112 // first slot of the buffer and stack address in the third. Following the
8113 // X86 target code, we'll store the jump address in the second slot. We also
8114 // need to save the TOC pointer (R2) to handle jumps between shared
8115 // libraries, and that will be stored in the fourth slot. The thread
8116 // identifier (R13) is not affected.
8117
8118 // thisMBB:
8119 const int64_t LabelOffset = 1 * PVT.getStoreSize();
8120 const int64_t TOCOffset = 3 * PVT.getStoreSize();
Hal Finkelf05d6c72013-07-17 23:50:51 +00008121 const int64_t BPOffset = 4 * PVT.getStoreSize();
Hal Finkel756810f2013-03-21 21:37:52 +00008122
8123 // Prepare IP either in reg.
8124 const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
8125 unsigned LabelReg = MRI.createVirtualRegister(PtrRC);
8126 unsigned BufReg = MI->getOperand(1).getReg();
8127
Eric Christopherb1aaebe2014-06-12 22:38:18 +00008128 if (Subtarget.isPPC64() && Subtarget.isSVR4ABI()) {
Hal Finkele6698d52015-02-01 15:03:28 +00008129 setUsesTOCBasePtr(*MBB->getParent());
Hal Finkel756810f2013-03-21 21:37:52 +00008130 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::STD))
8131 .addReg(PPC::X2)
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00008132 .addImm(TOCOffset)
Hal Finkel756810f2013-03-21 21:37:52 +00008133 .addReg(BufReg);
Hal Finkel756810f2013-03-21 21:37:52 +00008134 MIB.setMemRefs(MMOBegin, MMOEnd);
8135 }
8136
Hal Finkelf05d6c72013-07-17 23:50:51 +00008137 // Naked functions never have a base pointer, and so we use r1. For all
8138 // other functions, this decision must be delayed until during PEI.
8139 unsigned BaseReg;
Duncan P. N. Exon Smith5bedaf932015-02-14 02:54:07 +00008140 if (MF->getFunction()->hasFnAttribute(Attribute::Naked))
Eric Christopherb1aaebe2014-06-12 22:38:18 +00008141 BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
Hal Finkelf05d6c72013-07-17 23:50:51 +00008142 else
Eric Christopherb1aaebe2014-06-12 22:38:18 +00008143 BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP;
Hal Finkelf05d6c72013-07-17 23:50:51 +00008144
8145 MIB = BuildMI(*thisMBB, MI, DL,
Eric Christopherb1aaebe2014-06-12 22:38:18 +00008146 TII->get(Subtarget.isPPC64() ? PPC::STD : PPC::STW))
Eric Christophercccae792015-01-30 22:02:31 +00008147 .addReg(BaseReg)
8148 .addImm(BPOffset)
8149 .addReg(BufReg);
Hal Finkelf05d6c72013-07-17 23:50:51 +00008150 MIB.setMemRefs(MMOBegin, MMOEnd);
8151
Hal Finkel756810f2013-03-21 21:37:52 +00008152 // Setup
Hal Finkele5680b32013-04-04 22:55:54 +00008153 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::BCLalways)).addMBB(mainMBB);
Eric Christophercccae792015-01-30 22:02:31 +00008154 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo();
Bill Wendling5e7656b2013-06-07 07:55:53 +00008155 MIB.addRegMask(TRI->getNoPreservedMask());
Hal Finkel756810f2013-03-21 21:37:52 +00008156
8157 BuildMI(*thisMBB, MI, DL, TII->get(PPC::LI), restoreDstReg).addImm(1);
8158
8159 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::EH_SjLj_Setup))
8160 .addMBB(mainMBB);
8161 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::B)).addMBB(sinkMBB);
8162
8163 thisMBB->addSuccessor(mainMBB, /* weight */ 0);
8164 thisMBB->addSuccessor(sinkMBB, /* weight */ 1);
8165
8166 // mainMBB:
8167 // mainDstReg = 0
Eric Christophercccae792015-01-30 22:02:31 +00008168 MIB =
8169 BuildMI(mainMBB, DL,
8170 TII->get(Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), LabelReg);
Hal Finkel756810f2013-03-21 21:37:52 +00008171
8172 // Store IP
Eric Christopherb1aaebe2014-06-12 22:38:18 +00008173 if (Subtarget.isPPC64()) {
Hal Finkel756810f2013-03-21 21:37:52 +00008174 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STD))
8175 .addReg(LabelReg)
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00008176 .addImm(LabelOffset)
Hal Finkel756810f2013-03-21 21:37:52 +00008177 .addReg(BufReg);
8178 } else {
8179 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STW))
8180 .addReg(LabelReg)
8181 .addImm(LabelOffset)
8182 .addReg(BufReg);
8183 }
8184
8185 MIB.setMemRefs(MMOBegin, MMOEnd);
8186
8187 BuildMI(mainMBB, DL, TII->get(PPC::LI), mainDstReg).addImm(0);
8188 mainMBB->addSuccessor(sinkMBB);
8189
8190 // sinkMBB:
8191 BuildMI(*sinkMBB, sinkMBB->begin(), DL,
8192 TII->get(PPC::PHI), DstReg)
8193 .addReg(mainDstReg).addMBB(mainMBB)
8194 .addReg(restoreDstReg).addMBB(thisMBB);
8195
8196 MI->eraseFromParent();
8197 return sinkMBB;
8198}
8199
8200MachineBasicBlock *
8201PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr *MI,
8202 MachineBasicBlock *MBB) const {
8203 DebugLoc DL = MI->getDebugLoc();
Eric Christophercccae792015-01-30 22:02:31 +00008204 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
Hal Finkel756810f2013-03-21 21:37:52 +00008205
8206 MachineFunction *MF = MBB->getParent();
8207 MachineRegisterInfo &MRI = MF->getRegInfo();
8208
8209 // Memory Reference
8210 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
8211 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
8212
8213 MVT PVT = getPointerTy();
8214 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
8215 "Invalid Pointer Size!");
8216
8217 const TargetRegisterClass *RC =
8218 (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
8219 unsigned Tmp = MRI.createVirtualRegister(RC);
8220 // Since FP is only updated here but NOT referenced, it's treated as GPR.
8221 unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31;
8222 unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1;
Eric Christophercccae792015-01-30 22:02:31 +00008223 unsigned BP =
8224 (PVT == MVT::i64)
8225 ? PPC::X30
8226 : (Subtarget.isSVR4ABI() &&
8227 MF->getTarget().getRelocationModel() == Reloc::PIC_
8228 ? PPC::R29
8229 : PPC::R30);
Hal Finkel756810f2013-03-21 21:37:52 +00008230
8231 MachineInstrBuilder MIB;
8232
8233 const int64_t LabelOffset = 1 * PVT.getStoreSize();
8234 const int64_t SPOffset = 2 * PVT.getStoreSize();
8235 const int64_t TOCOffset = 3 * PVT.getStoreSize();
Hal Finkelf05d6c72013-07-17 23:50:51 +00008236 const int64_t BPOffset = 4 * PVT.getStoreSize();
Hal Finkel756810f2013-03-21 21:37:52 +00008237
8238 unsigned BufReg = MI->getOperand(0).getReg();
8239
8240 // Reload FP (the jumped-to function may not have had a
8241 // frame pointer, and if so, then its r31 will be restored
8242 // as necessary).
8243 if (PVT == MVT::i64) {
8244 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), FP)
8245 .addImm(0)
8246 .addReg(BufReg);
8247 } else {
8248 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), FP)
8249 .addImm(0)
8250 .addReg(BufReg);
8251 }
8252 MIB.setMemRefs(MMOBegin, MMOEnd);
8253
8254 // Reload IP
8255 if (PVT == MVT::i64) {
8256 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), Tmp)
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00008257 .addImm(LabelOffset)
Hal Finkel756810f2013-03-21 21:37:52 +00008258 .addReg(BufReg);
8259 } else {
8260 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), Tmp)
8261 .addImm(LabelOffset)
8262 .addReg(BufReg);
8263 }
8264 MIB.setMemRefs(MMOBegin, MMOEnd);
8265
8266 // Reload SP
8267 if (PVT == MVT::i64) {
8268 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), SP)
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00008269 .addImm(SPOffset)
Hal Finkel756810f2013-03-21 21:37:52 +00008270 .addReg(BufReg);
8271 } else {
8272 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), SP)
8273 .addImm(SPOffset)
8274 .addReg(BufReg);
8275 }
8276 MIB.setMemRefs(MMOBegin, MMOEnd);
8277
Hal Finkelf05d6c72013-07-17 23:50:51 +00008278 // Reload BP
8279 if (PVT == MVT::i64) {
8280 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), BP)
8281 .addImm(BPOffset)
8282 .addReg(BufReg);
8283 } else {
8284 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), BP)
8285 .addImm(BPOffset)
8286 .addReg(BufReg);
8287 }
8288 MIB.setMemRefs(MMOBegin, MMOEnd);
Hal Finkel756810f2013-03-21 21:37:52 +00008289
8290 // Reload TOC
Eric Christopherb1aaebe2014-06-12 22:38:18 +00008291 if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) {
Hal Finkele6698d52015-02-01 15:03:28 +00008292 setUsesTOCBasePtr(*MBB->getParent());
Hal Finkel756810f2013-03-21 21:37:52 +00008293 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), PPC::X2)
Ulrich Weigand9d980cb2013-05-16 17:58:02 +00008294 .addImm(TOCOffset)
Hal Finkel756810f2013-03-21 21:37:52 +00008295 .addReg(BufReg);
8296
8297 MIB.setMemRefs(MMOBegin, MMOEnd);
8298 }
8299
8300 // Jump
8301 BuildMI(*MBB, MI, DL,
8302 TII->get(PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(Tmp);
8303 BuildMI(*MBB, MI, DL, TII->get(PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR));
8304
8305 MI->eraseFromParent();
8306 return MBB;
8307}
8308
Dale Johannesena32affb2008-08-28 17:53:09 +00008309MachineBasicBlock *
Evan Cheng29cfb672008-01-30 18:18:23 +00008310PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
Dan Gohman25c16532010-05-01 00:01:06 +00008311 MachineBasicBlock *BB) const {
Hal Finkel934361a2015-01-14 01:07:51 +00008312 if (MI->getOpcode() == TargetOpcode::STACKMAP ||
Hal Finkelaf519932015-01-19 07:20:27 +00008313 MI->getOpcode() == TargetOpcode::PATCHPOINT) {
8314 if (Subtarget.isPPC64() && Subtarget.isSVR4ABI() &&
8315 MI->getOpcode() == TargetOpcode::PATCHPOINT) {
8316 // Call lowering should have added an r2 operand to indicate a dependence
8317 // on the TOC base pointer value. It can't however, because there is no
8318 // way to mark the dependence as implicit there, and so the stackmap code
8319 // will confuse it with a regular operand. Instead, add the dependence
8320 // here.
Hal Finkele6698d52015-02-01 15:03:28 +00008321 setUsesTOCBasePtr(*BB->getParent());
Hal Finkelaf519932015-01-19 07:20:27 +00008322 MI->addOperand(MachineOperand::CreateReg(PPC::X2, false, true));
8323 }
8324
Hal Finkel934361a2015-01-14 01:07:51 +00008325 return emitPatchPoint(MI, BB);
Hal Finkelaf519932015-01-19 07:20:27 +00008326 }
Hal Finkel934361a2015-01-14 01:07:51 +00008327
Hal Finkel756810f2013-03-21 21:37:52 +00008328 if (MI->getOpcode() == PPC::EH_SjLj_SetJmp32 ||
8329 MI->getOpcode() == PPC::EH_SjLj_SetJmp64) {
8330 return emitEHSjLjSetJmp(MI, BB);
8331 } else if (MI->getOpcode() == PPC::EH_SjLj_LongJmp32 ||
8332 MI->getOpcode() == PPC::EH_SjLj_LongJmp64) {
8333 return emitEHSjLjLongJmp(MI, BB);
8334 }
8335
Eric Christophercccae792015-01-30 22:02:31 +00008336 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
Evan Cheng32e376f2008-07-12 02:23:19 +00008337
8338 // To "insert" these instructions we actually have to insert their
8339 // control-flow patterns.
Chris Lattner9b577f12005-08-26 21:23:58 +00008340 const BasicBlock *LLVM_BB = BB->getBasicBlock();
Dan Gohman3b460302008-07-07 23:14:23 +00008341 MachineFunction::iterator It = BB;
Chris Lattner9b577f12005-08-26 21:23:58 +00008342 ++It;
Evan Cheng32e376f2008-07-12 02:23:19 +00008343
Dan Gohman3b460302008-07-07 23:14:23 +00008344 MachineFunction *F = BB->getParent();
Evan Cheng32e376f2008-07-12 02:23:19 +00008345
Eric Christopherb1aaebe2014-06-12 22:38:18 +00008346 if (Subtarget.hasISEL() && (MI->getOpcode() == PPC::SELECT_CC_I4 ||
Eric Christophercccae792015-01-30 22:02:31 +00008347 MI->getOpcode() == PPC::SELECT_CC_I8 ||
8348 MI->getOpcode() == PPC::SELECT_I4 ||
8349 MI->getOpcode() == PPC::SELECT_I8)) {
Hal Finkeled6a2852013-04-05 23:29:01 +00008350 SmallVector<MachineOperand, 2> Cond;
Hal Finkel940ab932014-02-28 00:27:01 +00008351 if (MI->getOpcode() == PPC::SELECT_CC_I4 ||
8352 MI->getOpcode() == PPC::SELECT_CC_I8)
8353 Cond.push_back(MI->getOperand(4));
8354 else
8355 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
Hal Finkeled6a2852013-04-05 23:29:01 +00008356 Cond.push_back(MI->getOperand(1));
8357
Hal Finkel460e94d2012-06-22 23:10:08 +00008358 DebugLoc dl = MI->getDebugLoc();
Bill Wendling5e7656b2013-06-07 07:55:53 +00008359 TII->insertSelect(*BB, MI, dl, MI->getOperand(0).getReg(),
8360 Cond, MI->getOperand(2).getReg(),
8361 MI->getOperand(3).getReg());
Hal Finkel460e94d2012-06-22 23:10:08 +00008362 } else if (MI->getOpcode() == PPC::SELECT_CC_I4 ||
8363 MI->getOpcode() == PPC::SELECT_CC_I8 ||
8364 MI->getOpcode() == PPC::SELECT_CC_F4 ||
8365 MI->getOpcode() == PPC::SELECT_CC_F8 ||
Hal Finkelc93a9a22015-02-25 01:06:45 +00008366 MI->getOpcode() == PPC::SELECT_CC_QFRC ||
8367 MI->getOpcode() == PPC::SELECT_CC_QSRC ||
8368 MI->getOpcode() == PPC::SELECT_CC_QBRC ||
Hal Finkel940ab932014-02-28 00:27:01 +00008369 MI->getOpcode() == PPC::SELECT_CC_VRRC ||
Bill Schmidt9c54bbd2014-10-22 16:58:20 +00008370 MI->getOpcode() == PPC::SELECT_CC_VSFRC ||
Bill Schmidt61e65232014-10-22 13:13:40 +00008371 MI->getOpcode() == PPC::SELECT_CC_VSRC ||
Hal Finkel940ab932014-02-28 00:27:01 +00008372 MI->getOpcode() == PPC::SELECT_I4 ||
8373 MI->getOpcode() == PPC::SELECT_I8 ||
8374 MI->getOpcode() == PPC::SELECT_F4 ||
8375 MI->getOpcode() == PPC::SELECT_F8 ||
Hal Finkelc93a9a22015-02-25 01:06:45 +00008376 MI->getOpcode() == PPC::SELECT_QFRC ||
8377 MI->getOpcode() == PPC::SELECT_QSRC ||
8378 MI->getOpcode() == PPC::SELECT_QBRC ||
Bill Schmidt61e65232014-10-22 13:13:40 +00008379 MI->getOpcode() == PPC::SELECT_VRRC ||
Bill Schmidt9c54bbd2014-10-22 16:58:20 +00008380 MI->getOpcode() == PPC::SELECT_VSFRC ||
Bill Schmidt61e65232014-10-22 13:13:40 +00008381 MI->getOpcode() == PPC::SELECT_VSRC) {
Evan Cheng32e376f2008-07-12 02:23:19 +00008382 // The incoming instruction knows the destination vreg to set, the
8383 // condition code register to branch on, the true/false values to
8384 // select between, and a branch opcode to use.
8385
8386 // thisMBB:
8387 // ...
8388 // TrueVal = ...
8389 // cmpTY ccX, r1, r2
8390 // bCC copy1MBB
8391 // fallthrough --> copy0MBB
8392 MachineBasicBlock *thisMBB = BB;
8393 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
8394 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008395 DebugLoc dl = MI->getDebugLoc();
Evan Cheng32e376f2008-07-12 02:23:19 +00008396 F->insert(It, copy0MBB);
8397 F->insert(It, sinkMBB);
Dan Gohman34396292010-07-06 20:24:04 +00008398
8399 // Transfer the remainder of BB and its successor edges to sinkMBB.
8400 sinkMBB->splice(sinkMBB->begin(), BB,
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00008401 std::next(MachineBasicBlock::iterator(MI)), BB->end());
Dan Gohman34396292010-07-06 20:24:04 +00008402 sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
8403
Evan Cheng32e376f2008-07-12 02:23:19 +00008404 // Next, add the true and fallthrough blocks as its successors.
8405 BB->addSuccessor(copy0MBB);
8406 BB->addSuccessor(sinkMBB);
Scott Michelcf0da6c2009-02-17 22:15:04 +00008407
Hal Finkel940ab932014-02-28 00:27:01 +00008408 if (MI->getOpcode() == PPC::SELECT_I4 ||
8409 MI->getOpcode() == PPC::SELECT_I8 ||
8410 MI->getOpcode() == PPC::SELECT_F4 ||
8411 MI->getOpcode() == PPC::SELECT_F8 ||
Hal Finkelc93a9a22015-02-25 01:06:45 +00008412 MI->getOpcode() == PPC::SELECT_QFRC ||
8413 MI->getOpcode() == PPC::SELECT_QSRC ||
8414 MI->getOpcode() == PPC::SELECT_QBRC ||
Bill Schmidt61e65232014-10-22 13:13:40 +00008415 MI->getOpcode() == PPC::SELECT_VRRC ||
Bill Schmidt9c54bbd2014-10-22 16:58:20 +00008416 MI->getOpcode() == PPC::SELECT_VSFRC ||
Bill Schmidt61e65232014-10-22 13:13:40 +00008417 MI->getOpcode() == PPC::SELECT_VSRC) {
Hal Finkel940ab932014-02-28 00:27:01 +00008418 BuildMI(BB, dl, TII->get(PPC::BC))
8419 .addReg(MI->getOperand(1).getReg()).addMBB(sinkMBB);
8420 } else {
8421 unsigned SelectPred = MI->getOperand(4).getImm();
8422 BuildMI(BB, dl, TII->get(PPC::BCC))
8423 .addImm(SelectPred).addReg(MI->getOperand(1).getReg()).addMBB(sinkMBB);
8424 }
Dan Gohman34396292010-07-06 20:24:04 +00008425
Evan Cheng32e376f2008-07-12 02:23:19 +00008426 // copy0MBB:
8427 // %FalseValue = ...
8428 // # fallthrough to sinkMBB
8429 BB = copy0MBB;
Scott Michelcf0da6c2009-02-17 22:15:04 +00008430
Evan Cheng32e376f2008-07-12 02:23:19 +00008431 // Update machine-CFG edges
8432 BB->addSuccessor(sinkMBB);
Scott Michelcf0da6c2009-02-17 22:15:04 +00008433
Evan Cheng32e376f2008-07-12 02:23:19 +00008434 // sinkMBB:
8435 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
8436 // ...
8437 BB = sinkMBB;
Dan Gohman34396292010-07-06 20:24:04 +00008438 BuildMI(*BB, BB->begin(), dl,
8439 TII->get(PPC::PHI), MI->getOperand(0).getReg())
Evan Cheng32e376f2008-07-12 02:23:19 +00008440 .addReg(MI->getOperand(3).getReg()).addMBB(copy0MBB)
8441 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
Hal Finkelbbdee932014-12-02 22:01:00 +00008442 } else if (MI->getOpcode() == PPC::ReadTB) {
8443 // To read the 64-bit time-base register on a 32-bit target, we read the
8444 // two halves. Should the counter have wrapped while it was being read, we
8445 // need to try again.
8446 // ...
8447 // readLoop:
8448 // mfspr Rx,TBU # load from TBU
8449 // mfspr Ry,TB # load from TB
8450 // mfspr Rz,TBU # load from TBU
8451 // cmpw crX,Rx,Rz # check if ‘old’=’new’
8452 // bne readLoop # branch if they're not equal
8453 // ...
8454
8455 MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(LLVM_BB);
8456 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
8457 DebugLoc dl = MI->getDebugLoc();
8458 F->insert(It, readMBB);
8459 F->insert(It, sinkMBB);
8460
8461 // Transfer the remainder of BB and its successor edges to sinkMBB.
8462 sinkMBB->splice(sinkMBB->begin(), BB,
8463 std::next(MachineBasicBlock::iterator(MI)), BB->end());
8464 sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
8465
8466 BB->addSuccessor(readMBB);
8467 BB = readMBB;
8468
8469 MachineRegisterInfo &RegInfo = F->getRegInfo();
8470 unsigned ReadAgainReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass);
8471 unsigned LoReg = MI->getOperand(0).getReg();
8472 unsigned HiReg = MI->getOperand(1).getReg();
8473
8474 BuildMI(BB, dl, TII->get(PPC::MFSPR), HiReg).addImm(269);
8475 BuildMI(BB, dl, TII->get(PPC::MFSPR), LoReg).addImm(268);
8476 BuildMI(BB, dl, TII->get(PPC::MFSPR), ReadAgainReg).addImm(269);
8477
8478 unsigned CmpReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass);
8479
8480 BuildMI(BB, dl, TII->get(PPC::CMPW), CmpReg)
8481 .addReg(HiReg).addReg(ReadAgainReg);
8482 BuildMI(BB, dl, TII->get(PPC::BCC))
8483 .addImm(PPC::PRED_NE).addReg(CmpReg).addMBB(readMBB);
8484
8485 BB->addSuccessor(readMBB);
8486 BB->addSuccessor(sinkMBB);
Evan Cheng32e376f2008-07-12 02:23:19 +00008487 }
Dale Johannesena32affb2008-08-28 17:53:09 +00008488 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I8)
8489 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::ADD4);
8490 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I16)
8491 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::ADD4);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008492 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I32)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008493 BB = EmitAtomicBinary(MI, BB, 4, PPC::ADD4);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008494 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_ADD_I64)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008495 BB = EmitAtomicBinary(MI, BB, 8, PPC::ADD8);
Dale Johannesena32affb2008-08-28 17:53:09 +00008496
8497 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I8)
8498 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::AND);
8499 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I16)
8500 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::AND);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008501 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I32)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008502 BB = EmitAtomicBinary(MI, BB, 4, PPC::AND);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008503 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_AND_I64)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008504 BB = EmitAtomicBinary(MI, BB, 8, PPC::AND8);
Dale Johannesena32affb2008-08-28 17:53:09 +00008505
8506 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I8)
8507 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::OR);
8508 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I16)
8509 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::OR);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008510 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I32)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008511 BB = EmitAtomicBinary(MI, BB, 4, PPC::OR);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008512 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_OR_I64)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008513 BB = EmitAtomicBinary(MI, BB, 8, PPC::OR8);
Dale Johannesena32affb2008-08-28 17:53:09 +00008514
8515 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I8)
8516 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::XOR);
8517 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I16)
8518 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::XOR);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008519 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I32)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008520 BB = EmitAtomicBinary(MI, BB, 4, PPC::XOR);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008521 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_XOR_I64)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008522 BB = EmitAtomicBinary(MI, BB, 8, PPC::XOR8);
Dale Johannesena32affb2008-08-28 17:53:09 +00008523
8524 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I8)
Ulrich Weigand862d8b82014-07-08 16:16:02 +00008525 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::NAND);
Dale Johannesena32affb2008-08-28 17:53:09 +00008526 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I16)
Ulrich Weigand862d8b82014-07-08 16:16:02 +00008527 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::NAND);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008528 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I32)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008529 BB = EmitAtomicBinary(MI, BB, 4, PPC::NAND);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008530 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_NAND_I64)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008531 BB = EmitAtomicBinary(MI, BB, 8, PPC::NAND8);
Dale Johannesena32affb2008-08-28 17:53:09 +00008532
8533 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I8)
8534 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::SUBF);
8535 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I16)
8536 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::SUBF);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008537 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I32)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008538 BB = EmitAtomicBinary(MI, BB, 4, PPC::SUBF);
Dale Johannesend4eb0522008-08-25 22:34:37 +00008539 else if (MI->getOpcode() == PPC::ATOMIC_LOAD_SUB_I64)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008540 BB = EmitAtomicBinary(MI, BB, 8, PPC::SUBF8);
Dale Johannesena32affb2008-08-28 17:53:09 +00008541
Dale Johannesenf0a88d62008-08-29 18:29:46 +00008542 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I8)
8543 BB = EmitPartwordAtomicBinary(MI, BB, true, 0);
8544 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I16)
8545 BB = EmitPartwordAtomicBinary(MI, BB, false, 0);
8546 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I32)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008547 BB = EmitAtomicBinary(MI, BB, 4, 0);
Dale Johannesenf0a88d62008-08-29 18:29:46 +00008548 else if (MI->getOpcode() == PPC::ATOMIC_SWAP_I64)
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008549 BB = EmitAtomicBinary(MI, BB, 8, 0);
Dale Johannesenf0a88d62008-08-29 18:29:46 +00008550
Evan Cheng32e376f2008-07-12 02:23:19 +00008551 else if (MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 ||
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008552 MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 ||
8553 (Subtarget.hasPartwordAtomics() &&
8554 MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I8) ||
8555 (Subtarget.hasPartwordAtomics() &&
8556 MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I16)) {
Evan Cheng32e376f2008-07-12 02:23:19 +00008557 bool is64bit = MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I64;
8558
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008559 auto LoadMnemonic = PPC::LDARX;
8560 auto StoreMnemonic = PPC::STDCX;
8561 switch(MI->getOpcode()) {
8562 default:
8563 llvm_unreachable("Compare and swap of unknown size");
8564 case PPC::ATOMIC_CMP_SWAP_I8:
8565 LoadMnemonic = PPC::LBARX;
8566 StoreMnemonic = PPC::STBCX;
8567 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
8568 break;
8569 case PPC::ATOMIC_CMP_SWAP_I16:
8570 LoadMnemonic = PPC::LHARX;
8571 StoreMnemonic = PPC::STHCX;
8572 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
8573 break;
8574 case PPC::ATOMIC_CMP_SWAP_I32:
8575 LoadMnemonic = PPC::LWARX;
8576 StoreMnemonic = PPC::STWCX;
8577 break;
8578 case PPC::ATOMIC_CMP_SWAP_I64:
8579 LoadMnemonic = PPC::LDARX;
8580 StoreMnemonic = PPC::STDCX;
8581 break;
8582 }
Evan Cheng32e376f2008-07-12 02:23:19 +00008583 unsigned dest = MI->getOperand(0).getReg();
8584 unsigned ptrA = MI->getOperand(1).getReg();
8585 unsigned ptrB = MI->getOperand(2).getReg();
8586 unsigned oldval = MI->getOperand(3).getReg();
8587 unsigned newval = MI->getOperand(4).getReg();
Dale Johannesene9f623e2009-02-13 02:27:39 +00008588 DebugLoc dl = MI->getDebugLoc();
Evan Cheng32e376f2008-07-12 02:23:19 +00008589
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008590 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB);
8591 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB);
8592 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB);
Evan Cheng32e376f2008-07-12 02:23:19 +00008593 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB);
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008594 F->insert(It, loop1MBB);
8595 F->insert(It, loop2MBB);
8596 F->insert(It, midMBB);
Evan Cheng32e376f2008-07-12 02:23:19 +00008597 F->insert(It, exitMBB);
Dan Gohman34396292010-07-06 20:24:04 +00008598 exitMBB->splice(exitMBB->begin(), BB,
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00008599 std::next(MachineBasicBlock::iterator(MI)), BB->end());
Dan Gohman34396292010-07-06 20:24:04 +00008600 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
Evan Cheng32e376f2008-07-12 02:23:19 +00008601
8602 // thisMBB:
8603 // ...
8604 // fallthrough --> loopMBB
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008605 BB->addSuccessor(loop1MBB);
Evan Cheng32e376f2008-07-12 02:23:19 +00008606
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008607 // loop1MBB:
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008608 // l[bhwd]arx dest, ptr
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008609 // cmp[wd] dest, oldval
8610 // bne- midMBB
8611 // loop2MBB:
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008612 // st[bhwd]cx. newval, ptr
Evan Cheng32e376f2008-07-12 02:23:19 +00008613 // bne- loopMBB
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008614 // b exitBB
8615 // midMBB:
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008616 // st[bhwd]cx. dest, ptr
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008617 // exitBB:
8618 BB = loop1MBB;
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008619 BuildMI(BB, dl, TII->get(LoadMnemonic), dest)
Evan Cheng32e376f2008-07-12 02:23:19 +00008620 .addReg(ptrA).addReg(ptrB);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008621 BuildMI(BB, dl, TII->get(is64bit ? PPC::CMPD : PPC::CMPW), PPC::CR0)
Evan Cheng32e376f2008-07-12 02:23:19 +00008622 .addReg(oldval).addReg(dest);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008623 BuildMI(BB, dl, TII->get(PPC::BCC))
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008624 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(midMBB);
8625 BB->addSuccessor(loop2MBB);
8626 BB->addSuccessor(midMBB);
8627
8628 BB = loop2MBB;
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008629 BuildMI(BB, dl, TII->get(StoreMnemonic))
Evan Cheng32e376f2008-07-12 02:23:19 +00008630 .addReg(newval).addReg(ptrA).addReg(ptrB);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008631 BuildMI(BB, dl, TII->get(PPC::BCC))
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008632 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loop1MBB);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008633 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB);
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008634 BB->addSuccessor(loop1MBB);
Evan Cheng32e376f2008-07-12 02:23:19 +00008635 BB->addSuccessor(exitMBB);
Scott Michelcf0da6c2009-02-17 22:15:04 +00008636
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008637 BB = midMBB;
Nemanja Ivanovic0adf26b2015-03-10 20:51:07 +00008638 BuildMI(BB, dl, TII->get(StoreMnemonic))
Dale Johannesen166d6cb2008-08-25 18:53:26 +00008639 .addReg(dest).addReg(ptrA).addReg(ptrB);
8640 BB->addSuccessor(exitMBB);
8641
Evan Cheng32e376f2008-07-12 02:23:19 +00008642 // exitMBB:
8643 // ...
8644 BB = exitMBB;
Dale Johannesen340d2642008-08-30 00:08:53 +00008645 } else if (MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 ||
8646 MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I16) {
8647 // We must use 64-bit registers for addresses when targeting 64-bit,
8648 // since we're actually doing arithmetic on them. Other registers
8649 // can be 32-bit.
Eric Christopherb1aaebe2014-06-12 22:38:18 +00008650 bool is64bit = Subtarget.isPPC64();
Dale Johannesen340d2642008-08-30 00:08:53 +00008651 bool is8bit = MI->getOpcode() == PPC::ATOMIC_CMP_SWAP_I8;
8652
8653 unsigned dest = MI->getOperand(0).getReg();
8654 unsigned ptrA = MI->getOperand(1).getReg();
8655 unsigned ptrB = MI->getOperand(2).getReg();
8656 unsigned oldval = MI->getOperand(3).getReg();
8657 unsigned newval = MI->getOperand(4).getReg();
Dale Johannesene9f623e2009-02-13 02:27:39 +00008658 DebugLoc dl = MI->getDebugLoc();
Dale Johannesen340d2642008-08-30 00:08:53 +00008659
8660 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB);
8661 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB);
8662 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB);
8663 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB);
8664 F->insert(It, loop1MBB);
8665 F->insert(It, loop2MBB);
8666 F->insert(It, midMBB);
8667 F->insert(It, exitMBB);
Dan Gohman34396292010-07-06 20:24:04 +00008668 exitMBB->splice(exitMBB->begin(), BB,
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00008669 std::next(MachineBasicBlock::iterator(MI)), BB->end());
Dan Gohman34396292010-07-06 20:24:04 +00008670 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
Dale Johannesen340d2642008-08-30 00:08:53 +00008671
8672 MachineRegisterInfo &RegInfo = F->getRegInfo();
Craig Topper61e88f42014-11-21 05:58:21 +00008673 const TargetRegisterClass *RC = is64bit ? &PPC::G8RCRegClass
8674 : &PPC::GPRCRegClass;
Dale Johannesen340d2642008-08-30 00:08:53 +00008675 unsigned PtrReg = RegInfo.createVirtualRegister(RC);
8676 unsigned Shift1Reg = RegInfo.createVirtualRegister(RC);
8677 unsigned ShiftReg = RegInfo.createVirtualRegister(RC);
8678 unsigned NewVal2Reg = RegInfo.createVirtualRegister(RC);
8679 unsigned NewVal3Reg = RegInfo.createVirtualRegister(RC);
8680 unsigned OldVal2Reg = RegInfo.createVirtualRegister(RC);
8681 unsigned OldVal3Reg = RegInfo.createVirtualRegister(RC);
8682 unsigned MaskReg = RegInfo.createVirtualRegister(RC);
8683 unsigned Mask2Reg = RegInfo.createVirtualRegister(RC);
8684 unsigned Mask3Reg = RegInfo.createVirtualRegister(RC);
8685 unsigned Tmp2Reg = RegInfo.createVirtualRegister(RC);
8686 unsigned Tmp4Reg = RegInfo.createVirtualRegister(RC);
8687 unsigned TmpDestReg = RegInfo.createVirtualRegister(RC);
8688 unsigned Ptr1Reg;
8689 unsigned TmpReg = RegInfo.createVirtualRegister(RC);
Hal Finkelf70c41e2013-03-21 23:45:03 +00008690 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
Dale Johannesen340d2642008-08-30 00:08:53 +00008691 // thisMBB:
8692 // ...
8693 // fallthrough --> loopMBB
8694 BB->addSuccessor(loop1MBB);
8695
8696 // The 4-byte load must be aligned, while a char or short may be
8697 // anywhere in the word. Hence all this nasty bookkeeping code.
8698 // add ptr1, ptrA, ptrB [copy if ptrA==0]
8699 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27]
Dale Johannesenbc698292008-09-02 20:30:23 +00008700 // xori shift, shift1, 24 [16]
Dale Johannesen340d2642008-08-30 00:08:53 +00008701 // rlwinm ptr, ptr1, 0, 0, 29
8702 // slw newval2, newval, shift
8703 // slw oldval2, oldval,shift
8704 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535]
8705 // slw mask, mask2, shift
8706 // and newval3, newval2, mask
8707 // and oldval3, oldval2, mask
8708 // loop1MBB:
8709 // lwarx tmpDest, ptr
8710 // and tmp, tmpDest, mask
8711 // cmpw tmp, oldval3
8712 // bne- midMBB
8713 // loop2MBB:
8714 // andc tmp2, tmpDest, mask
8715 // or tmp4, tmp2, newval3
8716 // stwcx. tmp4, ptr
8717 // bne- loop1MBB
8718 // b exitBB
8719 // midMBB:
8720 // stwcx. tmpDest, ptr
8721 // exitBB:
8722 // srw dest, tmpDest, shift
Jakob Stoklund Olesen7067bff2011-04-04 17:07:06 +00008723 if (ptrA != ZeroReg) {
Dale Johannesen340d2642008-08-30 00:08:53 +00008724 Ptr1Reg = RegInfo.createVirtualRegister(RC);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008725 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg)
Dale Johannesen340d2642008-08-30 00:08:53 +00008726 .addReg(ptrA).addReg(ptrB);
8727 } else {
8728 Ptr1Reg = ptrB;
8729 }
Dale Johannesene9f623e2009-02-13 02:27:39 +00008730 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg).addReg(Ptr1Reg)
Dale Johannesen340d2642008-08-30 00:08:53 +00008731 .addImm(3).addImm(27).addImm(is8bit ? 28 : 27);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008732 BuildMI(BB, dl, TII->get(is64bit ? PPC::XORI8 : PPC::XORI), ShiftReg)
Dale Johannesen340d2642008-08-30 00:08:53 +00008733 .addReg(Shift1Reg).addImm(is8bit ? 24 : 16);
8734 if (is64bit)
Dale Johannesene9f623e2009-02-13 02:27:39 +00008735 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg)
Dale Johannesen340d2642008-08-30 00:08:53 +00008736 .addReg(Ptr1Reg).addImm(0).addImm(61);
8737 else
Dale Johannesene9f623e2009-02-13 02:27:39 +00008738 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg)
Dale Johannesen340d2642008-08-30 00:08:53 +00008739 .addReg(Ptr1Reg).addImm(0).addImm(0).addImm(29);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008740 BuildMI(BB, dl, TII->get(PPC::SLW), NewVal2Reg)
Dale Johannesen340d2642008-08-30 00:08:53 +00008741 .addReg(newval).addReg(ShiftReg);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008742 BuildMI(BB, dl, TII->get(PPC::SLW), OldVal2Reg)
Dale Johannesen340d2642008-08-30 00:08:53 +00008743 .addReg(oldval).addReg(ShiftReg);
8744 if (is8bit)
Dale Johannesene9f623e2009-02-13 02:27:39 +00008745 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255);
Dale Johannesen340d2642008-08-30 00:08:53 +00008746 else {
Dale Johannesene9f623e2009-02-13 02:27:39 +00008747 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0);
8748 BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg)
8749 .addReg(Mask3Reg).addImm(65535);
Dale Johannesen340d2642008-08-30 00:08:53 +00008750 }
Dale Johannesene9f623e2009-02-13 02:27:39 +00008751 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg)
Dale Johannesen340d2642008-08-30 00:08:53 +00008752 .addReg(Mask2Reg).addReg(ShiftReg);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008753 BuildMI(BB, dl, TII->get(PPC::AND), NewVal3Reg)
Dale Johannesen340d2642008-08-30 00:08:53 +00008754 .addReg(NewVal2Reg).addReg(MaskReg);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008755 BuildMI(BB, dl, TII->get(PPC::AND), OldVal3Reg)
Dale Johannesen340d2642008-08-30 00:08:53 +00008756 .addReg(OldVal2Reg).addReg(MaskReg);
8757
8758 BB = loop1MBB;
Dale Johannesene9f623e2009-02-13 02:27:39 +00008759 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg)
Jakob Stoklund Olesen7067bff2011-04-04 17:07:06 +00008760 .addReg(ZeroReg).addReg(PtrReg);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008761 BuildMI(BB, dl, TII->get(PPC::AND),TmpReg)
8762 .addReg(TmpDestReg).addReg(MaskReg);
8763 BuildMI(BB, dl, TII->get(PPC::CMPW), PPC::CR0)
Dale Johannesen340d2642008-08-30 00:08:53 +00008764 .addReg(TmpReg).addReg(OldVal3Reg);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008765 BuildMI(BB, dl, TII->get(PPC::BCC))
Dale Johannesen340d2642008-08-30 00:08:53 +00008766 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(midMBB);
8767 BB->addSuccessor(loop2MBB);
8768 BB->addSuccessor(midMBB);
8769
8770 BB = loop2MBB;
Dale Johannesene9f623e2009-02-13 02:27:39 +00008771 BuildMI(BB, dl, TII->get(PPC::ANDC),Tmp2Reg)
8772 .addReg(TmpDestReg).addReg(MaskReg);
8773 BuildMI(BB, dl, TII->get(PPC::OR),Tmp4Reg)
8774 .addReg(Tmp2Reg).addReg(NewVal3Reg);
8775 BuildMI(BB, dl, TII->get(PPC::STWCX)).addReg(Tmp4Reg)
Jakob Stoklund Olesen7067bff2011-04-04 17:07:06 +00008776 .addReg(ZeroReg).addReg(PtrReg);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008777 BuildMI(BB, dl, TII->get(PPC::BCC))
Dale Johannesen340d2642008-08-30 00:08:53 +00008778 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loop1MBB);
Dale Johannesene9f623e2009-02-13 02:27:39 +00008779 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB);
Dale Johannesen340d2642008-08-30 00:08:53 +00008780 BB->addSuccessor(loop1MBB);
8781 BB->addSuccessor(exitMBB);
Scott Michelcf0da6c2009-02-17 22:15:04 +00008782
Dale Johannesen340d2642008-08-30 00:08:53 +00008783 BB = midMBB;
Dale Johannesene9f623e2009-02-13 02:27:39 +00008784 BuildMI(BB, dl, TII->get(PPC::STWCX)).addReg(TmpDestReg)
Jakob Stoklund Olesen7067bff2011-04-04 17:07:06 +00008785 .addReg(ZeroReg).addReg(PtrReg);
Dale Johannesen340d2642008-08-30 00:08:53 +00008786 BB->addSuccessor(exitMBB);
8787
8788 // exitMBB:
8789 // ...
8790 BB = exitMBB;
Jakob Stoklund Olesen13ce2362011-04-04 17:57:29 +00008791 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW),dest).addReg(TmpReg)
8792 .addReg(ShiftReg);
Ulrich Weigand874fc622013-03-26 10:56:22 +00008793 } else if (MI->getOpcode() == PPC::FADDrtz) {
8794 // This pseudo performs an FADD with rounding mode temporarily forced
8795 // to round-to-zero. We emit this via custom inserter since the FPSCR
8796 // is not modeled at the SelectionDAG level.
8797 unsigned Dest = MI->getOperand(0).getReg();
8798 unsigned Src1 = MI->getOperand(1).getReg();
8799 unsigned Src2 = MI->getOperand(2).getReg();
8800 DebugLoc dl = MI->getDebugLoc();
8801
8802 MachineRegisterInfo &RegInfo = F->getRegInfo();
8803 unsigned MFFSReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass);
8804
8805 // Save FPSCR value.
8806 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), MFFSReg);
8807
8808 // Set rounding mode to round-to-zero.
8809 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1)).addImm(31);
8810 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0)).addImm(30);
8811
8812 // Perform addition.
8813 BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest).addReg(Src1).addReg(Src2);
8814
8815 // Restore FPSCR value.
Hal Finkel64202162015-01-15 01:00:53 +00008816 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSFb)).addImm(1).addReg(MFFSReg);
Hal Finkel940ab932014-02-28 00:27:01 +00008817 } else if (MI->getOpcode() == PPC::ANDIo_1_EQ_BIT ||
8818 MI->getOpcode() == PPC::ANDIo_1_GT_BIT ||
8819 MI->getOpcode() == PPC::ANDIo_1_EQ_BIT8 ||
8820 MI->getOpcode() == PPC::ANDIo_1_GT_BIT8) {
8821 unsigned Opcode = (MI->getOpcode() == PPC::ANDIo_1_EQ_BIT8 ||
8822 MI->getOpcode() == PPC::ANDIo_1_GT_BIT8) ?
8823 PPC::ANDIo8 : PPC::ANDIo;
8824 bool isEQ = (MI->getOpcode() == PPC::ANDIo_1_EQ_BIT ||
8825 MI->getOpcode() == PPC::ANDIo_1_EQ_BIT8);
8826
8827 MachineRegisterInfo &RegInfo = F->getRegInfo();
8828 unsigned Dest = RegInfo.createVirtualRegister(Opcode == PPC::ANDIo ?
8829 &PPC::GPRCRegClass :
8830 &PPC::G8RCRegClass);
8831
8832 DebugLoc dl = MI->getDebugLoc();
8833 BuildMI(*BB, MI, dl, TII->get(Opcode), Dest)
8834 .addReg(MI->getOperand(1).getReg()).addImm(1);
8835 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY),
8836 MI->getOperand(0).getReg())
8837 .addReg(isEQ ? PPC::CR0EQ : PPC::CR0GT);
Kit Barton535e69d2015-03-25 19:36:23 +00008838 } else if (MI->getOpcode() == PPC::TCHECK_RET) {
8839 DebugLoc Dl = MI->getDebugLoc();
8840 MachineRegisterInfo &RegInfo = F->getRegInfo();
8841 unsigned CRReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass);
8842 BuildMI(*BB, MI, Dl, TII->get(PPC::TCHECK), CRReg);
8843 return BB;
Dale Johannesen340d2642008-08-30 00:08:53 +00008844 } else {
Torok Edwinfbcc6632009-07-14 16:55:14 +00008845 llvm_unreachable("Unexpected instr type to insert");
Evan Cheng32e376f2008-07-12 02:23:19 +00008846 }
Chris Lattner9b577f12005-08-26 21:23:58 +00008847
Dan Gohman34396292010-07-06 20:24:04 +00008848 MI->eraseFromParent(); // The pseudo instruction is gone now.
Chris Lattner9b577f12005-08-26 21:23:58 +00008849 return BB;
8850}
8851
Chris Lattner4211ca92006-04-14 06:01:58 +00008852//===----------------------------------------------------------------------===//
8853// Target Optimization Hooks
8854//===----------------------------------------------------------------------===//
8855
Sanjay Patel8fde95c2014-09-30 20:28:48 +00008856SDValue PPCTargetLowering::getRsqrtEstimate(SDValue Operand,
8857 DAGCombinerInfo &DCI,
Sanjay Patel957efc232014-10-24 17:02:16 +00008858 unsigned &RefinementSteps,
8859 bool &UseOneConstNR) const {
Sanjay Patelbdf1e382014-09-26 23:01:47 +00008860 EVT VT = Operand.getValueType();
Sanjay Patel8fde95c2014-09-30 20:28:48 +00008861 if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) ||
Eric Christophercccae792015-01-30 22:02:31 +00008862 (VT == MVT::f64 && Subtarget.hasFRSQRTE()) ||
Sanjay Patel8fde95c2014-09-30 20:28:48 +00008863 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
Hal Finkelc93a9a22015-02-25 01:06:45 +00008864 (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
8865 (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
8866 (VT == MVT::v4f64 && Subtarget.hasQPX())) {
Hal Finkel2e103312013-04-03 04:01:11 +00008867 // Convergence is quadratic, so we essentially double the number of digits
Sanjay Patelbdf1e382014-09-26 23:01:47 +00008868 // correct after every iteration. For both FRE and FRSQRTE, the minimum
8869 // architected relative accuracy is 2^-5. When hasRecipPrec(), this is
8870 // 2^-14. IEEE float has 23 digits and double has 52 digits.
8871 RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3;
Hal Finkelb0c810f2013-04-03 17:44:56 +00008872 if (VT.getScalarType() == MVT::f64)
Sanjay Patelbdf1e382014-09-26 23:01:47 +00008873 ++RefinementSteps;
Sanjay Patel957efc232014-10-24 17:02:16 +00008874 UseOneConstNR = true;
Sanjay Patel8fde95c2014-09-30 20:28:48 +00008875 return DCI.DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand);
Hal Finkel2e103312013-04-03 04:01:11 +00008876 }
Sanjay Patel8fde95c2014-09-30 20:28:48 +00008877 return SDValue();
8878}
8879
8880SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand,
8881 DAGCombinerInfo &DCI,
8882 unsigned &RefinementSteps) const {
8883 EVT VT = Operand.getValueType();
8884 if ((VT == MVT::f32 && Subtarget.hasFRES()) ||
Eric Christophercccae792015-01-30 22:02:31 +00008885 (VT == MVT::f64 && Subtarget.hasFRE()) ||
Sanjay Patel8fde95c2014-09-30 20:28:48 +00008886 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
Hal Finkelc93a9a22015-02-25 01:06:45 +00008887 (VT == MVT::v2f64 && Subtarget.hasVSX()) ||
8888 (VT == MVT::v4f32 && Subtarget.hasQPX()) ||
8889 (VT == MVT::v4f64 && Subtarget.hasQPX())) {
Sanjay Patel8fde95c2014-09-30 20:28:48 +00008890 // Convergence is quadratic, so we essentially double the number of digits
8891 // correct after every iteration. For both FRE and FRSQRTE, the minimum
8892 // architected relative accuracy is 2^-5. When hasRecipPrec(), this is
8893 // 2^-14. IEEE float has 23 digits and double has 52 digits.
8894 RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3;
8895 if (VT.getScalarType() == MVT::f64)
8896 ++RefinementSteps;
8897 return DCI.DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand);
8898 }
8899 return SDValue();
Hal Finkel2e103312013-04-03 04:01:11 +00008900}
8901
Hal Finkel360f2132014-11-24 23:45:21 +00008902bool PPCTargetLowering::combineRepeatedFPDivisors(unsigned NumUsers) const {
8903 // Note: This functionality is used only when unsafe-fp-math is enabled, and
8904 // on cores with reciprocal estimates (which are used when unsafe-fp-math is
8905 // enabled for division), this functionality is redundant with the default
8906 // combiner logic (once the division -> reciprocal/multiply transformation
8907 // has taken place). As a result, this matters more for older cores than for
8908 // newer ones.
8909
8910 // Combine multiple FDIVs with the same divisor into multiple FMULs by the
8911 // reciprocal if there are two or more FDIVs (for embedded cores with only
8912 // one FP pipeline) for three or more FDIVs (for generic OOO cores).
8913 switch (Subtarget.getDarwinDirective()) {
8914 default:
8915 return NumUsers > 2;
8916 case PPC::DIR_440:
8917 case PPC::DIR_A2:
8918 case PPC::DIR_E500mc:
8919 case PPC::DIR_E5500:
8920 return NumUsers > 1;
8921 }
8922}
8923
Hal Finkel3604bf72014-08-01 01:02:01 +00008924static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base,
Hal Finkel8ebfe6c2013-05-27 02:06:39 +00008925 unsigned Bytes, int Dist,
8926 SelectionDAG &DAG) {
Hal Finkel8ebfe6c2013-05-27 02:06:39 +00008927 if (VT.getSizeInBits() / 8 != Bytes)
8928 return false;
8929
Hal Finkel8ebfe6c2013-05-27 02:06:39 +00008930 SDValue BaseLoc = Base->getBasePtr();
8931 if (Loc.getOpcode() == ISD::FrameIndex) {
8932 if (BaseLoc.getOpcode() != ISD::FrameIndex)
8933 return false;
8934 const MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
8935 int FI = cast<FrameIndexSDNode>(Loc)->getIndex();
8936 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex();
8937 int FS = MFI->getObjectSize(FI);
8938 int BFS = MFI->getObjectSize(BFI);
8939 if (FS != BFS || FS != (int)Bytes) return false;
8940 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes);
8941 }
8942
8943 // Handle X+C
8944 if (DAG.isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc &&
8945 cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes)
8946 return true;
8947
8948 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Craig Topper062a2ba2014-04-25 05:30:21 +00008949 const GlobalValue *GV1 = nullptr;
8950 const GlobalValue *GV2 = nullptr;
Hal Finkel8ebfe6c2013-05-27 02:06:39 +00008951 int64_t Offset1 = 0;
8952 int64_t Offset2 = 0;
8953 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1);
8954 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2);
8955 if (isGA1 && isGA2 && GV1 == GV2)
8956 return Offset1 == (Offset2 + Dist*Bytes);
8957 return false;
8958}
8959
Hal Finkel3604bf72014-08-01 01:02:01 +00008960// Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does
8961// not enforce equality of the chain operands.
8962static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
8963 unsigned Bytes, int Dist,
8964 SelectionDAG &DAG) {
8965 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(N)) {
8966 EVT VT = LS->getMemoryVT();
8967 SDValue Loc = LS->getBasePtr();
8968 return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG);
8969 }
8970
8971 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
8972 EVT VT;
8973 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
8974 default: return false;
Hal Finkelc93a9a22015-02-25 01:06:45 +00008975 case Intrinsic::ppc_qpx_qvlfd:
8976 case Intrinsic::ppc_qpx_qvlfda:
8977 VT = MVT::v4f64;
8978 break;
8979 case Intrinsic::ppc_qpx_qvlfs:
8980 case Intrinsic::ppc_qpx_qvlfsa:
8981 VT = MVT::v4f32;
8982 break;
8983 case Intrinsic::ppc_qpx_qvlfcd:
8984 case Intrinsic::ppc_qpx_qvlfcda:
8985 VT = MVT::v2f64;
8986 break;
8987 case Intrinsic::ppc_qpx_qvlfcs:
8988 case Intrinsic::ppc_qpx_qvlfcsa:
8989 VT = MVT::v2f32;
8990 break;
8991 case Intrinsic::ppc_qpx_qvlfiwa:
8992 case Intrinsic::ppc_qpx_qvlfiwz:
Hal Finkel3604bf72014-08-01 01:02:01 +00008993 case Intrinsic::ppc_altivec_lvx:
8994 case Intrinsic::ppc_altivec_lvxl:
Bill Schmidt72954782014-11-12 04:19:40 +00008995 case Intrinsic::ppc_vsx_lxvw4x:
Hal Finkel3604bf72014-08-01 01:02:01 +00008996 VT = MVT::v4i32;
8997 break;
Bill Schmidt72954782014-11-12 04:19:40 +00008998 case Intrinsic::ppc_vsx_lxvd2x:
8999 VT = MVT::v2f64;
9000 break;
Hal Finkel3604bf72014-08-01 01:02:01 +00009001 case Intrinsic::ppc_altivec_lvebx:
9002 VT = MVT::i8;
9003 break;
9004 case Intrinsic::ppc_altivec_lvehx:
9005 VT = MVT::i16;
9006 break;
9007 case Intrinsic::ppc_altivec_lvewx:
9008 VT = MVT::i32;
9009 break;
9010 }
9011
9012 return isConsecutiveLSLoc(N->getOperand(2), VT, Base, Bytes, Dist, DAG);
9013 }
9014
9015 if (N->getOpcode() == ISD::INTRINSIC_VOID) {
9016 EVT VT;
9017 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
9018 default: return false;
Hal Finkelc93a9a22015-02-25 01:06:45 +00009019 case Intrinsic::ppc_qpx_qvstfd:
9020 case Intrinsic::ppc_qpx_qvstfda:
9021 VT = MVT::v4f64;
9022 break;
9023 case Intrinsic::ppc_qpx_qvstfs:
9024 case Intrinsic::ppc_qpx_qvstfsa:
9025 VT = MVT::v4f32;
9026 break;
9027 case Intrinsic::ppc_qpx_qvstfcd:
9028 case Intrinsic::ppc_qpx_qvstfcda:
9029 VT = MVT::v2f64;
9030 break;
9031 case Intrinsic::ppc_qpx_qvstfcs:
9032 case Intrinsic::ppc_qpx_qvstfcsa:
9033 VT = MVT::v2f32;
9034 break;
9035 case Intrinsic::ppc_qpx_qvstfiw:
9036 case Intrinsic::ppc_qpx_qvstfiwa:
Hal Finkel3604bf72014-08-01 01:02:01 +00009037 case Intrinsic::ppc_altivec_stvx:
9038 case Intrinsic::ppc_altivec_stvxl:
Bill Schmidt72954782014-11-12 04:19:40 +00009039 case Intrinsic::ppc_vsx_stxvw4x:
Hal Finkel3604bf72014-08-01 01:02:01 +00009040 VT = MVT::v4i32;
9041 break;
Bill Schmidt72954782014-11-12 04:19:40 +00009042 case Intrinsic::ppc_vsx_stxvd2x:
9043 VT = MVT::v2f64;
9044 break;
Hal Finkel3604bf72014-08-01 01:02:01 +00009045 case Intrinsic::ppc_altivec_stvebx:
9046 VT = MVT::i8;
9047 break;
9048 case Intrinsic::ppc_altivec_stvehx:
9049 VT = MVT::i16;
9050 break;
9051 case Intrinsic::ppc_altivec_stvewx:
9052 VT = MVT::i32;
9053 break;
9054 }
9055
9056 return isConsecutiveLSLoc(N->getOperand(3), VT, Base, Bytes, Dist, DAG);
9057 }
9058
9059 return false;
9060}
9061
Hal Finkel7d8a6912013-05-26 18:08:30 +00009062// Return true is there is a nearyby consecutive load to the one provided
9063// (regardless of alignment). We search up and down the chain, looking though
Matt Arsenault57e74d22014-07-29 00:02:40 +00009064// token factors and other loads (but nothing else). As a result, a true result
9065// indicates that it is safe to create a new consecutive load adjacent to the
9066// load provided.
Hal Finkel7d8a6912013-05-26 18:08:30 +00009067static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) {
9068 SDValue Chain = LD->getChain();
9069 EVT VT = LD->getMemoryVT();
9070
9071 SmallSet<SDNode *, 16> LoadRoots;
9072 SmallVector<SDNode *, 8> Queue(1, Chain.getNode());
9073 SmallSet<SDNode *, 16> Visited;
9074
9075 // First, search up the chain, branching to follow all token-factor operands.
9076 // If we find a consecutive load, then we're done, otherwise, record all
9077 // nodes just above the top-level loads and token factors.
9078 while (!Queue.empty()) {
9079 SDNode *ChainNext = Queue.pop_back_val();
David Blaikie70573dc2014-11-19 07:49:26 +00009080 if (!Visited.insert(ChainNext).second)
Hal Finkel7d8a6912013-05-26 18:08:30 +00009081 continue;
9082
Hal Finkel3604bf72014-08-01 01:02:01 +00009083 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(ChainNext)) {
Hal Finkel8ebfe6c2013-05-27 02:06:39 +00009084 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG))
Hal Finkel7d8a6912013-05-26 18:08:30 +00009085 return true;
9086
9087 if (!Visited.count(ChainLD->getChain().getNode()))
9088 Queue.push_back(ChainLD->getChain().getNode());
9089 } else if (ChainNext->getOpcode() == ISD::TokenFactor) {
Craig Topper66e588b2014-06-29 00:40:57 +00009090 for (const SDUse &O : ChainNext->ops())
9091 if (!Visited.count(O.getNode()))
9092 Queue.push_back(O.getNode());
Hal Finkel7d8a6912013-05-26 18:08:30 +00009093 } else
9094 LoadRoots.insert(ChainNext);
9095 }
9096
9097 // Second, search down the chain, starting from the top-level nodes recorded
9098 // in the first phase. These top-level nodes are the nodes just above all
9099 // loads and token factors. Starting with their uses, recursively look though
9100 // all loads (just the chain uses) and token factors to find a consecutive
9101 // load.
9102 Visited.clear();
9103 Queue.clear();
9104
9105 for (SmallSet<SDNode *, 16>::iterator I = LoadRoots.begin(),
9106 IE = LoadRoots.end(); I != IE; ++I) {
9107 Queue.push_back(*I);
9108
9109 while (!Queue.empty()) {
9110 SDNode *LoadRoot = Queue.pop_back_val();
David Blaikie70573dc2014-11-19 07:49:26 +00009111 if (!Visited.insert(LoadRoot).second)
Hal Finkel7d8a6912013-05-26 18:08:30 +00009112 continue;
9113
Hal Finkel3604bf72014-08-01 01:02:01 +00009114 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(LoadRoot))
Hal Finkel8ebfe6c2013-05-27 02:06:39 +00009115 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG))
Hal Finkel7d8a6912013-05-26 18:08:30 +00009116 return true;
9117
9118 for (SDNode::use_iterator UI = LoadRoot->use_begin(),
9119 UE = LoadRoot->use_end(); UI != UE; ++UI)
Hal Finkel3604bf72014-08-01 01:02:01 +00009120 if (((isa<MemSDNode>(*UI) &&
9121 cast<MemSDNode>(*UI)->getChain().getNode() == LoadRoot) ||
Hal Finkel7d8a6912013-05-26 18:08:30 +00009122 UI->getOpcode() == ISD::TokenFactor) && !Visited.count(*UI))
9123 Queue.push_back(*UI);
9124 }
9125 }
9126
9127 return false;
9128}
9129
Hal Finkel940ab932014-02-28 00:27:01 +00009130SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N,
9131 DAGCombinerInfo &DCI) const {
9132 SelectionDAG &DAG = DCI.DAG;
9133 SDLoc dl(N);
9134
Eric Christophercccae792015-01-30 22:02:31 +00009135 assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits");
Hal Finkel940ab932014-02-28 00:27:01 +00009136 // If we're tracking CR bits, we need to be careful that we don't have:
9137 // trunc(binary-ops(zext(x), zext(y)))
9138 // or
9139 // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...)
9140 // such that we're unnecessarily moving things into GPRs when it would be
9141 // better to keep them in CR bits.
9142
9143 // Note that trunc here can be an actual i1 trunc, or can be the effective
9144 // truncation that comes from a setcc or select_cc.
9145 if (N->getOpcode() == ISD::TRUNCATE &&
9146 N->getValueType(0) != MVT::i1)
9147 return SDValue();
9148
9149 if (N->getOperand(0).getValueType() != MVT::i32 &&
9150 N->getOperand(0).getValueType() != MVT::i64)
9151 return SDValue();
9152
9153 if (N->getOpcode() == ISD::SETCC ||
9154 N->getOpcode() == ISD::SELECT_CC) {
9155 // If we're looking at a comparison, then we need to make sure that the
9156 // high bits (all except for the first) don't matter the result.
9157 ISD::CondCode CC =
9158 cast<CondCodeSDNode>(N->getOperand(
9159 N->getOpcode() == ISD::SETCC ? 2 : 4))->get();
9160 unsigned OpBits = N->getOperand(0).getValueSizeInBits();
9161
9162 if (ISD::isSignedIntSetCC(CC)) {
9163 if (DAG.ComputeNumSignBits(N->getOperand(0)) != OpBits ||
9164 DAG.ComputeNumSignBits(N->getOperand(1)) != OpBits)
9165 return SDValue();
9166 } else if (ISD::isUnsignedIntSetCC(CC)) {
9167 if (!DAG.MaskedValueIsZero(N->getOperand(0),
9168 APInt::getHighBitsSet(OpBits, OpBits-1)) ||
9169 !DAG.MaskedValueIsZero(N->getOperand(1),
9170 APInt::getHighBitsSet(OpBits, OpBits-1)))
9171 return SDValue();
9172 } else {
9173 // This is neither a signed nor an unsigned comparison, just make sure
9174 // that the high bits are equal.
9175 APInt Op1Zero, Op1One;
9176 APInt Op2Zero, Op2One;
Jay Foada0653a32014-05-14 21:14:37 +00009177 DAG.computeKnownBits(N->getOperand(0), Op1Zero, Op1One);
9178 DAG.computeKnownBits(N->getOperand(1), Op2Zero, Op2One);
Hal Finkel940ab932014-02-28 00:27:01 +00009179
9180 // We don't really care about what is known about the first bit (if
9181 // anything), so clear it in all masks prior to comparing them.
9182 Op1Zero.clearBit(0); Op1One.clearBit(0);
9183 Op2Zero.clearBit(0); Op2One.clearBit(0);
9184
9185 if (Op1Zero != Op2Zero || Op1One != Op2One)
9186 return SDValue();
9187 }
9188 }
9189
9190 // We now know that the higher-order bits are irrelevant, we just need to
9191 // make sure that all of the intermediate operations are bit operations, and
9192 // all inputs are extensions.
9193 if (N->getOperand(0).getOpcode() != ISD::AND &&
9194 N->getOperand(0).getOpcode() != ISD::OR &&
9195 N->getOperand(0).getOpcode() != ISD::XOR &&
9196 N->getOperand(0).getOpcode() != ISD::SELECT &&
9197 N->getOperand(0).getOpcode() != ISD::SELECT_CC &&
9198 N->getOperand(0).getOpcode() != ISD::TRUNCATE &&
9199 N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND &&
9200 N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND &&
9201 N->getOperand(0).getOpcode() != ISD::ANY_EXTEND)
9202 return SDValue();
9203
9204 if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) &&
9205 N->getOperand(1).getOpcode() != ISD::AND &&
9206 N->getOperand(1).getOpcode() != ISD::OR &&
9207 N->getOperand(1).getOpcode() != ISD::XOR &&
9208 N->getOperand(1).getOpcode() != ISD::SELECT &&
9209 N->getOperand(1).getOpcode() != ISD::SELECT_CC &&
9210 N->getOperand(1).getOpcode() != ISD::TRUNCATE &&
9211 N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND &&
9212 N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND &&
9213 N->getOperand(1).getOpcode() != ISD::ANY_EXTEND)
9214 return SDValue();
9215
9216 SmallVector<SDValue, 4> Inputs;
9217 SmallVector<SDValue, 8> BinOps, PromOps;
9218 SmallPtrSet<SDNode *, 16> Visited;
9219
9220 for (unsigned i = 0; i < 2; ++i) {
9221 if (((N->getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
9222 N->getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
9223 N->getOperand(i).getOpcode() == ISD::ANY_EXTEND) &&
9224 N->getOperand(i).getOperand(0).getValueType() == MVT::i1) ||
9225 isa<ConstantSDNode>(N->getOperand(i)))
9226 Inputs.push_back(N->getOperand(i));
9227 else
9228 BinOps.push_back(N->getOperand(i));
9229
9230 if (N->getOpcode() == ISD::TRUNCATE)
9231 break;
9232 }
9233
9234 // Visit all inputs, collect all binary operations (and, or, xor and
9235 // select) that are all fed by extensions.
9236 while (!BinOps.empty()) {
9237 SDValue BinOp = BinOps.back();
9238 BinOps.pop_back();
9239
David Blaikie70573dc2014-11-19 07:49:26 +00009240 if (!Visited.insert(BinOp.getNode()).second)
Hal Finkel940ab932014-02-28 00:27:01 +00009241 continue;
9242
9243 PromOps.push_back(BinOp);
9244
9245 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
9246 // The condition of the select is not promoted.
9247 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
9248 continue;
9249 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
9250 continue;
9251
9252 if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
9253 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
9254 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) &&
9255 BinOp.getOperand(i).getOperand(0).getValueType() == MVT::i1) ||
9256 isa<ConstantSDNode>(BinOp.getOperand(i))) {
9257 Inputs.push_back(BinOp.getOperand(i));
9258 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
9259 BinOp.getOperand(i).getOpcode() == ISD::OR ||
9260 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
9261 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
9262 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC ||
9263 BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
9264 BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
9265 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
9266 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) {
9267 BinOps.push_back(BinOp.getOperand(i));
9268 } else {
9269 // We have an input that is not an extension or another binary
9270 // operation; we'll abort this transformation.
9271 return SDValue();
9272 }
9273 }
9274 }
9275
9276 // Make sure that this is a self-contained cluster of operations (which
9277 // is not quite the same thing as saying that everything has only one
9278 // use).
9279 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
9280 if (isa<ConstantSDNode>(Inputs[i]))
9281 continue;
9282
9283 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(),
9284 UE = Inputs[i].getNode()->use_end();
9285 UI != UE; ++UI) {
9286 SDNode *User = *UI;
9287 if (User != N && !Visited.count(User))
9288 return SDValue();
Hal Finkel46043ed2014-03-01 21:36:57 +00009289
9290 // Make sure that we're not going to promote the non-output-value
9291 // operand(s) or SELECT or SELECT_CC.
9292 // FIXME: Although we could sometimes handle this, and it does occur in
9293 // practice that one of the condition inputs to the select is also one of
9294 // the outputs, we currently can't deal with this.
9295 if (User->getOpcode() == ISD::SELECT) {
9296 if (User->getOperand(0) == Inputs[i])
9297 return SDValue();
9298 } else if (User->getOpcode() == ISD::SELECT_CC) {
9299 if (User->getOperand(0) == Inputs[i] ||
9300 User->getOperand(1) == Inputs[i])
9301 return SDValue();
9302 }
Hal Finkel940ab932014-02-28 00:27:01 +00009303 }
9304 }
9305
9306 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
9307 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(),
9308 UE = PromOps[i].getNode()->use_end();
9309 UI != UE; ++UI) {
9310 SDNode *User = *UI;
9311 if (User != N && !Visited.count(User))
9312 return SDValue();
Hal Finkel46043ed2014-03-01 21:36:57 +00009313
9314 // Make sure that we're not going to promote the non-output-value
9315 // operand(s) or SELECT or SELECT_CC.
9316 // FIXME: Although we could sometimes handle this, and it does occur in
9317 // practice that one of the condition inputs to the select is also one of
9318 // the outputs, we currently can't deal with this.
9319 if (User->getOpcode() == ISD::SELECT) {
9320 if (User->getOperand(0) == PromOps[i])
9321 return SDValue();
9322 } else if (User->getOpcode() == ISD::SELECT_CC) {
9323 if (User->getOperand(0) == PromOps[i] ||
9324 User->getOperand(1) == PromOps[i])
9325 return SDValue();
9326 }
Hal Finkel940ab932014-02-28 00:27:01 +00009327 }
9328 }
9329
9330 // Replace all inputs with the extension operand.
9331 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
9332 // Constants may have users outside the cluster of to-be-promoted nodes,
9333 // and so we need to replace those as we do the promotions.
9334 if (isa<ConstantSDNode>(Inputs[i]))
9335 continue;
9336 else
9337 DAG.ReplaceAllUsesOfValueWith(Inputs[i], Inputs[i].getOperand(0));
9338 }
9339
9340 // Replace all operations (these are all the same, but have a different
9341 // (i1) return type). DAG.getNode will validate that the types of
9342 // a binary operator match, so go through the list in reverse so that
9343 // we've likely promoted both operands first. Any intermediate truncations or
9344 // extensions disappear.
9345 while (!PromOps.empty()) {
9346 SDValue PromOp = PromOps.back();
9347 PromOps.pop_back();
9348
9349 if (PromOp.getOpcode() == ISD::TRUNCATE ||
9350 PromOp.getOpcode() == ISD::SIGN_EXTEND ||
9351 PromOp.getOpcode() == ISD::ZERO_EXTEND ||
9352 PromOp.getOpcode() == ISD::ANY_EXTEND) {
9353 if (!isa<ConstantSDNode>(PromOp.getOperand(0)) &&
9354 PromOp.getOperand(0).getValueType() != MVT::i1) {
9355 // The operand is not yet ready (see comment below).
9356 PromOps.insert(PromOps.begin(), PromOp);
9357 continue;
9358 }
9359
9360 SDValue RepValue = PromOp.getOperand(0);
9361 if (isa<ConstantSDNode>(RepValue))
9362 RepValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, RepValue);
9363
9364 DAG.ReplaceAllUsesOfValueWith(PromOp, RepValue);
9365 continue;
9366 }
9367
9368 unsigned C;
9369 switch (PromOp.getOpcode()) {
9370 default: C = 0; break;
9371 case ISD::SELECT: C = 1; break;
9372 case ISD::SELECT_CC: C = 2; break;
9373 }
9374
9375 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) &&
9376 PromOp.getOperand(C).getValueType() != MVT::i1) ||
9377 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) &&
9378 PromOp.getOperand(C+1).getValueType() != MVT::i1)) {
9379 // The to-be-promoted operands of this node have not yet been
9380 // promoted (this should be rare because we're going through the
9381 // list backward, but if one of the operands has several users in
9382 // this cluster of to-be-promoted nodes, it is possible).
9383 PromOps.insert(PromOps.begin(), PromOp);
9384 continue;
9385 }
9386
9387 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(),
9388 PromOp.getNode()->op_end());
9389
9390 // If there are any constant inputs, make sure they're replaced now.
9391 for (unsigned i = 0; i < 2; ++i)
9392 if (isa<ConstantSDNode>(Ops[C+i]))
9393 Ops[C+i] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ops[C+i]);
9394
9395 DAG.ReplaceAllUsesOfValueWith(PromOp,
Craig Topper48d114b2014-04-26 18:35:24 +00009396 DAG.getNode(PromOp.getOpcode(), dl, MVT::i1, Ops));
Hal Finkel940ab932014-02-28 00:27:01 +00009397 }
9398
9399 // Now we're left with the initial truncation itself.
9400 if (N->getOpcode() == ISD::TRUNCATE)
9401 return N->getOperand(0);
9402
9403 // Otherwise, this is a comparison. The operands to be compared have just
9404 // changed type (to i1), but everything else is the same.
9405 return SDValue(N, 0);
9406}
9407
9408SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N,
9409 DAGCombinerInfo &DCI) const {
9410 SelectionDAG &DAG = DCI.DAG;
9411 SDLoc dl(N);
9412
Hal Finkel940ab932014-02-28 00:27:01 +00009413 // If we're tracking CR bits, we need to be careful that we don't have:
9414 // zext(binary-ops(trunc(x), trunc(y)))
9415 // or
9416 // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...)
9417 // such that we're unnecessarily moving things into CR bits that can more
9418 // efficiently stay in GPRs. Note that if we're not certain that the high
9419 // bits are set as required by the final extension, we still may need to do
9420 // some masking to get the proper behavior.
9421
Hal Finkel46043ed2014-03-01 21:36:57 +00009422 // This same functionality is important on PPC64 when dealing with
9423 // 32-to-64-bit extensions; these occur often when 32-bit values are used as
9424 // the return values of functions. Because it is so similar, it is handled
9425 // here as well.
9426
Hal Finkel940ab932014-02-28 00:27:01 +00009427 if (N->getValueType(0) != MVT::i32 &&
9428 N->getValueType(0) != MVT::i64)
9429 return SDValue();
9430
Eric Christophercccae792015-01-30 22:02:31 +00009431 if (!((N->getOperand(0).getValueType() == MVT::i1 && Subtarget.useCRBits()) ||
9432 (N->getOperand(0).getValueType() == MVT::i32 && Subtarget.isPPC64())))
Hal Finkel940ab932014-02-28 00:27:01 +00009433 return SDValue();
9434
9435 if (N->getOperand(0).getOpcode() != ISD::AND &&
9436 N->getOperand(0).getOpcode() != ISD::OR &&
9437 N->getOperand(0).getOpcode() != ISD::XOR &&
9438 N->getOperand(0).getOpcode() != ISD::SELECT &&
9439 N->getOperand(0).getOpcode() != ISD::SELECT_CC)
9440 return SDValue();
9441
9442 SmallVector<SDValue, 4> Inputs;
9443 SmallVector<SDValue, 8> BinOps(1, N->getOperand(0)), PromOps;
9444 SmallPtrSet<SDNode *, 16> Visited;
9445
9446 // Visit all inputs, collect all binary operations (and, or, xor and
9447 // select) that are all fed by truncations.
9448 while (!BinOps.empty()) {
9449 SDValue BinOp = BinOps.back();
9450 BinOps.pop_back();
9451
David Blaikie70573dc2014-11-19 07:49:26 +00009452 if (!Visited.insert(BinOp.getNode()).second)
Hal Finkel940ab932014-02-28 00:27:01 +00009453 continue;
9454
9455 PromOps.push_back(BinOp);
9456
9457 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
9458 // The condition of the select is not promoted.
9459 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
9460 continue;
9461 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
9462 continue;
9463
9464 if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
9465 isa<ConstantSDNode>(BinOp.getOperand(i))) {
9466 Inputs.push_back(BinOp.getOperand(i));
9467 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
9468 BinOp.getOperand(i).getOpcode() == ISD::OR ||
9469 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
9470 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
9471 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) {
9472 BinOps.push_back(BinOp.getOperand(i));
9473 } else {
9474 // We have an input that is not a truncation or another binary
9475 // operation; we'll abort this transformation.
9476 return SDValue();
9477 }
9478 }
9479 }
9480
Hal Finkel4104a1a2014-12-14 05:53:19 +00009481 // The operands of a select that must be truncated when the select is
9482 // promoted because the operand is actually part of the to-be-promoted set.
9483 DenseMap<SDNode *, EVT> SelectTruncOp[2];
9484
Hal Finkel940ab932014-02-28 00:27:01 +00009485 // Make sure that this is a self-contained cluster of operations (which
9486 // is not quite the same thing as saying that everything has only one
9487 // use).
9488 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
9489 if (isa<ConstantSDNode>(Inputs[i]))
9490 continue;
9491
9492 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(),
9493 UE = Inputs[i].getNode()->use_end();
9494 UI != UE; ++UI) {
9495 SDNode *User = *UI;
9496 if (User != N && !Visited.count(User))
9497 return SDValue();
Hal Finkel46043ed2014-03-01 21:36:57 +00009498
Hal Finkel4104a1a2014-12-14 05:53:19 +00009499 // If we're going to promote the non-output-value operand(s) or SELECT or
9500 // SELECT_CC, record them for truncation.
Hal Finkel46043ed2014-03-01 21:36:57 +00009501 if (User->getOpcode() == ISD::SELECT) {
9502 if (User->getOperand(0) == Inputs[i])
Hal Finkel4104a1a2014-12-14 05:53:19 +00009503 SelectTruncOp[0].insert(std::make_pair(User,
9504 User->getOperand(0).getValueType()));
Hal Finkel46043ed2014-03-01 21:36:57 +00009505 } else if (User->getOpcode() == ISD::SELECT_CC) {
Hal Finkel4104a1a2014-12-14 05:53:19 +00009506 if (User->getOperand(0) == Inputs[i])
9507 SelectTruncOp[0].insert(std::make_pair(User,
9508 User->getOperand(0).getValueType()));
9509 if (User->getOperand(1) == Inputs[i])
9510 SelectTruncOp[1].insert(std::make_pair(User,
9511 User->getOperand(1).getValueType()));
Hal Finkel46043ed2014-03-01 21:36:57 +00009512 }
Hal Finkel940ab932014-02-28 00:27:01 +00009513 }
9514 }
9515
9516 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
9517 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(),
9518 UE = PromOps[i].getNode()->use_end();
9519 UI != UE; ++UI) {
9520 SDNode *User = *UI;
9521 if (User != N && !Visited.count(User))
9522 return SDValue();
Hal Finkel46043ed2014-03-01 21:36:57 +00009523
Hal Finkel4104a1a2014-12-14 05:53:19 +00009524 // If we're going to promote the non-output-value operand(s) or SELECT or
9525 // SELECT_CC, record them for truncation.
Hal Finkel46043ed2014-03-01 21:36:57 +00009526 if (User->getOpcode() == ISD::SELECT) {
9527 if (User->getOperand(0) == PromOps[i])
Hal Finkel4104a1a2014-12-14 05:53:19 +00009528 SelectTruncOp[0].insert(std::make_pair(User,
9529 User->getOperand(0).getValueType()));
Hal Finkel46043ed2014-03-01 21:36:57 +00009530 } else if (User->getOpcode() == ISD::SELECT_CC) {
Hal Finkel4104a1a2014-12-14 05:53:19 +00009531 if (User->getOperand(0) == PromOps[i])
9532 SelectTruncOp[0].insert(std::make_pair(User,
9533 User->getOperand(0).getValueType()));
9534 if (User->getOperand(1) == PromOps[i])
9535 SelectTruncOp[1].insert(std::make_pair(User,
9536 User->getOperand(1).getValueType()));
Hal Finkel46043ed2014-03-01 21:36:57 +00009537 }
Hal Finkel940ab932014-02-28 00:27:01 +00009538 }
9539 }
9540
Hal Finkel46043ed2014-03-01 21:36:57 +00009541 unsigned PromBits = N->getOperand(0).getValueSizeInBits();
Hal Finkel940ab932014-02-28 00:27:01 +00009542 bool ReallyNeedsExt = false;
9543 if (N->getOpcode() != ISD::ANY_EXTEND) {
9544 // If all of the inputs are not already sign/zero extended, then
9545 // we'll still need to do that at the end.
9546 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
9547 if (isa<ConstantSDNode>(Inputs[i]))
9548 continue;
9549
9550 unsigned OpBits =
9551 Inputs[i].getOperand(0).getValueSizeInBits();
Hal Finkel46043ed2014-03-01 21:36:57 +00009552 assert(PromBits < OpBits && "Truncation not to a smaller bit count?");
9553
Hal Finkel940ab932014-02-28 00:27:01 +00009554 if ((N->getOpcode() == ISD::ZERO_EXTEND &&
9555 !DAG.MaskedValueIsZero(Inputs[i].getOperand(0),
Hal Finkel46043ed2014-03-01 21:36:57 +00009556 APInt::getHighBitsSet(OpBits,
9557 OpBits-PromBits))) ||
Hal Finkel940ab932014-02-28 00:27:01 +00009558 (N->getOpcode() == ISD::SIGN_EXTEND &&
Hal Finkel46043ed2014-03-01 21:36:57 +00009559 DAG.ComputeNumSignBits(Inputs[i].getOperand(0)) <
9560 (OpBits-(PromBits-1)))) {
Hal Finkel940ab932014-02-28 00:27:01 +00009561 ReallyNeedsExt = true;
9562 break;
9563 }
9564 }
9565 }
9566
9567 // Replace all inputs, either with the truncation operand, or a
9568 // truncation or extension to the final output type.
9569 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
9570 // Constant inputs need to be replaced with the to-be-promoted nodes that
9571 // use them because they might have users outside of the cluster of
9572 // promoted nodes.
9573 if (isa<ConstantSDNode>(Inputs[i]))
9574 continue;
9575
9576 SDValue InSrc = Inputs[i].getOperand(0);
9577 if (Inputs[i].getValueType() == N->getValueType(0))
9578 DAG.ReplaceAllUsesOfValueWith(Inputs[i], InSrc);
9579 else if (N->getOpcode() == ISD::SIGN_EXTEND)
9580 DAG.ReplaceAllUsesOfValueWith(Inputs[i],
9581 DAG.getSExtOrTrunc(InSrc, dl, N->getValueType(0)));
9582 else if (N->getOpcode() == ISD::ZERO_EXTEND)
9583 DAG.ReplaceAllUsesOfValueWith(Inputs[i],
9584 DAG.getZExtOrTrunc(InSrc, dl, N->getValueType(0)));
9585 else
9586 DAG.ReplaceAllUsesOfValueWith(Inputs[i],
9587 DAG.getAnyExtOrTrunc(InSrc, dl, N->getValueType(0)));
9588 }
9589
9590 // Replace all operations (these are all the same, but have a different
9591 // (promoted) return type). DAG.getNode will validate that the types of
9592 // a binary operator match, so go through the list in reverse so that
9593 // we've likely promoted both operands first.
9594 while (!PromOps.empty()) {
9595 SDValue PromOp = PromOps.back();
9596 PromOps.pop_back();
9597
9598 unsigned C;
9599 switch (PromOp.getOpcode()) {
9600 default: C = 0; break;
9601 case ISD::SELECT: C = 1; break;
9602 case ISD::SELECT_CC: C = 2; break;
9603 }
9604
9605 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) &&
9606 PromOp.getOperand(C).getValueType() != N->getValueType(0)) ||
9607 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) &&
9608 PromOp.getOperand(C+1).getValueType() != N->getValueType(0))) {
9609 // The to-be-promoted operands of this node have not yet been
9610 // promoted (this should be rare because we're going through the
9611 // list backward, but if one of the operands has several users in
9612 // this cluster of to-be-promoted nodes, it is possible).
9613 PromOps.insert(PromOps.begin(), PromOp);
9614 continue;
9615 }
9616
Hal Finkel4104a1a2014-12-14 05:53:19 +00009617 // For SELECT and SELECT_CC nodes, we do a similar check for any
9618 // to-be-promoted comparison inputs.
9619 if (PromOp.getOpcode() == ISD::SELECT ||
9620 PromOp.getOpcode() == ISD::SELECT_CC) {
9621 if ((SelectTruncOp[0].count(PromOp.getNode()) &&
9622 PromOp.getOperand(0).getValueType() != N->getValueType(0)) ||
9623 (SelectTruncOp[1].count(PromOp.getNode()) &&
9624 PromOp.getOperand(1).getValueType() != N->getValueType(0))) {
9625 PromOps.insert(PromOps.begin(), PromOp);
9626 continue;
9627 }
9628 }
9629
Hal Finkel940ab932014-02-28 00:27:01 +00009630 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(),
9631 PromOp.getNode()->op_end());
9632
9633 // If this node has constant inputs, then they'll need to be promoted here.
9634 for (unsigned i = 0; i < 2; ++i) {
9635 if (!isa<ConstantSDNode>(Ops[C+i]))
9636 continue;
9637 if (Ops[C+i].getValueType() == N->getValueType(0))
9638 continue;
9639
9640 if (N->getOpcode() == ISD::SIGN_EXTEND)
9641 Ops[C+i] = DAG.getSExtOrTrunc(Ops[C+i], dl, N->getValueType(0));
9642 else if (N->getOpcode() == ISD::ZERO_EXTEND)
9643 Ops[C+i] = DAG.getZExtOrTrunc(Ops[C+i], dl, N->getValueType(0));
9644 else
9645 Ops[C+i] = DAG.getAnyExtOrTrunc(Ops[C+i], dl, N->getValueType(0));
9646 }
9647
Hal Finkel4104a1a2014-12-14 05:53:19 +00009648 // If we've promoted the comparison inputs of a SELECT or SELECT_CC,
9649 // truncate them again to the original value type.
9650 if (PromOp.getOpcode() == ISD::SELECT ||
9651 PromOp.getOpcode() == ISD::SELECT_CC) {
9652 auto SI0 = SelectTruncOp[0].find(PromOp.getNode());
9653 if (SI0 != SelectTruncOp[0].end())
9654 Ops[0] = DAG.getNode(ISD::TRUNCATE, dl, SI0->second, Ops[0]);
9655 auto SI1 = SelectTruncOp[1].find(PromOp.getNode());
9656 if (SI1 != SelectTruncOp[1].end())
9657 Ops[1] = DAG.getNode(ISD::TRUNCATE, dl, SI1->second, Ops[1]);
9658 }
9659
Hal Finkel940ab932014-02-28 00:27:01 +00009660 DAG.ReplaceAllUsesOfValueWith(PromOp,
Craig Topper48d114b2014-04-26 18:35:24 +00009661 DAG.getNode(PromOp.getOpcode(), dl, N->getValueType(0), Ops));
Hal Finkel940ab932014-02-28 00:27:01 +00009662 }
9663
9664 // Now we're left with the initial extension itself.
9665 if (!ReallyNeedsExt)
9666 return N->getOperand(0);
9667
Hal Finkel46043ed2014-03-01 21:36:57 +00009668 // To zero extend, just mask off everything except for the first bit (in the
9669 // i1 case).
Hal Finkel940ab932014-02-28 00:27:01 +00009670 if (N->getOpcode() == ISD::ZERO_EXTEND)
9671 return DAG.getNode(ISD::AND, dl, N->getValueType(0), N->getOperand(0),
Hal Finkel46043ed2014-03-01 21:36:57 +00009672 DAG.getConstant(APInt::getLowBitsSet(
9673 N->getValueSizeInBits(0), PromBits),
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00009674 dl, N->getValueType(0)));
Hal Finkel940ab932014-02-28 00:27:01 +00009675
9676 assert(N->getOpcode() == ISD::SIGN_EXTEND &&
9677 "Invalid extension type");
9678 EVT ShiftAmountTy = getShiftAmountTy(N->getValueType(0));
9679 SDValue ShiftCst =
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00009680 DAG.getConstant(N->getValueSizeInBits(0) - PromBits, dl, ShiftAmountTy);
Hal Finkel940ab932014-02-28 00:27:01 +00009681 return DAG.getNode(ISD::SRA, dl, N->getValueType(0),
9682 DAG.getNode(ISD::SHL, dl, N->getValueType(0),
9683 N->getOperand(0), ShiftCst), ShiftCst);
9684}
9685
Hal Finkel5efb9182015-01-06 06:01:57 +00009686SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N,
9687 DAGCombinerInfo &DCI) const {
9688 assert((N->getOpcode() == ISD::SINT_TO_FP ||
9689 N->getOpcode() == ISD::UINT_TO_FP) &&
9690 "Need an int -> FP conversion node here");
9691
9692 if (!Subtarget.has64BitSupport())
9693 return SDValue();
9694
9695 SelectionDAG &DAG = DCI.DAG;
9696 SDLoc dl(N);
9697 SDValue Op(N, 0);
9698
9699 // Don't handle ppc_fp128 here or i1 conversions.
9700 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
9701 return SDValue();
9702 if (Op.getOperand(0).getValueType() == MVT::i1)
9703 return SDValue();
9704
9705 // For i32 intermediate values, unfortunately, the conversion functions
9706 // leave the upper 32 bits of the value are undefined. Within the set of
9707 // scalar instructions, we have no method for zero- or sign-extending the
9708 // value. Thus, we cannot handle i32 intermediate values here.
9709 if (Op.getOperand(0).getValueType() == MVT::i32)
9710 return SDValue();
9711
9712 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) &&
9713 "UINT_TO_FP is supported only with FPCVT");
9714
9715 // If we have FCFIDS, then use it when converting to single-precision.
9716 // Otherwise, convert to double-precision and then round.
Eric Christophercccae792015-01-30 22:02:31 +00009717 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
9718 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS
9719 : PPCISD::FCFIDS)
9720 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU
9721 : PPCISD::FCFID);
9722 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
9723 ? MVT::f32
9724 : MVT::f64;
Hal Finkel5efb9182015-01-06 06:01:57 +00009725
9726 // If we're converting from a float, to an int, and back to a float again,
9727 // then we don't need the store/load pair at all.
9728 if ((Op.getOperand(0).getOpcode() == ISD::FP_TO_UINT &&
9729 Subtarget.hasFPCVT()) ||
9730 (Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT)) {
9731 SDValue Src = Op.getOperand(0).getOperand(0);
9732 if (Src.getValueType() == MVT::f32) {
9733 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src);
9734 DCI.AddToWorklist(Src.getNode());
9735 }
9736
9737 unsigned FCTOp =
9738 Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ :
9739 PPCISD::FCTIDUZ;
9740
9741 SDValue Tmp = DAG.getNode(FCTOp, dl, MVT::f64, Src);
9742 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Tmp);
9743
9744 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
9745 FP = DAG.getNode(ISD::FP_ROUND, dl,
Sergey Dmitroukadb4c692015-04-28 11:56:37 +00009746 MVT::f32, FP, DAG.getIntPtrConstant(0, dl));
Hal Finkel5efb9182015-01-06 06:01:57 +00009747 DCI.AddToWorklist(FP.getNode());
9748 }
9749
9750 return FP;
9751 }
9752
9753 return SDValue();
9754}
9755
Bill Schmidtfae5d712014-12-09 16:35:51 +00009756// expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for
9757// builtins) into loads with swaps.
9758SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N,
9759 DAGCombinerInfo &DCI) const {
9760 SelectionDAG &DAG = DCI.DAG;
9761 SDLoc dl(N);
9762 SDValue Chain;
9763 SDValue Base;
9764 MachineMemOperand *MMO;
9765
9766 switch (N->getOpcode()) {
9767 default:
9768 llvm_unreachable("Unexpected opcode for little endian VSX load");
9769 case ISD::LOAD: {
9770 LoadSDNode *LD = cast<LoadSDNode>(N);
9771 Chain = LD->getChain();
9772 Base = LD->getBasePtr();
9773 MMO = LD->getMemOperand();
9774 // If the MMO suggests this isn't a load of a full vector, leave
9775 // things alone. For a built-in, we have to make the change for
9776 // correctness, so if there is a size problem that will be a bug.
9777 if (MMO->getSize() < 16)
9778 return SDValue();
9779 break;
9780 }
9781 case ISD::INTRINSIC_W_CHAIN: {
9782 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N);
9783 Chain = Intrin->getChain();
9784 Base = Intrin->getBasePtr();
9785 MMO = Intrin->getMemOperand();
9786 break;
9787 }
9788 }
9789
9790 MVT VecTy = N->getValueType(0).getSimpleVT();
9791 SDValue LoadOps[] = { Chain, Base };
9792 SDValue Load = DAG.getMemIntrinsicNode(PPCISD::LXVD2X, dl,
9793 DAG.getVTList(VecTy, MVT::Other),
9794 LoadOps, VecTy, MMO);
9795 DCI.AddToWorklist(Load.getNode());
9796 Chain = Load.getValue(1);
9797 SDValue Swap = DAG.getNode(PPCISD::XXSWAPD, dl,
9798 DAG.getVTList(VecTy, MVT::Other), Chain, Load);
9799 DCI.AddToWorklist(Swap.getNode());
9800 return Swap;
9801}
9802
9803// expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for
9804// builtins) into stores with swaps.
9805SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N,
9806 DAGCombinerInfo &DCI) const {
9807 SelectionDAG &DAG = DCI.DAG;
9808 SDLoc dl(N);
9809 SDValue Chain;
9810 SDValue Base;
9811 unsigned SrcOpnd;
9812 MachineMemOperand *MMO;
9813
9814 switch (N->getOpcode()) {
9815 default:
9816 llvm_unreachable("Unexpected opcode for little endian VSX store");
9817 case ISD::STORE: {
9818 StoreSDNode *ST = cast<StoreSDNode>(N);
9819 Chain = ST->getChain();
9820 Base = ST->getBasePtr();
9821 MMO = ST->getMemOperand();
9822 SrcOpnd = 1;
9823 // If the MMO suggests this isn't a store of a full vector, leave
9824 // things alone. For a built-in, we have to make the change for
9825 // correctness, so if there is a size problem that will be a bug.
9826 if (MMO->getSize() < 16)
9827 return SDValue();
9828 break;
9829 }
9830 case ISD::INTRINSIC_VOID: {
9831 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N);
9832 Chain = Intrin->getChain();
9833 // Intrin->getBasePtr() oddly does not get what we want.
9834 Base = Intrin->getOperand(3);
9835 MMO = Intrin->getMemOperand();
9836 SrcOpnd = 2;
9837 break;
9838 }
9839 }
9840
9841 SDValue Src = N->getOperand(SrcOpnd);
9842 MVT VecTy = Src.getValueType().getSimpleVT();
9843 SDValue Swap = DAG.getNode(PPCISD::XXSWAPD, dl,
9844 DAG.getVTList(VecTy, MVT::Other), Chain, Src);
9845 DCI.AddToWorklist(Swap.getNode());
9846 Chain = Swap.getValue(1);
9847 SDValue StoreOps[] = { Chain, Swap, Base };
9848 SDValue Store = DAG.getMemIntrinsicNode(PPCISD::STXVD2X, dl,
9849 DAG.getVTList(MVT::Other),
9850 StoreOps, VecTy, MMO);
9851 DCI.AddToWorklist(Store.getNode());
9852 return Store;
9853}
9854
Duncan Sandsdc2dac12008-11-24 14:53:14 +00009855SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
9856 DAGCombinerInfo &DCI) const {
Chris Lattnerf4184352006-03-01 04:57:39 +00009857 SelectionDAG &DAG = DCI.DAG;
Andrew Trickef9de2a2013-05-25 02:42:55 +00009858 SDLoc dl(N);
Chris Lattnerf4184352006-03-01 04:57:39 +00009859 switch (N->getOpcode()) {
9860 default: break;
Chris Lattner3c48ea52006-09-19 05:22:59 +00009861 case PPCISD::SHL:
9862 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
Dan Gohmanf1d83042010-06-18 14:22:04 +00009863 if (C->isNullValue()) // 0 << V -> 0.
Chris Lattner3c48ea52006-09-19 05:22:59 +00009864 return N->getOperand(0);
9865 }
9866 break;
9867 case PPCISD::SRL:
9868 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
Dan Gohmanf1d83042010-06-18 14:22:04 +00009869 if (C->isNullValue()) // 0 >>u V -> 0.
Chris Lattner3c48ea52006-09-19 05:22:59 +00009870 return N->getOperand(0);
9871 }
9872 break;
9873 case PPCISD::SRA:
9874 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
Dan Gohmanf1d83042010-06-18 14:22:04 +00009875 if (C->isNullValue() || // 0 >>s V -> 0.
Chris Lattner3c48ea52006-09-19 05:22:59 +00009876 C->isAllOnesValue()) // -1 >>s V -> -1.
9877 return N->getOperand(0);
9878 }
9879 break;
Hal Finkel940ab932014-02-28 00:27:01 +00009880 case ISD::SIGN_EXTEND:
9881 case ISD::ZERO_EXTEND:
9882 case ISD::ANY_EXTEND:
9883 return DAGCombineExtBoolTrunc(N, DCI);
9884 case ISD::TRUNCATE:
9885 case ISD::SETCC:
9886 case ISD::SELECT_CC:
9887 return DAGCombineTruncBoolExt(N, DCI);
Chris Lattnerf4184352006-03-01 04:57:39 +00009888 case ISD::SINT_TO_FP:
Hal Finkel5efb9182015-01-06 06:01:57 +00009889 case ISD::UINT_TO_FP:
9890 return combineFPToIntToFP(N, DCI);
Bill Schmidtfae5d712014-12-09 16:35:51 +00009891 case ISD::STORE: {
Chris Lattner27f53452006-03-01 05:50:56 +00009892 // Turn STORE (FP_TO_SINT F) -> STFIWX(FCTIWZ(F)).
Eric Christophercccae792015-01-30 22:02:31 +00009893 if (Subtarget.hasSTFIWX() && !cast<StoreSDNode>(N)->isTruncatingStore() &&
Chris Lattner27f53452006-03-01 05:50:56 +00009894 N->getOperand(1).getOpcode() == ISD::FP_TO_SINT &&
Owen Anderson9f944592009-08-11 20:47:22 +00009895 N->getOperand(1).getValueType() == MVT::i32 &&
9896 N->getOperand(1).getOperand(0).getValueType() != MVT::ppcf128) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00009897 SDValue Val = N->getOperand(1).getOperand(0);
Owen Anderson9f944592009-08-11 20:47:22 +00009898 if (Val.getValueType() == MVT::f32) {
9899 Val = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Val);
Gabor Greiff304a7a2008-08-28 21:40:38 +00009900 DCI.AddToWorklist(Val.getNode());
Chris Lattner27f53452006-03-01 05:50:56 +00009901 }
Owen Anderson9f944592009-08-11 20:47:22 +00009902 Val = DAG.getNode(PPCISD::FCTIWZ, dl, MVT::f64, Val);
Gabor Greiff304a7a2008-08-28 21:40:38 +00009903 DCI.AddToWorklist(Val.getNode());
Chris Lattner27f53452006-03-01 05:50:56 +00009904
Hal Finkel60c75102013-04-01 15:37:53 +00009905 SDValue Ops[] = {
9906 N->getOperand(0), Val, N->getOperand(2),
9907 DAG.getValueType(N->getOperand(1).getValueType())
9908 };
9909
9910 Val = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl,
Craig Topper206fcd42014-04-26 19:29:41 +00009911 DAG.getVTList(MVT::Other), Ops,
Hal Finkel60c75102013-04-01 15:37:53 +00009912 cast<StoreSDNode>(N)->getMemoryVT(),
9913 cast<StoreSDNode>(N)->getMemOperand());
Gabor Greiff304a7a2008-08-28 21:40:38 +00009914 DCI.AddToWorklist(Val.getNode());
Chris Lattner27f53452006-03-01 05:50:56 +00009915 return Val;
9916 }
Scott Michelcf0da6c2009-02-17 22:15:04 +00009917
Chris Lattnera7976d32006-07-10 20:56:58 +00009918 // Turn STORE (BSWAP) -> sthbrx/stwbrx.
Dan Gohman28328db2009-09-25 00:57:30 +00009919 if (cast<StoreSDNode>(N)->isUnindexed() &&
9920 N->getOperand(1).getOpcode() == ISD::BSWAP &&
Gabor Greiff304a7a2008-08-28 21:40:38 +00009921 N->getOperand(1).getNode()->hasOneUse() &&
Owen Anderson9f944592009-08-11 20:47:22 +00009922 (N->getOperand(1).getValueType() == MVT::i32 ||
Hal Finkel31d29562013-03-28 19:25:55 +00009923 N->getOperand(1).getValueType() == MVT::i16 ||
Eric Christophercccae792015-01-30 22:02:31 +00009924 (Subtarget.hasLDBRX() && Subtarget.isPPC64() &&
Hal Finkel31d29562013-03-28 19:25:55 +00009925 N->getOperand(1).getValueType() == MVT::i64))) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00009926 SDValue BSwapOp = N->getOperand(1).getOperand(0);
Chris Lattnera7976d32006-07-10 20:56:58 +00009927 // Do an any-extend to 32-bits if this is a half-word input.
Owen Anderson9f944592009-08-11 20:47:22 +00009928 if (BSwapOp.getValueType() == MVT::i16)
9929 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, BSwapOp);
Chris Lattnera7976d32006-07-10 20:56:58 +00009930
Dan Gohman48b185d2009-09-25 20:36:54 +00009931 SDValue Ops[] = {
9932 N->getOperand(0), BSwapOp, N->getOperand(2),
9933 DAG.getValueType(N->getOperand(1).getValueType())
9934 };
9935 return
9936 DAG.getMemIntrinsicNode(PPCISD::STBRX, dl, DAG.getVTList(MVT::Other),
Craig Topper206fcd42014-04-26 19:29:41 +00009937 Ops, cast<StoreSDNode>(N)->getMemoryVT(),
Dan Gohman48b185d2009-09-25 20:36:54 +00009938 cast<StoreSDNode>(N)->getMemOperand());
Chris Lattnera7976d32006-07-10 20:56:58 +00009939 }
Bill Schmidtfae5d712014-12-09 16:35:51 +00009940
9941 // For little endian, VSX stores require generating xxswapd/lxvd2x.
9942 EVT VT = N->getOperand(1).getValueType();
9943 if (VT.isSimple()) {
9944 MVT StoreVT = VT.getSimpleVT();
Eric Christophercccae792015-01-30 22:02:31 +00009945 if (Subtarget.hasVSX() && Subtarget.isLittleEndian() &&
Bill Schmidtfae5d712014-12-09 16:35:51 +00009946 (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 ||
9947 StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32))
9948 return expandVSXStoreForLE(N, DCI);
9949 }
Chris Lattnera7976d32006-07-10 20:56:58 +00009950 break;
Bill Schmidtfae5d712014-12-09 16:35:51 +00009951 }
Hal Finkelcf2e9082013-05-24 23:00:14 +00009952 case ISD::LOAD: {
9953 LoadSDNode *LD = cast<LoadSDNode>(N);
9954 EVT VT = LD->getValueType(0);
Bill Schmidtfae5d712014-12-09 16:35:51 +00009955
9956 // For little endian, VSX loads require generating lxvd2x/xxswapd.
9957 if (VT.isSimple()) {
9958 MVT LoadVT = VT.getSimpleVT();
Eric Christophercccae792015-01-30 22:02:31 +00009959 if (Subtarget.hasVSX() && Subtarget.isLittleEndian() &&
Bill Schmidtfae5d712014-12-09 16:35:51 +00009960 (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 ||
9961 LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32))
9962 return expandVSXLoadForLE(N, DCI);
9963 }
9964
Hal Finkelc93a9a22015-02-25 01:06:45 +00009965 EVT MemVT = LD->getMemoryVT();
9966 Type *Ty = MemVT.getTypeForEVT(*DAG.getContext());
Hal Finkelcf2e9082013-05-24 23:00:14 +00009967 unsigned ABIAlignment = getDataLayout()->getABITypeAlignment(Ty);
Hal Finkelc93a9a22015-02-25 01:06:45 +00009968 Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext());
9969 unsigned ScalarABIAlignment = getDataLayout()->getABITypeAlignment(STy);
9970 if (LD->isUnindexed() && VT.isVector() &&
9971 ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) &&
9972 // P8 and later hardware should just use LOAD.
9973 !Subtarget.hasP8Vector() && (VT == MVT::v16i8 || VT == MVT::v8i16 ||
9974 VT == MVT::v4i32 || VT == MVT::v4f32)) ||
9975 (Subtarget.hasQPX() && (VT == MVT::v4f64 || VT == MVT::v4f32) &&
9976 LD->getAlignment() >= ScalarABIAlignment)) &&
Hal Finkelcf2e9082013-05-24 23:00:14 +00009977 LD->getAlignment() < ABIAlignment) {
Hal Finkelc93a9a22015-02-25 01:06:45 +00009978 // This is a type-legal unaligned Altivec or QPX load.
Hal Finkelcf2e9082013-05-24 23:00:14 +00009979 SDValue Chain = LD->getChain();
9980 SDValue Ptr = LD->getBasePtr();
Eric Christopherb1aaebe2014-06-12 22:38:18 +00009981 bool isLittleEndian = Subtarget.isLittleEndian();
Hal Finkelcf2e9082013-05-24 23:00:14 +00009982
9983 // This implements the loading of unaligned vectors as described in
9984 // the venerable Apple Velocity Engine overview. Specifically:
9985 // https://developer.apple.com/hardwaredrivers/ve/alignment.html
9986 // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html
9987 //
9988 // The general idea is to expand a sequence of one or more unaligned
Bill Schmidt6b5a7df2014-06-09 22:00:52 +00009989 // loads into an alignment-based permutation-control instruction (lvsl
9990 // or lvsr), a series of regular vector loads (which always truncate
9991 // their input address to an aligned address), and a series of
9992 // permutations. The results of these permutations are the requested
9993 // loaded values. The trick is that the last "extra" load is not taken
9994 // from the address you might suspect (sizeof(vector) bytes after the
9995 // last requested load), but rather sizeof(vector) - 1 bytes after the
9996 // last requested vector. The point of this is to avoid a page fault if
9997 // the base address happened to be aligned. This works because if the
9998 // base address is aligned, then adding less than a full vector length
9999 // will cause the last vector in the sequence to be (re)loaded.
10000 // Otherwise, the next vector will be fetched as you might suspect was
10001 // necessary.
Hal Finkelcf2e9082013-05-24 23:00:14 +000010002
Hal Finkelbc2ee4c2013-05-25 04:05:05 +000010003 // We might be able to reuse the permutation generation from
Hal Finkelcf2e9082013-05-24 23:00:14 +000010004 // a different base address offset from this one by an aligned amount.
Hal Finkelbc2ee4c2013-05-25 04:05:05 +000010005 // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this
10006 // optimization later.
Hal Finkelc93a9a22015-02-25 01:06:45 +000010007 Intrinsic::ID Intr, IntrLD, IntrPerm;
10008 MVT PermCntlTy, PermTy, LDTy;
10009 if (Subtarget.hasAltivec()) {
10010 Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr :
10011 Intrinsic::ppc_altivec_lvsl;
10012 IntrLD = Intrinsic::ppc_altivec_lvx;
10013 IntrPerm = Intrinsic::ppc_altivec_vperm;
10014 PermCntlTy = MVT::v16i8;
10015 PermTy = MVT::v4i32;
10016 LDTy = MVT::v4i32;
10017 } else {
10018 Intr = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlpcld :
10019 Intrinsic::ppc_qpx_qvlpcls;
10020 IntrLD = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlfd :
10021 Intrinsic::ppc_qpx_qvlfs;
10022 IntrPerm = Intrinsic::ppc_qpx_qvfperm;
10023 PermCntlTy = MVT::v4f64;
10024 PermTy = MVT::v4f64;
10025 LDTy = MemVT.getSimpleVT();
10026 }
10027
10028 SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy);
Hal Finkelcf2e9082013-05-24 23:00:14 +000010029
Hal Finkelb6d0d6b2014-08-01 05:20:41 +000010030 // Create the new MMO for the new base load. It is like the original MMO,
10031 // but represents an area in memory almost twice the vector size centered
10032 // on the original address. If the address is unaligned, we might start
10033 // reading up to (sizeof(vector)-1) bytes below the address of the
10034 // original unaligned load.
Hal Finkelcf2e9082013-05-24 23:00:14 +000010035 MachineFunction &MF = DAG.getMachineFunction();
Hal Finkelb6d0d6b2014-08-01 05:20:41 +000010036 MachineMemOperand *BaseMMO =
Hal Finkelc93a9a22015-02-25 01:06:45 +000010037 MF.getMachineMemOperand(LD->getMemOperand(), -MemVT.getStoreSize()+1,
10038 2*MemVT.getStoreSize()-1);
Hal Finkelb6d0d6b2014-08-01 05:20:41 +000010039
10040 // Create the new base load.
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010041 SDValue LDXIntID = DAG.getTargetConstant(IntrLD, dl, getPointerTy());
Hal Finkelb6d0d6b2014-08-01 05:20:41 +000010042 SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr };
10043 SDValue BaseLoad =
10044 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl,
Hal Finkelc93a9a22015-02-25 01:06:45 +000010045 DAG.getVTList(PermTy, MVT::Other),
10046 BaseLoadOps, LDTy, BaseMMO);
Hal Finkelcf2e9082013-05-24 23:00:14 +000010047
10048 // Note that the value of IncOffset (which is provided to the next
10049 // load's pointer info offset value, and thus used to calculate the
10050 // alignment), and the value of IncValue (which is actually used to
10051 // increment the pointer value) are different! This is because we
10052 // require the next load to appear to be aligned, even though it
10053 // is actually offset from the base pointer by a lesser amount.
10054 int IncOffset = VT.getSizeInBits() / 8;
Hal Finkel7d8a6912013-05-26 18:08:30 +000010055 int IncValue = IncOffset;
10056
10057 // Walk (both up and down) the chain looking for another load at the real
10058 // (aligned) offset (the alignment of the other load does not matter in
10059 // this case). If found, then do not use the offset reduction trick, as
10060 // that will prevent the loads from being later combined (as they would
10061 // otherwise be duplicates).
10062 if (!findConsecutiveLoad(LD, DAG))
10063 --IncValue;
10064
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010065 SDValue Increment = DAG.getConstant(IncValue, dl, getPointerTy());
Hal Finkelcf2e9082013-05-24 23:00:14 +000010066 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
10067
Hal Finkelb6d0d6b2014-08-01 05:20:41 +000010068 MachineMemOperand *ExtraMMO =
10069 MF.getMachineMemOperand(LD->getMemOperand(),
Hal Finkelc93a9a22015-02-25 01:06:45 +000010070 1, 2*MemVT.getStoreSize()-1);
Hal Finkelb6d0d6b2014-08-01 05:20:41 +000010071 SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr };
Hal Finkelcf2e9082013-05-24 23:00:14 +000010072 SDValue ExtraLoad =
Hal Finkelb6d0d6b2014-08-01 05:20:41 +000010073 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl,
Hal Finkelc93a9a22015-02-25 01:06:45 +000010074 DAG.getVTList(PermTy, MVT::Other),
10075 ExtraLoadOps, LDTy, ExtraMMO);
Hal Finkelcf2e9082013-05-24 23:00:14 +000010076
10077 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
10078 BaseLoad.getValue(1), ExtraLoad.getValue(1));
10079
Bill Schmidt6b5a7df2014-06-09 22:00:52 +000010080 // Because vperm has a big-endian bias, we must reverse the order
10081 // of the input vectors and complement the permute control vector
10082 // when generating little endian code. We have already handled the
10083 // latter by using lvsr instead of lvsl, so just reverse BaseLoad
10084 // and ExtraLoad here.
10085 SDValue Perm;
10086 if (isLittleEndian)
Hal Finkelc93a9a22015-02-25 01:06:45 +000010087 Perm = BuildIntrinsicOp(IntrPerm,
Bill Schmidt6b5a7df2014-06-09 22:00:52 +000010088 ExtraLoad, BaseLoad, PermCntl, DAG, dl);
10089 else
Hal Finkelc93a9a22015-02-25 01:06:45 +000010090 Perm = BuildIntrinsicOp(IntrPerm,
Bill Schmidt6b5a7df2014-06-09 22:00:52 +000010091 BaseLoad, ExtraLoad, PermCntl, DAG, dl);
Hal Finkelcf2e9082013-05-24 23:00:14 +000010092
Hal Finkelc93a9a22015-02-25 01:06:45 +000010093 if (VT != PermTy)
10094 Perm = Subtarget.hasAltivec() ?
10095 DAG.getNode(ISD::BITCAST, dl, VT, Perm) :
10096 DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, // QPX
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010097 DAG.getTargetConstant(1, dl, MVT::i64));
Hal Finkelc93a9a22015-02-25 01:06:45 +000010098 // second argument is 1 because this rounding
10099 // is always exact.
Hal Finkelcf2e9082013-05-24 23:00:14 +000010100
Hal Finkelb6d0d6b2014-08-01 05:20:41 +000010101 // The output of the permutation is our loaded result, the TokenFactor is
10102 // our new chain.
10103 DCI.CombineTo(N, Perm, TF);
Hal Finkelcf2e9082013-05-24 23:00:14 +000010104 return SDValue(N, 0);
10105 }
10106 }
10107 break;
Eric Christophercccae792015-01-30 22:02:31 +000010108 case ISD::INTRINSIC_WO_CHAIN: {
10109 bool isLittleEndian = Subtarget.isLittleEndian();
Hal Finkelc93a9a22015-02-25 01:06:45 +000010110 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
Eric Christophercccae792015-01-30 22:02:31 +000010111 Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr
10112 : Intrinsic::ppc_altivec_lvsl);
Hal Finkelc93a9a22015-02-25 01:06:45 +000010113 if ((IID == Intr ||
10114 IID == Intrinsic::ppc_qpx_qvlpcld ||
10115 IID == Intrinsic::ppc_qpx_qvlpcls) &&
10116 N->getOperand(1)->getOpcode() == ISD::ADD) {
Eric Christophercccae792015-01-30 22:02:31 +000010117 SDValue Add = N->getOperand(1);
Hal Finkelbc2ee4c2013-05-25 04:05:05 +000010118
Hal Finkelc93a9a22015-02-25 01:06:45 +000010119 int Bits = IID == Intrinsic::ppc_qpx_qvlpcld ?
10120 5 /* 32 byte alignment */ : 4 /* 16 byte alignment */;
10121
Eric Christophercccae792015-01-30 22:02:31 +000010122 if (DAG.MaskedValueIsZero(
10123 Add->getOperand(1),
Hal Finkelc93a9a22015-02-25 01:06:45 +000010124 APInt::getAllOnesValue(Bits /* alignment */)
Eric Christophercccae792015-01-30 22:02:31 +000010125 .zext(
10126 Add.getValueType().getScalarType().getSizeInBits()))) {
10127 SDNode *BasePtr = Add->getOperand(0).getNode();
10128 for (SDNode::use_iterator UI = BasePtr->use_begin(),
10129 UE = BasePtr->use_end();
10130 UI != UE; ++UI) {
10131 if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
Hal Finkelc93a9a22015-02-25 01:06:45 +000010132 cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() == IID) {
Eric Christophercccae792015-01-30 22:02:31 +000010133 // We've found another LVSL/LVSR, and this address is an aligned
10134 // multiple of that one. The results will be the same, so use the
10135 // one we've just found instead.
Hal Finkelbc2ee4c2013-05-25 04:05:05 +000010136
Eric Christophercccae792015-01-30 22:02:31 +000010137 return SDValue(*UI, 0);
10138 }
Hal Finkelbc2ee4c2013-05-25 04:05:05 +000010139 }
10140 }
Hal Finkelc93a9a22015-02-25 01:06:45 +000010141
10142 if (isa<ConstantSDNode>(Add->getOperand(1))) {
10143 SDNode *BasePtr = Add->getOperand(0).getNode();
10144 for (SDNode::use_iterator UI = BasePtr->use_begin(),
10145 UE = BasePtr->use_end(); UI != UE; ++UI) {
10146 if (UI->getOpcode() == ISD::ADD &&
10147 isa<ConstantSDNode>(UI->getOperand(1)) &&
10148 (cast<ConstantSDNode>(Add->getOperand(1))->getZExtValue() -
10149 cast<ConstantSDNode>(UI->getOperand(1))->getZExtValue()) %
Aaron Ballman5561ed42015-02-25 13:05:24 +000010150 (1ULL << Bits) == 0) {
Hal Finkelc93a9a22015-02-25 01:06:45 +000010151 SDNode *OtherAdd = *UI;
10152 for (SDNode::use_iterator VI = OtherAdd->use_begin(),
10153 VE = OtherAdd->use_end(); VI != VE; ++VI) {
10154 if (VI->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
10155 cast<ConstantSDNode>(VI->getOperand(0))->getZExtValue() == IID) {
10156 return SDValue(*VI, 0);
10157 }
10158 }
10159 }
10160 }
10161 }
Hal Finkelbc2ee4c2013-05-25 04:05:05 +000010162 }
10163 }
Hal Finkelc3cfbf82013-09-13 20:09:02 +000010164
10165 break;
Bill Schmidtfae5d712014-12-09 16:35:51 +000010166 case ISD::INTRINSIC_W_CHAIN: {
10167 // For little endian, VSX loads require generating lxvd2x/xxswapd.
Eric Christophercccae792015-01-30 22:02:31 +000010168 if (Subtarget.hasVSX() && Subtarget.isLittleEndian()) {
Bill Schmidtfae5d712014-12-09 16:35:51 +000010169 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
10170 default:
10171 break;
10172 case Intrinsic::ppc_vsx_lxvw4x:
10173 case Intrinsic::ppc_vsx_lxvd2x:
10174 return expandVSXLoadForLE(N, DCI);
10175 }
10176 }
10177 break;
10178 }
10179 case ISD::INTRINSIC_VOID: {
10180 // For little endian, VSX stores require generating xxswapd/stxvd2x.
Eric Christophercccae792015-01-30 22:02:31 +000010181 if (Subtarget.hasVSX() && Subtarget.isLittleEndian()) {
Bill Schmidtfae5d712014-12-09 16:35:51 +000010182 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
10183 default:
10184 break;
10185 case Intrinsic::ppc_vsx_stxvw4x:
10186 case Intrinsic::ppc_vsx_stxvd2x:
10187 return expandVSXStoreForLE(N, DCI);
10188 }
10189 }
10190 break;
10191 }
Chris Lattnera7976d32006-07-10 20:56:58 +000010192 case ISD::BSWAP:
10193 // Turn BSWAP (LOAD) -> lhbrx/lwbrx.
Gabor Greiff304a7a2008-08-28 21:40:38 +000010194 if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) &&
Chris Lattnera7976d32006-07-10 20:56:58 +000010195 N->getOperand(0).hasOneUse() &&
Hal Finkel31d29562013-03-28 19:25:55 +000010196 (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i16 ||
Eric Christophercccae792015-01-30 22:02:31 +000010197 (Subtarget.hasLDBRX() && Subtarget.isPPC64() &&
Hal Finkel31d29562013-03-28 19:25:55 +000010198 N->getValueType(0) == MVT::i64))) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000010199 SDValue Load = N->getOperand(0);
Evan Chenge71fe34d2006-10-09 20:57:25 +000010200 LoadSDNode *LD = cast<LoadSDNode>(Load);
Chris Lattnera7976d32006-07-10 20:56:58 +000010201 // Create the byte-swapping load.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000010202 SDValue Ops[] = {
Evan Chenge71fe34d2006-10-09 20:57:25 +000010203 LD->getChain(), // Chain
10204 LD->getBasePtr(), // Ptr
Chris Lattnerd66f14e2006-08-11 17:18:05 +000010205 DAG.getValueType(N->getValueType(0)) // VT
10206 };
Dan Gohman48b185d2009-09-25 20:36:54 +000010207 SDValue BSLoad =
10208 DAG.getMemIntrinsicNode(PPCISD::LBRX, dl,
Hal Finkel31d29562013-03-28 19:25:55 +000010209 DAG.getVTList(N->getValueType(0) == MVT::i64 ?
10210 MVT::i64 : MVT::i32, MVT::Other),
Craig Topper206fcd42014-04-26 19:29:41 +000010211 Ops, LD->getMemoryVT(), LD->getMemOperand());
Chris Lattnera7976d32006-07-10 20:56:58 +000010212
Scott Michelcf0da6c2009-02-17 22:15:04 +000010213 // If this is an i16 load, insert the truncate.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000010214 SDValue ResVal = BSLoad;
Owen Anderson9f944592009-08-11 20:47:22 +000010215 if (N->getValueType(0) == MVT::i16)
10216 ResVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, BSLoad);
Scott Michelcf0da6c2009-02-17 22:15:04 +000010217
Chris Lattnera7976d32006-07-10 20:56:58 +000010218 // First, combine the bswap away. This makes the value produced by the
10219 // load dead.
10220 DCI.CombineTo(N, ResVal);
10221
10222 // Next, combine the load away, we give it a bogus result value but a real
10223 // chain result. The result value is dead because the bswap is dead.
Gabor Greiff304a7a2008-08-28 21:40:38 +000010224 DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1));
Scott Michelcf0da6c2009-02-17 22:15:04 +000010225
Chris Lattnera7976d32006-07-10 20:56:58 +000010226 // Return N so it doesn't get rechecked!
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000010227 return SDValue(N, 0);
Chris Lattnera7976d32006-07-10 20:56:58 +000010228 }
Scott Michelcf0da6c2009-02-17 22:15:04 +000010229
Chris Lattner27f53452006-03-01 05:50:56 +000010230 break;
Chris Lattnerd4058a52006-03-31 06:02:07 +000010231 case PPCISD::VCMP: {
10232 // If a VCMPo node already exists with exactly the same operands as this
10233 // node, use its result instead of this node (VCMPo computes both a CR6 and
10234 // a normal output).
10235 //
10236 if (!N->getOperand(0).hasOneUse() &&
10237 !N->getOperand(1).hasOneUse() &&
10238 !N->getOperand(2).hasOneUse()) {
Scott Michelcf0da6c2009-02-17 22:15:04 +000010239
Chris Lattnerd4058a52006-03-31 06:02:07 +000010240 // Scan all of the users of the LHS, looking for VCMPo's that match.
Craig Topper062a2ba2014-04-25 05:30:21 +000010241 SDNode *VCMPoNode = nullptr;
Scott Michelcf0da6c2009-02-17 22:15:04 +000010242
Gabor Greiff304a7a2008-08-28 21:40:38 +000010243 SDNode *LHSN = N->getOperand(0).getNode();
Chris Lattnerd4058a52006-03-31 06:02:07 +000010244 for (SDNode::use_iterator UI = LHSN->use_begin(), E = LHSN->use_end();
10245 UI != E; ++UI)
Dan Gohman91e5dcb2008-07-27 20:43:25 +000010246 if (UI->getOpcode() == PPCISD::VCMPo &&
10247 UI->getOperand(1) == N->getOperand(1) &&
10248 UI->getOperand(2) == N->getOperand(2) &&
10249 UI->getOperand(0) == N->getOperand(0)) {
10250 VCMPoNode = *UI;
Chris Lattnerd4058a52006-03-31 06:02:07 +000010251 break;
10252 }
Scott Michelcf0da6c2009-02-17 22:15:04 +000010253
Chris Lattner518834c2006-04-18 18:28:22 +000010254 // If there is no VCMPo node, or if the flag value has a single use, don't
10255 // transform this.
10256 if (!VCMPoNode || VCMPoNode->hasNUsesOfValue(0, 1))
10257 break;
Scott Michelcf0da6c2009-02-17 22:15:04 +000010258
10259 // Look at the (necessarily single) use of the flag value. If it has a
Chris Lattner518834c2006-04-18 18:28:22 +000010260 // chain, this transformation is more complex. Note that multiple things
10261 // could use the value result, which we should ignore.
Craig Topper062a2ba2014-04-25 05:30:21 +000010262 SDNode *FlagUser = nullptr;
Scott Michelcf0da6c2009-02-17 22:15:04 +000010263 for (SDNode::use_iterator UI = VCMPoNode->use_begin();
Craig Topper062a2ba2014-04-25 05:30:21 +000010264 FlagUser == nullptr; ++UI) {
Chris Lattner518834c2006-04-18 18:28:22 +000010265 assert(UI != VCMPoNode->use_end() && "Didn't find user!");
Dan Gohman91e5dcb2008-07-27 20:43:25 +000010266 SDNode *User = *UI;
Chris Lattner518834c2006-04-18 18:28:22 +000010267 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000010268 if (User->getOperand(i) == SDValue(VCMPoNode, 1)) {
Chris Lattner518834c2006-04-18 18:28:22 +000010269 FlagUser = User;
10270 break;
10271 }
10272 }
10273 }
Scott Michelcf0da6c2009-02-17 22:15:04 +000010274
Ulrich Weigandd5ebc622013-07-03 17:05:42 +000010275 // If the user is a MFOCRF instruction, we know this is safe.
10276 // Otherwise we give up for right now.
10277 if (FlagUser->getOpcode() == PPCISD::MFOCRF)
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000010278 return SDValue(VCMPoNode, 0);
Chris Lattnerd4058a52006-03-31 06:02:07 +000010279 }
10280 break;
10281 }
Hal Finkel940ab932014-02-28 00:27:01 +000010282 case ISD::BRCOND: {
10283 SDValue Cond = N->getOperand(1);
10284 SDValue Target = N->getOperand(2);
10285
10286 if (Cond.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
10287 cast<ConstantSDNode>(Cond.getOperand(1))->getZExtValue() ==
10288 Intrinsic::ppc_is_decremented_ctr_nonzero) {
10289
10290 // We now need to make the intrinsic dead (it cannot be instruction
10291 // selected).
10292 DAG.ReplaceAllUsesOfValueWith(Cond.getValue(1), Cond.getOperand(0));
10293 assert(Cond.getNode()->hasOneUse() &&
10294 "Counter decrement has more than one use");
10295
10296 return DAG.getNode(PPCISD::BDNZ, dl, MVT::Other,
10297 N->getOperand(0), Target);
10298 }
10299 }
10300 break;
Chris Lattner9754d142006-04-18 17:59:36 +000010301 case ISD::BR_CC: {
10302 // If this is a branch on an altivec predicate comparison, lower this so
Ulrich Weigandd5ebc622013-07-03 17:05:42 +000010303 // that we don't have to do a MFOCRF: instead, branch directly on CR6. This
Chris Lattner9754d142006-04-18 17:59:36 +000010304 // lowering is done pre-legalize, because the legalizer lowers the predicate
10305 // compare down to code that is difficult to reassemble.
10306 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000010307 SDValue LHS = N->getOperand(2), RHS = N->getOperand(3);
Hal Finkel25c19922013-05-15 21:37:41 +000010308
10309 // Sometimes the promoted value of the intrinsic is ANDed by some non-zero
10310 // value. If so, pass-through the AND to get to the intrinsic.
10311 if (LHS.getOpcode() == ISD::AND &&
10312 LHS.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN &&
10313 cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1))->getZExtValue() ==
10314 Intrinsic::ppc_is_decremented_ctr_nonzero &&
10315 isa<ConstantSDNode>(LHS.getOperand(1)) &&
10316 !cast<ConstantSDNode>(LHS.getOperand(1))->getConstantIntValue()->
10317 isZero())
10318 LHS = LHS.getOperand(0);
10319
10320 if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
10321 cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue() ==
10322 Intrinsic::ppc_is_decremented_ctr_nonzero &&
10323 isa<ConstantSDNode>(RHS)) {
10324 assert((CC == ISD::SETEQ || CC == ISD::SETNE) &&
10325 "Counter decrement comparison is not EQ or NE");
10326
10327 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue();
10328 bool isBDNZ = (CC == ISD::SETEQ && Val) ||
10329 (CC == ISD::SETNE && !Val);
10330
10331 // We now need to make the intrinsic dead (it cannot be instruction
10332 // selected).
10333 DAG.ReplaceAllUsesOfValueWith(LHS.getValue(1), LHS.getOperand(0));
10334 assert(LHS.getNode()->hasOneUse() &&
10335 "Counter decrement has more than one use");
10336
10337 return DAG.getNode(isBDNZ ? PPCISD::BDNZ : PPCISD::BDZ, dl, MVT::Other,
10338 N->getOperand(0), N->getOperand(4));
10339 }
10340
Chris Lattner9754d142006-04-18 17:59:36 +000010341 int CompareOpc;
10342 bool isDot;
Scott Michelcf0da6c2009-02-17 22:15:04 +000010343
Chris Lattner9754d142006-04-18 17:59:36 +000010344 if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
10345 isa<ConstantSDNode>(RHS) && (CC == ISD::SETEQ || CC == ISD::SETNE) &&
Kit Barton0cfa7b72015-03-03 19:55:45 +000010346 getAltivecCompareInfo(LHS, CompareOpc, isDot, Subtarget)) {
Chris Lattner9754d142006-04-18 17:59:36 +000010347 assert(isDot && "Can't compare against a vector result!");
Scott Michelcf0da6c2009-02-17 22:15:04 +000010348
Chris Lattner9754d142006-04-18 17:59:36 +000010349 // If this is a comparison against something other than 0/1, then we know
10350 // that the condition is never/always true.
Dan Gohmaneffb8942008-09-12 16:56:44 +000010351 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue();
Chris Lattner9754d142006-04-18 17:59:36 +000010352 if (Val != 0 && Val != 1) {
10353 if (CC == ISD::SETEQ) // Cond never true, remove branch.
10354 return N->getOperand(0);
10355 // Always !=, turn it into an unconditional branch.
Owen Anderson9f944592009-08-11 20:47:22 +000010356 return DAG.getNode(ISD::BR, dl, MVT::Other,
Chris Lattner9754d142006-04-18 17:59:36 +000010357 N->getOperand(0), N->getOperand(4));
10358 }
Scott Michelcf0da6c2009-02-17 22:15:04 +000010359
Chris Lattner9754d142006-04-18 17:59:36 +000010360 bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0);
Scott Michelcf0da6c2009-02-17 22:15:04 +000010361
Chris Lattner9754d142006-04-18 17:59:36 +000010362 // Create the PPCISD altivec 'dot' comparison node.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000010363 SDValue Ops[] = {
Chris Lattnerd66f14e2006-08-11 17:18:05 +000010364 LHS.getOperand(2), // LHS of compare
10365 LHS.getOperand(3), // RHS of compare
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010366 DAG.getConstant(CompareOpc, dl, MVT::i32)
Chris Lattnerd66f14e2006-08-11 17:18:05 +000010367 };
Benjamin Kramerfdf362b2013-03-07 20:33:29 +000010368 EVT VTs[] = { LHS.getOperand(2).getValueType(), MVT::Glue };
Craig Topper48d114b2014-04-26 18:35:24 +000010369 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops);
Scott Michelcf0da6c2009-02-17 22:15:04 +000010370
Chris Lattner9754d142006-04-18 17:59:36 +000010371 // Unpack the result based on how the target uses it.
Chris Lattner8c6a41e2006-11-17 22:10:59 +000010372 PPC::Predicate CompOpc;
Dan Gohmaneffb8942008-09-12 16:56:44 +000010373 switch (cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue()) {
Chris Lattner9754d142006-04-18 17:59:36 +000010374 default: // Can't happen, don't crash on invalid number though.
10375 case 0: // Branch on the value of the EQ bit of CR6.
Chris Lattner8c6a41e2006-11-17 22:10:59 +000010376 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE;
Chris Lattner9754d142006-04-18 17:59:36 +000010377 break;
10378 case 1: // Branch on the inverted value of the EQ bit of CR6.
Chris Lattner8c6a41e2006-11-17 22:10:59 +000010379 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ;
Chris Lattner9754d142006-04-18 17:59:36 +000010380 break;
10381 case 2: // Branch on the value of the LT bit of CR6.
Chris Lattner8c6a41e2006-11-17 22:10:59 +000010382 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE;
Chris Lattner9754d142006-04-18 17:59:36 +000010383 break;
10384 case 3: // Branch on the inverted value of the LT bit of CR6.
Chris Lattner8c6a41e2006-11-17 22:10:59 +000010385 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT;
Chris Lattner9754d142006-04-18 17:59:36 +000010386 break;
10387 }
10388
Owen Anderson9f944592009-08-11 20:47:22 +000010389 return DAG.getNode(PPCISD::COND_BRANCH, dl, MVT::Other, N->getOperand(0),
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010390 DAG.getConstant(CompOpc, dl, MVT::i32),
Owen Anderson9f944592009-08-11 20:47:22 +000010391 DAG.getRegister(PPC::CR6, MVT::i32),
Chris Lattner9754d142006-04-18 17:59:36 +000010392 N->getOperand(4), CompNode.getValue(1));
10393 }
10394 break;
10395 }
Chris Lattnerf4184352006-03-01 04:57:39 +000010396 }
Scott Michelcf0da6c2009-02-17 22:15:04 +000010397
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000010398 return SDValue();
Chris Lattnerf4184352006-03-01 04:57:39 +000010399}
10400
Hal Finkel13d104b2014-12-11 18:37:52 +000010401SDValue
10402PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
10403 SelectionDAG &DAG,
10404 std::vector<SDNode *> *Created) const {
10405 // fold (sdiv X, pow2)
10406 EVT VT = N->getValueType(0);
Hal Finkel04b16b52014-12-23 08:38:50 +000010407 if (VT == MVT::i64 && !Subtarget.isPPC64())
10408 return SDValue();
Hal Finkel13d104b2014-12-11 18:37:52 +000010409 if ((VT != MVT::i32 && VT != MVT::i64) ||
10410 !(Divisor.isPowerOf2() || (-Divisor).isPowerOf2()))
10411 return SDValue();
10412
10413 SDLoc DL(N);
10414 SDValue N0 = N->getOperand(0);
10415
10416 bool IsNegPow2 = (-Divisor).isPowerOf2();
10417 unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countTrailingZeros();
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010418 SDValue ShiftAmt = DAG.getConstant(Lg2, DL, VT);
Hal Finkel13d104b2014-12-11 18:37:52 +000010419
10420 SDValue Op = DAG.getNode(PPCISD::SRA_ADDZE, DL, VT, N0, ShiftAmt);
10421 if (Created)
10422 Created->push_back(Op.getNode());
10423
10424 if (IsNegPow2) {
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010425 Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op);
Hal Finkel13d104b2014-12-11 18:37:52 +000010426 if (Created)
10427 Created->push_back(Op.getNode());
10428 }
10429
10430 return Op;
10431}
10432
Chris Lattner4211ca92006-04-14 06:01:58 +000010433//===----------------------------------------------------------------------===//
10434// Inline Assembly Support
10435//===----------------------------------------------------------------------===//
10436
Jay Foada0653a32014-05-14 21:14:37 +000010437void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
10438 APInt &KnownZero,
10439 APInt &KnownOne,
10440 const SelectionDAG &DAG,
10441 unsigned Depth) const {
Rafael Espindolaba0a6ca2012-04-04 12:51:34 +000010442 KnownZero = KnownOne = APInt(KnownZero.getBitWidth(), 0);
Chris Lattnerc5287c02006-04-02 06:26:07 +000010443 switch (Op.getOpcode()) {
10444 default: break;
Chris Lattnera7976d32006-07-10 20:56:58 +000010445 case PPCISD::LBRX: {
10446 // lhbrx is known to have the top bits cleared out.
Dan Gohmana5fc0352009-09-27 23:17:47 +000010447 if (cast<VTSDNode>(Op.getOperand(2))->getVT() == MVT::i16)
Chris Lattnera7976d32006-07-10 20:56:58 +000010448 KnownZero = 0xFFFF0000;
10449 break;
10450 }
Chris Lattnerc5287c02006-04-02 06:26:07 +000010451 case ISD::INTRINSIC_WO_CHAIN: {
Dan Gohmaneffb8942008-09-12 16:56:44 +000010452 switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) {
Chris Lattnerc5287c02006-04-02 06:26:07 +000010453 default: break;
10454 case Intrinsic::ppc_altivec_vcmpbfp_p:
10455 case Intrinsic::ppc_altivec_vcmpeqfp_p:
10456 case Intrinsic::ppc_altivec_vcmpequb_p:
10457 case Intrinsic::ppc_altivec_vcmpequh_p:
10458 case Intrinsic::ppc_altivec_vcmpequw_p:
Kit Barton0cfa7b72015-03-03 19:55:45 +000010459 case Intrinsic::ppc_altivec_vcmpequd_p:
Chris Lattnerc5287c02006-04-02 06:26:07 +000010460 case Intrinsic::ppc_altivec_vcmpgefp_p:
10461 case Intrinsic::ppc_altivec_vcmpgtfp_p:
10462 case Intrinsic::ppc_altivec_vcmpgtsb_p:
10463 case Intrinsic::ppc_altivec_vcmpgtsh_p:
10464 case Intrinsic::ppc_altivec_vcmpgtsw_p:
Kit Barton0cfa7b72015-03-03 19:55:45 +000010465 case Intrinsic::ppc_altivec_vcmpgtsd_p:
Chris Lattnerc5287c02006-04-02 06:26:07 +000010466 case Intrinsic::ppc_altivec_vcmpgtub_p:
10467 case Intrinsic::ppc_altivec_vcmpgtuh_p:
10468 case Intrinsic::ppc_altivec_vcmpgtuw_p:
Kit Barton0cfa7b72015-03-03 19:55:45 +000010469 case Intrinsic::ppc_altivec_vcmpgtud_p:
Chris Lattnerc5287c02006-04-02 06:26:07 +000010470 KnownZero = ~1U; // All bits but the low one are known to be zero.
10471 break;
Scott Michelcf0da6c2009-02-17 22:15:04 +000010472 }
Chris Lattnerc5287c02006-04-02 06:26:07 +000010473 }
10474 }
10475}
10476
Hal Finkel57725662015-01-03 17:58:24 +000010477unsigned PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
10478 switch (Subtarget.getDarwinDirective()) {
10479 default: break;
10480 case PPC::DIR_970:
10481 case PPC::DIR_PWR4:
10482 case PPC::DIR_PWR5:
10483 case PPC::DIR_PWR5X:
10484 case PPC::DIR_PWR6:
10485 case PPC::DIR_PWR6X:
10486 case PPC::DIR_PWR7:
10487 case PPC::DIR_PWR8: {
10488 if (!ML)
10489 break;
10490
Eric Christophercccae792015-01-30 22:02:31 +000010491 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
Hal Finkel57725662015-01-03 17:58:24 +000010492
10493 // For small loops (between 5 and 8 instructions), align to a 32-byte
10494 // boundary so that the entire loop fits in one instruction-cache line.
10495 uint64_t LoopSize = 0;
10496 for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I)
10497 for (auto J = (*I)->begin(), JE = (*I)->end(); J != JE; ++J)
10498 LoopSize += TII->GetInstSizeInBytes(J);
10499
10500 if (LoopSize > 16 && LoopSize <= 32)
10501 return 5;
10502
10503 break;
10504 }
10505 }
10506
10507 return TargetLowering::getPrefLoopAlignment(ML);
10508}
Chris Lattnerc5287c02006-04-02 06:26:07 +000010509
Chris Lattnerd6855142007-03-25 02:14:49 +000010510/// getConstraintType - Given a constraint, return the type of
Chris Lattner203b2f12006-02-07 20:16:30 +000010511/// constraint it is for this target.
Scott Michelcf0da6c2009-02-17 22:15:04 +000010512PPCTargetLowering::ConstraintType
Chris Lattnerd6855142007-03-25 02:14:49 +000010513PPCTargetLowering::getConstraintType(const std::string &Constraint) const {
10514 if (Constraint.size() == 1) {
10515 switch (Constraint[0]) {
10516 default: break;
10517 case 'b':
10518 case 'r':
10519 case 'f':
10520 case 'v':
10521 case 'y':
10522 return C_RegisterClass;
Hal Finkel4f24c622012-11-05 18:18:42 +000010523 case 'Z':
10524 // FIXME: While Z does indicate a memory constraint, it specifically
10525 // indicates an r+r address (used in conjunction with the 'y' modifier
10526 // in the replacement string). Currently, we're forcing the base
10527 // register to be r0 in the asm printer (which is interpreted as zero)
10528 // and forming the complete address in the second register. This is
10529 // suboptimal.
10530 return C_Memory;
Chris Lattnerd6855142007-03-25 02:14:49 +000010531 }
Hal Finkel6aca2372014-03-02 18:23:39 +000010532 } else if (Constraint == "wc") { // individual CR bits.
10533 return C_RegisterClass;
Hal Finkel27774d92014-03-13 07:58:58 +000010534 } else if (Constraint == "wa" || Constraint == "wd" ||
10535 Constraint == "wf" || Constraint == "ws") {
10536 return C_RegisterClass; // VSX registers.
Chris Lattnerd6855142007-03-25 02:14:49 +000010537 }
10538 return TargetLowering::getConstraintType(Constraint);
Chris Lattner203b2f12006-02-07 20:16:30 +000010539}
10540
John Thompsone8360b72010-10-29 17:29:13 +000010541/// Examine constraint type and operand type and determine a weight value.
10542/// This object must already have been set up with the operand type
10543/// and the current alternative constraint selected.
10544TargetLowering::ConstraintWeight
10545PPCTargetLowering::getSingleConstraintMatchWeight(
10546 AsmOperandInfo &info, const char *constraint) const {
10547 ConstraintWeight weight = CW_Invalid;
10548 Value *CallOperandVal = info.CallOperandVal;
10549 // If we don't have a value, we can't do a match,
10550 // but allow it at the lowest weight.
Craig Topper062a2ba2014-04-25 05:30:21 +000010551 if (!CallOperandVal)
John Thompsone8360b72010-10-29 17:29:13 +000010552 return CW_Default;
Chris Lattner229907c2011-07-18 04:54:35 +000010553 Type *type = CallOperandVal->getType();
Hal Finkel6aca2372014-03-02 18:23:39 +000010554
John Thompsone8360b72010-10-29 17:29:13 +000010555 // Look at the constraint type.
Hal Finkel6aca2372014-03-02 18:23:39 +000010556 if (StringRef(constraint) == "wc" && type->isIntegerTy(1))
10557 return CW_Register; // an individual CR bit.
Hal Finkel27774d92014-03-13 07:58:58 +000010558 else if ((StringRef(constraint) == "wa" ||
10559 StringRef(constraint) == "wd" ||
10560 StringRef(constraint) == "wf") &&
10561 type->isVectorTy())
10562 return CW_Register;
10563 else if (StringRef(constraint) == "ws" && type->isDoubleTy())
10564 return CW_Register;
Hal Finkel6aca2372014-03-02 18:23:39 +000010565
John Thompsone8360b72010-10-29 17:29:13 +000010566 switch (*constraint) {
10567 default:
10568 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
10569 break;
10570 case 'b':
10571 if (type->isIntegerTy())
10572 weight = CW_Register;
10573 break;
10574 case 'f':
10575 if (type->isFloatTy())
10576 weight = CW_Register;
10577 break;
10578 case 'd':
10579 if (type->isDoubleTy())
10580 weight = CW_Register;
10581 break;
10582 case 'v':
10583 if (type->isVectorTy())
10584 weight = CW_Register;
10585 break;
10586 case 'y':
10587 weight = CW_Register;
10588 break;
Hal Finkel4f24c622012-11-05 18:18:42 +000010589 case 'Z':
10590 weight = CW_Memory;
10591 break;
John Thompsone8360b72010-10-29 17:29:13 +000010592 }
10593 return weight;
10594}
10595
Eric Christopher11e4df72015-02-26 22:38:43 +000010596std::pair<unsigned, const TargetRegisterClass *>
10597PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
10598 const std::string &Constraint,
Chad Rosier295bd432013-06-22 18:37:38 +000010599 MVT VT) const {
Chris Lattner01513612006-01-31 19:20:21 +000010600 if (Constraint.size() == 1) {
Chris Lattner584a11a2006-11-02 01:44:04 +000010601 // GCC RS6000 Constraint Letters
10602 switch (Constraint[0]) {
10603 case 'b': // R1-R31
Eric Christopherb1aaebe2014-06-12 22:38:18 +000010604 if (VT == MVT::i64 && Subtarget.isPPC64())
Hal Finkel638a9fa2013-03-19 18:51:05 +000010605 return std::make_pair(0U, &PPC::G8RC_NOX0RegClass);
10606 return std::make_pair(0U, &PPC::GPRC_NOR0RegClass);
Chris Lattner584a11a2006-11-02 01:44:04 +000010607 case 'r': // R0-R31
Eric Christopherb1aaebe2014-06-12 22:38:18 +000010608 if (VT == MVT::i64 && Subtarget.isPPC64())
Craig Topperabadc662012-04-20 06:31:50 +000010609 return std::make_pair(0U, &PPC::G8RCRegClass);
10610 return std::make_pair(0U, &PPC::GPRCRegClass);
Chris Lattner584a11a2006-11-02 01:44:04 +000010611 case 'f':
Ulrich Weigand0de4a1e2012-10-29 17:49:34 +000010612 if (VT == MVT::f32 || VT == MVT::i32)
Craig Topperabadc662012-04-20 06:31:50 +000010613 return std::make_pair(0U, &PPC::F4RCRegClass);
Ulrich Weigand0de4a1e2012-10-29 17:49:34 +000010614 if (VT == MVT::f64 || VT == MVT::i64)
Craig Topperabadc662012-04-20 06:31:50 +000010615 return std::make_pair(0U, &PPC::F8RCRegClass);
Hal Finkelc93a9a22015-02-25 01:06:45 +000010616 if (VT == MVT::v4f64 && Subtarget.hasQPX())
10617 return std::make_pair(0U, &PPC::QFRCRegClass);
10618 if (VT == MVT::v4f32 && Subtarget.hasQPX())
10619 return std::make_pair(0U, &PPC::QSRCRegClass);
Chris Lattner584a11a2006-11-02 01:44:04 +000010620 break;
Scott Michelcf0da6c2009-02-17 22:15:04 +000010621 case 'v':
Hal Finkelc93a9a22015-02-25 01:06:45 +000010622 if (VT == MVT::v4f64 && Subtarget.hasQPX())
10623 return std::make_pair(0U, &PPC::QFRCRegClass);
10624 if (VT == MVT::v4f32 && Subtarget.hasQPX())
10625 return std::make_pair(0U, &PPC::QSRCRegClass);
Craig Topperabadc662012-04-20 06:31:50 +000010626 return std::make_pair(0U, &PPC::VRRCRegClass);
Chris Lattner584a11a2006-11-02 01:44:04 +000010627 case 'y': // crrc
Craig Topperabadc662012-04-20 06:31:50 +000010628 return std::make_pair(0U, &PPC::CRRCRegClass);
Chris Lattner01513612006-01-31 19:20:21 +000010629 }
Hal Finkel6aca2372014-03-02 18:23:39 +000010630 } else if (Constraint == "wc") { // an individual CR bit.
10631 return std::make_pair(0U, &PPC::CRBITRCRegClass);
Hal Finkel27774d92014-03-13 07:58:58 +000010632 } else if (Constraint == "wa" || Constraint == "wd" ||
Hal Finkel19be5062014-03-29 05:29:01 +000010633 Constraint == "wf") {
Hal Finkel27774d92014-03-13 07:58:58 +000010634 return std::make_pair(0U, &PPC::VSRCRegClass);
Hal Finkel19be5062014-03-29 05:29:01 +000010635 } else if (Constraint == "ws") {
10636 return std::make_pair(0U, &PPC::VSFRCRegClass);
Chris Lattner01513612006-01-31 19:20:21 +000010637 }
Scott Michelcf0da6c2009-02-17 22:15:04 +000010638
Eric Christopher11e4df72015-02-26 22:38:43 +000010639 std::pair<unsigned, const TargetRegisterClass *> R =
10640 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
Hal Finkelb176acb2013-08-03 12:25:10 +000010641
10642 // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers
10643 // (which we call X[0-9]+). If a 64-bit value has been requested, and a
10644 // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent
10645 // register.
10646 // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use
10647 // the AsmName field from *RegisterInfo.td, then this would not be necessary.
Eric Christopherb1aaebe2014-06-12 22:38:18 +000010648 if (R.first && VT == MVT::i64 && Subtarget.isPPC64() &&
Eric Christopher11e4df72015-02-26 22:38:43 +000010649 PPC::GPRCRegClass.contains(R.first))
Hal Finkelb176acb2013-08-03 12:25:10 +000010650 return std::make_pair(TRI->getMatchingSuperReg(R.first,
Hal Finkelb3ca00d2013-08-14 20:05:04 +000010651 PPC::sub_32, &PPC::G8RCRegClass),
Hal Finkelb176acb2013-08-03 12:25:10 +000010652 &PPC::G8RCRegClass);
Hal Finkelb176acb2013-08-03 12:25:10 +000010653
Hal Finkelaa10b3c2014-12-08 22:54:22 +000010654 // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same.
10655 if (!R.second && StringRef("{cc}").equals_lower(Constraint)) {
10656 R.first = PPC::CR0;
10657 R.second = &PPC::CRRCRegClass;
10658 }
10659
Hal Finkelb176acb2013-08-03 12:25:10 +000010660 return R;
Chris Lattner01513612006-01-31 19:20:21 +000010661}
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010662
Chris Lattner584a11a2006-11-02 01:44:04 +000010663
Chris Lattnerd8c9cb92007-08-25 00:47:38 +000010664/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
Dale Johannesence97d552010-06-25 21:55:36 +000010665/// vector. If it is invalid, don't add anything to Ops.
Eric Christopher0713a9d2011-06-08 23:55:35 +000010666void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
Eric Christopherde9399b2011-06-02 23:16:42 +000010667 std::string &Constraint,
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000010668 std::vector<SDValue>&Ops,
Chris Lattner724539c2008-04-26 23:02:14 +000010669 SelectionDAG &DAG) const {
Craig Topper062a2ba2014-04-25 05:30:21 +000010670 SDValue Result;
Eric Christopher0713a9d2011-06-08 23:55:35 +000010671
Eric Christopherde9399b2011-06-02 23:16:42 +000010672 // Only support length 1 constraints.
10673 if (Constraint.length() > 1) return;
Eric Christopher0713a9d2011-06-08 23:55:35 +000010674
Eric Christopherde9399b2011-06-02 23:16:42 +000010675 char Letter = Constraint[0];
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010676 switch (Letter) {
10677 default: break;
10678 case 'I':
10679 case 'J':
10680 case 'K':
10681 case 'L':
10682 case 'M':
10683 case 'N':
10684 case 'O':
10685 case 'P': {
Chris Lattner0b7472d2007-05-15 01:31:05 +000010686 ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op);
Chris Lattnerd8c9cb92007-08-25 00:47:38 +000010687 if (!CST) return; // Must be an immediate to match.
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010688 SDLoc dl(Op);
Hal Finkelc91fc112014-12-03 09:37:50 +000010689 int64_t Value = CST->getSExtValue();
10690 EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative
10691 // numbers are printed as such.
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010692 switch (Letter) {
Torok Edwinfbcc6632009-07-14 16:55:14 +000010693 default: llvm_unreachable("Unknown constraint letter!");
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010694 case 'I': // "I" is a signed 16-bit constant.
Hal Finkelc91fc112014-12-03 09:37:50 +000010695 if (isInt<16>(Value))
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010696 Result = DAG.getTargetConstant(Value, dl, TCVT);
Chris Lattner8c6949e2006-10-31 19:40:43 +000010697 break;
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010698 case 'J': // "J" is a constant with only the high-order 16 bits nonzero.
Hal Finkelc91fc112014-12-03 09:37:50 +000010699 if (isShiftedUInt<16, 16>(Value))
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010700 Result = DAG.getTargetConstant(Value, dl, TCVT);
Hal Finkelc91fc112014-12-03 09:37:50 +000010701 break;
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010702 case 'L': // "L" is a signed 16-bit constant shifted left 16 bits.
Hal Finkelc91fc112014-12-03 09:37:50 +000010703 if (isShiftedInt<16, 16>(Value))
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010704 Result = DAG.getTargetConstant(Value, dl, TCVT);
Chris Lattner8c6949e2006-10-31 19:40:43 +000010705 break;
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010706 case 'K': // "K" is a constant with only the low-order 16 bits nonzero.
Hal Finkelc91fc112014-12-03 09:37:50 +000010707 if (isUInt<16>(Value))
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010708 Result = DAG.getTargetConstant(Value, dl, TCVT);
Chris Lattner8c6949e2006-10-31 19:40:43 +000010709 break;
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010710 case 'M': // "M" is a constant that is greater than 31.
Chris Lattner0b7472d2007-05-15 01:31:05 +000010711 if (Value > 31)
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010712 Result = DAG.getTargetConstant(Value, dl, TCVT);
Chris Lattner8c6949e2006-10-31 19:40:43 +000010713 break;
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010714 case 'N': // "N" is a positive constant that is an exact power of two.
Hal Finkelc91fc112014-12-03 09:37:50 +000010715 if (Value > 0 && isPowerOf2_64(Value))
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010716 Result = DAG.getTargetConstant(Value, dl, TCVT);
Chris Lattner8c6949e2006-10-31 19:40:43 +000010717 break;
Scott Michelcf0da6c2009-02-17 22:15:04 +000010718 case 'O': // "O" is the constant zero.
Chris Lattner0b7472d2007-05-15 01:31:05 +000010719 if (Value == 0)
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010720 Result = DAG.getTargetConstant(Value, dl, TCVT);
Chris Lattner8c6949e2006-10-31 19:40:43 +000010721 break;
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010722 case 'P': // "P" is a constant whose negation is a signed 16-bit constant.
Hal Finkelc91fc112014-12-03 09:37:50 +000010723 if (isInt<16>(-Value))
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010724 Result = DAG.getTargetConstant(Value, dl, TCVT);
Chris Lattner8c6949e2006-10-31 19:40:43 +000010725 break;
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010726 }
10727 break;
10728 }
10729 }
Scott Michelcf0da6c2009-02-17 22:15:04 +000010730
Gabor Greiff304a7a2008-08-28 21:40:38 +000010731 if (Result.getNode()) {
Chris Lattnerd8c9cb92007-08-25 00:47:38 +000010732 Ops.push_back(Result);
10733 return;
10734 }
Scott Michelcf0da6c2009-02-17 22:15:04 +000010735
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010736 // Handle standard constraint letters.
Eric Christopherde9399b2011-06-02 23:16:42 +000010737 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
Chris Lattner15a6c4c2006-02-07 00:47:13 +000010738}
Evan Cheng2dd2c652006-03-13 23:20:37 +000010739
Chris Lattner1eb94d92007-03-30 23:15:24 +000010740// isLegalAddressingMode - Return true if the addressing mode represented
10741// by AM is legal for this target, for a load/store of the specified type.
Scott Michelcf0da6c2009-02-17 22:15:04 +000010742bool PPCTargetLowering::isLegalAddressingMode(const AddrMode &AM,
Chris Lattner229907c2011-07-18 04:54:35 +000010743 Type *Ty) const {
Hal Finkelc93a9a22015-02-25 01:06:45 +000010744 // PPC does not allow r+i addressing modes for vectors!
10745 if (Ty->isVectorTy() && AM.BaseOffs != 0)
10746 return false;
Scott Michelcf0da6c2009-02-17 22:15:04 +000010747
Chris Lattner1eb94d92007-03-30 23:15:24 +000010748 // PPC allows a sign-extended 16-bit immediate field.
10749 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
10750 return false;
Scott Michelcf0da6c2009-02-17 22:15:04 +000010751
Chris Lattner1eb94d92007-03-30 23:15:24 +000010752 // No global is ever allowed as a base.
10753 if (AM.BaseGV)
10754 return false;
Scott Michelcf0da6c2009-02-17 22:15:04 +000010755
10756 // PPC only support r+r,
Chris Lattner1eb94d92007-03-30 23:15:24 +000010757 switch (AM.Scale) {
10758 case 0: // "r+i" or just "i", depending on HasBaseReg.
10759 break;
10760 case 1:
10761 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
10762 return false;
10763 // Otherwise we have r+r or r+i.
10764 break;
10765 case 2:
10766 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
10767 return false;
10768 // Allow 2*r as r+r.
10769 break;
Chris Lattner19ccd622007-04-09 22:10:05 +000010770 default:
10771 // No other scales are supported.
10772 return false;
Chris Lattner1eb94d92007-03-30 23:15:24 +000010773 }
Scott Michelcf0da6c2009-02-17 22:15:04 +000010774
Chris Lattner1eb94d92007-03-30 23:15:24 +000010775 return true;
10776}
10777
Dan Gohman21cea8a2010-04-17 15:26:15 +000010778SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op,
10779 SelectionDAG &DAG) const {
Evan Cheng168ced92010-05-22 01:47:14 +000010780 MachineFunction &MF = DAG.getMachineFunction();
10781 MachineFrameInfo *MFI = MF.getFrameInfo();
10782 MFI->setReturnAddressIsTaken(true);
10783
Bill Wendling908bf812014-01-06 00:43:20 +000010784 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
Bill Wendlingdf7dd282014-01-05 01:47:20 +000010785 return SDValue();
Bill Wendlingdf7dd282014-01-05 01:47:20 +000010786
Andrew Trickef9de2a2013-05-25 02:42:55 +000010787 SDLoc dl(Op);
Dale Johannesen81bfca72010-05-03 22:59:34 +000010788 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
Chris Lattnerf6a81562007-12-08 06:59:59 +000010789
Dale Johannesen81bfca72010-05-03 22:59:34 +000010790 // Make sure the function does not optimize away the store of the RA to
10791 // the stack.
Chris Lattnerf6a81562007-12-08 06:59:59 +000010792 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
Dale Johannesen81bfca72010-05-03 22:59:34 +000010793 FuncInfo->setLRStoreRequired();
Eric Christopherb1aaebe2014-06-12 22:38:18 +000010794 bool isPPC64 = Subtarget.isPPC64();
Dale Johannesen81bfca72010-05-03 22:59:34 +000010795
10796 if (Depth > 0) {
10797 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
10798 SDValue Offset =
Sergey Dmitroukadb4c692015-04-28 11:56:37 +000010799 DAG.getConstant(Subtarget.getFrameLowering()->getReturnSaveOffset(), dl,
Eric Christopherf71609b2015-02-13 00:39:27 +000010800 isPPC64 ? MVT::i64 : MVT::i32);
Dale Johannesen81bfca72010-05-03 22:59:34 +000010801 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
10802 DAG.getNode(ISD::ADD, dl, getPointerTy(),
10803 FrameAddr, Offset),
Pete Cooper82cd9e82011-11-08 18:42:53 +000010804 MachinePointerInfo(), false, false, false, 0);
Dale Johannesen81bfca72010-05-03 22:59:34 +000010805 }
Chris Lattnerf6a81562007-12-08 06:59:59 +000010806
Chris Lattnerf6a81562007-12-08 06:59:59 +000010807 // Just load the return address off the stack.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000010808 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG);
Dale Johannesen81bfca72010-05-03 22:59:34 +000010809 return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
Pete Cooper82cd9e82011-11-08 18:42:53 +000010810 RetAddrFI, MachinePointerInfo(), false, false, false, 0);
Chris Lattnerf6a81562007-12-08 06:59:59 +000010811}
10812
Dan Gohman21cea8a2010-04-17 15:26:15 +000010813SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
10814 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +000010815 SDLoc dl(Op);
Dale Johannesen81bfca72010-05-03 22:59:34 +000010816 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
Scott Michelcf0da6c2009-02-17 22:15:04 +000010817
Owen Anderson53aa7a92009-08-10 22:56:29 +000010818 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
Owen Anderson9f944592009-08-11 20:47:22 +000010819 bool isPPC64 = PtrVT == MVT::i64;
Scott Michelcf0da6c2009-02-17 22:15:04 +000010820
Nicolas Geoffray75ab9792007-03-01 13:11:38 +000010821 MachineFunction &MF = DAG.getMachineFunction();
10822 MachineFrameInfo *MFI = MF.getFrameInfo();
Dale Johannesen81bfca72010-05-03 22:59:34 +000010823 MFI->setFrameAddressIsTaken(true);
Hal Finkelaa03c032013-03-21 19:03:19 +000010824
10825 // Naked functions never have a frame pointer, and so we use r1. For all
10826 // other functions, this decision must be delayed until during PEI.
10827 unsigned FrameReg;
Duncan P. N. Exon Smith5bedaf932015-02-14 02:54:07 +000010828 if (MF.getFunction()->hasFnAttribute(Attribute::Naked))
Hal Finkelaa03c032013-03-21 19:03:19 +000010829 FrameReg = isPPC64 ? PPC::X1 : PPC::R1;
10830 else
10831 FrameReg = isPPC64 ? PPC::FP8 : PPC::FP;
10832
Dale Johannesen81bfca72010-05-03 22:59:34 +000010833 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg,
10834 PtrVT);
10835 while (Depth--)
10836 FrameAddr = DAG.getLoad(Op.getValueType(), dl, DAG.getEntryNode(),
Pete Cooper82cd9e82011-11-08 18:42:53 +000010837 FrameAddr, MachinePointerInfo(), false, false,
10838 false, 0);
Dale Johannesen81bfca72010-05-03 22:59:34 +000010839 return FrameAddr;
Nicolas Geoffray75ab9792007-03-01 13:11:38 +000010840}
Dan Gohmanc14e5222008-10-21 03:41:46 +000010841
Hal Finkel0d8db462014-05-11 19:29:11 +000010842// FIXME? Maybe this could be a TableGen attribute on some registers and
10843// this table could be generated automatically from RegInfo.
10844unsigned PPCTargetLowering::getRegisterByName(const char* RegName,
10845 EVT VT) const {
Eric Christopherb1aaebe2014-06-12 22:38:18 +000010846 bool isPPC64 = Subtarget.isPPC64();
10847 bool isDarwinABI = Subtarget.isDarwinABI();
Hal Finkel0d8db462014-05-11 19:29:11 +000010848
10849 if ((isPPC64 && VT != MVT::i64 && VT != MVT::i32) ||
10850 (!isPPC64 && VT != MVT::i32))
10851 report_fatal_error("Invalid register global variable type");
10852
10853 bool is64Bit = isPPC64 && VT == MVT::i64;
10854 unsigned Reg = StringSwitch<unsigned>(RegName)
10855 .Case("r1", is64Bit ? PPC::X1 : PPC::R1)
Hal Finkele6698d52015-02-01 15:03:28 +000010856 .Case("r2", (isDarwinABI || isPPC64) ? 0 : PPC::R2)
Hal Finkel0d8db462014-05-11 19:29:11 +000010857 .Case("r13", (!isPPC64 && isDarwinABI) ? 0 :
10858 (is64Bit ? PPC::X13 : PPC::R13))
10859 .Default(0);
10860
10861 if (Reg)
10862 return Reg;
10863 report_fatal_error("Invalid register name global variable");
10864}
10865
Dan Gohmanc14e5222008-10-21 03:41:46 +000010866bool
10867PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
10868 // The PowerPC target isn't yet aware of offsets.
10869 return false;
10870}
Tilmann Schellerb93960d2009-07-03 06:45:56 +000010871
Hal Finkel46ef7ce2014-08-13 01:15:40 +000010872bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
10873 const CallInst &I,
10874 unsigned Intrinsic) const {
10875
10876 switch (Intrinsic) {
Hal Finkelc93a9a22015-02-25 01:06:45 +000010877 case Intrinsic::ppc_qpx_qvlfd:
10878 case Intrinsic::ppc_qpx_qvlfs:
10879 case Intrinsic::ppc_qpx_qvlfcd:
10880 case Intrinsic::ppc_qpx_qvlfcs:
10881 case Intrinsic::ppc_qpx_qvlfiwa:
10882 case Intrinsic::ppc_qpx_qvlfiwz:
Hal Finkel46ef7ce2014-08-13 01:15:40 +000010883 case Intrinsic::ppc_altivec_lvx:
10884 case Intrinsic::ppc_altivec_lvxl:
10885 case Intrinsic::ppc_altivec_lvebx:
10886 case Intrinsic::ppc_altivec_lvehx:
Bill Schmidt72954782014-11-12 04:19:40 +000010887 case Intrinsic::ppc_altivec_lvewx:
10888 case Intrinsic::ppc_vsx_lxvd2x:
10889 case Intrinsic::ppc_vsx_lxvw4x: {
Hal Finkel46ef7ce2014-08-13 01:15:40 +000010890 EVT VT;
10891 switch (Intrinsic) {
10892 case Intrinsic::ppc_altivec_lvebx:
10893 VT = MVT::i8;
10894 break;
10895 case Intrinsic::ppc_altivec_lvehx:
10896 VT = MVT::i16;
10897 break;
10898 case Intrinsic::ppc_altivec_lvewx:
10899 VT = MVT::i32;
10900 break;
Bill Schmidt72954782014-11-12 04:19:40 +000010901 case Intrinsic::ppc_vsx_lxvd2x:
10902 VT = MVT::v2f64;
10903 break;
Hal Finkelc93a9a22015-02-25 01:06:45 +000010904 case Intrinsic::ppc_qpx_qvlfd:
10905 VT = MVT::v4f64;
10906 break;
10907 case Intrinsic::ppc_qpx_qvlfs:
10908 VT = MVT::v4f32;
10909 break;
10910 case Intrinsic::ppc_qpx_qvlfcd:
10911 VT = MVT::v2f64;
10912 break;
10913 case Intrinsic::ppc_qpx_qvlfcs:
10914 VT = MVT::v2f32;
10915 break;
Hal Finkel46ef7ce2014-08-13 01:15:40 +000010916 default:
10917 VT = MVT::v4i32;
10918 break;
10919 }
10920
10921 Info.opc = ISD::INTRINSIC_W_CHAIN;
10922 Info.memVT = VT;
10923 Info.ptrVal = I.getArgOperand(0);
10924 Info.offset = -VT.getStoreSize()+1;
10925 Info.size = 2*VT.getStoreSize()-1;
10926 Info.align = 1;
10927 Info.vol = false;
10928 Info.readMem = true;
10929 Info.writeMem = false;
10930 return true;
10931 }
Hal Finkelc93a9a22015-02-25 01:06:45 +000010932 case Intrinsic::ppc_qpx_qvlfda:
10933 case Intrinsic::ppc_qpx_qvlfsa:
10934 case Intrinsic::ppc_qpx_qvlfcda:
10935 case Intrinsic::ppc_qpx_qvlfcsa:
10936 case Intrinsic::ppc_qpx_qvlfiwaa:
10937 case Intrinsic::ppc_qpx_qvlfiwza: {
10938 EVT VT;
10939 switch (Intrinsic) {
10940 case Intrinsic::ppc_qpx_qvlfda:
10941 VT = MVT::v4f64;
10942 break;
10943 case Intrinsic::ppc_qpx_qvlfsa:
10944 VT = MVT::v4f32;
10945 break;
10946 case Intrinsic::ppc_qpx_qvlfcda:
10947 VT = MVT::v2f64;
10948 break;
10949 case Intrinsic::ppc_qpx_qvlfcsa:
10950 VT = MVT::v2f32;
10951 break;
10952 default:
10953 VT = MVT::v4i32;
10954 break;
10955 }
10956
10957 Info.opc = ISD::INTRINSIC_W_CHAIN;
10958 Info.memVT = VT;
10959 Info.ptrVal = I.getArgOperand(0);
10960 Info.offset = 0;
10961 Info.size = VT.getStoreSize();
10962 Info.align = 1;
10963 Info.vol = false;
10964 Info.readMem = true;
10965 Info.writeMem = false;
10966 return true;
10967 }
10968 case Intrinsic::ppc_qpx_qvstfd:
10969 case Intrinsic::ppc_qpx_qvstfs:
10970 case Intrinsic::ppc_qpx_qvstfcd:
10971 case Intrinsic::ppc_qpx_qvstfcs:
10972 case Intrinsic::ppc_qpx_qvstfiw:
Hal Finkel46ef7ce2014-08-13 01:15:40 +000010973 case Intrinsic::ppc_altivec_stvx:
10974 case Intrinsic::ppc_altivec_stvxl:
10975 case Intrinsic::ppc_altivec_stvebx:
10976 case Intrinsic::ppc_altivec_stvehx:
Bill Schmidt72954782014-11-12 04:19:40 +000010977 case Intrinsic::ppc_altivec_stvewx:
10978 case Intrinsic::ppc_vsx_stxvd2x:
10979 case Intrinsic::ppc_vsx_stxvw4x: {
Hal Finkel46ef7ce2014-08-13 01:15:40 +000010980 EVT VT;
10981 switch (Intrinsic) {
10982 case Intrinsic::ppc_altivec_stvebx:
10983 VT = MVT::i8;
10984 break;
10985 case Intrinsic::ppc_altivec_stvehx:
10986 VT = MVT::i16;
10987 break;
10988 case Intrinsic::ppc_altivec_stvewx:
10989 VT = MVT::i32;
10990 break;
Bill Schmidt72954782014-11-12 04:19:40 +000010991 case Intrinsic::ppc_vsx_stxvd2x:
10992 VT = MVT::v2f64;
10993 break;
Hal Finkelc93a9a22015-02-25 01:06:45 +000010994 case Intrinsic::ppc_qpx_qvstfd:
10995 VT = MVT::v4f64;
10996 break;
10997 case Intrinsic::ppc_qpx_qvstfs:
10998 VT = MVT::v4f32;
10999 break;
11000 case Intrinsic::ppc_qpx_qvstfcd:
11001 VT = MVT::v2f64;
11002 break;
11003 case Intrinsic::ppc_qpx_qvstfcs:
11004 VT = MVT::v2f32;
11005 break;
Hal Finkel46ef7ce2014-08-13 01:15:40 +000011006 default:
11007 VT = MVT::v4i32;
11008 break;
11009 }
11010
11011 Info.opc = ISD::INTRINSIC_VOID;
11012 Info.memVT = VT;
11013 Info.ptrVal = I.getArgOperand(1);
11014 Info.offset = -VT.getStoreSize()+1;
11015 Info.size = 2*VT.getStoreSize()-1;
11016 Info.align = 1;
11017 Info.vol = false;
11018 Info.readMem = false;
11019 Info.writeMem = true;
11020 return true;
11021 }
Hal Finkelc93a9a22015-02-25 01:06:45 +000011022 case Intrinsic::ppc_qpx_qvstfda:
11023 case Intrinsic::ppc_qpx_qvstfsa:
11024 case Intrinsic::ppc_qpx_qvstfcda:
11025 case Intrinsic::ppc_qpx_qvstfcsa:
11026 case Intrinsic::ppc_qpx_qvstfiwa: {
11027 EVT VT;
11028 switch (Intrinsic) {
11029 case Intrinsic::ppc_qpx_qvstfda:
11030 VT = MVT::v4f64;
11031 break;
11032 case Intrinsic::ppc_qpx_qvstfsa:
11033 VT = MVT::v4f32;
11034 break;
11035 case Intrinsic::ppc_qpx_qvstfcda:
11036 VT = MVT::v2f64;
11037 break;
11038 case Intrinsic::ppc_qpx_qvstfcsa:
11039 VT = MVT::v2f32;
11040 break;
11041 default:
11042 VT = MVT::v4i32;
11043 break;
11044 }
11045
11046 Info.opc = ISD::INTRINSIC_VOID;
11047 Info.memVT = VT;
11048 Info.ptrVal = I.getArgOperand(1);
11049 Info.offset = 0;
11050 Info.size = VT.getStoreSize();
11051 Info.align = 1;
11052 Info.vol = false;
11053 Info.readMem = false;
11054 Info.writeMem = true;
11055 return true;
11056 }
Hal Finkel46ef7ce2014-08-13 01:15:40 +000011057 default:
11058 break;
11059 }
11060
11061 return false;
11062}
11063
Evan Chengd9929f02010-04-01 20:10:42 +000011064/// getOptimalMemOpType - Returns the target specific optimal type for load
Evan Cheng61399372010-04-02 19:36:14 +000011065/// and store operations as a result of memset, memcpy, and memmove
11066/// lowering. If DstAlign is zero that means it's safe to destination
11067/// alignment can satisfy any constraint. Similarly if SrcAlign is zero it
11068/// means there isn't a need to check it against alignment requirement,
Evan Cheng962711e2012-12-12 02:34:41 +000011069/// probably because the source does not need to be loaded. If 'IsMemset' is
11070/// true, that means it's expanding a memset. If 'ZeroMemset' is true, that
11071/// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy
11072/// source is constant so it does not need to be loaded.
Dan Gohman148c69a2010-04-16 20:11:05 +000011073/// It returns EVT::Other if the type should be determined using generic
11074/// target-independent logic.
Evan Cheng43cd9e32010-04-01 06:04:33 +000011075EVT PPCTargetLowering::getOptimalMemOpType(uint64_t Size,
11076 unsigned DstAlign, unsigned SrcAlign,
Evan Cheng962711e2012-12-12 02:34:41 +000011077 bool IsMemset, bool ZeroMemset,
Evan Chengebe47c82010-04-08 07:37:57 +000011078 bool MemcpyStrSrc,
Dan Gohman148c69a2010-04-16 20:11:05 +000011079 MachineFunction &MF) const {
Hal Finkel52368d42015-03-31 20:56:09 +000011080 if (getTargetMachine().getOptLevel() != CodeGenOpt::None) {
11081 const Function *F = MF.getFunction();
11082 // When expanding a memset, require at least two QPX instructions to cover
11083 // the cost of loading the value to be stored from the constant pool.
11084 if (Subtarget.hasQPX() && Size >= 32 && (!IsMemset || Size >= 64) &&
11085 (!SrcAlign || SrcAlign >= 32) && (!DstAlign || DstAlign >= 32) &&
11086 !F->hasFnAttribute(Attribute::NoImplicitFloat)) {
11087 return MVT::v4f64;
11088 }
Hal Finkel5c3cacf2015-02-27 19:58:28 +000011089
Hal Finkel52368d42015-03-31 20:56:09 +000011090 // We should use Altivec/VSX loads and stores when available. For unaligned
11091 // addresses, unaligned VSX loads are only fast starting with the P8.
11092 if (Subtarget.hasAltivec() && Size >= 16 &&
11093 (((!SrcAlign || SrcAlign >= 16) && (!DstAlign || DstAlign >= 16)) ||
11094 ((IsMemset && Subtarget.hasVSX()) || Subtarget.hasP8Vector())))
11095 return MVT::v4i32;
11096 }
Hal Finkel5c3cacf2015-02-27 19:58:28 +000011097
Eric Christopherd90a8742014-06-12 22:38:20 +000011098 if (Subtarget.isPPC64()) {
Owen Anderson9f944592009-08-11 20:47:22 +000011099 return MVT::i64;
Tilmann Schellerb93960d2009-07-03 06:45:56 +000011100 }
Hal Finkel5c3cacf2015-02-27 19:58:28 +000011101
11102 return MVT::i32;
Tilmann Schellerb93960d2009-07-03 06:45:56 +000011103}
Hal Finkel88ed4e32012-04-01 19:23:08 +000011104
Hal Finkel34974ed2014-04-12 21:52:38 +000011105/// \brief Returns true if it is beneficial to convert a load of a constant
11106/// to just the constant itself.
11107bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
11108 Type *Ty) const {
11109 assert(Ty->isIntegerTy());
11110
11111 unsigned BitSize = Ty->getPrimitiveSizeInBits();
11112 if (BitSize == 0 || BitSize > 64)
11113 return false;
11114 return true;
11115}
11116
11117bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
11118 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
11119 return false;
11120 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
11121 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
11122 return NumBits1 == 64 && NumBits2 == 32;
11123}
11124
11125bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
11126 if (!VT1.isInteger() || !VT2.isInteger())
11127 return false;
11128 unsigned NumBits1 = VT1.getSizeInBits();
11129 unsigned NumBits2 = VT2.getSizeInBits();
11130 return NumBits1 == 64 && NumBits2 == 32;
11131}
11132
Hal Finkel5d5d1532015-01-10 08:21:59 +000011133bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
11134 // Generally speaking, zexts are not free, but they are free when they can be
11135 // folded with other operations.
11136 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val)) {
11137 EVT MemVT = LD->getMemoryVT();
11138 if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 ||
11139 (Subtarget.isPPC64() && MemVT == MVT::i32)) &&
11140 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
11141 LD->getExtensionType() == ISD::ZEXTLOAD))
11142 return true;
11143 }
11144
11145 // FIXME: Add other cases...
11146 // - 32-bit shifts with a zext to i64
11147 // - zext after ctlz, bswap, etc.
11148 // - zext after and by a constant mask
11149
11150 return TargetLowering::isZExtFree(Val, VT2);
11151}
11152
Olivier Sallenave32509692015-01-13 15:06:36 +000011153bool PPCTargetLowering::isFPExtFree(EVT VT) const {
11154 assert(VT.isFloatingPoint());
11155 return true;
11156}
11157
Hal Finkel34974ed2014-04-12 21:52:38 +000011158bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
11159 return isInt<16>(Imm) || isUInt<16>(Imm);
11160}
11161
11162bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const {
11163 return isInt<16>(Imm) || isUInt<16>(Imm);
11164}
11165
Matt Arsenault6f2a5262014-07-27 17:46:40 +000011166bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
11167 unsigned,
11168 unsigned,
11169 bool *Fast) const {
Hal Finkel8d7fbc92013-03-15 15:27:13 +000011170 if (DisablePPCUnaligned)
11171 return false;
11172
11173 // PowerPC supports unaligned memory access for simple non-vector types.
11174 // Although accessing unaligned addresses is not as efficient as accessing
11175 // aligned addresses, it is generally more efficient than manual expansion,
11176 // and generally only traps for software emulation when crossing page
11177 // boundaries.
11178
11179 if (!VT.isSimple())
11180 return false;
11181
Hal Finkel6e28e6a2014-03-26 19:39:09 +000011182 if (VT.getSimpleVT().isVector()) {
Eric Christopherb1aaebe2014-06-12 22:38:18 +000011183 if (Subtarget.hasVSX()) {
Bill Schmidt2d1128a2014-10-17 15:13:38 +000011184 if (VT != MVT::v2f64 && VT != MVT::v2i64 &&
11185 VT != MVT::v4f32 && VT != MVT::v4i32)
Hal Finkel6e28e6a2014-03-26 19:39:09 +000011186 return false;
11187 } else {
11188 return false;
11189 }
11190 }
Hal Finkel8d7fbc92013-03-15 15:27:13 +000011191
11192 if (VT == MVT::ppcf128)
11193 return false;
11194
11195 if (Fast)
11196 *Fast = true;
11197
11198 return true;
11199}
11200
Stephen Lin73de7bf2013-07-09 18:16:56 +000011201bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const {
11202 VT = VT.getScalarType();
11203
Hal Finkel0a479ae2012-06-22 00:49:52 +000011204 if (!VT.isSimple())
11205 return false;
11206
11207 switch (VT.getSimpleVT().SimpleTy) {
11208 case MVT::f32:
11209 case MVT::f64:
Hal Finkel0a479ae2012-06-22 00:49:52 +000011210 return true;
11211 default:
11212 break;
11213 }
11214
11215 return false;
11216}
11217
Hal Finkel934361a2015-01-14 01:07:51 +000011218const MCPhysReg *
11219PPCTargetLowering::getScratchRegisters(CallingConv::ID) const {
11220 // LR is a callee-save register, but we must treat it as clobbered by any call
11221 // site. Hence we include LR in the scratch registers, which are in turn added
11222 // as implicit-defs for stackmaps and patchpoints. The same reasoning applies
11223 // to CTR, which is used by any indirect call.
11224 static const MCPhysReg ScratchRegs[] = {
Hal Finkelc19805a2015-01-17 03:57:34 +000011225 PPC::X12, PPC::LR8, PPC::CTR8, 0
Hal Finkel934361a2015-01-14 01:07:51 +000011226 };
11227
11228 return ScratchRegs;
11229}
11230
Hal Finkelb4240ca2014-03-31 17:48:16 +000011231bool
11232PPCTargetLowering::shouldExpandBuildVectorWithShuffles(
11233 EVT VT , unsigned DefinedValues) const {
11234 if (VT == MVT::v2i64)
11235 return false;
11236
Hal Finkelc93a9a22015-02-25 01:06:45 +000011237 if (Subtarget.hasQPX()) {
11238 if (VT == MVT::v4f32 || VT == MVT::v4f64 || VT == MVT::v4i1)
11239 return true;
11240 }
11241
Hal Finkelb4240ca2014-03-31 17:48:16 +000011242 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues);
11243}
11244
Hal Finkel88ed4e32012-04-01 19:23:08 +000011245Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const {
Eric Christopherb1aaebe2014-06-12 22:38:18 +000011246 if (DisableILPPref || Subtarget.enableMachineScheduler())
Hal Finkel4e9f1a82012-06-10 19:32:29 +000011247 return TargetLowering::getSchedulingPreference(N);
Hal Finkel88ed4e32012-04-01 19:23:08 +000011248
Hal Finkel4e9f1a82012-06-10 19:32:29 +000011249 return Sched::ILP;
Hal Finkel88ed4e32012-04-01 19:23:08 +000011250}
11251
Bill Schmidt0cf702f2013-07-30 00:50:39 +000011252// Create a fast isel object.
11253FastISel *
11254PPCTargetLowering::createFastISel(FunctionLoweringInfo &FuncInfo,
11255 const TargetLibraryInfo *LibInfo) const {
11256 return PPC::createFastISel(FuncInfo, LibInfo);
11257}