blob: 33d5ddad957a8997fa4d520e435d7e9edd11adb4 [file] [log] [blame]
Nate Begeman4ebd8052005-09-01 23:24:04 +00001//===-- DAGCombiner.cpp - Implement a DAG node combiner -------------------===//
Nate Begeman1d4d4142005-09-01 00:19:25 +00002//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner4ee451d2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Nate Begeman1d4d4142005-09-01 00:19:25 +00007//
8//===----------------------------------------------------------------------===//
9//
10// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
11// both before and after the DAG is legalized.
Scott Michelfdc40a02009-02-17 22:15:04 +000012//
Dan Gohman41287002009-04-25 17:09:45 +000013// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
14// primarily intended to handle simplification opportunities that are implicit
15// in the LLVM IR and exposed by the various codegen lowering phases.
16//
Nate Begeman1d4d4142005-09-01 00:19:25 +000017//===----------------------------------------------------------------------===//
18
19#define DEBUG_TYPE "dagcombine"
Nate Begeman1d4d4142005-09-01 00:19:25 +000020#include "llvm/CodeGen/SelectionDAG.h"
Chris Lattner600fec32009-03-11 05:08:08 +000021#include "llvm/DerivedTypes.h"
Owen Andersona90b3dc2009-07-15 21:51:10 +000022#include "llvm/LLVMContext.h"
Chris Lattner00161a62008-01-25 07:20:16 +000023#include "llvm/CodeGen/MachineFunction.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000025#include "llvm/Analysis/AliasAnalysis.h"
Evan Cheng59d5b682007-05-07 21:27:48 +000026#include "llvm/Target/TargetData.h"
Nate Begeman1d4d4142005-09-01 00:19:25 +000027#include "llvm/Target/TargetLowering.h"
Evan Cheng59d5b682007-05-07 21:27:48 +000028#include "llvm/Target/TargetMachine.h"
Chris Lattnerddae4bd2007-01-08 23:04:05 +000029#include "llvm/Target/TargetOptions.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000030#include "llvm/ADT/SmallPtrSet.h"
31#include "llvm/ADT/Statistic.h"
Jim Laskeyd1aed7a2006-09-21 16:28:59 +000032#include "llvm/Support/CommandLine.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000033#include "llvm/Support/Debug.h"
Torok Edwin7d696d82009-07-11 13:10:19 +000034#include "llvm/Support/ErrorHandling.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000035#include "llvm/Support/MathExtras.h"
Chris Lattnerbbbfa992009-08-23 06:35:02 +000036#include "llvm/Support/raw_ostream.h"
Chris Lattnera500fc62005-09-09 23:53:39 +000037#include <algorithm>
Nate Begeman1d4d4142005-09-01 00:19:25 +000038using namespace llvm;
39
Chris Lattnercd3245a2006-12-19 22:41:21 +000040STATISTIC(NodesCombined , "Number of dag nodes combined");
41STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
42STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
Evan Cheng8b944d32009-05-28 00:35:15 +000043STATISTIC(OpsNarrowed , "Number of load/op/store narrowed");
Evan Cheng31959b12011-02-02 01:06:55 +000044STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int");
Chris Lattnercd3245a2006-12-19 22:41:21 +000045
Nate Begeman1d4d4142005-09-01 00:19:25 +000046namespace {
Jim Laskey71382342006-10-07 23:37:56 +000047 static cl::opt<bool>
Owen Anderson0dcc8142010-09-19 21:01:26 +000048 CombinerAA("combiner-alias-analysis", cl::Hidden,
Jim Laskey26f7fa72006-10-17 19:33:52 +000049 cl::desc("Turn on alias analysis during testing"));
Jim Laskey3ad175b2006-10-12 15:22:24 +000050
Jim Laskey07a27092006-10-18 19:08:31 +000051 static cl::opt<bool>
52 CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
53 cl::desc("Include global information in alias analysis"));
54
Jim Laskeybc588b82006-10-05 15:07:25 +000055//------------------------------ DAGCombiner ---------------------------------//
56
Nick Lewycky6726b6d2009-10-25 06:33:48 +000057 class DAGCombiner {
Nate Begeman1d4d4142005-09-01 00:19:25 +000058 SelectionDAG &DAG;
Dan Gohman79ce2762009-01-15 19:20:50 +000059 const TargetLowering &TLI;
Duncan Sands25cf2272008-11-24 14:53:14 +000060 CombineLevel Level;
Bill Wendling98a366d2009-04-29 23:29:43 +000061 CodeGenOpt::Level OptLevel;
Duncan Sands25cf2272008-11-24 14:53:14 +000062 bool LegalOperations;
63 bool LegalTypes;
Nate Begeman1d4d4142005-09-01 00:19:25 +000064
65 // Worklist of all of the nodes that need to be simplified.
James Molloy6660c052012-02-16 09:17:04 +000066 //
67 // This has the semantics that when adding to the worklist,
68 // the item added must be next to be processed. It should
69 // also only appear once. The naive approach to this takes
70 // linear time.
71 //
72 // To reduce the insert/remove time to logarithmic, we use
73 // a set and a vector to maintain our worklist.
74 //
75 // The set contains the items on the worklist, but does not
76 // maintain the order they should be visited.
77 //
78 // The vector maintains the order nodes should be visited, but may
79 // contain duplicate or removed nodes. When choosing a node to
80 // visit, we pop off the order stack until we find an item that is
81 // also in the contents set. All operations are O(log N).
82 SmallPtrSet<SDNode*, 64> WorkListContents;
Benjamin Kramerd5f76902012-03-10 00:23:58 +000083 SmallVector<SDNode*, 64> WorkListOrder;
Nate Begeman1d4d4142005-09-01 00:19:25 +000084
Jim Laskeyc7c3f112006-10-16 20:52:31 +000085 // AA - Used for DAG load/store alias analysis.
86 AliasAnalysis &AA;
87
Nate Begeman1d4d4142005-09-01 00:19:25 +000088 /// AddUsersToWorkList - When an instruction is simplified, add all users of
89 /// the instruction to the work lists because they might get more simplified
90 /// now.
91 ///
92 void AddUsersToWorkList(SDNode *N) {
93 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
Nate Begeman4ebd8052005-09-01 23:24:04 +000094 UI != UE; ++UI)
Dan Gohman89684502008-07-27 20:43:25 +000095 AddToWorkList(*UI);
Nate Begeman1d4d4142005-09-01 00:19:25 +000096 }
97
Dan Gohman389079b2007-10-08 17:57:15 +000098 /// visit - call the node-specific routine that knows how to fold each
99 /// particular type of node.
Dan Gohman475871a2008-07-27 21:46:04 +0000100 SDValue visit(SDNode *N);
Dan Gohman389079b2007-10-08 17:57:15 +0000101
Chris Lattner24664722006-03-01 04:53:38 +0000102 public:
James Molloy6afa3f72012-02-16 09:48:07 +0000103 /// AddToWorkList - Add to the work list making sure its instance is at the
James Molloy6660c052012-02-16 09:17:04 +0000104 /// back (next to be processed.)
Chris Lattner5750df92006-03-01 04:03:14 +0000105 void AddToWorkList(SDNode *N) {
James Molloy6660c052012-02-16 09:17:04 +0000106 WorkListContents.insert(N);
107 WorkListOrder.push_back(N);
Chris Lattner5750df92006-03-01 04:03:14 +0000108 }
Jim Laskey6ff23e52006-10-04 16:53:27 +0000109
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000110 /// removeFromWorkList - remove all instances of N from the worklist.
111 ///
112 void removeFromWorkList(SDNode *N) {
James Molloy6660c052012-02-16 09:17:04 +0000113 WorkListContents.erase(N);
Chris Lattner01a22022005-10-10 22:04:48 +0000114 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000115
Dan Gohman475871a2008-07-27 21:46:04 +0000116 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
Evan Cheng0b0cd912009-03-28 05:57:29 +0000117 bool AddTo = true);
Scott Michelfdc40a02009-02-17 22:15:04 +0000118
Dan Gohman475871a2008-07-27 21:46:04 +0000119 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
Jim Laskey274062c2006-10-13 23:32:28 +0000120 return CombineTo(N, &Res, 1, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000121 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000122
Dan Gohman475871a2008-07-27 21:46:04 +0000123 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
Evan Cheng0b0cd912009-03-28 05:57:29 +0000124 bool AddTo = true) {
Dan Gohman475871a2008-07-27 21:46:04 +0000125 SDValue To[] = { Res0, Res1 };
Jim Laskey274062c2006-10-13 23:32:28 +0000126 return CombineTo(N, To, 2, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000127 }
Dan Gohmane5af2d32009-01-29 01:59:02 +0000128
129 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
Scott Michelfdc40a02009-02-17 22:15:04 +0000130
131 private:
132
Chris Lattner012f2412006-02-17 21:58:01 +0000133 /// SimplifyDemandedBits - Check the specified integer node value to see if
Chris Lattnerb2742f42006-03-01 19:55:35 +0000134 /// it can be simplified or if things it uses can be simplified by bit
Chris Lattner012f2412006-02-17 21:58:01 +0000135 /// propagation. If so, return true.
Dan Gohman475871a2008-07-27 21:46:04 +0000136 bool SimplifyDemandedBits(SDValue Op) {
Dan Gohman87862e72009-12-11 21:31:27 +0000137 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
138 APInt Demanded = APInt::getAllOnesValue(BitWidth);
Dan Gohman7b8d4a92008-02-27 00:25:32 +0000139 return SimplifyDemandedBits(Op, Demanded);
140 }
141
Dan Gohman475871a2008-07-27 21:46:04 +0000142 bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
Chris Lattner87514ca2005-10-10 22:31:19 +0000143
Chris Lattner448f2192006-11-11 00:39:41 +0000144 bool CombineToPreIndexedLoadStore(SDNode *N);
145 bool CombineToPostIndexedLoadStore(SDNode *N);
Scott Michelfdc40a02009-02-17 22:15:04 +0000146
Evan Cheng95c57ea2010-04-24 04:43:44 +0000147 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
148 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
149 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
150 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
Evan Cheng64b7bf72010-04-16 06:14:10 +0000151 SDValue PromoteIntBinOp(SDValue Op);
Evan Cheng07c4e102010-04-22 20:19:46 +0000152 SDValue PromoteIntShiftOp(SDValue Op);
Evan Cheng4c26e932010-04-19 19:29:22 +0000153 SDValue PromoteExtend(SDValue Op);
154 bool PromoteLoad(SDValue Op);
Scott Michelfdc40a02009-02-17 22:15:04 +0000155
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +0000156 void ExtendSetCCUses(SmallVector<SDNode*, 4> SetCCs,
157 SDValue Trunc, SDValue ExtLoad, DebugLoc DL,
158 ISD::NodeType ExtType);
159
Dan Gohman389079b2007-10-08 17:57:15 +0000160 /// combine - call the node-specific routine that knows how to fold each
161 /// particular type of node. If that doesn't do anything, try the
162 /// target-specific DAG combines.
Dan Gohman475871a2008-07-27 21:46:04 +0000163 SDValue combine(SDNode *N);
Nate Begeman1d4d4142005-09-01 00:19:25 +0000164
165 // Visitation implementation - Implement dag node combining for different
166 // node types. The semantics are as follows:
167 // Return Value:
Evan Cheng17a568b2008-08-29 22:21:44 +0000168 // SDValue.getNode() == 0 - No change was made
169 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
170 // otherwise - N should be replaced by the returned Operand.
Nate Begeman1d4d4142005-09-01 00:19:25 +0000171 //
Dan Gohman475871a2008-07-27 21:46:04 +0000172 SDValue visitTokenFactor(SDNode *N);
173 SDValue visitMERGE_VALUES(SDNode *N);
174 SDValue visitADD(SDNode *N);
175 SDValue visitSUB(SDNode *N);
176 SDValue visitADDC(SDNode *N);
Craig Toppercc274522012-01-07 09:06:39 +0000177 SDValue visitSUBC(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000178 SDValue visitADDE(SDNode *N);
Craig Toppercc274522012-01-07 09:06:39 +0000179 SDValue visitSUBE(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000180 SDValue visitMUL(SDNode *N);
181 SDValue visitSDIV(SDNode *N);
182 SDValue visitUDIV(SDNode *N);
183 SDValue visitSREM(SDNode *N);
184 SDValue visitUREM(SDNode *N);
185 SDValue visitMULHU(SDNode *N);
186 SDValue visitMULHS(SDNode *N);
187 SDValue visitSMUL_LOHI(SDNode *N);
188 SDValue visitUMUL_LOHI(SDNode *N);
Benjamin Kramerf55d26e2011-05-21 18:31:55 +0000189 SDValue visitSMULO(SDNode *N);
190 SDValue visitUMULO(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000191 SDValue visitSDIVREM(SDNode *N);
192 SDValue visitUDIVREM(SDNode *N);
193 SDValue visitAND(SDNode *N);
194 SDValue visitOR(SDNode *N);
195 SDValue visitXOR(SDNode *N);
196 SDValue SimplifyVBinOp(SDNode *N);
Craig Topperdd201ff2012-09-11 01:45:21 +0000197 SDValue SimplifyVUnaryOp(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000198 SDValue visitSHL(SDNode *N);
199 SDValue visitSRA(SDNode *N);
200 SDValue visitSRL(SDNode *N);
201 SDValue visitCTLZ(SDNode *N);
Chandler Carruth63974b22011-12-13 01:56:10 +0000202 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000203 SDValue visitCTTZ(SDNode *N);
Chandler Carruth63974b22011-12-13 01:56:10 +0000204 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000205 SDValue visitCTPOP(SDNode *N);
206 SDValue visitSELECT(SDNode *N);
207 SDValue visitSELECT_CC(SDNode *N);
208 SDValue visitSETCC(SDNode *N);
209 SDValue visitSIGN_EXTEND(SDNode *N);
210 SDValue visitZERO_EXTEND(SDNode *N);
211 SDValue visitANY_EXTEND(SDNode *N);
212 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
213 SDValue visitTRUNCATE(SDNode *N);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000214 SDValue visitBITCAST(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000215 SDValue visitBUILD_PAIR(SDNode *N);
216 SDValue visitFADD(SDNode *N);
217 SDValue visitFSUB(SDNode *N);
218 SDValue visitFMUL(SDNode *N);
Owen Anderson062c0a52012-05-02 22:17:40 +0000219 SDValue visitFMA(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000220 SDValue visitFDIV(SDNode *N);
221 SDValue visitFREM(SDNode *N);
222 SDValue visitFCOPYSIGN(SDNode *N);
223 SDValue visitSINT_TO_FP(SDNode *N);
224 SDValue visitUINT_TO_FP(SDNode *N);
225 SDValue visitFP_TO_SINT(SDNode *N);
226 SDValue visitFP_TO_UINT(SDNode *N);
227 SDValue visitFP_ROUND(SDNode *N);
228 SDValue visitFP_ROUND_INREG(SDNode *N);
229 SDValue visitFP_EXTEND(SDNode *N);
230 SDValue visitFNEG(SDNode *N);
231 SDValue visitFABS(SDNode *N);
Owen Anderson7c626d32012-08-13 23:32:49 +0000232 SDValue visitFCEIL(SDNode *N);
233 SDValue visitFTRUNC(SDNode *N);
234 SDValue visitFFLOOR(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000235 SDValue visitBRCOND(SDNode *N);
236 SDValue visitBR_CC(SDNode *N);
237 SDValue visitLOAD(SDNode *N);
238 SDValue visitSTORE(SDNode *N);
239 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
240 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
241 SDValue visitBUILD_VECTOR(SDNode *N);
242 SDValue visitCONCAT_VECTORS(SDNode *N);
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +0000243 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000244 SDValue visitVECTOR_SHUFFLE(SDNode *N);
Jim Grosbach9a526492010-06-23 16:07:42 +0000245 SDValue visitMEMBARRIER(SDNode *N);
Chris Lattner01a22022005-10-10 22:04:48 +0000246
Dan Gohman475871a2008-07-27 21:46:04 +0000247 SDValue XformToShuffleWithZero(SDNode *N);
Bill Wendling35247c32009-01-30 00:45:56 +0000248 SDValue ReassociateOps(unsigned Opc, DebugLoc DL, SDValue LHS, SDValue RHS);
Scott Michelfdc40a02009-02-17 22:15:04 +0000249
Dan Gohman475871a2008-07-27 21:46:04 +0000250 SDValue visitShiftByConstant(SDNode *N, unsigned Amt);
Chris Lattnere70da202007-12-06 07:33:36 +0000251
Dan Gohman475871a2008-07-27 21:46:04 +0000252 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
253 SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
Bill Wendling836ca7d2009-01-30 23:59:18 +0000254 SDValue SimplifySelect(DebugLoc DL, SDValue N0, SDValue N1, SDValue N2);
Scott Michelfdc40a02009-02-17 22:15:04 +0000255 SDValue SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1, SDValue N2,
256 SDValue N3, ISD::CondCode CC,
Bill Wendling836ca7d2009-01-30 23:59:18 +0000257 bool NotExtCompare = false);
Owen Andersone50ed302009-08-10 22:56:29 +0000258 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
Dale Johannesenff97d4f2009-02-03 00:47:48 +0000259 DebugLoc DL, bool foldBooleans = true);
Scott Michelfdc40a02009-02-17 22:15:04 +0000260 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
Chris Lattner5eee4272008-01-26 01:09:19 +0000261 unsigned HiOp);
Owen Andersone50ed302009-08-10 22:56:29 +0000262 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000263 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
Dan Gohman475871a2008-07-27 21:46:04 +0000264 SDValue BuildSDIV(SDNode *N);
265 SDValue BuildUDIV(SDNode *N);
Evan Cheng9568e5c2011-06-21 06:01:08 +0000266 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
267 bool DemandHighBits = true);
268 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
Bill Wendling317bd702009-01-30 21:14:50 +0000269 SDNode *MatchRotate(SDValue LHS, SDValue RHS, DebugLoc DL);
Dan Gohman475871a2008-07-27 21:46:04 +0000270 SDValue ReduceLoadWidth(SDNode *N);
Evan Cheng8b944d32009-05-28 00:35:15 +0000271 SDValue ReduceLoadOpStoreWidth(SDNode *N);
Evan Cheng31959b12011-02-02 01:06:55 +0000272 SDValue TransformFPLoadStorePair(SDNode *N);
Scott Michelfdc40a02009-02-17 22:15:04 +0000273
Dan Gohman475871a2008-07-27 21:46:04 +0000274 SDValue GetDemandedBits(SDValue V, const APInt &Mask);
Scott Michelfdc40a02009-02-17 22:15:04 +0000275
Jim Laskey6ff23e52006-10-04 16:53:27 +0000276 /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
277 /// looking for aliasing nodes and adding them to the Aliases vector.
Dan Gohman475871a2008-07-27 21:46:04 +0000278 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
279 SmallVector<SDValue, 8> &Aliases);
Jim Laskey6ff23e52006-10-04 16:53:27 +0000280
Jim Laskey096c22e2006-10-18 12:29:57 +0000281 /// isAlias - Return true if there is any possibility that the two addresses
282 /// overlap.
Dan Gohman475871a2008-07-27 21:46:04 +0000283 bool isAlias(SDValue Ptr1, int64_t Size1,
Jim Laskey096c22e2006-10-18 12:29:57 +0000284 const Value *SrcValue1, int SrcValueOffset1,
Nate Begemanb6aef5c2009-09-15 00:18:30 +0000285 unsigned SrcValueAlign1,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +0000286 const MDNode *TBAAInfo1,
Dan Gohman475871a2008-07-27 21:46:04 +0000287 SDValue Ptr2, int64_t Size2,
Nate Begemanb6aef5c2009-09-15 00:18:30 +0000288 const Value *SrcValue2, int SrcValueOffset2,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +0000289 unsigned SrcValueAlign2,
290 const MDNode *TBAAInfo2) const;
Scott Michelfdc40a02009-02-17 22:15:04 +0000291
Jim Laskey7ca56af2006-10-11 13:47:09 +0000292 /// FindAliasInfo - Extracts the relevant alias information from the memory
293 /// node. Returns true if the operand was a load.
294 bool FindAliasInfo(SDNode *N,
Dan Gohman475871a2008-07-27 21:46:04 +0000295 SDValue &Ptr, int64_t &Size,
Nate Begemanb6aef5c2009-09-15 00:18:30 +0000296 const Value *&SrcValue, int &SrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +0000297 unsigned &SrcValueAlignment,
298 const MDNode *&TBAAInfo) const;
Scott Michelfdc40a02009-02-17 22:15:04 +0000299
Jim Laskey279f0532006-09-25 16:29:54 +0000300 /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes,
Jim Laskey6ff23e52006-10-04 16:53:27 +0000301 /// looking for a better chain (aliasing node.)
Dan Gohman475871a2008-07-27 21:46:04 +0000302 SDValue FindBetterChain(SDNode *N, SDValue Chain);
Duncan Sands92abc622009-01-31 15:50:11 +0000303
Nadav Rotem72f7b082012-09-29 06:33:25 +0000304 /// Merge consecutive store operations into a wide store.
305 /// \return True if some memory operations were changed.
306 bool MergeConsecutiveStores(StoreSDNode *N);
307
Chris Lattner2392ae72010-04-15 04:48:01 +0000308 public:
Bill Wendling98a366d2009-04-29 23:29:43 +0000309 DAGCombiner(SelectionDAG &D, AliasAnalysis &A, CodeGenOpt::Level OL)
Eli Friedman50185242011-11-12 00:35:34 +0000310 : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
Chris Lattner2392ae72010-04-15 04:48:01 +0000311 OptLevel(OL), LegalOperations(false), LegalTypes(false), AA(A) {}
Scott Michelfdc40a02009-02-17 22:15:04 +0000312
Nate Begeman1d4d4142005-09-01 00:19:25 +0000313 /// Run - runs the dag combiner on all nodes in the work list
Duncan Sands25cf2272008-11-24 14:53:14 +0000314 void Run(CombineLevel AtLevel);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000315
Chris Lattner2392ae72010-04-15 04:48:01 +0000316 SelectionDAG &getDAG() const { return DAG; }
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000317
Chris Lattner2392ae72010-04-15 04:48:01 +0000318 /// getShiftAmountTy - Returns a type large enough to hold any valid
319 /// shift amount - before type legalization these can be huge.
Owen Anderson95771af2011-02-25 21:41:48 +0000320 EVT getShiftAmountTy(EVT LHSTy) {
321 return LegalTypes ? TLI.getShiftAmountTy(LHSTy) : TLI.getPointerTy();
Chris Lattner2392ae72010-04-15 04:48:01 +0000322 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000323
Chris Lattner2392ae72010-04-15 04:48:01 +0000324 /// isTypeLegal - This method returns true if we are running before type
325 /// legalization or if the specified VT is legal.
326 bool isTypeLegal(const EVT &VT) {
327 if (!LegalTypes) return true;
328 return TLI.isTypeLegal(VT);
329 }
Nate Begeman1d4d4142005-09-01 00:19:25 +0000330 };
331}
332
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000333
334namespace {
335/// WorkListRemover - This class is a DAGUpdateListener that removes any deleted
336/// nodes from the worklist.
Nick Lewycky6726b6d2009-10-25 06:33:48 +0000337class WorkListRemover : public SelectionDAG::DAGUpdateListener {
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000338 DAGCombiner &DC;
339public:
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000340 explicit WorkListRemover(DAGCombiner &dc)
341 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
Scott Michelfdc40a02009-02-17 22:15:04 +0000342
Duncan Sandsedfcf592008-06-11 11:42:12 +0000343 virtual void NodeDeleted(SDNode *N, SDNode *E) {
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000344 DC.removeFromWorkList(N);
345 }
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000346};
347}
348
Chris Lattner24664722006-03-01 04:53:38 +0000349//===----------------------------------------------------------------------===//
350// TargetLowering::DAGCombinerInfo implementation
351//===----------------------------------------------------------------------===//
352
353void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
354 ((DAGCombiner*)DC)->AddToWorkList(N);
355}
356
Cameron Zwariched3caf92011-04-02 02:40:26 +0000357void TargetLowering::DAGCombinerInfo::RemoveFromWorklist(SDNode *N) {
358 ((DAGCombiner*)DC)->removeFromWorkList(N);
359}
360
Dan Gohman475871a2008-07-27 21:46:04 +0000361SDValue TargetLowering::DAGCombinerInfo::
Evan Cheng0b0cd912009-03-28 05:57:29 +0000362CombineTo(SDNode *N, const std::vector<SDValue> &To, bool AddTo) {
363 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000364}
365
Dan Gohman475871a2008-07-27 21:46:04 +0000366SDValue TargetLowering::DAGCombinerInfo::
Evan Cheng0b0cd912009-03-28 05:57:29 +0000367CombineTo(SDNode *N, SDValue Res, bool AddTo) {
368 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000369}
370
371
Dan Gohman475871a2008-07-27 21:46:04 +0000372SDValue TargetLowering::DAGCombinerInfo::
Evan Cheng0b0cd912009-03-28 05:57:29 +0000373CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
374 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000375}
376
Dan Gohmane5af2d32009-01-29 01:59:02 +0000377void TargetLowering::DAGCombinerInfo::
378CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
379 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
380}
Chris Lattner24664722006-03-01 04:53:38 +0000381
Chris Lattner24664722006-03-01 04:53:38 +0000382//===----------------------------------------------------------------------===//
Chris Lattner29446522007-05-14 22:04:50 +0000383// Helper Functions
384//===----------------------------------------------------------------------===//
385
386/// isNegatibleForFree - Return 1 if we can compute the negated form of the
387/// specified expression for the same cost as the expression itself, or 2 if we
388/// can compute the negated form more cheaply than the expression itself.
Duncan Sands25cf2272008-11-24 14:53:14 +0000389static char isNegatibleForFree(SDValue Op, bool LegalOperations,
Owen Andersonafd3d562012-03-06 00:29:31 +0000390 const TargetLowering &TLI,
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000391 const TargetOptions *Options,
Chris Lattner0254e702008-02-26 07:04:54 +0000392 unsigned Depth = 0) {
Dale Johannesendb44bf82007-10-16 23:38:29 +0000393 // No compile time optimizations on this type.
Owen Anderson825b72b2009-08-11 20:47:22 +0000394 if (Op.getValueType() == MVT::ppcf128)
Dale Johannesendb44bf82007-10-16 23:38:29 +0000395 return 0;
396
Chris Lattner29446522007-05-14 22:04:50 +0000397 // fneg is removable even if it has multiple uses.
398 if (Op.getOpcode() == ISD::FNEG) return 2;
Scott Michelfdc40a02009-02-17 22:15:04 +0000399
Chris Lattner29446522007-05-14 22:04:50 +0000400 // Don't allow anything with multiple uses.
401 if (!Op.hasOneUse()) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000402
Chris Lattner3adf9512007-05-25 02:19:06 +0000403 // Don't recurse exponentially.
404 if (Depth > 6) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000405
Chris Lattner29446522007-05-14 22:04:50 +0000406 switch (Op.getOpcode()) {
407 default: return false;
408 case ISD::ConstantFP:
Chris Lattner0254e702008-02-26 07:04:54 +0000409 // Don't invert constant FP values after legalize. The negated constant
410 // isn't necessarily legal.
Duncan Sands25cf2272008-11-24 14:53:14 +0000411 return LegalOperations ? 0 : 1;
Chris Lattner29446522007-05-14 22:04:50 +0000412 case ISD::FADD:
413 // FIXME: determine better conditions for this xform.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000414 if (!Options->UnsafeFPMath) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000415
Owen Andersonafd3d562012-03-06 00:29:31 +0000416 // After operation legalization, it might not be legal to create new FSUBs.
417 if (LegalOperations &&
418 !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType()))
419 return 0;
420
Craig Topper956342b2012-09-09 22:58:45 +0000421 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
Owen Andersonafd3d562012-03-06 00:29:31 +0000422 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
423 Options, Depth + 1))
Chris Lattner29446522007-05-14 22:04:50 +0000424 return V;
Bill Wendlingd34470c2009-01-30 23:10:18 +0000425 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
Owen Andersonafd3d562012-03-06 00:29:31 +0000426 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000427 Depth + 1);
Chris Lattner29446522007-05-14 22:04:50 +0000428 case ISD::FSUB:
Scott Michelfdc40a02009-02-17 22:15:04 +0000429 // We can't turn -(A-B) into B-A when we honor signed zeros.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000430 if (!Options->UnsafeFPMath) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000431
Bill Wendlingd34470c2009-01-30 23:10:18 +0000432 // fold (fneg (fsub A, B)) -> (fsub B, A)
Chris Lattner29446522007-05-14 22:04:50 +0000433 return 1;
Scott Michelfdc40a02009-02-17 22:15:04 +0000434
Chris Lattner29446522007-05-14 22:04:50 +0000435 case ISD::FMUL:
436 case ISD::FDIV:
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000437 if (Options->HonorSignDependentRoundingFPMath()) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000438
Bill Wendlingd34470c2009-01-30 23:10:18 +0000439 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
Owen Andersonafd3d562012-03-06 00:29:31 +0000440 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
441 Options, Depth + 1))
Chris Lattner29446522007-05-14 22:04:50 +0000442 return V;
Scott Michelfdc40a02009-02-17 22:15:04 +0000443
Owen Andersonafd3d562012-03-06 00:29:31 +0000444 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000445 Depth + 1);
Scott Michelfdc40a02009-02-17 22:15:04 +0000446
Chris Lattner29446522007-05-14 22:04:50 +0000447 case ISD::FP_EXTEND:
448 case ISD::FP_ROUND:
449 case ISD::FSIN:
Owen Andersonafd3d562012-03-06 00:29:31 +0000450 return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000451 Depth + 1);
Chris Lattner29446522007-05-14 22:04:50 +0000452 }
453}
454
455/// GetNegatedExpression - If isNegatibleForFree returns true, this function
456/// returns the newly negated expression.
Dan Gohman475871a2008-07-27 21:46:04 +0000457static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000458 bool LegalOperations, unsigned Depth = 0) {
Chris Lattner29446522007-05-14 22:04:50 +0000459 // fneg is removable even if it has multiple uses.
460 if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
Scott Michelfdc40a02009-02-17 22:15:04 +0000461
Chris Lattner29446522007-05-14 22:04:50 +0000462 // Don't allow anything with multiple uses.
463 assert(Op.hasOneUse() && "Unknown reuse!");
Scott Michelfdc40a02009-02-17 22:15:04 +0000464
Chris Lattner3adf9512007-05-25 02:19:06 +0000465 assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
Chris Lattner29446522007-05-14 22:04:50 +0000466 switch (Op.getOpcode()) {
Torok Edwinc23197a2009-07-14 16:55:14 +0000467 default: llvm_unreachable("Unknown code");
Dale Johannesenc4dd3c32007-08-31 23:34:27 +0000468 case ISD::ConstantFP: {
469 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
470 V.changeSign();
471 return DAG.getConstantFP(V, Op.getValueType());
472 }
Chris Lattner29446522007-05-14 22:04:50 +0000473 case ISD::FADD:
474 // FIXME: determine better conditions for this xform.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000475 assert(DAG.getTarget().Options.UnsafeFPMath);
Scott Michelfdc40a02009-02-17 22:15:04 +0000476
Bill Wendlingd34470c2009-01-30 23:10:18 +0000477 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000478 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
Owen Andersonafd3d562012-03-06 00:29:31 +0000479 DAG.getTargetLoweringInfo(),
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000480 &DAG.getTarget().Options, Depth+1))
Bill Wendling35247c32009-01-30 00:45:56 +0000481 return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000482 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000483 LegalOperations, Depth+1),
Chris Lattner29446522007-05-14 22:04:50 +0000484 Op.getOperand(1));
Bill Wendlingd34470c2009-01-30 23:10:18 +0000485 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
Bill Wendling35247c32009-01-30 00:45:56 +0000486 return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000487 GetNegatedExpression(Op.getOperand(1), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000488 LegalOperations, Depth+1),
Chris Lattner29446522007-05-14 22:04:50 +0000489 Op.getOperand(0));
490 case ISD::FSUB:
Scott Michelfdc40a02009-02-17 22:15:04 +0000491 // We can't turn -(A-B) into B-A when we honor signed zeros.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000492 assert(DAG.getTarget().Options.UnsafeFPMath);
Dan Gohman23ff1822007-07-02 15:48:56 +0000493
Bill Wendlingd34470c2009-01-30 23:10:18 +0000494 // fold (fneg (fsub 0, B)) -> B
Dan Gohman23ff1822007-07-02 15:48:56 +0000495 if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
Dale Johannesenc4dd3c32007-08-31 23:34:27 +0000496 if (N0CFP->getValueAPF().isZero())
Dan Gohman23ff1822007-07-02 15:48:56 +0000497 return Op.getOperand(1);
Scott Michelfdc40a02009-02-17 22:15:04 +0000498
Bill Wendlingd34470c2009-01-30 23:10:18 +0000499 // fold (fneg (fsub A, B)) -> (fsub B, A)
Bill Wendling35247c32009-01-30 00:45:56 +0000500 return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(),
501 Op.getOperand(1), Op.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +0000502
Chris Lattner29446522007-05-14 22:04:50 +0000503 case ISD::FMUL:
504 case ISD::FDIV:
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000505 assert(!DAG.getTarget().Options.HonorSignDependentRoundingFPMath());
Scott Michelfdc40a02009-02-17 22:15:04 +0000506
Bill Wendlingd34470c2009-01-30 23:10:18 +0000507 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000508 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
Owen Andersonafd3d562012-03-06 00:29:31 +0000509 DAG.getTargetLoweringInfo(),
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000510 &DAG.getTarget().Options, Depth+1))
Bill Wendling35247c32009-01-30 00:45:56 +0000511 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000512 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000513 LegalOperations, Depth+1),
Chris Lattner29446522007-05-14 22:04:50 +0000514 Op.getOperand(1));
Scott Michelfdc40a02009-02-17 22:15:04 +0000515
Bill Wendlingd34470c2009-01-30 23:10:18 +0000516 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
Bill Wendling35247c32009-01-30 00:45:56 +0000517 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(),
Chris Lattner29446522007-05-14 22:04:50 +0000518 Op.getOperand(0),
Chris Lattner0254e702008-02-26 07:04:54 +0000519 GetNegatedExpression(Op.getOperand(1), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000520 LegalOperations, Depth+1));
Scott Michelfdc40a02009-02-17 22:15:04 +0000521
Chris Lattner29446522007-05-14 22:04:50 +0000522 case ISD::FP_EXTEND:
Chris Lattner29446522007-05-14 22:04:50 +0000523 case ISD::FSIN:
Bill Wendling35247c32009-01-30 00:45:56 +0000524 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000525 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000526 LegalOperations, Depth+1));
Chris Lattner0bd48932008-01-17 07:00:52 +0000527 case ISD::FP_ROUND:
Bill Wendling35247c32009-01-30 00:45:56 +0000528 return DAG.getNode(ISD::FP_ROUND, Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000529 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000530 LegalOperations, Depth+1),
Chris Lattner0bd48932008-01-17 07:00:52 +0000531 Op.getOperand(1));
Chris Lattner29446522007-05-14 22:04:50 +0000532 }
533}
Chris Lattner24664722006-03-01 04:53:38 +0000534
535
Nate Begeman4ebd8052005-09-01 23:24:04 +0000536// isSetCCEquivalent - Return true if this node is a setcc, or is a select_cc
537// that selects between the values 1 and 0, making it equivalent to a setcc.
Scott Michelfdc40a02009-02-17 22:15:04 +0000538// Also, set the incoming LHS, RHS, and CC references to the appropriate
Nate Begeman646d7e22005-09-02 21:18:40 +0000539// nodes based on the type of node we are checking. This simplifies life a
540// bit for the callers.
Dan Gohman475871a2008-07-27 21:46:04 +0000541static bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
542 SDValue &CC) {
Nate Begeman646d7e22005-09-02 21:18:40 +0000543 if (N.getOpcode() == ISD::SETCC) {
544 LHS = N.getOperand(0);
545 RHS = N.getOperand(1);
546 CC = N.getOperand(2);
Nate Begeman4ebd8052005-09-01 23:24:04 +0000547 return true;
Nate Begeman646d7e22005-09-02 21:18:40 +0000548 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000549 if (N.getOpcode() == ISD::SELECT_CC &&
Nate Begeman1d4d4142005-09-01 00:19:25 +0000550 N.getOperand(2).getOpcode() == ISD::Constant &&
551 N.getOperand(3).getOpcode() == ISD::Constant &&
Dan Gohman002e5d02008-03-13 22:13:53 +0000552 cast<ConstantSDNode>(N.getOperand(2))->getAPIntValue() == 1 &&
Nate Begeman646d7e22005-09-02 21:18:40 +0000553 cast<ConstantSDNode>(N.getOperand(3))->isNullValue()) {
554 LHS = N.getOperand(0);
555 RHS = N.getOperand(1);
556 CC = N.getOperand(4);
Nate Begeman1d4d4142005-09-01 00:19:25 +0000557 return true;
Nate Begeman646d7e22005-09-02 21:18:40 +0000558 }
Nate Begeman1d4d4142005-09-01 00:19:25 +0000559 return false;
560}
561
Nate Begeman99801192005-09-07 23:25:52 +0000562// isOneUseSetCC - Return true if this is a SetCC-equivalent operation with only
563// one use. If this is true, it allows the users to invert the operation for
564// free when it is profitable to do so.
Dan Gohman475871a2008-07-27 21:46:04 +0000565static bool isOneUseSetCC(SDValue N) {
566 SDValue N0, N1, N2;
Gabor Greifba36cb52008-08-28 21:40:38 +0000567 if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
Nate Begeman4ebd8052005-09-01 23:24:04 +0000568 return true;
569 return false;
570}
571
Bill Wendling35247c32009-01-30 00:45:56 +0000572SDValue DAGCombiner::ReassociateOps(unsigned Opc, DebugLoc DL,
573 SDValue N0, SDValue N1) {
Owen Andersone50ed302009-08-10 22:56:29 +0000574 EVT VT = N0.getValueType();
Nate Begemancd4d58c2006-02-03 06:46:56 +0000575 if (N0.getOpcode() == Opc && isa<ConstantSDNode>(N0.getOperand(1))) {
576 if (isa<ConstantSDNode>(N1)) {
Bill Wendling35247c32009-01-30 00:45:56 +0000577 // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
Bill Wendling6af76182009-01-30 20:50:00 +0000578 SDValue OpNode =
579 DAG.FoldConstantArithmetic(Opc, VT,
580 cast<ConstantSDNode>(N0.getOperand(1)),
581 cast<ConstantSDNode>(N1));
Bill Wendlingd69c3142009-01-30 02:23:43 +0000582 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
Dan Gohman71dc7c92011-05-17 22:20:36 +0000583 }
584 if (N0.hasOneUse()) {
Sylvestre Ledru94c22712012-09-27 10:14:43 +0000585 // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one use
Bill Wendling35247c32009-01-30 00:45:56 +0000586 SDValue OpNode = DAG.getNode(Opc, N0.getDebugLoc(), VT,
587 N0.getOperand(0), N1);
Gabor Greifba36cb52008-08-28 21:40:38 +0000588 AddToWorkList(OpNode.getNode());
Bill Wendling35247c32009-01-30 00:45:56 +0000589 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
Nate Begemancd4d58c2006-02-03 06:46:56 +0000590 }
591 }
Bill Wendling35247c32009-01-30 00:45:56 +0000592
Nate Begemancd4d58c2006-02-03 06:46:56 +0000593 if (N1.getOpcode() == Opc && isa<ConstantSDNode>(N1.getOperand(1))) {
594 if (isa<ConstantSDNode>(N0)) {
Bill Wendling35247c32009-01-30 00:45:56 +0000595 // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
Bill Wendling6af76182009-01-30 20:50:00 +0000596 SDValue OpNode =
597 DAG.FoldConstantArithmetic(Opc, VT,
598 cast<ConstantSDNode>(N1.getOperand(1)),
599 cast<ConstantSDNode>(N0));
Bill Wendlingd69c3142009-01-30 02:23:43 +0000600 return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
Dan Gohman71dc7c92011-05-17 22:20:36 +0000601 }
602 if (N1.hasOneUse()) {
Sylvestre Ledru94c22712012-09-27 10:14:43 +0000603 // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one use
Bill Wendlingd69c3142009-01-30 02:23:43 +0000604 SDValue OpNode = DAG.getNode(Opc, N0.getDebugLoc(), VT,
Bill Wendling35247c32009-01-30 00:45:56 +0000605 N1.getOperand(0), N0);
Gabor Greifba36cb52008-08-28 21:40:38 +0000606 AddToWorkList(OpNode.getNode());
Bill Wendling35247c32009-01-30 00:45:56 +0000607 return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
Nate Begemancd4d58c2006-02-03 06:46:56 +0000608 }
609 }
Bill Wendling35247c32009-01-30 00:45:56 +0000610
Dan Gohman475871a2008-07-27 21:46:04 +0000611 return SDValue();
Nate Begemancd4d58c2006-02-03 06:46:56 +0000612}
613
Dan Gohman475871a2008-07-27 21:46:04 +0000614SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
615 bool AddTo) {
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000616 assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
617 ++NodesCombined;
David Greenef1090292010-01-05 01:25:00 +0000618 DEBUG(dbgs() << "\nReplacing.1 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000619 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000620 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000621 To[0].getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000622 dbgs() << " and " << NumTo-1 << " other values\n";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000623 for (unsigned i = 0, e = NumTo; i != e; ++i)
Jakob Stoklund Olesen9f0d4e62009-12-03 05:15:35 +0000624 assert((!To[i].getNode() ||
625 N->getValueType(i) == To[i].getValueType()) &&
Dan Gohman764fd0c2009-01-21 15:17:51 +0000626 "Cannot combine value to value of different type!"));
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000627 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000628 DAG.ReplaceAllUsesWith(N, To);
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000629 if (AddTo) {
630 // Push the new nodes and any users onto the worklist
631 for (unsigned i = 0, e = NumTo; i != e; ++i) {
Chris Lattnerd1980a52009-03-12 06:52:53 +0000632 if (To[i].getNode()) {
633 AddToWorkList(To[i].getNode());
634 AddUsersToWorkList(To[i].getNode());
635 }
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000636 }
637 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000638
Dan Gohmandbe664a2009-01-19 21:44:21 +0000639 // Finally, if the node is now dead, remove it from the graph. The node
640 // may not be dead if the replacement process recursively simplified to
641 // something else needing this node.
642 if (N->use_empty()) {
643 // Nodes can be reintroduced into the worklist. Make sure we do not
644 // process a node that has been replaced.
645 removeFromWorkList(N);
Scott Michelfdc40a02009-02-17 22:15:04 +0000646
Dan Gohmandbe664a2009-01-19 21:44:21 +0000647 // Finally, since the node is now dead, remove it from the graph.
648 DAG.DeleteNode(N);
649 }
Dan Gohman475871a2008-07-27 21:46:04 +0000650 return SDValue(N, 0);
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000651}
652
Evan Chenge5b51ac2010-04-17 06:13:15 +0000653void DAGCombiner::
654CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
Scott Michelfdc40a02009-02-17 22:15:04 +0000655 // Replace all uses. If any nodes become isomorphic to other nodes and
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000656 // are deleted, make sure to remove them from our worklist.
657 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000658 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
Dan Gohmane5af2d32009-01-29 01:59:02 +0000659
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000660 // Push the new node and any (possibly new) users onto the worklist.
Gabor Greifba36cb52008-08-28 21:40:38 +0000661 AddToWorkList(TLO.New.getNode());
662 AddUsersToWorkList(TLO.New.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000663
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000664 // Finally, if the node is now dead, remove it from the graph. The node
665 // may not be dead if the replacement process recursively simplified to
666 // something else needing this node.
Gabor Greifba36cb52008-08-28 21:40:38 +0000667 if (TLO.Old.getNode()->use_empty()) {
668 removeFromWorkList(TLO.Old.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000669
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000670 // If the operands of this node are only used by the node, they will now
671 // be dead. Make sure to visit them first to delete dead nodes early.
Gabor Greifba36cb52008-08-28 21:40:38 +0000672 for (unsigned i = 0, e = TLO.Old.getNode()->getNumOperands(); i != e; ++i)
673 if (TLO.Old.getNode()->getOperand(i).getNode()->hasOneUse())
674 AddToWorkList(TLO.Old.getNode()->getOperand(i).getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000675
Gabor Greifba36cb52008-08-28 21:40:38 +0000676 DAG.DeleteNode(TLO.Old.getNode());
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000677 }
Dan Gohmane5af2d32009-01-29 01:59:02 +0000678}
679
680/// SimplifyDemandedBits - Check the specified integer node value to see if
681/// it can be simplified or if things it uses can be simplified by bit
682/// propagation. If so, return true.
683bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
Evan Chenge5b51ac2010-04-17 06:13:15 +0000684 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
Dan Gohmane5af2d32009-01-29 01:59:02 +0000685 APInt KnownZero, KnownOne;
686 if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
687 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +0000688
Dan Gohmane5af2d32009-01-29 01:59:02 +0000689 // Revisit the node.
690 AddToWorkList(Op.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000691
Dan Gohmane5af2d32009-01-29 01:59:02 +0000692 // Replace the old value with the new one.
693 ++NodesCombined;
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000694 DEBUG(dbgs() << "\nReplacing.2 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000695 TLO.Old.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000696 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000697 TLO.New.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000698 dbgs() << '\n');
Scott Michelfdc40a02009-02-17 22:15:04 +0000699
Dan Gohmane5af2d32009-01-29 01:59:02 +0000700 CommitTargetLoweringOpt(TLO);
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000701 return true;
702}
703
Evan Cheng95c57ea2010-04-24 04:43:44 +0000704void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
705 DebugLoc dl = Load->getDebugLoc();
706 EVT VT = Load->getValueType(0);
707 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0));
Evan Cheng4c26e932010-04-19 19:29:22 +0000708
Evan Cheng95c57ea2010-04-24 04:43:44 +0000709 DEBUG(dbgs() << "\nReplacing.9 ";
710 Load->dump(&DAG);
711 dbgs() << "\nWith: ";
712 Trunc.getNode()->dump(&DAG);
713 dbgs() << '\n');
714 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000715 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
716 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
Evan Cheng95c57ea2010-04-24 04:43:44 +0000717 removeFromWorkList(Load);
718 DAG.DeleteNode(Load);
Evan Chengac7eae52010-04-27 19:48:13 +0000719 AddToWorkList(Trunc.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000720}
721
722SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
723 Replace = false;
Evan Cheng4c26e932010-04-19 19:29:22 +0000724 DebugLoc dl = Op.getDebugLoc();
Evan Chenge5b51ac2010-04-17 06:13:15 +0000725 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
Evan Chengac7eae52010-04-27 19:48:13 +0000726 EVT MemVT = LD->getMemoryVT();
727 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
Owen Anderson95771af2011-02-25 21:41:48 +0000728 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
Eric Christopher503a64d2010-12-09 04:48:06 +0000729 : ISD::EXTLOAD)
Evan Chengac7eae52010-04-27 19:48:13 +0000730 : LD->getExtensionType();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000731 Replace = true;
Stuart Hastingsa9011292011-02-16 16:23:55 +0000732 return DAG.getExtLoad(ExtType, dl, PVT,
Evan Chenge5b51ac2010-04-17 06:13:15 +0000733 LD->getChain(), LD->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +0000734 LD->getPointerInfo(),
Evan Chengac7eae52010-04-27 19:48:13 +0000735 MemVT, LD->isVolatile(),
Evan Chenge5b51ac2010-04-17 06:13:15 +0000736 LD->isNonTemporal(), LD->getAlignment());
737 }
738
Evan Cheng4c26e932010-04-19 19:29:22 +0000739 unsigned Opc = Op.getOpcode();
Evan Chengcaf77402010-04-23 19:10:30 +0000740 switch (Opc) {
741 default: break;
742 case ISD::AssertSext:
Evan Cheng4c26e932010-04-19 19:29:22 +0000743 return DAG.getNode(ISD::AssertSext, dl, PVT,
Evan Cheng95c57ea2010-04-24 04:43:44 +0000744 SExtPromoteOperand(Op.getOperand(0), PVT),
Evan Cheng4c26e932010-04-19 19:29:22 +0000745 Op.getOperand(1));
Evan Chengcaf77402010-04-23 19:10:30 +0000746 case ISD::AssertZext:
Evan Cheng4c26e932010-04-19 19:29:22 +0000747 return DAG.getNode(ISD::AssertZext, dl, PVT,
Evan Cheng95c57ea2010-04-24 04:43:44 +0000748 ZExtPromoteOperand(Op.getOperand(0), PVT),
Evan Cheng4c26e932010-04-19 19:29:22 +0000749 Op.getOperand(1));
Evan Chengcaf77402010-04-23 19:10:30 +0000750 case ISD::Constant: {
751 unsigned ExtOpc =
Evan Cheng4c26e932010-04-19 19:29:22 +0000752 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
Evan Chengcaf77402010-04-23 19:10:30 +0000753 return DAG.getNode(ExtOpc, dl, PVT, Op);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000754 }
Evan Chengcaf77402010-04-23 19:10:30 +0000755 }
756
757 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
Evan Chenge5b51ac2010-04-17 06:13:15 +0000758 return SDValue();
Evan Chengcaf77402010-04-23 19:10:30 +0000759 return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op);
Evan Cheng64b7bf72010-04-16 06:14:10 +0000760}
761
Evan Cheng95c57ea2010-04-24 04:43:44 +0000762SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
Evan Chenge5b51ac2010-04-17 06:13:15 +0000763 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
764 return SDValue();
765 EVT OldVT = Op.getValueType();
766 DebugLoc dl = Op.getDebugLoc();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000767 bool Replace = false;
768 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
769 if (NewOp.getNode() == 0)
Evan Chenge5b51ac2010-04-17 06:13:15 +0000770 return SDValue();
Evan Chengac7eae52010-04-27 19:48:13 +0000771 AddToWorkList(NewOp.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000772
773 if (Replace)
774 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
775 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp,
Evan Chenge5b51ac2010-04-17 06:13:15 +0000776 DAG.getValueType(OldVT));
777}
778
Evan Cheng95c57ea2010-04-24 04:43:44 +0000779SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
Evan Chenge5b51ac2010-04-17 06:13:15 +0000780 EVT OldVT = Op.getValueType();
781 DebugLoc dl = Op.getDebugLoc();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000782 bool Replace = false;
783 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
784 if (NewOp.getNode() == 0)
Evan Chenge5b51ac2010-04-17 06:13:15 +0000785 return SDValue();
Evan Chengac7eae52010-04-27 19:48:13 +0000786 AddToWorkList(NewOp.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000787
788 if (Replace)
789 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
790 return DAG.getZeroExtendInReg(NewOp, dl, OldVT);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000791}
792
Evan Cheng64b7bf72010-04-16 06:14:10 +0000793/// PromoteIntBinOp - Promote the specified integer binary operation if the
794/// target indicates it is beneficial. e.g. On x86, it's usually better to
795/// promote i16 operations to i32 since i16 instructions are longer.
796SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
797 if (!LegalOperations)
798 return SDValue();
799
800 EVT VT = Op.getValueType();
801 if (VT.isVector() || !VT.isInteger())
802 return SDValue();
803
Evan Chenge5b51ac2010-04-17 06:13:15 +0000804 // If operation type is 'undesirable', e.g. i16 on x86, consider
805 // promoting it.
806 unsigned Opc = Op.getOpcode();
807 if (TLI.isTypeDesirableForOp(Opc, VT))
808 return SDValue();
809
Evan Cheng64b7bf72010-04-16 06:14:10 +0000810 EVT PVT = VT;
Evan Chenge5b51ac2010-04-17 06:13:15 +0000811 // Consult target whether it is a good idea to promote this operation and
812 // what's the right type to promote it to.
813 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
Evan Cheng64b7bf72010-04-16 06:14:10 +0000814 assert(PVT != VT && "Don't know what type to promote to!");
815
Evan Cheng95c57ea2010-04-24 04:43:44 +0000816 bool Replace0 = false;
817 SDValue N0 = Op.getOperand(0);
818 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
819 if (NN0.getNode() == 0)
Evan Cheng07c4e102010-04-22 20:19:46 +0000820 return SDValue();
821
Evan Cheng95c57ea2010-04-24 04:43:44 +0000822 bool Replace1 = false;
823 SDValue N1 = Op.getOperand(1);
Evan Chengaad753b2010-05-10 19:03:57 +0000824 SDValue NN1;
825 if (N0 == N1)
826 NN1 = NN0;
827 else {
828 NN1 = PromoteOperand(N1, PVT, Replace1);
829 if (NN1.getNode() == 0)
830 return SDValue();
831 }
Evan Cheng07c4e102010-04-22 20:19:46 +0000832
Evan Cheng95c57ea2010-04-24 04:43:44 +0000833 AddToWorkList(NN0.getNode());
Evan Chengaad753b2010-05-10 19:03:57 +0000834 if (NN1.getNode())
835 AddToWorkList(NN1.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000836
837 if (Replace0)
838 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
839 if (Replace1)
840 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
Evan Cheng07c4e102010-04-22 20:19:46 +0000841
Evan Chengac7eae52010-04-27 19:48:13 +0000842 DEBUG(dbgs() << "\nPromoting ";
843 Op.getNode()->dump(&DAG));
Evan Cheng07c4e102010-04-22 20:19:46 +0000844 DebugLoc dl = Op.getDebugLoc();
845 return DAG.getNode(ISD::TRUNCATE, dl, VT,
Evan Cheng95c57ea2010-04-24 04:43:44 +0000846 DAG.getNode(Opc, dl, PVT, NN0, NN1));
Evan Cheng07c4e102010-04-22 20:19:46 +0000847 }
848 return SDValue();
849}
850
851/// PromoteIntShiftOp - Promote the specified integer shift operation if the
852/// target indicates it is beneficial. e.g. On x86, it's usually better to
853/// promote i16 operations to i32 since i16 instructions are longer.
854SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
855 if (!LegalOperations)
856 return SDValue();
857
858 EVT VT = Op.getValueType();
859 if (VT.isVector() || !VT.isInteger())
860 return SDValue();
861
862 // If operation type is 'undesirable', e.g. i16 on x86, consider
863 // promoting it.
864 unsigned Opc = Op.getOpcode();
865 if (TLI.isTypeDesirableForOp(Opc, VT))
866 return SDValue();
867
868 EVT PVT = VT;
869 // Consult target whether it is a good idea to promote this operation and
870 // what's the right type to promote it to.
871 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
872 assert(PVT != VT && "Don't know what type to promote to!");
873
Evan Cheng95c57ea2010-04-24 04:43:44 +0000874 bool Replace = false;
Evan Chenge5b51ac2010-04-17 06:13:15 +0000875 SDValue N0 = Op.getOperand(0);
876 if (Opc == ISD::SRA)
Evan Cheng95c57ea2010-04-24 04:43:44 +0000877 N0 = SExtPromoteOperand(Op.getOperand(0), PVT);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000878 else if (Opc == ISD::SRL)
Evan Cheng95c57ea2010-04-24 04:43:44 +0000879 N0 = ZExtPromoteOperand(Op.getOperand(0), PVT);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000880 else
Evan Cheng95c57ea2010-04-24 04:43:44 +0000881 N0 = PromoteOperand(N0, PVT, Replace);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000882 if (N0.getNode() == 0)
883 return SDValue();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000884
Evan Chenge5b51ac2010-04-17 06:13:15 +0000885 AddToWorkList(N0.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000886 if (Replace)
887 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
Evan Cheng64b7bf72010-04-16 06:14:10 +0000888
Evan Chengac7eae52010-04-27 19:48:13 +0000889 DEBUG(dbgs() << "\nPromoting ";
890 Op.getNode()->dump(&DAG));
Evan Cheng64b7bf72010-04-16 06:14:10 +0000891 DebugLoc dl = Op.getDebugLoc();
892 return DAG.getNode(ISD::TRUNCATE, dl, VT,
Evan Cheng07c4e102010-04-22 20:19:46 +0000893 DAG.getNode(Opc, dl, PVT, N0, Op.getOperand(1)));
Evan Cheng64b7bf72010-04-16 06:14:10 +0000894 }
895 return SDValue();
896}
897
Evan Cheng4c26e932010-04-19 19:29:22 +0000898SDValue DAGCombiner::PromoteExtend(SDValue Op) {
899 if (!LegalOperations)
900 return SDValue();
901
902 EVT VT = Op.getValueType();
903 if (VT.isVector() || !VT.isInteger())
904 return SDValue();
905
906 // If operation type is 'undesirable', e.g. i16 on x86, consider
907 // promoting it.
908 unsigned Opc = Op.getOpcode();
909 if (TLI.isTypeDesirableForOp(Opc, VT))
910 return SDValue();
911
912 EVT PVT = VT;
913 // Consult target whether it is a good idea to promote this operation and
914 // what's the right type to promote it to.
915 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
916 assert(PVT != VT && "Don't know what type to promote to!");
917 // fold (aext (aext x)) -> (aext x)
918 // fold (aext (zext x)) -> (zext x)
919 // fold (aext (sext x)) -> (sext x)
Evan Chengac7eae52010-04-27 19:48:13 +0000920 DEBUG(dbgs() << "\nPromoting ";
921 Op.getNode()->dump(&DAG));
Evan Cheng4c26e932010-04-19 19:29:22 +0000922 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), VT, Op.getOperand(0));
923 }
924 return SDValue();
925}
926
927bool DAGCombiner::PromoteLoad(SDValue Op) {
928 if (!LegalOperations)
929 return false;
930
931 EVT VT = Op.getValueType();
932 if (VT.isVector() || !VT.isInteger())
933 return false;
934
935 // If operation type is 'undesirable', e.g. i16 on x86, consider
936 // promoting it.
937 unsigned Opc = Op.getOpcode();
938 if (TLI.isTypeDesirableForOp(Opc, VT))
939 return false;
940
941 EVT PVT = VT;
942 // Consult target whether it is a good idea to promote this operation and
943 // what's the right type to promote it to.
944 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
945 assert(PVT != VT && "Don't know what type to promote to!");
946
947 DebugLoc dl = Op.getDebugLoc();
948 SDNode *N = Op.getNode();
949 LoadSDNode *LD = cast<LoadSDNode>(N);
Evan Chengac7eae52010-04-27 19:48:13 +0000950 EVT MemVT = LD->getMemoryVT();
951 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
Owen Anderson95771af2011-02-25 21:41:48 +0000952 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
Eric Christopher503a64d2010-12-09 04:48:06 +0000953 : ISD::EXTLOAD)
Evan Chengac7eae52010-04-27 19:48:13 +0000954 : LD->getExtensionType();
Stuart Hastingsa9011292011-02-16 16:23:55 +0000955 SDValue NewLD = DAG.getExtLoad(ExtType, dl, PVT,
Evan Cheng4c26e932010-04-19 19:29:22 +0000956 LD->getChain(), LD->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +0000957 LD->getPointerInfo(),
Evan Chengac7eae52010-04-27 19:48:13 +0000958 MemVT, LD->isVolatile(),
Evan Cheng4c26e932010-04-19 19:29:22 +0000959 LD->isNonTemporal(), LD->getAlignment());
960 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD);
961
Evan Cheng95c57ea2010-04-24 04:43:44 +0000962 DEBUG(dbgs() << "\nPromoting ";
Evan Cheng4c26e932010-04-19 19:29:22 +0000963 N->dump(&DAG);
Evan Cheng95c57ea2010-04-24 04:43:44 +0000964 dbgs() << "\nTo: ";
Evan Cheng4c26e932010-04-19 19:29:22 +0000965 Result.getNode()->dump(&DAG);
966 dbgs() << '\n');
967 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000968 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
969 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
Evan Cheng4c26e932010-04-19 19:29:22 +0000970 removeFromWorkList(N);
971 DAG.DeleteNode(N);
Evan Chengac7eae52010-04-27 19:48:13 +0000972 AddToWorkList(Result.getNode());
Evan Cheng4c26e932010-04-19 19:29:22 +0000973 return true;
974 }
975 return false;
976}
977
Evan Chenge5b51ac2010-04-17 06:13:15 +0000978
Chris Lattner29446522007-05-14 22:04:50 +0000979//===----------------------------------------------------------------------===//
980// Main DAG Combiner implementation
981//===----------------------------------------------------------------------===//
982
Duncan Sands25cf2272008-11-24 14:53:14 +0000983void DAGCombiner::Run(CombineLevel AtLevel) {
984 // set the instance variables, so that the various visit routines may use it.
985 Level = AtLevel;
Eli Friedman50185242011-11-12 00:35:34 +0000986 LegalOperations = Level >= AfterLegalizeVectorOps;
987 LegalTypes = Level >= AfterLegalizeTypes;
Nate Begeman4ebd8052005-09-01 23:24:04 +0000988
Evan Cheng17a568b2008-08-29 22:21:44 +0000989 // Add all the dag nodes to the worklist.
Evan Cheng17a568b2008-08-29 22:21:44 +0000990 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
991 E = DAG.allnodes_end(); I != E; ++I)
James Molloy6660c052012-02-16 09:17:04 +0000992 AddToWorkList(I);
Duncan Sands25cf2272008-11-24 14:53:14 +0000993
Evan Cheng17a568b2008-08-29 22:21:44 +0000994 // Create a dummy node (which is not added to allnodes), that adds a reference
995 // to the root node, preventing it from being deleted, and tracking any
996 // changes of the root.
997 HandleSDNode Dummy(DAG.getRoot());
Scott Michelfdc40a02009-02-17 22:15:04 +0000998
Jim Laskey26f7fa72006-10-17 19:33:52 +0000999 // The root of the dag may dangle to deleted nodes until the dag combiner is
1000 // done. Set it to null to avoid confusion.
Dan Gohman475871a2008-07-27 21:46:04 +00001001 DAG.setRoot(SDValue());
Scott Michelfdc40a02009-02-17 22:15:04 +00001002
James Molloy6660c052012-02-16 09:17:04 +00001003 // while the worklist isn't empty, find a node and
Evan Cheng17a568b2008-08-29 22:21:44 +00001004 // try and combine it.
James Molloy6660c052012-02-16 09:17:04 +00001005 while (!WorkListContents.empty()) {
1006 SDNode *N;
1007 // The WorkListOrder holds the SDNodes in order, but it may contain duplicates.
1008 // In order to avoid a linear scan, we use a set (O(log N)) to hold what the
1009 // worklist *should* contain, and check the node we want to visit is should
1010 // actually be visited.
1011 do {
Benjamin Kramerd5f76902012-03-10 00:23:58 +00001012 N = WorkListOrder.pop_back_val();
James Molloy6660c052012-02-16 09:17:04 +00001013 } while (!WorkListContents.erase(N));
Scott Michelfdc40a02009-02-17 22:15:04 +00001014
Evan Cheng17a568b2008-08-29 22:21:44 +00001015 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1016 // N is deleted from the DAG, since they too may now be dead or may have a
1017 // reduced number of uses, allowing other xforms.
1018 if (N->use_empty() && N != &Dummy) {
1019 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1020 AddToWorkList(N->getOperand(i).getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00001021
Evan Cheng17a568b2008-08-29 22:21:44 +00001022 DAG.DeleteNode(N);
1023 continue;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001024 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001025
Evan Cheng17a568b2008-08-29 22:21:44 +00001026 SDValue RV = combine(N);
Scott Michelfdc40a02009-02-17 22:15:04 +00001027
Evan Cheng17a568b2008-08-29 22:21:44 +00001028 if (RV.getNode() == 0)
1029 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00001030
Evan Cheng17a568b2008-08-29 22:21:44 +00001031 ++NodesCombined;
Scott Michelfdc40a02009-02-17 22:15:04 +00001032
Evan Cheng17a568b2008-08-29 22:21:44 +00001033 // If we get back the same node we passed in, rather than a new node or
1034 // zero, we know that the node must have defined multiple values and
Scott Michelfdc40a02009-02-17 22:15:04 +00001035 // CombineTo was used. Since CombineTo takes care of the worklist
Evan Cheng17a568b2008-08-29 22:21:44 +00001036 // mechanics for us, we have no work to do in this case.
1037 if (RV.getNode() == N)
1038 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00001039
Evan Cheng17a568b2008-08-29 22:21:44 +00001040 assert(N->getOpcode() != ISD::DELETED_NODE &&
1041 RV.getNode()->getOpcode() != ISD::DELETED_NODE &&
1042 "Node was deleted but visit returned new node!");
Chris Lattner729c6d12006-05-27 00:43:02 +00001043
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001044 DEBUG(dbgs() << "\nReplacing.3 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00001045 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00001046 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00001047 RV.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00001048 dbgs() << '\n');
Eric Christopher7332e6e2011-07-14 01:12:15 +00001049
Devang Patel9728ea22011-05-23 22:04:42 +00001050 // Transfer debug value.
1051 DAG.TransferDbgValues(SDValue(N, 0), RV);
Evan Cheng17a568b2008-08-29 22:21:44 +00001052 WorkListRemover DeadNodes(*this);
1053 if (N->getNumValues() == RV.getNode()->getNumValues())
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00001054 DAG.ReplaceAllUsesWith(N, RV.getNode());
Evan Cheng17a568b2008-08-29 22:21:44 +00001055 else {
1056 assert(N->getValueType(0) == RV.getValueType() &&
1057 N->getNumValues() == 1 && "Type mismatch");
1058 SDValue OpV = RV;
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00001059 DAG.ReplaceAllUsesWith(N, &OpV);
Evan Cheng17a568b2008-08-29 22:21:44 +00001060 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001061
Evan Cheng17a568b2008-08-29 22:21:44 +00001062 // Push the new node and any users onto the worklist
1063 AddToWorkList(RV.getNode());
1064 AddUsersToWorkList(RV.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00001065
Evan Cheng17a568b2008-08-29 22:21:44 +00001066 // Add any uses of the old node to the worklist in case this node is the
1067 // last one that uses them. They may become dead after this node is
1068 // deleted.
1069 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1070 AddToWorkList(N->getOperand(i).getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00001071
Dan Gohmandbe664a2009-01-19 21:44:21 +00001072 // Finally, if the node is now dead, remove it from the graph. The node
1073 // may not be dead if the replacement process recursively simplified to
1074 // something else needing this node.
1075 if (N->use_empty()) {
1076 // Nodes can be reintroduced into the worklist. Make sure we do not
1077 // process a node that has been replaced.
1078 removeFromWorkList(N);
Scott Michelfdc40a02009-02-17 22:15:04 +00001079
Dan Gohmandbe664a2009-01-19 21:44:21 +00001080 // Finally, since the node is now dead, remove it from the graph.
1081 DAG.DeleteNode(N);
1082 }
Evan Cheng17a568b2008-08-29 22:21:44 +00001083 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001084
Chris Lattner95038592005-10-05 06:35:28 +00001085 // If the root changed (e.g. it was a dead load, update the root).
1086 DAG.setRoot(Dummy.getValue());
Hal Finkel31490ba2012-04-16 03:33:22 +00001087 DAG.RemoveDeadNodes();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001088}
1089
Dan Gohman475871a2008-07-27 21:46:04 +00001090SDValue DAGCombiner::visit(SDNode *N) {
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001091 switch (N->getOpcode()) {
Nate Begeman1d4d4142005-09-01 00:19:25 +00001092 default: break;
Nate Begeman4942a962005-09-01 00:33:32 +00001093 case ISD::TokenFactor: return visitTokenFactor(N);
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001094 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001095 case ISD::ADD: return visitADD(N);
1096 case ISD::SUB: return visitSUB(N);
Chris Lattner91153682007-03-04 20:03:15 +00001097 case ISD::ADDC: return visitADDC(N);
Craig Toppercc274522012-01-07 09:06:39 +00001098 case ISD::SUBC: return visitSUBC(N);
Chris Lattner91153682007-03-04 20:03:15 +00001099 case ISD::ADDE: return visitADDE(N);
Craig Toppercc274522012-01-07 09:06:39 +00001100 case ISD::SUBE: return visitSUBE(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001101 case ISD::MUL: return visitMUL(N);
1102 case ISD::SDIV: return visitSDIV(N);
1103 case ISD::UDIV: return visitUDIV(N);
1104 case ISD::SREM: return visitSREM(N);
1105 case ISD::UREM: return visitUREM(N);
1106 case ISD::MULHU: return visitMULHU(N);
1107 case ISD::MULHS: return visitMULHS(N);
Dan Gohman389079b2007-10-08 17:57:15 +00001108 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1109 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
Benjamin Kramerf55d26e2011-05-21 18:31:55 +00001110 case ISD::SMULO: return visitSMULO(N);
1111 case ISD::UMULO: return visitUMULO(N);
Dan Gohman389079b2007-10-08 17:57:15 +00001112 case ISD::SDIVREM: return visitSDIVREM(N);
1113 case ISD::UDIVREM: return visitUDIVREM(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001114 case ISD::AND: return visitAND(N);
1115 case ISD::OR: return visitOR(N);
1116 case ISD::XOR: return visitXOR(N);
1117 case ISD::SHL: return visitSHL(N);
1118 case ISD::SRA: return visitSRA(N);
1119 case ISD::SRL: return visitSRL(N);
1120 case ISD::CTLZ: return visitCTLZ(N);
Chandler Carruth63974b22011-12-13 01:56:10 +00001121 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001122 case ISD::CTTZ: return visitCTTZ(N);
Chandler Carruth63974b22011-12-13 01:56:10 +00001123 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001124 case ISD::CTPOP: return visitCTPOP(N);
Nate Begeman452d7beb2005-09-16 00:54:12 +00001125 case ISD::SELECT: return visitSELECT(N);
1126 case ISD::SELECT_CC: return visitSELECT_CC(N);
1127 case ISD::SETCC: return visitSETCC(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001128 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1129 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
Chris Lattner5ffc0662006-05-05 05:58:59 +00001130 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001131 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1132 case ISD::TRUNCATE: return visitTRUNCATE(N);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001133 case ISD::BITCAST: return visitBITCAST(N);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00001134 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
Chris Lattner01b3d732005-09-28 22:28:18 +00001135 case ISD::FADD: return visitFADD(N);
1136 case ISD::FSUB: return visitFSUB(N);
1137 case ISD::FMUL: return visitFMUL(N);
Owen Anderson062c0a52012-05-02 22:17:40 +00001138 case ISD::FMA: return visitFMA(N);
Chris Lattner01b3d732005-09-28 22:28:18 +00001139 case ISD::FDIV: return visitFDIV(N);
1140 case ISD::FREM: return visitFREM(N);
Chris Lattner12d83032006-03-05 05:30:57 +00001141 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001142 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1143 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1144 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1145 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1146 case ISD::FP_ROUND: return visitFP_ROUND(N);
1147 case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
1148 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1149 case ISD::FNEG: return visitFNEG(N);
1150 case ISD::FABS: return visitFABS(N);
Owen Anderson7c626d32012-08-13 23:32:49 +00001151 case ISD::FFLOOR: return visitFFLOOR(N);
1152 case ISD::FCEIL: return visitFCEIL(N);
1153 case ISD::FTRUNC: return visitFTRUNC(N);
Nate Begeman44728a72005-09-19 22:34:01 +00001154 case ISD::BRCOND: return visitBRCOND(N);
Nate Begeman44728a72005-09-19 22:34:01 +00001155 case ISD::BR_CC: return visitBR_CC(N);
Chris Lattner01a22022005-10-10 22:04:48 +00001156 case ISD::LOAD: return visitLOAD(N);
Chris Lattner87514ca2005-10-10 22:31:19 +00001157 case ISD::STORE: return visitSTORE(N);
Chris Lattnerca242442006-03-19 01:27:56 +00001158 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
Evan Cheng513da432007-10-06 08:19:55 +00001159 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
Dan Gohman7f321562007-06-25 16:23:39 +00001160 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1161 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00001162 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
Chris Lattner66445d32006-03-28 22:11:53 +00001163 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
Jim Grosbach9a526492010-06-23 16:07:42 +00001164 case ISD::MEMBARRIER: return visitMEMBARRIER(N);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001165 }
Dan Gohman475871a2008-07-27 21:46:04 +00001166 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001167}
1168
Dan Gohman475871a2008-07-27 21:46:04 +00001169SDValue DAGCombiner::combine(SDNode *N) {
Dan Gohman475871a2008-07-27 21:46:04 +00001170 SDValue RV = visit(N);
Dan Gohman389079b2007-10-08 17:57:15 +00001171
1172 // If nothing happened, try a target-specific DAG combine.
Gabor Greifba36cb52008-08-28 21:40:38 +00001173 if (RV.getNode() == 0) {
Dan Gohman389079b2007-10-08 17:57:15 +00001174 assert(N->getOpcode() != ISD::DELETED_NODE &&
1175 "Node was deleted but visit returned NULL!");
1176
1177 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1178 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1179
1180 // Expose the DAG combiner to the target combiner impls.
Scott Michelfdc40a02009-02-17 22:15:04 +00001181 TargetLowering::DAGCombinerInfo
Jakob Stoklund Olesen78d12642009-07-24 18:22:59 +00001182 DagCombineInfo(DAG, !LegalTypes, !LegalOperations, false, this);
Dan Gohman389079b2007-10-08 17:57:15 +00001183
1184 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1185 }
1186 }
1187
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001188 // If nothing happened still, try promoting the operation.
1189 if (RV.getNode() == 0) {
1190 switch (N->getOpcode()) {
1191 default: break;
1192 case ISD::ADD:
1193 case ISD::SUB:
1194 case ISD::MUL:
1195 case ISD::AND:
1196 case ISD::OR:
1197 case ISD::XOR:
1198 RV = PromoteIntBinOp(SDValue(N, 0));
1199 break;
1200 case ISD::SHL:
1201 case ISD::SRA:
1202 case ISD::SRL:
1203 RV = PromoteIntShiftOp(SDValue(N, 0));
1204 break;
1205 case ISD::SIGN_EXTEND:
1206 case ISD::ZERO_EXTEND:
1207 case ISD::ANY_EXTEND:
1208 RV = PromoteExtend(SDValue(N, 0));
1209 break;
1210 case ISD::LOAD:
1211 if (PromoteLoad(SDValue(N, 0)))
1212 RV = SDValue(N, 0);
1213 break;
1214 }
1215 }
1216
Scott Michelfdc40a02009-02-17 22:15:04 +00001217 // If N is a commutative binary node, try commuting it to enable more
Evan Cheng08b11732008-03-22 01:55:50 +00001218 // sdisel CSE.
Scott Michelfdc40a02009-02-17 22:15:04 +00001219 if (RV.getNode() == 0 &&
Evan Cheng08b11732008-03-22 01:55:50 +00001220 SelectionDAG::isCommutativeBinOp(N->getOpcode()) &&
1221 N->getNumValues() == 1) {
Dan Gohman475871a2008-07-27 21:46:04 +00001222 SDValue N0 = N->getOperand(0);
1223 SDValue N1 = N->getOperand(1);
Bill Wendling5c71acf2009-01-30 01:13:16 +00001224
Evan Cheng08b11732008-03-22 01:55:50 +00001225 // Constant operands are canonicalized to RHS.
1226 if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
Dan Gohman475871a2008-07-27 21:46:04 +00001227 SDValue Ops[] = { N1, N0 };
Evan Cheng08b11732008-03-22 01:55:50 +00001228 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(),
1229 Ops, 2);
Evan Chengea100462008-03-24 23:55:16 +00001230 if (CSENode)
Dan Gohman475871a2008-07-27 21:46:04 +00001231 return SDValue(CSENode, 0);
Evan Cheng08b11732008-03-22 01:55:50 +00001232 }
1233 }
1234
Dan Gohman389079b2007-10-08 17:57:15 +00001235 return RV;
Scott Michelfdc40a02009-02-17 22:15:04 +00001236}
Dan Gohman389079b2007-10-08 17:57:15 +00001237
Chris Lattner6270f682006-10-08 22:57:01 +00001238/// getInputChainForNode - Given a node, return its input chain if it has one,
1239/// otherwise return a null sd operand.
Dan Gohman475871a2008-07-27 21:46:04 +00001240static SDValue getInputChainForNode(SDNode *N) {
Chris Lattner6270f682006-10-08 22:57:01 +00001241 if (unsigned NumOps = N->getNumOperands()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00001242 if (N->getOperand(0).getValueType() == MVT::Other)
Chris Lattner6270f682006-10-08 22:57:01 +00001243 return N->getOperand(0);
Owen Anderson825b72b2009-08-11 20:47:22 +00001244 else if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
Chris Lattner6270f682006-10-08 22:57:01 +00001245 return N->getOperand(NumOps-1);
1246 for (unsigned i = 1; i < NumOps-1; ++i)
Owen Anderson825b72b2009-08-11 20:47:22 +00001247 if (N->getOperand(i).getValueType() == MVT::Other)
Chris Lattner6270f682006-10-08 22:57:01 +00001248 return N->getOperand(i);
1249 }
Bill Wendling5c71acf2009-01-30 01:13:16 +00001250 return SDValue();
Chris Lattner6270f682006-10-08 22:57:01 +00001251}
1252
Dan Gohman475871a2008-07-27 21:46:04 +00001253SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
Chris Lattner6270f682006-10-08 22:57:01 +00001254 // If N has two operands, where one has an input chain equal to the other,
1255 // the 'other' chain is redundant.
1256 if (N->getNumOperands() == 2) {
Gabor Greifba36cb52008-08-28 21:40:38 +00001257 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
Chris Lattner6270f682006-10-08 22:57:01 +00001258 return N->getOperand(0);
Gabor Greifba36cb52008-08-28 21:40:38 +00001259 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
Chris Lattner6270f682006-10-08 22:57:01 +00001260 return N->getOperand(1);
1261 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001262
Chris Lattnerc76d4412007-05-16 06:37:59 +00001263 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
Dan Gohman475871a2008-07-27 21:46:04 +00001264 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
Scott Michelfdc40a02009-02-17 22:15:04 +00001265 SmallPtrSet<SDNode*, 16> SeenOps;
Chris Lattnerc76d4412007-05-16 06:37:59 +00001266 bool Changed = false; // If we should replace this token factor.
Scott Michelfdc40a02009-02-17 22:15:04 +00001267
Jim Laskey6ff23e52006-10-04 16:53:27 +00001268 // Start out with this token factor.
Jim Laskey279f0532006-09-25 16:29:54 +00001269 TFs.push_back(N);
Scott Michelfdc40a02009-02-17 22:15:04 +00001270
Jim Laskey71382342006-10-07 23:37:56 +00001271 // Iterate through token factors. The TFs grows when new token factors are
Jim Laskeybc588b82006-10-05 15:07:25 +00001272 // encountered.
1273 for (unsigned i = 0; i < TFs.size(); ++i) {
1274 SDNode *TF = TFs[i];
Scott Michelfdc40a02009-02-17 22:15:04 +00001275
Jim Laskey6ff23e52006-10-04 16:53:27 +00001276 // Check each of the operands.
1277 for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00001278 SDValue Op = TF->getOperand(i);
Scott Michelfdc40a02009-02-17 22:15:04 +00001279
Jim Laskey6ff23e52006-10-04 16:53:27 +00001280 switch (Op.getOpcode()) {
1281 case ISD::EntryToken:
Jim Laskeybc588b82006-10-05 15:07:25 +00001282 // Entry tokens don't need to be added to the list. They are
1283 // rededundant.
1284 Changed = true;
Jim Laskey6ff23e52006-10-04 16:53:27 +00001285 break;
Scott Michelfdc40a02009-02-17 22:15:04 +00001286
Jim Laskey6ff23e52006-10-04 16:53:27 +00001287 case ISD::TokenFactor:
Nate Begemanb6aef5c2009-09-15 00:18:30 +00001288 if (Op.hasOneUse() &&
Gabor Greifba36cb52008-08-28 21:40:38 +00001289 std::find(TFs.begin(), TFs.end(), Op.getNode()) == TFs.end()) {
Jim Laskey6ff23e52006-10-04 16:53:27 +00001290 // Queue up for processing.
Gabor Greifba36cb52008-08-28 21:40:38 +00001291 TFs.push_back(Op.getNode());
Jim Laskey6ff23e52006-10-04 16:53:27 +00001292 // Clean up in case the token factor is removed.
Gabor Greifba36cb52008-08-28 21:40:38 +00001293 AddToWorkList(Op.getNode());
Jim Laskey6ff23e52006-10-04 16:53:27 +00001294 Changed = true;
1295 break;
Jim Laskey279f0532006-09-25 16:29:54 +00001296 }
Jim Laskey6ff23e52006-10-04 16:53:27 +00001297 // Fall thru
Scott Michelfdc40a02009-02-17 22:15:04 +00001298
Jim Laskey6ff23e52006-10-04 16:53:27 +00001299 default:
Chris Lattnerc76d4412007-05-16 06:37:59 +00001300 // Only add if it isn't already in the list.
Gabor Greifba36cb52008-08-28 21:40:38 +00001301 if (SeenOps.insert(Op.getNode()))
Jim Laskeybc588b82006-10-05 15:07:25 +00001302 Ops.push_back(Op);
Chris Lattnerc76d4412007-05-16 06:37:59 +00001303 else
1304 Changed = true;
Jim Laskey6ff23e52006-10-04 16:53:27 +00001305 break;
Jim Laskey279f0532006-09-25 16:29:54 +00001306 }
1307 }
Jim Laskey6ff23e52006-10-04 16:53:27 +00001308 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001309
Dan Gohman475871a2008-07-27 21:46:04 +00001310 SDValue Result;
Jim Laskey6ff23e52006-10-04 16:53:27 +00001311
1312 // If we've change things around then replace token factor.
1313 if (Changed) {
Dan Gohman30359592008-01-29 13:02:09 +00001314 if (Ops.empty()) {
Jim Laskey6ff23e52006-10-04 16:53:27 +00001315 // The entry token is the only possible outcome.
1316 Result = DAG.getEntryNode();
1317 } else {
1318 // New and improved token factor.
Bill Wendling5c71acf2009-01-30 01:13:16 +00001319 Result = DAG.getNode(ISD::TokenFactor, N->getDebugLoc(),
Owen Anderson825b72b2009-08-11 20:47:22 +00001320 MVT::Other, &Ops[0], Ops.size());
Nate Begemanded49632005-10-13 03:11:28 +00001321 }
Bill Wendling5c71acf2009-01-30 01:13:16 +00001322
Jim Laskey274062c2006-10-13 23:32:28 +00001323 // Don't add users to work list.
1324 return CombineTo(N, Result, false);
Nate Begemanded49632005-10-13 03:11:28 +00001325 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001326
Jim Laskey6ff23e52006-10-04 16:53:27 +00001327 return Result;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001328}
1329
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001330/// MERGE_VALUES can always be eliminated.
Dan Gohman475871a2008-07-27 21:46:04 +00001331SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001332 WorkListRemover DeadNodes(*this);
Dan Gohman00edf392009-08-10 23:43:19 +00001333 // Replacing results may cause a different MERGE_VALUES to suddenly
1334 // be CSE'd with N, and carry its uses with it. Iterate until no
1335 // uses remain, to ensure that the node can be safely deleted.
Pete Cooper3affd9e2012-06-20 19:35:43 +00001336 // First add the users of this node to the work list so that they
1337 // can be tried again once they have new operands.
1338 AddUsersToWorkList(N);
Dan Gohman00edf392009-08-10 23:43:19 +00001339 do {
1340 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00001341 DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i));
Dan Gohman00edf392009-08-10 23:43:19 +00001342 } while (!N->use_empty());
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001343 removeFromWorkList(N);
1344 DAG.DeleteNode(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001345 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001346}
1347
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001348static
Bill Wendlingd69c3142009-01-30 02:23:43 +00001349SDValue combineShlAddConstant(DebugLoc DL, SDValue N0, SDValue N1,
1350 SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00001351 EVT VT = N0.getValueType();
Dan Gohman475871a2008-07-27 21:46:04 +00001352 SDValue N00 = N0.getOperand(0);
1353 SDValue N01 = N0.getOperand(1);
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001354 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N01);
Bill Wendlingd69c3142009-01-30 02:23:43 +00001355
Gabor Greifba36cb52008-08-28 21:40:38 +00001356 if (N01C && N00.getOpcode() == ISD::ADD && N00.getNode()->hasOneUse() &&
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001357 isa<ConstantSDNode>(N00.getOperand(1))) {
Bill Wendlingd69c3142009-01-30 02:23:43 +00001358 // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<<c2), )
1359 N0 = DAG.getNode(ISD::ADD, N0.getDebugLoc(), VT,
1360 DAG.getNode(ISD::SHL, N00.getDebugLoc(), VT,
1361 N00.getOperand(0), N01),
1362 DAG.getNode(ISD::SHL, N01.getDebugLoc(), VT,
1363 N00.getOperand(1), N01));
1364 return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001365 }
Bill Wendlingd69c3142009-01-30 02:23:43 +00001366
Dan Gohman475871a2008-07-27 21:46:04 +00001367 return SDValue();
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001368}
1369
Dan Gohman475871a2008-07-27 21:46:04 +00001370SDValue DAGCombiner::visitADD(SDNode *N) {
1371 SDValue N0 = N->getOperand(0);
1372 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00001373 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1374 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001375 EVT VT = N0.getValueType();
Dan Gohman7f321562007-06-25 16:23:39 +00001376
1377 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001378 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001379 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001380 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001381 }
Bill Wendling2476e5d2008-12-10 22:36:00 +00001382
Dan Gohman613e0d82007-07-03 14:03:57 +00001383 // fold (add x, undef) -> undef
Dan Gohman70fb1ae2007-07-10 15:19:29 +00001384 if (N0.getOpcode() == ISD::UNDEF)
1385 return N0;
1386 if (N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00001387 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001388 // fold (add c1, c2) -> c1+c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001389 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001390 return DAG.FoldConstantArithmetic(ISD::ADD, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00001391 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00001392 if (N0C && !N1C)
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001393 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001394 // fold (add x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00001395 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00001396 return N0;
Dan Gohman6520e202008-10-18 02:06:02 +00001397 // fold (add Sym, c) -> Sym+c
1398 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
Duncan Sands25cf2272008-11-24 14:53:14 +00001399 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA) && N1C &&
Dan Gohman6520e202008-10-18 02:06:02 +00001400 GA->getOpcode() == ISD::GlobalAddress)
Devang Patel0d881da2010-07-06 22:08:15 +00001401 return DAG.getGlobalAddress(GA->getGlobal(), N1C->getDebugLoc(), VT,
Dan Gohman6520e202008-10-18 02:06:02 +00001402 GA->getOffset() +
1403 (uint64_t)N1C->getSExtValue());
Chris Lattner4aafb4f2006-01-12 20:22:43 +00001404 // fold ((c1-A)+c2) -> (c1+c2)-A
1405 if (N1C && N0.getOpcode() == ISD::SUB)
1406 if (ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getOperand(0)))
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001407 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
Dan Gohman002e5d02008-03-13 22:13:53 +00001408 DAG.getConstant(N1C->getAPIntValue()+
1409 N0C->getAPIntValue(), VT),
Chris Lattner4aafb4f2006-01-12 20:22:43 +00001410 N0.getOperand(1));
Nate Begemancd4d58c2006-02-03 06:46:56 +00001411 // reassociate add
Bill Wendling35247c32009-01-30 00:45:56 +00001412 SDValue RADD = ReassociateOps(ISD::ADD, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001413 if (RADD.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00001414 return RADD;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001415 // fold ((0-A) + B) -> B-A
1416 if (N0.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N0.getOperand(0)) &&
1417 cast<ConstantSDNode>(N0.getOperand(0))->isNullValue())
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001418 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1, N0.getOperand(1));
Nate Begeman1d4d4142005-09-01 00:19:25 +00001419 // fold (A + (0-B)) -> A-B
1420 if (N1.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N1.getOperand(0)) &&
1421 cast<ConstantSDNode>(N1.getOperand(0))->isNullValue())
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001422 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, N1.getOperand(1));
Chris Lattner01b3d732005-09-28 22:28:18 +00001423 // fold (A+(B-A)) -> B
1424 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
Nate Begeman83e75ec2005-09-06 04:43:02 +00001425 return N1.getOperand(0);
Dale Johannesen56eca912008-11-27 00:43:21 +00001426 // fold ((B-A)+A) -> B
1427 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
1428 return N0.getOperand(0);
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001429 // fold (A+(B-(A+C))) to (B-C)
1430 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001431 N0 == N1.getOperand(1).getOperand(0))
1432 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1.getOperand(0),
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001433 N1.getOperand(1).getOperand(1));
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001434 // fold (A+(B-(C+A))) to (B-C)
1435 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001436 N0 == N1.getOperand(1).getOperand(1))
1437 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1.getOperand(0),
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001438 N1.getOperand(1).getOperand(0));
Dale Johannesen7c7bc722008-12-23 23:47:22 +00001439 // fold (A+((B-A)+or-C)) to (B+or-C)
Dale Johannesen34d79852008-12-02 18:40:40 +00001440 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
1441 N1.getOperand(0).getOpcode() == ISD::SUB &&
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001442 N0 == N1.getOperand(0).getOperand(1))
1443 return DAG.getNode(N1.getOpcode(), N->getDebugLoc(), VT,
1444 N1.getOperand(0).getOperand(0), N1.getOperand(1));
Dale Johannesen34d79852008-12-02 18:40:40 +00001445
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001446 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
1447 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
1448 SDValue N00 = N0.getOperand(0);
1449 SDValue N01 = N0.getOperand(1);
1450 SDValue N10 = N1.getOperand(0);
1451 SDValue N11 = N1.getOperand(1);
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001452
1453 if (isa<ConstantSDNode>(N00) || isa<ConstantSDNode>(N10))
1454 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1455 DAG.getNode(ISD::ADD, N0.getDebugLoc(), VT, N00, N10),
1456 DAG.getNode(ISD::ADD, N1.getDebugLoc(), VT, N01, N11));
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001457 }
Chris Lattner947c2892006-03-13 06:51:27 +00001458
Dan Gohman475871a2008-07-27 21:46:04 +00001459 if (!VT.isVector() && SimplifyDemandedBits(SDValue(N, 0)))
1460 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001461
Sylvestre Ledru94c22712012-09-27 10:14:43 +00001462 // fold (a+b) -> (a|b) iff a and b share no bits.
Duncan Sands83ec4b62008-06-06 12:08:01 +00001463 if (VT.isInteger() && !VT.isVector()) {
Dan Gohman948d8ea2008-02-20 16:33:30 +00001464 APInt LHSZero, LHSOne;
1465 APInt RHSZero, RHSOne;
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001466 DAG.ComputeMaskedBits(N0, LHSZero, LHSOne);
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001467
Dan Gohman948d8ea2008-02-20 16:33:30 +00001468 if (LHSZero.getBoolValue()) {
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001469 DAG.ComputeMaskedBits(N1, RHSZero, RHSOne);
Scott Michelfdc40a02009-02-17 22:15:04 +00001470
Chris Lattner947c2892006-03-13 06:51:27 +00001471 // If all possibly-set bits on the LHS are clear on the RHS, return an OR.
1472 // If all possibly-set bits on the RHS are clear on the LHS, return an OR.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001473 if ((RHSZero & ~LHSZero) == ~LHSZero || (LHSZero & ~RHSZero) == ~RHSZero)
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001474 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N1);
Chris Lattner947c2892006-03-13 06:51:27 +00001475 }
1476 }
Evan Cheng3ef554d2006-11-06 08:14:30 +00001477
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001478 // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<<c2), )
Gabor Greifba36cb52008-08-28 21:40:38 +00001479 if (N0.getOpcode() == ISD::SHL && N0.getNode()->hasOneUse()) {
Bill Wendlingd69c3142009-01-30 02:23:43 +00001480 SDValue Result = combineShlAddConstant(N->getDebugLoc(), N0, N1, DAG);
Gabor Greifba36cb52008-08-28 21:40:38 +00001481 if (Result.getNode()) return Result;
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001482 }
Gabor Greifba36cb52008-08-28 21:40:38 +00001483 if (N1.getOpcode() == ISD::SHL && N1.getNode()->hasOneUse()) {
Bill Wendlingd69c3142009-01-30 02:23:43 +00001484 SDValue Result = combineShlAddConstant(N->getDebugLoc(), N1, N0, DAG);
Gabor Greifba36cb52008-08-28 21:40:38 +00001485 if (Result.getNode()) return Result;
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001486 }
1487
Dan Gohmancd9e1552010-01-19 23:30:49 +00001488 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
1489 if (N1.getOpcode() == ISD::SHL &&
1490 N1.getOperand(0).getOpcode() == ISD::SUB)
1491 if (ConstantSDNode *C =
1492 dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(0)))
1493 if (C->getAPIntValue() == 0)
1494 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0,
1495 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1496 N1.getOperand(0).getOperand(1),
1497 N1.getOperand(1)));
1498 if (N0.getOpcode() == ISD::SHL &&
1499 N0.getOperand(0).getOpcode() == ISD::SUB)
1500 if (ConstantSDNode *C =
1501 dyn_cast<ConstantSDNode>(N0.getOperand(0).getOperand(0)))
1502 if (C->getAPIntValue() == 0)
1503 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1,
1504 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1505 N0.getOperand(0).getOperand(1),
1506 N0.getOperand(1)));
1507
Owen Andersonbc146b02010-09-21 20:42:50 +00001508 if (N1.getOpcode() == ISD::AND) {
1509 SDValue AndOp0 = N1.getOperand(0);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001510 ConstantSDNode *AndOp1 = dyn_cast<ConstantSDNode>(N1->getOperand(1));
Owen Andersonbc146b02010-09-21 20:42:50 +00001511 unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
1512 unsigned DestBits = VT.getScalarType().getSizeInBits();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001513
Owen Andersonbc146b02010-09-21 20:42:50 +00001514 // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
1515 // and similar xforms where the inner op is either ~0 or 0.
1516 if (NumSignBits == DestBits && AndOp1 && AndOp1->isOne()) {
1517 DebugLoc DL = N->getDebugLoc();
1518 return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), AndOp0);
1519 }
1520 }
1521
Benjamin Kramerf50125e2010-12-22 23:17:45 +00001522 // add (sext i1), X -> sub X, (zext i1)
1523 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
1524 N0.getOperand(0).getValueType() == MVT::i1 &&
1525 !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
1526 DebugLoc DL = N->getDebugLoc();
1527 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
1528 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
1529 }
1530
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001531 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001532}
1533
Dan Gohman475871a2008-07-27 21:46:04 +00001534SDValue DAGCombiner::visitADDC(SDNode *N) {
1535 SDValue N0 = N->getOperand(0);
1536 SDValue N1 = N->getOperand(1);
Chris Lattner91153682007-03-04 20:03:15 +00001537 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1538 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001539 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00001540
Chris Lattner91153682007-03-04 20:03:15 +00001541 // If the flag result is dead, turn this into an ADD.
Craig Topper704e1a02012-01-07 18:31:09 +00001542 if (!N->hasAnyUseOfValue(1))
Craig Toppercc274522012-01-07 09:06:39 +00001543 return CombineTo(N, DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N0, N1),
Dale Johannesen874ae252009-06-02 03:12:52 +00001544 DAG.getNode(ISD::CARRY_FALSE,
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00001545 N->getDebugLoc(), MVT::Glue));
Scott Michelfdc40a02009-02-17 22:15:04 +00001546
Chris Lattner91153682007-03-04 20:03:15 +00001547 // canonicalize constant to RHS.
Dan Gohman0a4627d2008-06-23 15:29:14 +00001548 if (N0C && !N1C)
Bill Wendling14036c02009-01-30 02:38:00 +00001549 return DAG.getNode(ISD::ADDC, N->getDebugLoc(), N->getVTList(), N1, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001550
Chris Lattnerb6541762007-03-04 20:40:38 +00001551 // fold (addc x, 0) -> x + no carry out
1552 if (N1C && N1C->isNullValue())
Dale Johannesen874ae252009-06-02 03:12:52 +00001553 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00001554 N->getDebugLoc(), MVT::Glue));
Scott Michelfdc40a02009-02-17 22:15:04 +00001555
Sylvestre Ledru94c22712012-09-27 10:14:43 +00001556 // fold (addc a, b) -> (or a, b), CARRY_FALSE iff a and b share no bits.
Dan Gohman948d8ea2008-02-20 16:33:30 +00001557 APInt LHSZero, LHSOne;
1558 APInt RHSZero, RHSOne;
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001559 DAG.ComputeMaskedBits(N0, LHSZero, LHSOne);
Bill Wendling14036c02009-01-30 02:38:00 +00001560
Dan Gohman948d8ea2008-02-20 16:33:30 +00001561 if (LHSZero.getBoolValue()) {
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001562 DAG.ComputeMaskedBits(N1, RHSZero, RHSOne);
Scott Michelfdc40a02009-02-17 22:15:04 +00001563
Chris Lattnerb6541762007-03-04 20:40:38 +00001564 // If all possibly-set bits on the LHS are clear on the RHS, return an OR.
1565 // If all possibly-set bits on the RHS are clear on the LHS, return an OR.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001566 if ((RHSZero & ~LHSZero) == ~LHSZero || (LHSZero & ~RHSZero) == ~RHSZero)
Bill Wendling14036c02009-01-30 02:38:00 +00001567 return CombineTo(N, DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N1),
Dale Johannesen874ae252009-06-02 03:12:52 +00001568 DAG.getNode(ISD::CARRY_FALSE,
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00001569 N->getDebugLoc(), MVT::Glue));
Chris Lattnerb6541762007-03-04 20:40:38 +00001570 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001571
Dan Gohman475871a2008-07-27 21:46:04 +00001572 return SDValue();
Chris Lattner91153682007-03-04 20:03:15 +00001573}
1574
Dan Gohman475871a2008-07-27 21:46:04 +00001575SDValue DAGCombiner::visitADDE(SDNode *N) {
1576 SDValue N0 = N->getOperand(0);
1577 SDValue N1 = N->getOperand(1);
1578 SDValue CarryIn = N->getOperand(2);
Chris Lattner91153682007-03-04 20:03:15 +00001579 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1580 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001581
Chris Lattner91153682007-03-04 20:03:15 +00001582 // canonicalize constant to RHS
Dan Gohman0a4627d2008-06-23 15:29:14 +00001583 if (N0C && !N1C)
Bill Wendling14036c02009-01-30 02:38:00 +00001584 return DAG.getNode(ISD::ADDE, N->getDebugLoc(), N->getVTList(),
1585 N1, N0, CarryIn);
Scott Michelfdc40a02009-02-17 22:15:04 +00001586
Chris Lattnerb6541762007-03-04 20:40:38 +00001587 // fold (adde x, y, false) -> (addc x, y)
Dale Johannesen874ae252009-06-02 03:12:52 +00001588 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
Craig Toppercc274522012-01-07 09:06:39 +00001589 return DAG.getNode(ISD::ADDC, N->getDebugLoc(), N->getVTList(), N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001590
Dan Gohman475871a2008-07-27 21:46:04 +00001591 return SDValue();
Chris Lattner91153682007-03-04 20:03:15 +00001592}
1593
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001594// Since it may not be valid to emit a fold to zero for vector initializers
1595// check if we can before folding.
1596static SDValue tryFoldToZero(DebugLoc DL, const TargetLowering &TLI, EVT VT,
Owen Anderson95771af2011-02-25 21:41:48 +00001597 SelectionDAG &DAG, bool LegalOperations) {
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001598 if (!VT.isVector()) {
1599 return DAG.getConstant(0, VT);
Dan Gohman71dc7c92011-05-17 22:20:36 +00001600 }
1601 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) {
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001602 // Produce a vector of zeros.
1603 SDValue El = DAG.getConstant(0, VT.getVectorElementType());
1604 std::vector<SDValue> Ops(VT.getVectorNumElements(), El);
1605 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT,
1606 &Ops[0], Ops.size());
1607 }
1608 return SDValue();
1609}
1610
Dan Gohman475871a2008-07-27 21:46:04 +00001611SDValue DAGCombiner::visitSUB(SDNode *N) {
1612 SDValue N0 = N->getOperand(0);
1613 SDValue N1 = N->getOperand(1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001614 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode());
1615 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
Eric Christopher7332e6e2011-07-14 01:12:15 +00001616 ConstantSDNode *N1C1 = N1.getOpcode() != ISD::ADD ? 0 :
1617 dyn_cast<ConstantSDNode>(N1.getOperand(1).getNode());
Owen Andersone50ed302009-08-10 22:56:29 +00001618 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00001619
Dan Gohman7f321562007-06-25 16:23:39 +00001620 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001621 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001622 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001623 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001624 }
Bill Wendling2476e5d2008-12-10 22:36:00 +00001625
Chris Lattner854077d2005-10-17 01:07:11 +00001626 // fold (sub x, x) -> 0
Eric Christopher169e1552011-02-16 01:10:03 +00001627 // FIXME: Refactor this and xor and other similar operations together.
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001628 if (N0 == N1)
1629 return tryFoldToZero(N->getDebugLoc(), TLI, VT, DAG, LegalOperations);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001630 // fold (sub c1, c2) -> c1-c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001631 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001632 return DAG.FoldConstantArithmetic(ISD::SUB, VT, N0C, N1C);
Chris Lattner05b57432005-10-11 06:07:15 +00001633 // fold (sub x, c) -> (add x, -c)
1634 if (N1C)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001635 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00001636 DAG.getConstant(-N1C->getAPIntValue(), VT));
Evan Cheng1ad0e8b2010-01-18 21:38:44 +00001637 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
1638 if (N0C && N0C->isAllOnesValue())
1639 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N1, N0);
Benjamin Kramer2c94b422011-01-29 12:34:05 +00001640 // fold A-(A-B) -> B
1641 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
1642 return N1.getOperand(1);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001643 // fold (A+B)-A -> B
Chris Lattner01b3d732005-09-28 22:28:18 +00001644 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
Nate Begeman83e75ec2005-09-06 04:43:02 +00001645 return N0.getOperand(1);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001646 // fold (A+B)-B -> A
Chris Lattner01b3d732005-09-28 22:28:18 +00001647 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
Scott Michelfdc40a02009-02-17 22:15:04 +00001648 return N0.getOperand(0);
Eric Christopher7332e6e2011-07-14 01:12:15 +00001649 // fold C2-(A+C1) -> (C2-C1)-A
1650 if (N1.getOpcode() == ISD::ADD && N0C && N1C1) {
Nadav Rotem6dfabb62012-09-20 08:53:31 +00001651 SDValue NewC = DAG.getConstant(N0C->getAPIntValue() - N1C1->getAPIntValue(),
1652 VT);
Eric Christopher7332e6e2011-07-14 01:12:15 +00001653 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, NewC,
Bill Wendling96cb1122012-07-19 00:04:14 +00001654 N1.getOperand(0));
Eric Christopher7332e6e2011-07-14 01:12:15 +00001655 }
Dale Johannesen7c7bc722008-12-23 23:47:22 +00001656 // fold ((A+(B+or-C))-B) -> A+or-C
Dale Johannesenfd3b7b72008-12-16 22:13:49 +00001657 if (N0.getOpcode() == ISD::ADD &&
Dale Johannesenf9cbc1f2008-12-23 23:01:27 +00001658 (N0.getOperand(1).getOpcode() == ISD::SUB ||
1659 N0.getOperand(1).getOpcode() == ISD::ADD) &&
Dale Johannesenfd3b7b72008-12-16 22:13:49 +00001660 N0.getOperand(1).getOperand(0) == N1)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001661 return DAG.getNode(N0.getOperand(1).getOpcode(), N->getDebugLoc(), VT,
1662 N0.getOperand(0), N0.getOperand(1).getOperand(1));
Dale Johannesenf9cbc1f2008-12-23 23:01:27 +00001663 // fold ((A+(C+B))-B) -> A+C
1664 if (N0.getOpcode() == ISD::ADD &&
1665 N0.getOperand(1).getOpcode() == ISD::ADD &&
1666 N0.getOperand(1).getOperand(1) == N1)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001667 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT,
1668 N0.getOperand(0), N0.getOperand(1).getOperand(0));
Dale Johannesen58e39b02008-12-23 01:59:54 +00001669 // fold ((A-(B-C))-C) -> A-B
1670 if (N0.getOpcode() == ISD::SUB &&
1671 N0.getOperand(1).getOpcode() == ISD::SUB &&
1672 N0.getOperand(1).getOperand(1) == N1)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001673 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1674 N0.getOperand(0), N0.getOperand(1).getOperand(0));
Bill Wendlingb0702e02009-01-30 02:42:10 +00001675
Dan Gohman613e0d82007-07-03 14:03:57 +00001676 // If either operand of a sub is undef, the result is undef
Dan Gohman70fb1ae2007-07-10 15:19:29 +00001677 if (N0.getOpcode() == ISD::UNDEF)
1678 return N0;
1679 if (N1.getOpcode() == ISD::UNDEF)
1680 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001681
Dan Gohman6520e202008-10-18 02:06:02 +00001682 // If the relocation model supports it, consider symbol offsets.
1683 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
Duncan Sands25cf2272008-11-24 14:53:14 +00001684 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
Dan Gohman6520e202008-10-18 02:06:02 +00001685 // fold (sub Sym, c) -> Sym-c
1686 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
Devang Patel0d881da2010-07-06 22:08:15 +00001687 return DAG.getGlobalAddress(GA->getGlobal(), N1C->getDebugLoc(), VT,
Dan Gohman6520e202008-10-18 02:06:02 +00001688 GA->getOffset() -
1689 (uint64_t)N1C->getSExtValue());
1690 // fold (sub Sym+c1, Sym+c2) -> c1-c2
1691 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
1692 if (GA->getGlobal() == GB->getGlobal())
1693 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
1694 VT);
1695 }
1696
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001697 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001698}
1699
Craig Toppercc274522012-01-07 09:06:39 +00001700SDValue DAGCombiner::visitSUBC(SDNode *N) {
1701 SDValue N0 = N->getOperand(0);
1702 SDValue N1 = N->getOperand(1);
1703 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1704 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1705 EVT VT = N0.getValueType();
1706
1707 // If the flag result is dead, turn this into an SUB.
Craig Topper704e1a02012-01-07 18:31:09 +00001708 if (!N->hasAnyUseOfValue(1))
Craig Toppercc274522012-01-07 09:06:39 +00001709 return CombineTo(N, DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, N1),
1710 DAG.getNode(ISD::CARRY_FALSE, N->getDebugLoc(),
1711 MVT::Glue));
1712
1713 // fold (subc x, x) -> 0 + no borrow
1714 if (N0 == N1)
1715 return CombineTo(N, DAG.getConstant(0, VT),
1716 DAG.getNode(ISD::CARRY_FALSE, N->getDebugLoc(),
1717 MVT::Glue));
1718
1719 // fold (subc x, 0) -> x + no borrow
1720 if (N1C && N1C->isNullValue())
1721 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, N->getDebugLoc(),
1722 MVT::Glue));
1723
1724 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
1725 if (N0C && N0C->isAllOnesValue())
1726 return CombineTo(N, DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N1, N0),
1727 DAG.getNode(ISD::CARRY_FALSE, N->getDebugLoc(),
1728 MVT::Glue));
1729
1730 return SDValue();
1731}
1732
1733SDValue DAGCombiner::visitSUBE(SDNode *N) {
1734 SDValue N0 = N->getOperand(0);
1735 SDValue N1 = N->getOperand(1);
1736 SDValue CarryIn = N->getOperand(2);
1737
1738 // fold (sube x, y, false) -> (subc x, y)
1739 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
1740 return DAG.getNode(ISD::SUBC, N->getDebugLoc(), N->getVTList(), N0, N1);
1741
1742 return SDValue();
1743}
1744
Dan Gohman475871a2008-07-27 21:46:04 +00001745SDValue DAGCombiner::visitMUL(SDNode *N) {
1746 SDValue N0 = N->getOperand(0);
1747 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00001748 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1749 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001750 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00001751
Dan Gohman7f321562007-06-25 16:23:39 +00001752 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001753 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001754 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001755 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001756 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001757
Dan Gohman613e0d82007-07-03 14:03:57 +00001758 // fold (mul x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00001759 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00001760 return DAG.getConstant(0, VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001761 // fold (mul c1, c2) -> c1*c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001762 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001763 return DAG.FoldConstantArithmetic(ISD::MUL, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00001764 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00001765 if (N0C && !N1C)
Bill Wendling9c8148a2009-01-30 02:45:56 +00001766 return DAG.getNode(ISD::MUL, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001767 // fold (mul x, 0) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00001768 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00001769 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001770 // fold (mul x, -1) -> 0-x
Nate Begeman646d7e22005-09-02 21:18:40 +00001771 if (N1C && N1C->isAllOnesValue())
Bill Wendling9c8148a2009-01-30 02:45:56 +00001772 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1773 DAG.getConstant(0, VT), N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001774 // fold (mul x, (1 << c)) -> x << c
Dan Gohman002e5d02008-03-13 22:13:53 +00001775 if (N1C && N1C->getAPIntValue().isPowerOf2())
Bill Wendling9c8148a2009-01-30 02:45:56 +00001776 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00001777 DAG.getConstant(N1C->getAPIntValue().logBase2(),
Owen Anderson95771af2011-02-25 21:41:48 +00001778 getShiftAmountTy(N0.getValueType())));
Chris Lattner3e6099b2005-10-30 06:41:49 +00001779 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
Chris Lattner66b8bc32009-03-09 20:22:18 +00001780 if (N1C && (-N1C->getAPIntValue()).isPowerOf2()) {
1781 unsigned Log2Val = (-N1C->getAPIntValue()).logBase2();
Scott Michelfdc40a02009-02-17 22:15:04 +00001782 // FIXME: If the input is something that is easily negated (e.g. a
Chris Lattner3e6099b2005-10-30 06:41:49 +00001783 // single-use add), we should put the negate there.
Bill Wendling9c8148a2009-01-30 02:45:56 +00001784 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1785 DAG.getConstant(0, VT),
Bill Wendling73e16b22009-01-30 02:49:26 +00001786 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0,
Owen Anderson95771af2011-02-25 21:41:48 +00001787 DAG.getConstant(Log2Val,
1788 getShiftAmountTy(N0.getValueType()))));
Chris Lattner66b8bc32009-03-09 20:22:18 +00001789 }
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001790 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
Bill Wendling73e16b22009-01-30 02:49:26 +00001791 if (N1C && N0.getOpcode() == ISD::SHL &&
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001792 isa<ConstantSDNode>(N0.getOperand(1))) {
Bill Wendling9c8148a2009-01-30 02:45:56 +00001793 SDValue C3 = DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1794 N1, N0.getOperand(1));
Gabor Greifba36cb52008-08-28 21:40:38 +00001795 AddToWorkList(C3.getNode());
Bill Wendling9c8148a2009-01-30 02:45:56 +00001796 return DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
1797 N0.getOperand(0), C3);
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001798 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001799
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001800 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
1801 // use.
1802 {
Dan Gohman475871a2008-07-27 21:46:04 +00001803 SDValue Sh(0,0), Y(0,0);
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001804 // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
1805 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
Gabor Greifba36cb52008-08-28 21:40:38 +00001806 N0.getNode()->hasOneUse()) {
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001807 Sh = N0; Y = N1;
Scott Michelfdc40a02009-02-17 22:15:04 +00001808 } else if (N1.getOpcode() == ISD::SHL &&
Gabor Greif12632d22008-08-30 19:29:20 +00001809 isa<ConstantSDNode>(N1.getOperand(1)) &&
1810 N1.getNode()->hasOneUse()) {
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001811 Sh = N1; Y = N0;
1812 }
Bill Wendling73e16b22009-01-30 02:49:26 +00001813
Gabor Greifba36cb52008-08-28 21:40:38 +00001814 if (Sh.getNode()) {
Bill Wendling9c8148a2009-01-30 02:45:56 +00001815 SDValue Mul = DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
1816 Sh.getOperand(0), Y);
1817 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1818 Mul, Sh.getOperand(1));
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001819 }
1820 }
Bill Wendling73e16b22009-01-30 02:49:26 +00001821
Chris Lattnera1deca32006-03-04 23:33:26 +00001822 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
Scott Michelfdc40a02009-02-17 22:15:04 +00001823 if (N1C && N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse() &&
Bill Wendling9c8148a2009-01-30 02:45:56 +00001824 isa<ConstantSDNode>(N0.getOperand(1)))
1825 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT,
1826 DAG.getNode(ISD::MUL, N0.getDebugLoc(), VT,
1827 N0.getOperand(0), N1),
1828 DAG.getNode(ISD::MUL, N1.getDebugLoc(), VT,
1829 N0.getOperand(1), N1));
Scott Michelfdc40a02009-02-17 22:15:04 +00001830
Nate Begemancd4d58c2006-02-03 06:46:56 +00001831 // reassociate mul
Bill Wendling35247c32009-01-30 00:45:56 +00001832 SDValue RMUL = ReassociateOps(ISD::MUL, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001833 if (RMUL.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00001834 return RMUL;
Dan Gohman7f321562007-06-25 16:23:39 +00001835
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001836 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001837}
1838
Dan Gohman475871a2008-07-27 21:46:04 +00001839SDValue DAGCombiner::visitSDIV(SDNode *N) {
1840 SDValue N0 = N->getOperand(0);
1841 SDValue N1 = N->getOperand(1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001842 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode());
1843 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
Owen Andersone50ed302009-08-10 22:56:29 +00001844 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001845
Dan Gohman7f321562007-06-25 16:23:39 +00001846 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001847 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001848 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001849 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001850 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001851
Nate Begeman1d4d4142005-09-01 00:19:25 +00001852 // fold (sdiv c1, c2) -> c1/c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001853 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001854 return DAG.FoldConstantArithmetic(ISD::SDIV, VT, N0C, N1C);
Nate Begeman405e3ec2005-10-21 00:02:42 +00001855 // fold (sdiv X, 1) -> X
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001856 if (N1C && N1C->getAPIntValue() == 1LL)
Nate Begeman405e3ec2005-10-21 00:02:42 +00001857 return N0;
1858 // fold (sdiv X, -1) -> 0-X
1859 if (N1C && N1C->isAllOnesValue())
Bill Wendling944d34b2009-01-30 02:52:17 +00001860 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1861 DAG.getConstant(0, VT), N0);
Chris Lattner094c8fc2005-10-07 06:10:46 +00001862 // If we know the sign bits of both operands are zero, strength reduce to a
1863 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
Duncan Sands83ec4b62008-06-06 12:08:01 +00001864 if (!VT.isVector()) {
Dan Gohman2e68b6f2008-02-25 21:11:39 +00001865 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
Bill Wendling944d34b2009-01-30 02:52:17 +00001866 return DAG.getNode(ISD::UDIV, N->getDebugLoc(), N1.getValueType(),
1867 N0, N1);
Chris Lattnerf32aac32008-01-27 23:32:17 +00001868 }
Nate Begemancd6a6ed2006-02-17 07:26:20 +00001869 // fold (sdiv X, pow2) -> simple ops after legalize
Eli Friedman1c663fe2011-12-07 03:55:52 +00001870 if (N1C && !N1C->isNullValue() &&
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001871 (N1C->getAPIntValue().isPowerOf2() ||
1872 (-N1C->getAPIntValue()).isPowerOf2())) {
Nate Begeman405e3ec2005-10-21 00:02:42 +00001873 // If dividing by powers of two is cheap, then don't perform the following
1874 // fold.
1875 if (TLI.isPow2DivCheap())
Dan Gohman475871a2008-07-27 21:46:04 +00001876 return SDValue();
Bill Wendling944d34b2009-01-30 02:52:17 +00001877
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001878 unsigned lg2 = N1C->getAPIntValue().countTrailingZeros();
Bill Wendling944d34b2009-01-30 02:52:17 +00001879
Chris Lattner8f4880b2006-02-16 08:02:36 +00001880 // Splat the sign bit into the register
Bill Wendling944d34b2009-01-30 02:52:17 +00001881 SDValue SGN = DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, N0,
1882 DAG.getConstant(VT.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00001883 getShiftAmountTy(N0.getValueType())));
Gabor Greifba36cb52008-08-28 21:40:38 +00001884 AddToWorkList(SGN.getNode());
Bill Wendling944d34b2009-01-30 02:52:17 +00001885
Chris Lattner8f4880b2006-02-16 08:02:36 +00001886 // Add (N0 < 0) ? abs2 - 1 : 0;
Bill Wendling944d34b2009-01-30 02:52:17 +00001887 SDValue SRL = DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, SGN,
1888 DAG.getConstant(VT.getSizeInBits() - lg2,
Owen Anderson95771af2011-02-25 21:41:48 +00001889 getShiftAmountTy(SGN.getValueType())));
Bill Wendling944d34b2009-01-30 02:52:17 +00001890 SDValue ADD = DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N0, SRL);
Gabor Greifba36cb52008-08-28 21:40:38 +00001891 AddToWorkList(SRL.getNode());
1892 AddToWorkList(ADD.getNode()); // Divide by pow2
Bill Wendling944d34b2009-01-30 02:52:17 +00001893 SDValue SRA = DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, ADD,
Owen Anderson95771af2011-02-25 21:41:48 +00001894 DAG.getConstant(lg2, getShiftAmountTy(ADD.getValueType())));
Bill Wendling944d34b2009-01-30 02:52:17 +00001895
Nate Begeman405e3ec2005-10-21 00:02:42 +00001896 // If we're dividing by a positive value, we're done. Otherwise, we must
1897 // negate the result.
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001898 if (N1C->getAPIntValue().isNonNegative())
Nate Begeman405e3ec2005-10-21 00:02:42 +00001899 return SRA;
Bill Wendling944d34b2009-01-30 02:52:17 +00001900
Gabor Greifba36cb52008-08-28 21:40:38 +00001901 AddToWorkList(SRA.getNode());
Bill Wendling944d34b2009-01-30 02:52:17 +00001902 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1903 DAG.getConstant(0, VT), SRA);
Nate Begeman405e3ec2005-10-21 00:02:42 +00001904 }
Bill Wendling944d34b2009-01-30 02:52:17 +00001905
Nate Begeman69575232005-10-20 02:15:44 +00001906 // if integer divide is expensive and we satisfy the requirements, emit an
1907 // alternate sequence.
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001908 if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001909 SDValue Op = BuildSDIV(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001910 if (Op.getNode()) return Op;
Nate Begeman69575232005-10-20 02:15:44 +00001911 }
Dan Gohman7f321562007-06-25 16:23:39 +00001912
Dan Gohman613e0d82007-07-03 14:03:57 +00001913 // undef / X -> 0
1914 if (N0.getOpcode() == ISD::UNDEF)
1915 return DAG.getConstant(0, VT);
1916 // X / undef -> undef
1917 if (N1.getOpcode() == ISD::UNDEF)
1918 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001919
Dan Gohman475871a2008-07-27 21:46:04 +00001920 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001921}
1922
Dan Gohman475871a2008-07-27 21:46:04 +00001923SDValue DAGCombiner::visitUDIV(SDNode *N) {
1924 SDValue N0 = N->getOperand(0);
1925 SDValue N1 = N->getOperand(1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001926 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode());
1927 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
Owen Andersone50ed302009-08-10 22:56:29 +00001928 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001929
Dan Gohman7f321562007-06-25 16:23:39 +00001930 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001931 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001932 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001933 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001934 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001935
Nate Begeman1d4d4142005-09-01 00:19:25 +00001936 // fold (udiv c1, c2) -> c1/c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001937 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001938 return DAG.FoldConstantArithmetic(ISD::UDIV, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001939 // fold (udiv x, (1 << c)) -> x >>u c
Dan Gohman002e5d02008-03-13 22:13:53 +00001940 if (N1C && N1C->getAPIntValue().isPowerOf2())
Scott Michelfdc40a02009-02-17 22:15:04 +00001941 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00001942 DAG.getConstant(N1C->getAPIntValue().logBase2(),
Owen Anderson95771af2011-02-25 21:41:48 +00001943 getShiftAmountTy(N0.getValueType())));
Sylvestre Ledru94c22712012-09-27 10:14:43 +00001944 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
Nate Begemanfb5e4bd2006-02-05 07:20:23 +00001945 if (N1.getOpcode() == ISD::SHL) {
1946 if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00001947 if (SHC->getAPIntValue().isPowerOf2()) {
Owen Andersone50ed302009-08-10 22:56:29 +00001948 EVT ADDVT = N1.getOperand(1).getValueType();
Bill Wendling07d85142009-01-30 02:55:25 +00001949 SDValue Add = DAG.getNode(ISD::ADD, N->getDebugLoc(), ADDVT,
1950 N1.getOperand(1),
1951 DAG.getConstant(SHC->getAPIntValue()
1952 .logBase2(),
1953 ADDVT));
Gabor Greifba36cb52008-08-28 21:40:38 +00001954 AddToWorkList(Add.getNode());
Bill Wendling07d85142009-01-30 02:55:25 +00001955 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0, Add);
Nate Begemanfb5e4bd2006-02-05 07:20:23 +00001956 }
1957 }
1958 }
Nate Begeman69575232005-10-20 02:15:44 +00001959 // fold (udiv x, c) -> alternate
Dan Gohman002e5d02008-03-13 22:13:53 +00001960 if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001961 SDValue Op = BuildUDIV(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001962 if (Op.getNode()) return Op;
Chris Lattnere9936d12005-10-22 18:50:15 +00001963 }
Dan Gohman7f321562007-06-25 16:23:39 +00001964
Dan Gohman613e0d82007-07-03 14:03:57 +00001965 // undef / X -> 0
1966 if (N0.getOpcode() == ISD::UNDEF)
1967 return DAG.getConstant(0, VT);
1968 // X / undef -> undef
1969 if (N1.getOpcode() == ISD::UNDEF)
1970 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001971
Dan Gohman475871a2008-07-27 21:46:04 +00001972 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001973}
1974
Dan Gohman475871a2008-07-27 21:46:04 +00001975SDValue DAGCombiner::visitSREM(SDNode *N) {
1976 SDValue N0 = N->getOperand(0);
1977 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00001978 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1979 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001980 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001981
Nate Begeman1d4d4142005-09-01 00:19:25 +00001982 // fold (srem c1, c2) -> c1%c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001983 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001984 return DAG.FoldConstantArithmetic(ISD::SREM, VT, N0C, N1C);
Nate Begeman07ed4172005-10-10 21:26:48 +00001985 // If we know the sign bits of both operands are zero, strength reduce to a
1986 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
Duncan Sands83ec4b62008-06-06 12:08:01 +00001987 if (!VT.isVector()) {
Dan Gohman2e68b6f2008-02-25 21:11:39 +00001988 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001989 return DAG.getNode(ISD::UREM, N->getDebugLoc(), VT, N0, N1);
Chris Lattneree339f42008-01-27 23:21:58 +00001990 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001991
Dan Gohman77003042007-11-26 23:46:11 +00001992 // If X/C can be simplified by the division-by-constant logic, lower
1993 // X%C to the equivalent of X-X/C*C.
Chris Lattner26d29902006-10-12 20:58:32 +00001994 if (N1C && !N1C->isNullValue()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001995 SDValue Div = DAG.getNode(ISD::SDIV, N->getDebugLoc(), VT, N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001996 AddToWorkList(Div.getNode());
1997 SDValue OptimizedDiv = combine(Div.getNode());
1998 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001999 SDValue Mul = DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
2000 OptimizedDiv, N1);
2001 SDValue Sub = DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, Mul);
Gabor Greifba36cb52008-08-28 21:40:38 +00002002 AddToWorkList(Mul.getNode());
Dan Gohman77003042007-11-26 23:46:11 +00002003 return Sub;
2004 }
Chris Lattner26d29902006-10-12 20:58:32 +00002005 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002006
Dan Gohman613e0d82007-07-03 14:03:57 +00002007 // undef % X -> 0
2008 if (N0.getOpcode() == ISD::UNDEF)
2009 return DAG.getConstant(0, VT);
2010 // X % undef -> undef
2011 if (N1.getOpcode() == ISD::UNDEF)
2012 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00002013
Dan Gohman475871a2008-07-27 21:46:04 +00002014 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002015}
2016
Dan Gohman475871a2008-07-27 21:46:04 +00002017SDValue DAGCombiner::visitUREM(SDNode *N) {
2018 SDValue N0 = N->getOperand(0);
2019 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00002020 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2021 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002022 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00002023
Nate Begeman1d4d4142005-09-01 00:19:25 +00002024 // fold (urem c1, c2) -> c1%c2
Nate Begeman646d7e22005-09-02 21:18:40 +00002025 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00002026 return DAG.FoldConstantArithmetic(ISD::UREM, VT, N0C, N1C);
Nate Begeman07ed4172005-10-10 21:26:48 +00002027 // fold (urem x, pow2) -> (and x, pow2-1)
Dan Gohman002e5d02008-03-13 22:13:53 +00002028 if (N1C && !N1C->isNullValue() && N1C->getAPIntValue().isPowerOf2())
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00002029 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00002030 DAG.getConstant(N1C->getAPIntValue()-1,VT));
Nate Begemanc031e332006-02-05 07:36:48 +00002031 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
2032 if (N1.getOpcode() == ISD::SHL) {
2033 if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00002034 if (SHC->getAPIntValue().isPowerOf2()) {
Dan Gohman475871a2008-07-27 21:46:04 +00002035 SDValue Add =
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00002036 DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N1,
Duncan Sands83ec4b62008-06-06 12:08:01 +00002037 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()),
Dan Gohman002e5d02008-03-13 22:13:53 +00002038 VT));
Gabor Greifba36cb52008-08-28 21:40:38 +00002039 AddToWorkList(Add.getNode());
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00002040 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0, Add);
Nate Begemanc031e332006-02-05 07:36:48 +00002041 }
2042 }
2043 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002044
Dan Gohman77003042007-11-26 23:46:11 +00002045 // If X/C can be simplified by the division-by-constant logic, lower
2046 // X%C to the equivalent of X-X/C*C.
Chris Lattner26d29902006-10-12 20:58:32 +00002047 if (N1C && !N1C->isNullValue()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00002048 SDValue Div = DAG.getNode(ISD::UDIV, N->getDebugLoc(), VT, N0, N1);
Dan Gohman942ca7f2008-09-08 16:59:01 +00002049 AddToWorkList(Div.getNode());
Gabor Greifba36cb52008-08-28 21:40:38 +00002050 SDValue OptimizedDiv = combine(Div.getNode());
2051 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00002052 SDValue Mul = DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
2053 OptimizedDiv, N1);
2054 SDValue Sub = DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, Mul);
Gabor Greifba36cb52008-08-28 21:40:38 +00002055 AddToWorkList(Mul.getNode());
Dan Gohman77003042007-11-26 23:46:11 +00002056 return Sub;
2057 }
Chris Lattner26d29902006-10-12 20:58:32 +00002058 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002059
Dan Gohman613e0d82007-07-03 14:03:57 +00002060 // undef % X -> 0
2061 if (N0.getOpcode() == ISD::UNDEF)
2062 return DAG.getConstant(0, VT);
2063 // X % undef -> undef
2064 if (N1.getOpcode() == ISD::UNDEF)
2065 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00002066
Dan Gohman475871a2008-07-27 21:46:04 +00002067 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002068}
2069
Dan Gohman475871a2008-07-27 21:46:04 +00002070SDValue DAGCombiner::visitMULHS(SDNode *N) {
2071 SDValue N0 = N->getOperand(0);
2072 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00002073 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002074 EVT VT = N->getValueType(0);
Chris Lattnerde1c3602010-12-13 08:39:01 +00002075 DebugLoc DL = N->getDebugLoc();
Scott Michelfdc40a02009-02-17 22:15:04 +00002076
Nate Begeman1d4d4142005-09-01 00:19:25 +00002077 // fold (mulhs x, 0) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00002078 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002079 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00002080 // fold (mulhs x, 1) -> (sra x, size(x)-1)
Dan Gohman002e5d02008-03-13 22:13:53 +00002081 if (N1C && N1C->getAPIntValue() == 1)
Bill Wendling326411d2009-01-30 03:00:18 +00002082 return DAG.getNode(ISD::SRA, N->getDebugLoc(), N0.getValueType(), N0,
2083 DAG.getConstant(N0.getValueType().getSizeInBits() - 1,
Owen Anderson95771af2011-02-25 21:41:48 +00002084 getShiftAmountTy(N0.getValueType())));
Dan Gohman613e0d82007-07-03 14:03:57 +00002085 // fold (mulhs x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00002086 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00002087 return DAG.getConstant(0, VT);
Dan Gohman7f321562007-06-25 16:23:39 +00002088
Chris Lattnerde1c3602010-12-13 08:39:01 +00002089 // If the type twice as wide is legal, transform the mulhs to a wider multiply
2090 // plus a shift.
2091 if (VT.isSimple() && !VT.isVector()) {
2092 MVT Simple = VT.getSimpleVT();
2093 unsigned SimpleSize = Simple.getSizeInBits();
2094 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2095 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2096 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
2097 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
2098 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
Chris Lattner1a0fbe22010-12-15 05:51:39 +00002099 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
Owen Anderson95771af2011-02-25 21:41:48 +00002100 DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType())));
Chris Lattnerde1c3602010-12-13 08:39:01 +00002101 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
2102 }
2103 }
Owen Anderson95771af2011-02-25 21:41:48 +00002104
Dan Gohman475871a2008-07-27 21:46:04 +00002105 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002106}
2107
Dan Gohman475871a2008-07-27 21:46:04 +00002108SDValue DAGCombiner::visitMULHU(SDNode *N) {
2109 SDValue N0 = N->getOperand(0);
2110 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00002111 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002112 EVT VT = N->getValueType(0);
Chris Lattnerde1c3602010-12-13 08:39:01 +00002113 DebugLoc DL = N->getDebugLoc();
Scott Michelfdc40a02009-02-17 22:15:04 +00002114
Nate Begeman1d4d4142005-09-01 00:19:25 +00002115 // fold (mulhu x, 0) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00002116 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002117 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00002118 // fold (mulhu x, 1) -> 0
Dan Gohman002e5d02008-03-13 22:13:53 +00002119 if (N1C && N1C->getAPIntValue() == 1)
Nate Begeman83e75ec2005-09-06 04:43:02 +00002120 return DAG.getConstant(0, N0.getValueType());
Dan Gohman613e0d82007-07-03 14:03:57 +00002121 // fold (mulhu x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00002122 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00002123 return DAG.getConstant(0, VT);
Dan Gohman7f321562007-06-25 16:23:39 +00002124
Chris Lattnerde1c3602010-12-13 08:39:01 +00002125 // If the type twice as wide is legal, transform the mulhu to a wider multiply
2126 // plus a shift.
2127 if (VT.isSimple() && !VT.isVector()) {
2128 MVT Simple = VT.getSimpleVT();
2129 unsigned SimpleSize = Simple.getSizeInBits();
2130 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2131 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2132 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
2133 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
2134 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
2135 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
Owen Anderson95771af2011-02-25 21:41:48 +00002136 DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType())));
Chris Lattnerde1c3602010-12-13 08:39:01 +00002137 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
2138 }
2139 }
Owen Anderson95771af2011-02-25 21:41:48 +00002140
Dan Gohman475871a2008-07-27 21:46:04 +00002141 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002142}
2143
Dan Gohman389079b2007-10-08 17:57:15 +00002144/// SimplifyNodeWithTwoResults - Perform optimizations common to nodes that
2145/// compute two values. LoOp and HiOp give the opcodes for the two computations
2146/// that are being performed. Return true if a simplification was made.
2147///
Scott Michelfdc40a02009-02-17 22:15:04 +00002148SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
Dan Gohman475871a2008-07-27 21:46:04 +00002149 unsigned HiOp) {
Dan Gohman389079b2007-10-08 17:57:15 +00002150 // If the high half is not needed, just compute the low half.
Evan Cheng44711942007-11-08 09:25:29 +00002151 bool HiExists = N->hasAnyUseOfValue(1);
2152 if (!HiExists &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002153 (!LegalOperations ||
Dan Gohman389079b2007-10-08 17:57:15 +00002154 TLI.isOperationLegal(LoOp, N->getValueType(0)))) {
Bill Wendling826d1142009-01-30 03:08:40 +00002155 SDValue Res = DAG.getNode(LoOp, N->getDebugLoc(), N->getValueType(0),
2156 N->op_begin(), N->getNumOperands());
Chris Lattner5eee4272008-01-26 01:09:19 +00002157 return CombineTo(N, Res, Res);
Dan Gohman389079b2007-10-08 17:57:15 +00002158 }
2159
2160 // If the low half is not needed, just compute the high half.
Evan Cheng44711942007-11-08 09:25:29 +00002161 bool LoExists = N->hasAnyUseOfValue(0);
2162 if (!LoExists &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002163 (!LegalOperations ||
Dan Gohman389079b2007-10-08 17:57:15 +00002164 TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
Bill Wendling826d1142009-01-30 03:08:40 +00002165 SDValue Res = DAG.getNode(HiOp, N->getDebugLoc(), N->getValueType(1),
2166 N->op_begin(), N->getNumOperands());
Chris Lattner5eee4272008-01-26 01:09:19 +00002167 return CombineTo(N, Res, Res);
Dan Gohman389079b2007-10-08 17:57:15 +00002168 }
2169
Evan Cheng44711942007-11-08 09:25:29 +00002170 // If both halves are used, return as it is.
2171 if (LoExists && HiExists)
Dan Gohman475871a2008-07-27 21:46:04 +00002172 return SDValue();
Evan Cheng44711942007-11-08 09:25:29 +00002173
2174 // If the two computed results can be simplified separately, separate them.
Evan Cheng44711942007-11-08 09:25:29 +00002175 if (LoExists) {
Bill Wendling826d1142009-01-30 03:08:40 +00002176 SDValue Lo = DAG.getNode(LoOp, N->getDebugLoc(), N->getValueType(0),
2177 N->op_begin(), N->getNumOperands());
Gabor Greifba36cb52008-08-28 21:40:38 +00002178 AddToWorkList(Lo.getNode());
2179 SDValue LoOpt = combine(Lo.getNode());
2180 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002181 (!LegalOperations ||
Duncan Sandsd4b9c172008-06-13 19:07:40 +00002182 TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
Chris Lattner5eee4272008-01-26 01:09:19 +00002183 return CombineTo(N, LoOpt, LoOpt);
Dan Gohman389079b2007-10-08 17:57:15 +00002184 }
2185
Evan Cheng44711942007-11-08 09:25:29 +00002186 if (HiExists) {
Bill Wendling826d1142009-01-30 03:08:40 +00002187 SDValue Hi = DAG.getNode(HiOp, N->getDebugLoc(), N->getValueType(1),
Duncan Sands25cf2272008-11-24 14:53:14 +00002188 N->op_begin(), N->getNumOperands());
Gabor Greifba36cb52008-08-28 21:40:38 +00002189 AddToWorkList(Hi.getNode());
2190 SDValue HiOpt = combine(Hi.getNode());
2191 if (HiOpt.getNode() && HiOpt != Hi &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002192 (!LegalOperations ||
Duncan Sandsd4b9c172008-06-13 19:07:40 +00002193 TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
Chris Lattner5eee4272008-01-26 01:09:19 +00002194 return CombineTo(N, HiOpt, HiOpt);
Evan Cheng44711942007-11-08 09:25:29 +00002195 }
Bill Wendling826d1142009-01-30 03:08:40 +00002196
Dan Gohman475871a2008-07-27 21:46:04 +00002197 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002198}
2199
Dan Gohman475871a2008-07-27 21:46:04 +00002200SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
2201 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS);
Gabor Greifba36cb52008-08-28 21:40:38 +00002202 if (Res.getNode()) return Res;
Dan Gohman389079b2007-10-08 17:57:15 +00002203
Chris Lattner33e77d32010-12-15 06:04:19 +00002204 EVT VT = N->getValueType(0);
2205 DebugLoc DL = N->getDebugLoc();
2206
2207 // If the type twice as wide is legal, transform the mulhu to a wider multiply
2208 // plus a shift.
2209 if (VT.isSimple() && !VT.isVector()) {
2210 MVT Simple = VT.getSimpleVT();
2211 unsigned SimpleSize = Simple.getSizeInBits();
2212 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2213 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2214 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
2215 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
2216 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
2217 // Compute the high part as N1.
2218 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
Owen Anderson95771af2011-02-25 21:41:48 +00002219 DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType())));
Chris Lattner33e77d32010-12-15 06:04:19 +00002220 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
2221 // Compute the low part as N0.
2222 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
2223 return CombineTo(N, Lo, Hi);
2224 }
2225 }
Owen Anderson95771af2011-02-25 21:41:48 +00002226
Dan Gohman475871a2008-07-27 21:46:04 +00002227 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002228}
2229
Dan Gohman475871a2008-07-27 21:46:04 +00002230SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
2231 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU);
Gabor Greifba36cb52008-08-28 21:40:38 +00002232 if (Res.getNode()) return Res;
Dan Gohman389079b2007-10-08 17:57:15 +00002233
Chris Lattner33e77d32010-12-15 06:04:19 +00002234 EVT VT = N->getValueType(0);
2235 DebugLoc DL = N->getDebugLoc();
Owen Anderson95771af2011-02-25 21:41:48 +00002236
Chris Lattner33e77d32010-12-15 06:04:19 +00002237 // If the type twice as wide is legal, transform the mulhu to a wider multiply
2238 // plus a shift.
2239 if (VT.isSimple() && !VT.isVector()) {
2240 MVT Simple = VT.getSimpleVT();
2241 unsigned SimpleSize = Simple.getSizeInBits();
2242 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2243 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2244 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
2245 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
2246 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
2247 // Compute the high part as N1.
2248 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
Owen Anderson95771af2011-02-25 21:41:48 +00002249 DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType())));
Chris Lattner33e77d32010-12-15 06:04:19 +00002250 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
2251 // Compute the low part as N0.
2252 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
2253 return CombineTo(N, Lo, Hi);
2254 }
2255 }
Owen Anderson95771af2011-02-25 21:41:48 +00002256
Dan Gohman475871a2008-07-27 21:46:04 +00002257 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002258}
2259
Benjamin Kramerf55d26e2011-05-21 18:31:55 +00002260SDValue DAGCombiner::visitSMULO(SDNode *N) {
2261 // (smulo x, 2) -> (saddo x, x)
2262 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2263 if (C2->getAPIntValue() == 2)
2264 return DAG.getNode(ISD::SADDO, N->getDebugLoc(), N->getVTList(),
2265 N->getOperand(0), N->getOperand(0));
2266
2267 return SDValue();
2268}
2269
2270SDValue DAGCombiner::visitUMULO(SDNode *N) {
2271 // (umulo x, 2) -> (uaddo x, x)
2272 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2273 if (C2->getAPIntValue() == 2)
2274 return DAG.getNode(ISD::UADDO, N->getDebugLoc(), N->getVTList(),
2275 N->getOperand(0), N->getOperand(0));
2276
2277 return SDValue();
2278}
2279
Dan Gohman475871a2008-07-27 21:46:04 +00002280SDValue DAGCombiner::visitSDIVREM(SDNode *N) {
2281 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::SDIV, ISD::SREM);
Gabor Greifba36cb52008-08-28 21:40:38 +00002282 if (Res.getNode()) return Res;
Scott Michelfdc40a02009-02-17 22:15:04 +00002283
Dan Gohman475871a2008-07-27 21:46:04 +00002284 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002285}
2286
Dan Gohman475871a2008-07-27 21:46:04 +00002287SDValue DAGCombiner::visitUDIVREM(SDNode *N) {
2288 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::UDIV, ISD::UREM);
Gabor Greifba36cb52008-08-28 21:40:38 +00002289 if (Res.getNode()) return Res;
Scott Michelfdc40a02009-02-17 22:15:04 +00002290
Dan Gohman475871a2008-07-27 21:46:04 +00002291 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002292}
2293
Chris Lattner35e5c142006-05-05 05:51:50 +00002294/// SimplifyBinOpWithSameOpcodeHands - If this is a binary operator with
2295/// two operands of the same opcode, try to simplify it.
Dan Gohman475871a2008-07-27 21:46:04 +00002296SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
2297 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
Owen Andersone50ed302009-08-10 22:56:29 +00002298 EVT VT = N0.getValueType();
Chris Lattner35e5c142006-05-05 05:51:50 +00002299 assert(N0.getOpcode() == N1.getOpcode() && "Bad input!");
Scott Michelfdc40a02009-02-17 22:15:04 +00002300
Dan Gohmanff00a552010-01-14 03:08:49 +00002301 // Bail early if none of these transforms apply.
2302 if (N0.getNode()->getNumOperands() == 0) return SDValue();
2303
Chris Lattner540121f2006-05-05 06:31:05 +00002304 // For each of OP in AND/OR/XOR:
2305 // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
2306 // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
2307 // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
Dan Gohman4e39e9d2010-06-24 14:30:44 +00002308 // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
Nate Begeman93e0ed32009-12-03 07:11:29 +00002309 //
2310 // do not sink logical op inside of a vector extend, since it may combine
2311 // into a vsetcc.
Evan Chengd40d03e2010-01-06 19:38:29 +00002312 EVT Op0VT = N0.getOperand(0).getValueType();
2313 if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
Dan Gohman97121ba2009-04-08 00:15:30 +00002314 N0.getOpcode() == ISD::SIGN_EXTEND ||
Evan Chenge5b51ac2010-04-17 06:13:15 +00002315 // Avoid infinite looping with PromoteIntBinOp.
2316 (N0.getOpcode() == ISD::ANY_EXTEND &&
2317 (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
Dan Gohman4e39e9d2010-06-24 14:30:44 +00002318 (N0.getOpcode() == ISD::TRUNCATE &&
2319 (!TLI.isZExtFree(VT, Op0VT) ||
2320 !TLI.isTruncateFree(Op0VT, VT)) &&
2321 TLI.isTypeLegal(Op0VT))) &&
Nate Begeman93e0ed32009-12-03 07:11:29 +00002322 !VT.isVector() &&
Evan Chengd40d03e2010-01-06 19:38:29 +00002323 Op0VT == N1.getOperand(0).getValueType() &&
2324 (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
Bill Wendlingb74c8672009-01-30 19:25:47 +00002325 SDValue ORNode = DAG.getNode(N->getOpcode(), N0.getDebugLoc(),
2326 N0.getOperand(0).getValueType(),
2327 N0.getOperand(0), N1.getOperand(0));
Gabor Greifba36cb52008-08-28 21:40:38 +00002328 AddToWorkList(ORNode.getNode());
Bill Wendlingb74c8672009-01-30 19:25:47 +00002329 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT, ORNode);
Chris Lattner35e5c142006-05-05 05:51:50 +00002330 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002331
Chris Lattnera3dc3f62006-05-05 06:10:43 +00002332 // For each of OP in SHL/SRL/SRA/AND...
2333 // fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
2334 // fold (or (OP x, z), (OP y, z)) -> (OP (or x, y), z)
2335 // fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
Chris Lattner35e5c142006-05-05 05:51:50 +00002336 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
Chris Lattnera3dc3f62006-05-05 06:10:43 +00002337 N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
Chris Lattner35e5c142006-05-05 05:51:50 +00002338 N0.getOperand(1) == N1.getOperand(1)) {
Bill Wendlingb74c8672009-01-30 19:25:47 +00002339 SDValue ORNode = DAG.getNode(N->getOpcode(), N0.getDebugLoc(),
2340 N0.getOperand(0).getValueType(),
2341 N0.getOperand(0), N1.getOperand(0));
Gabor Greifba36cb52008-08-28 21:40:38 +00002342 AddToWorkList(ORNode.getNode());
Bill Wendlingb74c8672009-01-30 19:25:47 +00002343 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
2344 ORNode, N0.getOperand(1));
Chris Lattner35e5c142006-05-05 05:51:50 +00002345 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002346
Nadav Rotem4ac90812012-04-01 19:31:22 +00002347 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
2348 // Only perform this optimization after type legalization and before
2349 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
2350 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
2351 // we don't want to undo this promotion.
2352 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
2353 // on scalars.
Nadav Rotem6dfabb62012-09-20 08:53:31 +00002354 if ((N0.getOpcode() == ISD::BITCAST ||
2355 N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
2356 Level == AfterLegalizeTypes) {
Nadav Rotem4ac90812012-04-01 19:31:22 +00002357 SDValue In0 = N0.getOperand(0);
2358 SDValue In1 = N1.getOperand(0);
2359 EVT In0Ty = In0.getValueType();
2360 EVT In1Ty = In1.getValueType();
Nadav Rotem6dfabb62012-09-20 08:53:31 +00002361 DebugLoc DL = N->getDebugLoc();
2362 // If both incoming values are integers, and the original types are the
2363 // same.
Nadav Rotem4ac90812012-04-01 19:31:22 +00002364 if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
Nadav Rotem6dfabb62012-09-20 08:53:31 +00002365 SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1);
2366 SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op);
Nadav Rotem4ac90812012-04-01 19:31:22 +00002367 AddToWorkList(Op.getNode());
2368 return BC;
2369 }
2370 }
2371
2372 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
2373 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
2374 // If both shuffles use the same mask, and both shuffle within a single
2375 // vector, then it is worthwhile to move the swizzle after the operation.
2376 // The type-legalizer generates this pattern when loading illegal
2377 // vector types from memory. In many cases this allows additional shuffle
2378 // optimizations.
Craig Topperf9204232012-04-09 07:19:09 +00002379 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG &&
2380 N0.getOperand(1).getOpcode() == ISD::UNDEF &&
2381 N1.getOperand(1).getOpcode() == ISD::UNDEF) {
Nadav Rotem4ac90812012-04-01 19:31:22 +00002382 ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0);
2383 ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1);
Craig Topperf9204232012-04-09 07:19:09 +00002384
2385 assert(N0.getOperand(0).getValueType() == N1.getOperand(1).getValueType() &&
2386 "Inputs to shuffles are not the same type");
Nadav Rotem4ac90812012-04-01 19:31:22 +00002387
2388 unsigned NumElts = VT.getVectorNumElements();
Nadav Rotem4ac90812012-04-01 19:31:22 +00002389
2390 // Check that both shuffles use the same mask. The masks are known to be of
2391 // the same length because the result vector type is the same.
2392 bool SameMask = true;
2393 for (unsigned i = 0; i != NumElts; ++i) {
2394 int Idx0 = SVN0->getMaskElt(i);
2395 int Idx1 = SVN1->getMaskElt(i);
2396 if (Idx0 != Idx1) {
2397 SameMask = false;
2398 break;
2399 }
2400 }
2401
Craig Topperf9204232012-04-09 07:19:09 +00002402 if (SameMask) {
2403 SDValue Op = DAG.getNode(N->getOpcode(), N->getDebugLoc(), VT,
2404 N0.getOperand(0), N1.getOperand(0));
Nadav Rotem4ac90812012-04-01 19:31:22 +00002405 AddToWorkList(Op.getNode());
Craig Topperf9204232012-04-09 07:19:09 +00002406 return DAG.getVectorShuffle(VT, N->getDebugLoc(), Op,
2407 DAG.getUNDEF(VT), &SVN0->getMask()[0]);
Nadav Rotem4ac90812012-04-01 19:31:22 +00002408 }
2409 }
Craig Topperf9204232012-04-09 07:19:09 +00002410
Dan Gohman475871a2008-07-27 21:46:04 +00002411 return SDValue();
Chris Lattner35e5c142006-05-05 05:51:50 +00002412}
2413
Dan Gohman475871a2008-07-27 21:46:04 +00002414SDValue DAGCombiner::visitAND(SDNode *N) {
2415 SDValue N0 = N->getOperand(0);
2416 SDValue N1 = N->getOperand(1);
2417 SDValue LL, LR, RL, RR, CC0, CC1;
Nate Begeman646d7e22005-09-02 21:18:40 +00002418 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2419 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002420 EVT VT = N1.getValueType();
Dan Gohman6900a392010-03-04 00:23:16 +00002421 unsigned BitWidth = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00002422
Dan Gohman7f321562007-06-25 16:23:39 +00002423 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00002424 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00002425 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002426 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00002427 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002428
Dan Gohman613e0d82007-07-03 14:03:57 +00002429 // fold (and x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00002430 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00002431 return DAG.getConstant(0, VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00002432 // fold (and c1, c2) -> c1&c2
Nate Begeman646d7e22005-09-02 21:18:40 +00002433 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00002434 return DAG.FoldConstantArithmetic(ISD::AND, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00002435 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00002436 if (N0C && !N1C)
Bill Wendlingfc4b6772009-02-01 11:19:36 +00002437 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00002438 // fold (and x, -1) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00002439 if (N1C && N1C->isAllOnesValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002440 return N0;
2441 // if (and x, c) is known to be zero, return 0
Dan Gohman475871a2008-07-27 21:46:04 +00002442 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002443 APInt::getAllOnesValue(BitWidth)))
Nate Begeman83e75ec2005-09-06 04:43:02 +00002444 return DAG.getConstant(0, VT);
Nate Begemancd4d58c2006-02-03 06:46:56 +00002445 // reassociate and
Bill Wendling35247c32009-01-30 00:45:56 +00002446 SDValue RAND = ReassociateOps(ISD::AND, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00002447 if (RAND.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00002448 return RAND;
Bill Wendling7d9f2b92010-03-03 00:35:56 +00002449 // fold (and (or x, C), D) -> D if (C & D) == D
Nate Begeman5dc7e862005-11-02 18:42:59 +00002450 if (N1C && N0.getOpcode() == ISD::OR)
Nate Begeman1d4d4142005-09-01 00:19:25 +00002451 if (ConstantSDNode *ORI = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
Dan Gohman002e5d02008-03-13 22:13:53 +00002452 if ((ORI->getAPIntValue() & N1C->getAPIntValue()) == N1C->getAPIntValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002453 return N1;
Chris Lattner3603cd62006-02-02 07:17:31 +00002454 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
2455 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
Dan Gohman475871a2008-07-27 21:46:04 +00002456 SDValue N0Op0 = N0.getOperand(0);
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002457 APInt Mask = ~N1C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00002458 Mask = Mask.trunc(N0Op0.getValueSizeInBits());
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002459 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
Bill Wendling2627a882009-01-30 20:43:18 +00002460 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(),
2461 N0.getValueType(), N0Op0);
Scott Michelfdc40a02009-02-17 22:15:04 +00002462
Chris Lattner1ec05d12006-03-01 21:47:21 +00002463 // Replace uses of the AND with uses of the Zero extend node.
2464 CombineTo(N, Zext);
Scott Michelfdc40a02009-02-17 22:15:04 +00002465
Chris Lattner3603cd62006-02-02 07:17:31 +00002466 // We actually want to replace all uses of the any_extend with the
2467 // zero_extend, to avoid duplicating things. This will later cause this
2468 // AND to be folded.
Gabor Greifba36cb52008-08-28 21:40:38 +00002469 CombineTo(N0.getNode(), Zext);
Dan Gohman475871a2008-07-27 21:46:04 +00002470 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattner3603cd62006-02-02 07:17:31 +00002471 }
2472 }
James Molloy6259dcd2012-02-20 12:02:38 +00002473 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
2474 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
2475 // already be zero by virtue of the width of the base type of the load.
2476 //
2477 // the 'X' node here can either be nothing or an extract_vector_elt to catch
2478 // more cases.
2479 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
2480 N0.getOperand(0).getOpcode() == ISD::LOAD) ||
2481 N0.getOpcode() == ISD::LOAD) {
2482 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
2483 N0 : N0.getOperand(0) );
2484
2485 // Get the constant (if applicable) the zero'th operand is being ANDed with.
2486 // This can be a pure constant or a vector splat, in which case we treat the
2487 // vector as a scalar and use the splat value.
2488 APInt Constant = APInt::getNullValue(1);
2489 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
2490 Constant = C->getAPIntValue();
2491 } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
2492 APInt SplatValue, SplatUndef;
2493 unsigned SplatBitSize;
2494 bool HasAnyUndefs;
2495 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
2496 SplatBitSize, HasAnyUndefs);
2497 if (IsSplat) {
2498 // Undef bits can contribute to a possible optimisation if set, so
2499 // set them.
2500 SplatValue |= SplatUndef;
2501
2502 // The splat value may be something like "0x00FFFFFF", which means 0 for
2503 // the first vector value and FF for the rest, repeating. We need a mask
2504 // that will apply equally to all members of the vector, so AND all the
2505 // lanes of the constant together.
2506 EVT VT = Vector->getValueType(0);
2507 unsigned BitWidth = VT.getVectorElementType().getSizeInBits();
Silviu Baranga3d5e1612012-09-05 08:57:21 +00002508
2509 // If the splat value has been compressed to a bitlength lower
2510 // than the size of the vector lane, we need to re-expand it to
2511 // the lane size.
2512 if (BitWidth > SplatBitSize)
2513 for (SplatValue = SplatValue.zextOrTrunc(BitWidth);
2514 SplatBitSize < BitWidth;
2515 SplatBitSize = SplatBitSize * 2)
2516 SplatValue |= SplatValue.shl(SplatBitSize);
2517
James Molloy6259dcd2012-02-20 12:02:38 +00002518 Constant = APInt::getAllOnesValue(BitWidth);
Silviu Baranga3d5e1612012-09-05 08:57:21 +00002519 for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
James Molloy6259dcd2012-02-20 12:02:38 +00002520 Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
2521 }
2522 }
2523
2524 // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
2525 // actually legal and isn't going to get expanded, else this is a false
2526 // optimisation.
2527 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
2528 Load->getMemoryVT());
2529
2530 // Resize the constant to the same size as the original memory access before
2531 // extension. If it is still the AllOnesValue then this AND is completely
2532 // unneeded.
2533 Constant =
2534 Constant.zextOrTrunc(Load->getMemoryVT().getScalarType().getSizeInBits());
2535
2536 bool B;
2537 switch (Load->getExtensionType()) {
2538 default: B = false; break;
2539 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
2540 case ISD::ZEXTLOAD:
2541 case ISD::NON_EXTLOAD: B = true; break;
2542 }
2543
2544 if (B && Constant.isAllOnesValue()) {
2545 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
2546 // preserve semantics once we get rid of the AND.
2547 SDValue NewLoad(Load, 0);
2548 if (Load->getExtensionType() == ISD::EXTLOAD) {
2549 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
2550 Load->getValueType(0), Load->getDebugLoc(),
2551 Load->getChain(), Load->getBasePtr(),
2552 Load->getOffset(), Load->getMemoryVT(),
2553 Load->getMemOperand());
2554 // Replace uses of the EXTLOAD with the new ZEXTLOAD.
Hal Finkeld65e4632012-06-20 15:42:48 +00002555 if (Load->getNumValues() == 3) {
2556 // PRE/POST_INC loads have 3 values.
2557 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
2558 NewLoad.getValue(2) };
2559 CombineTo(Load, To, 3, true);
2560 } else {
2561 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
2562 }
James Molloy6259dcd2012-02-20 12:02:38 +00002563 }
2564
2565 // Fold the AND away, taking care not to fold to the old load node if we
2566 // replaced it.
2567 CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
2568
2569 return SDValue(N, 0); // Return N so it doesn't get rechecked!
2570 }
2571 }
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002572 // fold (and (setcc x), (setcc y)) -> (setcc (and x, y))
2573 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
2574 ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
2575 ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
Scott Michelfdc40a02009-02-17 22:15:04 +00002576
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002577 if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00002578 LL.getValueType().isInteger()) {
Bill Wendling2627a882009-01-30 20:43:18 +00002579 // fold (and (seteq X, 0), (seteq Y, 0)) -> (seteq (or X, Y), 0)
Dan Gohman002e5d02008-03-13 22:13:53 +00002580 if (cast<ConstantSDNode>(LR)->isNullValue() && Op1 == ISD::SETEQ) {
Bill Wendling2627a882009-01-30 20:43:18 +00002581 SDValue ORNode = DAG.getNode(ISD::OR, N0.getDebugLoc(),
2582 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00002583 AddToWorkList(ORNode.getNode());
Bill Wendling2627a882009-01-30 20:43:18 +00002584 return DAG.getSetCC(N->getDebugLoc(), VT, ORNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002585 }
Bill Wendling2627a882009-01-30 20:43:18 +00002586 // fold (and (seteq X, -1), (seteq Y, -1)) -> (seteq (and X, Y), -1)
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002587 if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETEQ) {
Bill Wendling2627a882009-01-30 20:43:18 +00002588 SDValue ANDNode = DAG.getNode(ISD::AND, N0.getDebugLoc(),
2589 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00002590 AddToWorkList(ANDNode.getNode());
Bill Wendling2627a882009-01-30 20:43:18 +00002591 return DAG.getSetCC(N->getDebugLoc(), VT, ANDNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002592 }
Bill Wendling2627a882009-01-30 20:43:18 +00002593 // fold (and (setgt X, -1), (setgt Y, -1)) -> (setgt (or X, Y), -1)
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002594 if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETGT) {
Bill Wendling2627a882009-01-30 20:43:18 +00002595 SDValue ORNode = DAG.getNode(ISD::OR, N0.getDebugLoc(),
2596 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00002597 AddToWorkList(ORNode.getNode());
Bill Wendling2627a882009-01-30 20:43:18 +00002598 return DAG.getSetCC(N->getDebugLoc(), VT, ORNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002599 }
2600 }
2601 // canonicalize equivalent to ll == rl
2602 if (LL == RR && LR == RL) {
2603 Op1 = ISD::getSetCCSwappedOperands(Op1);
2604 std::swap(RL, RR);
2605 }
2606 if (LL == RL && LR == RR) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00002607 bool isInteger = LL.getValueType().isInteger();
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002608 ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger);
Chris Lattner6e1c6232008-10-28 07:11:07 +00002609 if (Result != ISD::SETCC_INVALID &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002610 (!LegalOperations || TLI.isCondCodeLegal(Result, LL.getValueType())))
Bill Wendling2627a882009-01-30 20:43:18 +00002611 return DAG.getSetCC(N->getDebugLoc(), N0.getValueType(),
2612 LL, LR, Result);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002613 }
2614 }
Chris Lattner35e5c142006-05-05 05:51:50 +00002615
Bill Wendling2627a882009-01-30 20:43:18 +00002616 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
Chris Lattner35e5c142006-05-05 05:51:50 +00002617 if (N0.getOpcode() == N1.getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00002618 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002619 if (Tmp.getNode()) return Tmp;
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002620 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002621
Nate Begemande996292006-02-03 22:24:05 +00002622 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
2623 // fold (and (sra)) -> (and (srl)) when possible.
Duncan Sands83ec4b62008-06-06 12:08:01 +00002624 if (!VT.isVector() &&
Dan Gohman475871a2008-07-27 21:46:04 +00002625 SimplifyDemandedBits(SDValue(N, 0)))
2626 return SDValue(N, 0);
Evan Chengd40d03e2010-01-06 19:38:29 +00002627
Nate Begemanded49632005-10-13 03:11:28 +00002628 // fold (zext_inreg (extload x)) -> (zextload x)
Gabor Greifba36cb52008-08-28 21:40:38 +00002629 if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
Evan Cheng466685d2006-10-09 20:57:25 +00002630 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00002631 EVT MemVT = LN0->getMemoryVT();
Nate Begemanbfd65a02005-10-13 18:34:58 +00002632 // If we zero all the possible extended bits, then we can turn this into
2633 // a zextload if we are running before legalize or the operation is legal.
Dan Gohman6900a392010-03-04 00:23:16 +00002634 unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits();
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002635 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
Dan Gohman6900a392010-03-04 00:23:16 +00002636 BitWidth - MemVT.getScalarType().getSizeInBits())) &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002637 ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00002638 TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT))) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00002639 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N0.getDebugLoc(), VT,
Bill Wendling2627a882009-01-30 20:43:18 +00002640 LN0->getChain(), LN0->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002641 LN0->getPointerInfo(), MemVT,
David Greene1e559442010-02-15 17:00:31 +00002642 LN0->isVolatile(), LN0->isNonTemporal(),
2643 LN0->getAlignment());
Chris Lattner5750df92006-03-01 04:03:14 +00002644 AddToWorkList(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002645 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00002646 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00002647 }
2648 }
Sylvestre Ledru94c22712012-09-27 10:14:43 +00002649 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
Gabor Greifba36cb52008-08-28 21:40:38 +00002650 if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
Evan Cheng83060c52007-03-07 08:07:03 +00002651 N0.hasOneUse()) {
Evan Cheng466685d2006-10-09 20:57:25 +00002652 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00002653 EVT MemVT = LN0->getMemoryVT();
Nate Begemanbfd65a02005-10-13 18:34:58 +00002654 // If we zero all the possible extended bits, then we can turn this into
2655 // a zextload if we are running before legalize or the operation is legal.
Dan Gohman6900a392010-03-04 00:23:16 +00002656 unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits();
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002657 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
Dan Gohman6900a392010-03-04 00:23:16 +00002658 BitWidth - MemVT.getScalarType().getSizeInBits())) &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002659 ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00002660 TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT))) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00002661 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N0.getDebugLoc(), VT,
Bill Wendling2627a882009-01-30 20:43:18 +00002662 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002663 LN0->getBasePtr(), LN0->getPointerInfo(),
2664 MemVT,
David Greene1e559442010-02-15 17:00:31 +00002665 LN0->isVolatile(), LN0->isNonTemporal(),
2666 LN0->getAlignment());
Chris Lattner5750df92006-03-01 04:03:14 +00002667 AddToWorkList(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002668 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00002669 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00002670 }
2671 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002672
Chris Lattner35a9f5a2006-02-28 06:49:37 +00002673 // fold (and (load x), 255) -> (zextload x, i8)
2674 // fold (and (extload x, i16), 255) -> (zextload x, i8)
Evan Chengd40d03e2010-01-06 19:38:29 +00002675 // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
2676 if (N1C && (N0.getOpcode() == ISD::LOAD ||
2677 (N0.getOpcode() == ISD::ANY_EXTEND &&
2678 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
2679 bool HasAnyExt = N0.getOpcode() == ISD::ANY_EXTEND;
2680 LoadSDNode *LN0 = HasAnyExt
2681 ? cast<LoadSDNode>(N0.getOperand(0))
2682 : cast<LoadSDNode>(N0);
Evan Cheng466685d2006-10-09 20:57:25 +00002683 if (LN0->getExtensionType() != ISD::SEXTLOAD &&
Chris Lattnerbd1fccf2010-01-07 21:59:23 +00002684 LN0->isUnindexed() && N0.hasOneUse() && LN0->hasOneUse()) {
Duncan Sands8eab8a22008-06-09 11:32:28 +00002685 uint32_t ActiveBits = N1C->getAPIntValue().getActiveBits();
Evan Chengd40d03e2010-01-06 19:38:29 +00002686 if (ActiveBits > 0 && APIntOps::isMask(ActiveBits, N1C->getAPIntValue())){
2687 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
2688 EVT LoadedVT = LN0->getMemoryVT();
Duncan Sands8eab8a22008-06-09 11:32:28 +00002689
Evan Chengd40d03e2010-01-06 19:38:29 +00002690 if (ExtVT == LoadedVT &&
2691 (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT))) {
Chris Lattneref7634c2010-01-07 21:53:27 +00002692 EVT LoadResultTy = HasAnyExt ? LN0->getValueType(0) : VT;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002693
2694 SDValue NewLoad =
Stuart Hastingsa9011292011-02-16 16:23:55 +00002695 DAG.getExtLoad(ISD::ZEXTLOAD, LN0->getDebugLoc(), LoadResultTy,
Chris Lattneref7634c2010-01-07 21:53:27 +00002696 LN0->getChain(), LN0->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002697 LN0->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00002698 ExtVT, LN0->isVolatile(), LN0->isNonTemporal(),
2699 LN0->getAlignment());
Chris Lattneref7634c2010-01-07 21:53:27 +00002700 AddToWorkList(N);
2701 CombineTo(LN0, NewLoad, NewLoad.getValue(1));
2702 return SDValue(N, 0); // Return N so it doesn't get rechecked!
2703 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002704
Chris Lattneref7634c2010-01-07 21:53:27 +00002705 // Do not change the width of a volatile load.
2706 // Do not generate loads of non-round integer types since these can
2707 // be expensive (and would be wrong if the type is not byte sized).
2708 if (!LN0->isVolatile() && LoadedVT.bitsGT(ExtVT) && ExtVT.isRound() &&
2709 (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT))) {
2710 EVT PtrType = LN0->getOperand(1).getValueType();
Bill Wendling2627a882009-01-30 20:43:18 +00002711
Chris Lattneref7634c2010-01-07 21:53:27 +00002712 unsigned Alignment = LN0->getAlignment();
2713 SDValue NewPtr = LN0->getBasePtr();
2714
2715 // For big endian targets, we need to add an offset to the pointer
2716 // to load the correct bytes. For little endian systems, we merely
2717 // need to read fewer bytes from the same pointer.
2718 if (TLI.isBigEndian()) {
Evan Chengd40d03e2010-01-06 19:38:29 +00002719 unsigned LVTStoreBytes = LoadedVT.getStoreSize();
2720 unsigned EVTStoreBytes = ExtVT.getStoreSize();
2721 unsigned PtrOff = LVTStoreBytes - EVTStoreBytes;
Chris Lattneref7634c2010-01-07 21:53:27 +00002722 NewPtr = DAG.getNode(ISD::ADD, LN0->getDebugLoc(), PtrType,
2723 NewPtr, DAG.getConstant(PtrOff, PtrType));
2724 Alignment = MinAlign(Alignment, PtrOff);
Evan Chengd40d03e2010-01-06 19:38:29 +00002725 }
Chris Lattneref7634c2010-01-07 21:53:27 +00002726
2727 AddToWorkList(NewPtr.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002728
Chris Lattneref7634c2010-01-07 21:53:27 +00002729 EVT LoadResultTy = HasAnyExt ? LN0->getValueType(0) : VT;
2730 SDValue Load =
Stuart Hastingsa9011292011-02-16 16:23:55 +00002731 DAG.getExtLoad(ISD::ZEXTLOAD, LN0->getDebugLoc(), LoadResultTy,
Chris Lattneref7634c2010-01-07 21:53:27 +00002732 LN0->getChain(), NewPtr,
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002733 LN0->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00002734 ExtVT, LN0->isVolatile(), LN0->isNonTemporal(),
2735 Alignment);
Chris Lattneref7634c2010-01-07 21:53:27 +00002736 AddToWorkList(N);
2737 CombineTo(LN0, Load, Load.getValue(1));
2738 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Duncan Sandsdc846502007-10-28 12:59:45 +00002739 }
Evan Cheng466685d2006-10-09 20:57:25 +00002740 }
Chris Lattner15045b62006-02-28 06:35:35 +00002741 }
2742 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002743
Evan Chenga9e13ba2012-07-17 18:54:11 +00002744 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
2745 VT.getSizeInBits() <= 64) {
2746 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2747 APInt ADDC = ADDI->getAPIntValue();
2748 if (!TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
2749 // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
2750 // immediate for an add, but it is legal if its top c2 bits are set,
2751 // transform the ADD so the immediate doesn't need to be materialized
2752 // in a register.
2753 if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
2754 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
2755 SRLI->getZExtValue());
2756 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
2757 ADDC |= Mask;
2758 if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
2759 SDValue NewAdd =
2760 DAG.getNode(ISD::ADD, N0.getDebugLoc(), VT,
2761 N0.getOperand(0), DAG.getConstant(ADDC, VT));
2762 CombineTo(N0.getNode(), NewAdd);
2763 return SDValue(N, 0); // Return N so it doesn't get rechecked!
2764 }
2765 }
2766 }
2767 }
2768 }
2769 }
2770
2771
Evan Chengb3a3d5e2010-04-28 07:10:39 +00002772 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002773}
2774
Evan Cheng9568e5c2011-06-21 06:01:08 +00002775/// MatchBSwapHWord - Match (a >> 8) | (a << 8) as (bswap a) >> 16
2776///
2777SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
2778 bool DemandHighBits) {
2779 if (!LegalOperations)
2780 return SDValue();
2781
2782 EVT VT = N->getValueType(0);
2783 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
2784 return SDValue();
2785 if (!TLI.isOperationLegal(ISD::BSWAP, VT))
2786 return SDValue();
2787
2788 // Recognize (and (shl a, 8), 0xff), (and (srl a, 8), 0xff00)
2789 bool LookPassAnd0 = false;
2790 bool LookPassAnd1 = false;
2791 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
2792 std::swap(N0, N1);
2793 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
2794 std::swap(N0, N1);
2795 if (N0.getOpcode() == ISD::AND) {
2796 if (!N0.getNode()->hasOneUse())
2797 return SDValue();
2798 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2799 if (!N01C || N01C->getZExtValue() != 0xFF00)
2800 return SDValue();
2801 N0 = N0.getOperand(0);
2802 LookPassAnd0 = true;
2803 }
2804
2805 if (N1.getOpcode() == ISD::AND) {
2806 if (!N1.getNode()->hasOneUse())
2807 return SDValue();
2808 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
2809 if (!N11C || N11C->getZExtValue() != 0xFF)
2810 return SDValue();
2811 N1 = N1.getOperand(0);
2812 LookPassAnd1 = true;
2813 }
2814
2815 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
2816 std::swap(N0, N1);
2817 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
2818 return SDValue();
2819 if (!N0.getNode()->hasOneUse() ||
2820 !N1.getNode()->hasOneUse())
2821 return SDValue();
2822
2823 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2824 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
2825 if (!N01C || !N11C)
2826 return SDValue();
2827 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
2828 return SDValue();
2829
2830 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
2831 SDValue N00 = N0->getOperand(0);
2832 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
2833 if (!N00.getNode()->hasOneUse())
2834 return SDValue();
2835 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
2836 if (!N001C || N001C->getZExtValue() != 0xFF)
2837 return SDValue();
2838 N00 = N00.getOperand(0);
2839 LookPassAnd0 = true;
2840 }
2841
2842 SDValue N10 = N1->getOperand(0);
2843 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
2844 if (!N10.getNode()->hasOneUse())
2845 return SDValue();
2846 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
2847 if (!N101C || N101C->getZExtValue() != 0xFF00)
2848 return SDValue();
2849 N10 = N10.getOperand(0);
2850 LookPassAnd1 = true;
2851 }
2852
2853 if (N00 != N10)
2854 return SDValue();
2855
2856 // Make sure everything beyond the low halfword is zero since the SRL 16
2857 // will clear the top bits.
2858 unsigned OpSizeInBits = VT.getSizeInBits();
2859 if (DemandHighBits && OpSizeInBits > 16 &&
2860 (!LookPassAnd0 || !LookPassAnd1) &&
2861 !DAG.MaskedValueIsZero(N10, APInt::getHighBitsSet(OpSizeInBits, 16)))
2862 return SDValue();
Eric Christopher7332e6e2011-07-14 01:12:15 +00002863
Evan Cheng9568e5c2011-06-21 06:01:08 +00002864 SDValue Res = DAG.getNode(ISD::BSWAP, N->getDebugLoc(), VT, N00);
2865 if (OpSizeInBits > 16)
2866 Res = DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, Res,
2867 DAG.getConstant(OpSizeInBits-16, getShiftAmountTy(VT)));
2868 return Res;
2869}
2870
2871/// isBSwapHWordElement - Return true if the specified node is an element
2872/// that makes up a 32-bit packed halfword byteswap. i.e.
2873/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8)
2874static bool isBSwapHWordElement(SDValue N, SmallVector<SDNode*,4> &Parts) {
2875 if (!N.getNode()->hasOneUse())
2876 return false;
2877
2878 unsigned Opc = N.getOpcode();
2879 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
2880 return false;
2881
2882 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2883 if (!N1C)
2884 return false;
2885
2886 unsigned Num;
2887 switch (N1C->getZExtValue()) {
2888 default:
2889 return false;
2890 case 0xFF: Num = 0; break;
2891 case 0xFF00: Num = 1; break;
2892 case 0xFF0000: Num = 2; break;
2893 case 0xFF000000: Num = 3; break;
2894 }
2895
2896 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
2897 SDValue N0 = N.getOperand(0);
2898 if (Opc == ISD::AND) {
2899 if (Num == 0 || Num == 2) {
2900 // (x >> 8) & 0xff
2901 // (x >> 8) & 0xff0000
2902 if (N0.getOpcode() != ISD::SRL)
2903 return false;
2904 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2905 if (!C || C->getZExtValue() != 8)
2906 return false;
2907 } else {
2908 // (x << 8) & 0xff00
2909 // (x << 8) & 0xff000000
2910 if (N0.getOpcode() != ISD::SHL)
2911 return false;
2912 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2913 if (!C || C->getZExtValue() != 8)
2914 return false;
2915 }
2916 } else if (Opc == ISD::SHL) {
2917 // (x & 0xff) << 8
2918 // (x & 0xff0000) << 8
2919 if (Num != 0 && Num != 2)
2920 return false;
2921 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2922 if (!C || C->getZExtValue() != 8)
2923 return false;
2924 } else { // Opc == ISD::SRL
2925 // (x & 0xff00) >> 8
2926 // (x & 0xff000000) >> 8
2927 if (Num != 1 && Num != 3)
2928 return false;
2929 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2930 if (!C || C->getZExtValue() != 8)
2931 return false;
2932 }
2933
2934 if (Parts[Num])
2935 return false;
2936
2937 Parts[Num] = N0.getOperand(0).getNode();
2938 return true;
2939}
2940
2941/// MatchBSwapHWord - Match a 32-bit packed halfword bswap. That is
2942/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8)
2943/// => (rotl (bswap x), 16)
2944SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
2945 if (!LegalOperations)
2946 return SDValue();
2947
2948 EVT VT = N->getValueType(0);
2949 if (VT != MVT::i32)
2950 return SDValue();
2951 if (!TLI.isOperationLegal(ISD::BSWAP, VT))
2952 return SDValue();
2953
2954 SmallVector<SDNode*,4> Parts(4, (SDNode*)0);
2955 // Look for either
2956 // (or (or (and), (and)), (or (and), (and)))
2957 // (or (or (or (and), (and)), (and)), (and))
2958 if (N0.getOpcode() != ISD::OR)
2959 return SDValue();
2960 SDValue N00 = N0.getOperand(0);
2961 SDValue N01 = N0.getOperand(1);
2962
2963 if (N1.getOpcode() == ISD::OR) {
2964 // (or (or (and), (and)), (or (and), (and)))
2965 SDValue N000 = N00.getOperand(0);
2966 if (!isBSwapHWordElement(N000, Parts))
2967 return SDValue();
2968
2969 SDValue N001 = N00.getOperand(1);
2970 if (!isBSwapHWordElement(N001, Parts))
2971 return SDValue();
2972 SDValue N010 = N01.getOperand(0);
2973 if (!isBSwapHWordElement(N010, Parts))
2974 return SDValue();
2975 SDValue N011 = N01.getOperand(1);
2976 if (!isBSwapHWordElement(N011, Parts))
2977 return SDValue();
2978 } else {
2979 // (or (or (or (and), (and)), (and)), (and))
2980 if (!isBSwapHWordElement(N1, Parts))
2981 return SDValue();
2982 if (!isBSwapHWordElement(N01, Parts))
2983 return SDValue();
2984 if (N00.getOpcode() != ISD::OR)
2985 return SDValue();
2986 SDValue N000 = N00.getOperand(0);
2987 if (!isBSwapHWordElement(N000, Parts))
2988 return SDValue();
2989 SDValue N001 = N00.getOperand(1);
2990 if (!isBSwapHWordElement(N001, Parts))
2991 return SDValue();
2992 }
2993
2994 // Make sure the parts are all coming from the same node.
2995 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
2996 return SDValue();
2997
2998 SDValue BSwap = DAG.getNode(ISD::BSWAP, N->getDebugLoc(), VT,
2999 SDValue(Parts[0],0));
3000
3001 // Result of the bswap should be rotated by 16. If it's not legal, than
3002 // do (x << 16) | (x >> 16).
3003 SDValue ShAmt = DAG.getConstant(16, getShiftAmountTy(VT));
3004 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
3005 return DAG.getNode(ISD::ROTL, N->getDebugLoc(), VT, BSwap, ShAmt);
3006 else if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
3007 return DAG.getNode(ISD::ROTR, N->getDebugLoc(), VT, BSwap, ShAmt);
3008 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT,
3009 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, BSwap, ShAmt),
3010 DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, BSwap, ShAmt));
3011}
3012
Dan Gohman475871a2008-07-27 21:46:04 +00003013SDValue DAGCombiner::visitOR(SDNode *N) {
3014 SDValue N0 = N->getOperand(0);
3015 SDValue N1 = N->getOperand(1);
3016 SDValue LL, LR, RL, RR, CC0, CC1;
Nate Begeman646d7e22005-09-02 21:18:40 +00003017 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3018 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003019 EVT VT = N1.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00003020
Dan Gohman7f321562007-06-25 16:23:39 +00003021 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00003022 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003023 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00003024 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00003025 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003026
Dan Gohman613e0d82007-07-03 14:03:57 +00003027 // fold (or x, undef) -> -1
Bob Wilson86749492010-06-28 23:40:25 +00003028 if (!LegalOperations &&
3029 (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)) {
Nate Begeman93e0ed32009-12-03 07:11:29 +00003030 EVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT;
3031 return DAG.getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
3032 }
Nate Begeman1d4d4142005-09-01 00:19:25 +00003033 // fold (or c1, c2) -> c1|c2
Nate Begeman646d7e22005-09-02 21:18:40 +00003034 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003035 return DAG.FoldConstantArithmetic(ISD::OR, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00003036 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00003037 if (N0C && !N1C)
Bill Wendling09025642009-01-30 20:59:34 +00003038 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003039 // fold (or x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003040 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003041 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003042 // fold (or x, -1) -> -1
Nate Begeman646d7e22005-09-02 21:18:40 +00003043 if (N1C && N1C->isAllOnesValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003044 return N1;
Sylvestre Ledru94c22712012-09-27 10:14:43 +00003045 // fold (or x, c) -> c iff (x & ~c) == 0
Dan Gohman2e68b6f2008-02-25 21:11:39 +00003046 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
Nate Begeman83e75ec2005-09-06 04:43:02 +00003047 return N1;
Evan Cheng9568e5c2011-06-21 06:01:08 +00003048
3049 // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
3050 SDValue BSwap = MatchBSwapHWord(N, N0, N1);
3051 if (BSwap.getNode() != 0)
3052 return BSwap;
3053 BSwap = MatchBSwapHWordLow(N, N0, N1);
3054 if (BSwap.getNode() != 0)
3055 return BSwap;
3056
Nate Begemancd4d58c2006-02-03 06:46:56 +00003057 // reassociate or
Bill Wendling35247c32009-01-30 00:45:56 +00003058 SDValue ROR = ReassociateOps(ISD::OR, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00003059 if (ROR.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00003060 return ROR;
3061 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
Sylvestre Ledru94c22712012-09-27 10:14:43 +00003062 // iff (c1 & c2) == 0.
Gabor Greifba36cb52008-08-28 21:40:38 +00003063 if (N1C && N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
Chris Lattner731d3482005-10-27 05:06:38 +00003064 isa<ConstantSDNode>(N0.getOperand(1))) {
Chris Lattner731d3482005-10-27 05:06:38 +00003065 ConstantSDNode *C1 = cast<ConstantSDNode>(N0.getOperand(1));
Bill Wendling32f9eb22010-03-03 01:58:01 +00003066 if ((C1->getAPIntValue() & N1C->getAPIntValue()) != 0)
Bill Wendling7d9f2b92010-03-03 00:35:56 +00003067 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
3068 DAG.getNode(ISD::OR, N0.getDebugLoc(), VT,
3069 N0.getOperand(0), N1),
3070 DAG.FoldConstantArithmetic(ISD::OR, VT, N1C, C1));
Nate Begeman223df222005-09-08 20:18:10 +00003071 }
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003072 // fold (or (setcc x), (setcc y)) -> (setcc (or x, y))
3073 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
3074 ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
3075 ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
Scott Michelfdc40a02009-02-17 22:15:04 +00003076
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003077 if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00003078 LL.getValueType().isInteger()) {
Bill Wendling09025642009-01-30 20:59:34 +00003079 // fold (or (setne X, 0), (setne Y, 0)) -> (setne (or X, Y), 0)
3080 // fold (or (setlt X, 0), (setlt Y, 0)) -> (setne (or X, Y), 0)
Scott Michelfdc40a02009-02-17 22:15:04 +00003081 if (cast<ConstantSDNode>(LR)->isNullValue() &&
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003082 (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) {
Bill Wendling09025642009-01-30 20:59:34 +00003083 SDValue ORNode = DAG.getNode(ISD::OR, LR.getDebugLoc(),
3084 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00003085 AddToWorkList(ORNode.getNode());
Bill Wendling09025642009-01-30 20:59:34 +00003086 return DAG.getSetCC(N->getDebugLoc(), VT, ORNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003087 }
Bill Wendling09025642009-01-30 20:59:34 +00003088 // fold (or (setne X, -1), (setne Y, -1)) -> (setne (and X, Y), -1)
3089 // fold (or (setgt X, -1), (setgt Y -1)) -> (setgt (and X, Y), -1)
Scott Michelfdc40a02009-02-17 22:15:04 +00003090 if (cast<ConstantSDNode>(LR)->isAllOnesValue() &&
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003091 (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) {
Bill Wendling09025642009-01-30 20:59:34 +00003092 SDValue ANDNode = DAG.getNode(ISD::AND, LR.getDebugLoc(),
3093 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00003094 AddToWorkList(ANDNode.getNode());
Bill Wendling09025642009-01-30 20:59:34 +00003095 return DAG.getSetCC(N->getDebugLoc(), VT, ANDNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003096 }
3097 }
3098 // canonicalize equivalent to ll == rl
3099 if (LL == RR && LR == RL) {
3100 Op1 = ISD::getSetCCSwappedOperands(Op1);
3101 std::swap(RL, RR);
3102 }
3103 if (LL == RL && LR == RR) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00003104 bool isInteger = LL.getValueType().isInteger();
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003105 ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger);
Chris Lattner6e1c6232008-10-28 07:11:07 +00003106 if (Result != ISD::SETCC_INVALID &&
Duncan Sands25cf2272008-11-24 14:53:14 +00003107 (!LegalOperations || TLI.isCondCodeLegal(Result, LL.getValueType())))
Bill Wendling09025642009-01-30 20:59:34 +00003108 return DAG.getSetCC(N->getDebugLoc(), N0.getValueType(),
3109 LL, LR, Result);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003110 }
3111 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003112
Bill Wendling09025642009-01-30 20:59:34 +00003113 // Simplify: (or (op x...), (op y...)) -> (op (or x, y))
Chris Lattner35e5c142006-05-05 05:51:50 +00003114 if (N0.getOpcode() == N1.getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003115 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00003116 if (Tmp.getNode()) return Tmp;
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003117 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003118
Bill Wendling09025642009-01-30 20:59:34 +00003119 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible.
Chris Lattner1ec72732006-09-14 21:11:37 +00003120 if (N0.getOpcode() == ISD::AND &&
3121 N1.getOpcode() == ISD::AND &&
3122 N0.getOperand(1).getOpcode() == ISD::Constant &&
3123 N1.getOperand(1).getOpcode() == ISD::Constant &&
3124 // Don't increase # computations.
Gabor Greifba36cb52008-08-28 21:40:38 +00003125 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
Chris Lattner1ec72732006-09-14 21:11:37 +00003126 // We can only do this xform if we know that bits from X that are set in C2
3127 // but not in C1 are already zero. Likewise for Y.
Dan Gohman2e68b6f2008-02-25 21:11:39 +00003128 const APInt &LHSMask =
3129 cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
3130 const APInt &RHSMask =
3131 cast<ConstantSDNode>(N1.getOperand(1))->getAPIntValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00003132
Dan Gohmanea859be2007-06-22 14:59:07 +00003133 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
3134 DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
Bill Wendling09025642009-01-30 20:59:34 +00003135 SDValue X = DAG.getNode(ISD::OR, N0.getDebugLoc(), VT,
3136 N0.getOperand(0), N1.getOperand(0));
3137 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, X,
3138 DAG.getConstant(LHSMask | RHSMask, VT));
Chris Lattner1ec72732006-09-14 21:11:37 +00003139 }
3140 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003141
Chris Lattner516b9622006-09-14 20:50:57 +00003142 // See if this is some rotate idiom.
Bill Wendling317bd702009-01-30 21:14:50 +00003143 if (SDNode *Rot = MatchRotate(N0, N1, N->getDebugLoc()))
Dan Gohman475871a2008-07-27 21:46:04 +00003144 return SDValue(Rot, 0);
Chris Lattner35e5c142006-05-05 05:51:50 +00003145
Dan Gohman4e39e9d2010-06-24 14:30:44 +00003146 // Simplify the operands using demanded-bits information.
3147 if (!VT.isVector() &&
3148 SimplifyDemandedBits(SDValue(N, 0)))
3149 return SDValue(N, 0);
3150
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003151 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003152}
3153
Chris Lattner516b9622006-09-14 20:50:57 +00003154/// MatchRotateHalf - Match "(X shl/srl V1) & V2" where V2 may not be present.
Dan Gohman475871a2008-07-27 21:46:04 +00003155static bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) {
Chris Lattner516b9622006-09-14 20:50:57 +00003156 if (Op.getOpcode() == ISD::AND) {
Reid Spencer3ed469c2006-11-02 20:25:50 +00003157 if (isa<ConstantSDNode>(Op.getOperand(1))) {
Chris Lattner516b9622006-09-14 20:50:57 +00003158 Mask = Op.getOperand(1);
3159 Op = Op.getOperand(0);
3160 } else {
3161 return false;
3162 }
3163 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003164
Chris Lattner516b9622006-09-14 20:50:57 +00003165 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
3166 Shift = Op;
3167 return true;
3168 }
Bill Wendling09025642009-01-30 20:59:34 +00003169
Scott Michelfdc40a02009-02-17 22:15:04 +00003170 return false;
Chris Lattner516b9622006-09-14 20:50:57 +00003171}
3172
Chris Lattner516b9622006-09-14 20:50:57 +00003173// MatchRotate - Handle an 'or' of two operands. If this is one of the many
3174// idioms for rotate, and if the target supports rotation instructions, generate
3175// a rot[lr].
Bill Wendling317bd702009-01-30 21:14:50 +00003176SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, DebugLoc DL) {
Duncan Sandsd4b9c172008-06-13 19:07:40 +00003177 // Must be a legal type. Expanded 'n promoted things won't work with rotates.
Owen Andersone50ed302009-08-10 22:56:29 +00003178 EVT VT = LHS.getValueType();
Chris Lattner516b9622006-09-14 20:50:57 +00003179 if (!TLI.isTypeLegal(VT)) return 0;
3180
3181 // The target must have at least one rotate flavor.
Dan Gohmanf560ffa2009-01-28 17:46:25 +00003182 bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT);
3183 bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT);
Chris Lattner516b9622006-09-14 20:50:57 +00003184 if (!HasROTL && !HasROTR) return 0;
Duncan Sandsd4b9c172008-06-13 19:07:40 +00003185
Chris Lattner516b9622006-09-14 20:50:57 +00003186 // Match "(X shl/srl V1) & V2" where V2 may not be present.
Dan Gohman475871a2008-07-27 21:46:04 +00003187 SDValue LHSShift; // The shift.
3188 SDValue LHSMask; // AND value if any.
Chris Lattner516b9622006-09-14 20:50:57 +00003189 if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
3190 return 0; // Not part of a rotate.
3191
Dan Gohman475871a2008-07-27 21:46:04 +00003192 SDValue RHSShift; // The shift.
3193 SDValue RHSMask; // AND value if any.
Chris Lattner516b9622006-09-14 20:50:57 +00003194 if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
3195 return 0; // Not part of a rotate.
Scott Michelfdc40a02009-02-17 22:15:04 +00003196
Chris Lattner516b9622006-09-14 20:50:57 +00003197 if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
3198 return 0; // Not shifting the same value.
3199
3200 if (LHSShift.getOpcode() == RHSShift.getOpcode())
3201 return 0; // Shifts must disagree.
Scott Michelfdc40a02009-02-17 22:15:04 +00003202
Chris Lattner516b9622006-09-14 20:50:57 +00003203 // Canonicalize shl to left side in a shl/srl pair.
3204 if (RHSShift.getOpcode() == ISD::SHL) {
3205 std::swap(LHS, RHS);
3206 std::swap(LHSShift, RHSShift);
3207 std::swap(LHSMask , RHSMask );
3208 }
3209
Duncan Sands83ec4b62008-06-06 12:08:01 +00003210 unsigned OpSizeInBits = VT.getSizeInBits();
Dan Gohman475871a2008-07-27 21:46:04 +00003211 SDValue LHSShiftArg = LHSShift.getOperand(0);
3212 SDValue LHSShiftAmt = LHSShift.getOperand(1);
3213 SDValue RHSShiftAmt = RHSShift.getOperand(1);
Chris Lattner516b9622006-09-14 20:50:57 +00003214
3215 // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
3216 // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
Scott Michelc9dc1142007-04-02 21:36:32 +00003217 if (LHSShiftAmt.getOpcode() == ISD::Constant &&
3218 RHSShiftAmt.getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003219 uint64_t LShVal = cast<ConstantSDNode>(LHSShiftAmt)->getZExtValue();
3220 uint64_t RShVal = cast<ConstantSDNode>(RHSShiftAmt)->getZExtValue();
Chris Lattner516b9622006-09-14 20:50:57 +00003221 if ((LShVal + RShVal) != OpSizeInBits)
3222 return 0;
3223
Craig Topper32b73432012-09-29 06:54:22 +00003224 SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
3225 LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
Scott Michelfdc40a02009-02-17 22:15:04 +00003226
Chris Lattner516b9622006-09-14 20:50:57 +00003227 // If there is an AND of either shifted operand, apply it to the result.
Gabor Greifba36cb52008-08-28 21:40:38 +00003228 if (LHSMask.getNode() || RHSMask.getNode()) {
Dan Gohman220a8232008-03-03 23:51:38 +00003229 APInt Mask = APInt::getAllOnesValue(OpSizeInBits);
Scott Michelfdc40a02009-02-17 22:15:04 +00003230
Gabor Greifba36cb52008-08-28 21:40:38 +00003231 if (LHSMask.getNode()) {
Dan Gohman220a8232008-03-03 23:51:38 +00003232 APInt RHSBits = APInt::getLowBitsSet(OpSizeInBits, LShVal);
3233 Mask &= cast<ConstantSDNode>(LHSMask)->getAPIntValue() | RHSBits;
Chris Lattner516b9622006-09-14 20:50:57 +00003234 }
Gabor Greifba36cb52008-08-28 21:40:38 +00003235 if (RHSMask.getNode()) {
Dan Gohman220a8232008-03-03 23:51:38 +00003236 APInt LHSBits = APInt::getHighBitsSet(OpSizeInBits, RShVal);
3237 Mask &= cast<ConstantSDNode>(RHSMask)->getAPIntValue() | LHSBits;
Chris Lattner516b9622006-09-14 20:50:57 +00003238 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003239
Bill Wendling317bd702009-01-30 21:14:50 +00003240 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, DAG.getConstant(Mask, VT));
Chris Lattner516b9622006-09-14 20:50:57 +00003241 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003242
Gabor Greifba36cb52008-08-28 21:40:38 +00003243 return Rot.getNode();
Chris Lattner516b9622006-09-14 20:50:57 +00003244 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003245
Chris Lattner516b9622006-09-14 20:50:57 +00003246 // If there is a mask here, and we have a variable shift, we can't be sure
3247 // that we're masking out the right stuff.
Gabor Greifba36cb52008-08-28 21:40:38 +00003248 if (LHSMask.getNode() || RHSMask.getNode())
Chris Lattner516b9622006-09-14 20:50:57 +00003249 return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +00003250
Chris Lattner516b9622006-09-14 20:50:57 +00003251 // fold (or (shl x, y), (srl x, (sub 32, y))) -> (rotl x, y)
3252 // fold (or (shl x, y), (srl x, (sub 32, y))) -> (rotr x, (sub 32, y))
Scott Michelc9dc1142007-04-02 21:36:32 +00003253 if (RHSShiftAmt.getOpcode() == ISD::SUB &&
3254 LHSShiftAmt == RHSShiftAmt.getOperand(1)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00003255 if (ConstantSDNode *SUBC =
Scott Michelc9dc1142007-04-02 21:36:32 +00003256 dyn_cast<ConstantSDNode>(RHSShiftAmt.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003257 if (SUBC->getAPIntValue() == OpSizeInBits) {
Craig Topper32b73432012-09-29 06:54:22 +00003258 return DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg,
3259 HasROTL ? LHSShiftAmt : RHSShiftAmt).getNode();
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00003260 }
Chris Lattner516b9622006-09-14 20:50:57 +00003261 }
3262 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003263
Chris Lattner516b9622006-09-14 20:50:57 +00003264 // fold (or (shl x, (sub 32, y)), (srl x, r)) -> (rotr x, y)
3265 // fold (or (shl x, (sub 32, y)), (srl x, r)) -> (rotl x, (sub 32, y))
Scott Michelc9dc1142007-04-02 21:36:32 +00003266 if (LHSShiftAmt.getOpcode() == ISD::SUB &&
3267 RHSShiftAmt == LHSShiftAmt.getOperand(1)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00003268 if (ConstantSDNode *SUBC =
Scott Michelc9dc1142007-04-02 21:36:32 +00003269 dyn_cast<ConstantSDNode>(LHSShiftAmt.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003270 if (SUBC->getAPIntValue() == OpSizeInBits) {
Craig Topper32b73432012-09-29 06:54:22 +00003271 return DAG.getNode(HasROTR ? ISD::ROTR : ISD::ROTL, DL, VT, LHSShiftArg,
3272 HasROTR ? RHSShiftAmt : LHSShiftAmt).getNode();
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00003273 }
Scott Michelc9dc1142007-04-02 21:36:32 +00003274 }
3275 }
3276
Dan Gohman74feef22008-10-17 01:23:35 +00003277 // Look for sign/zext/any-extended or truncate cases:
Scott Michelc9dc1142007-04-02 21:36:32 +00003278 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND
3279 || LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND
Dan Gohman74feef22008-10-17 01:23:35 +00003280 || LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND
3281 || LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
Scott Michelc9dc1142007-04-02 21:36:32 +00003282 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND
3283 || RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND
Dan Gohman74feef22008-10-17 01:23:35 +00003284 || RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND
3285 || RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
Dan Gohman475871a2008-07-27 21:46:04 +00003286 SDValue LExtOp0 = LHSShiftAmt.getOperand(0);
3287 SDValue RExtOp0 = RHSShiftAmt.getOperand(0);
Scott Michelc9dc1142007-04-02 21:36:32 +00003288 if (RExtOp0.getOpcode() == ISD::SUB &&
3289 RExtOp0.getOperand(1) == LExtOp0) {
3290 // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003291 // (rotl x, y)
Scott Michelc9dc1142007-04-02 21:36:32 +00003292 // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003293 // (rotr x, (sub 32, y))
Dan Gohman74feef22008-10-17 01:23:35 +00003294 if (ConstantSDNode *SUBC =
3295 dyn_cast<ConstantSDNode>(RExtOp0.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003296 if (SUBC->getAPIntValue() == OpSizeInBits) {
Bill Wendling317bd702009-01-30 21:14:50 +00003297 return DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
3298 LHSShiftArg,
Gabor Greif12632d22008-08-30 19:29:20 +00003299 HasROTL ? LHSShiftAmt : RHSShiftAmt).getNode();
Scott Michelc9dc1142007-04-02 21:36:32 +00003300 }
3301 }
3302 } else if (LExtOp0.getOpcode() == ISD::SUB &&
3303 RExtOp0 == LExtOp0.getOperand(1)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00003304 // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext y))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003305 // (rotr x, y)
Bill Wendling353dea22008-08-31 01:04:56 +00003306 // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext y))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003307 // (rotl x, (sub 32, y))
Dan Gohman74feef22008-10-17 01:23:35 +00003308 if (ConstantSDNode *SUBC =
3309 dyn_cast<ConstantSDNode>(LExtOp0.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003310 if (SUBC->getAPIntValue() == OpSizeInBits) {
Bill Wendling317bd702009-01-30 21:14:50 +00003311 return DAG.getNode(HasROTR ? ISD::ROTR : ISD::ROTL, DL, VT,
3312 LHSShiftArg,
Bill Wendling353dea22008-08-31 01:04:56 +00003313 HasROTR ? RHSShiftAmt : LHSShiftAmt).getNode();
Scott Michelc9dc1142007-04-02 21:36:32 +00003314 }
3315 }
Chris Lattner516b9622006-09-14 20:50:57 +00003316 }
3317 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003318
Chris Lattner516b9622006-09-14 20:50:57 +00003319 return 0;
3320}
3321
Dan Gohman475871a2008-07-27 21:46:04 +00003322SDValue DAGCombiner::visitXOR(SDNode *N) {
3323 SDValue N0 = N->getOperand(0);
3324 SDValue N1 = N->getOperand(1);
3325 SDValue LHS, RHS, CC;
Nate Begeman646d7e22005-09-02 21:18:40 +00003326 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3327 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003328 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00003329
Dan Gohman7f321562007-06-25 16:23:39 +00003330 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00003331 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003332 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00003333 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00003334 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003335
Evan Cheng26471c42008-03-25 20:08:07 +00003336 // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
3337 if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF)
3338 return DAG.getConstant(0, VT);
Dan Gohman613e0d82007-07-03 14:03:57 +00003339 // fold (xor x, undef) -> undef
Dan Gohman70fb1ae2007-07-10 15:19:29 +00003340 if (N0.getOpcode() == ISD::UNDEF)
3341 return N0;
3342 if (N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00003343 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003344 // fold (xor c1, c2) -> c1^c2
Nate Begeman646d7e22005-09-02 21:18:40 +00003345 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003346 return DAG.FoldConstantArithmetic(ISD::XOR, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00003347 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00003348 if (N0C && !N1C)
Bill Wendling317bd702009-01-30 21:14:50 +00003349 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003350 // fold (xor x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003351 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003352 return N0;
Nate Begemancd4d58c2006-02-03 06:46:56 +00003353 // reassociate xor
Bill Wendling35247c32009-01-30 00:45:56 +00003354 SDValue RXOR = ReassociateOps(ISD::XOR, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00003355 if (RXOR.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00003356 return RXOR;
Bill Wendlingae89bb12008-11-11 08:25:46 +00003357
Nate Begeman1d4d4142005-09-01 00:19:25 +00003358 // fold !(x cc y) -> (x !cc y)
Dan Gohman002e5d02008-03-13 22:13:53 +00003359 if (N1C && N1C->getAPIntValue() == 1 && isSetCCEquivalent(N0, LHS, RHS, CC)) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00003360 bool isInt = LHS.getValueType().isInteger();
Nate Begeman646d7e22005-09-02 21:18:40 +00003361 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
3362 isInt);
Bill Wendlingae89bb12008-11-11 08:25:46 +00003363
Duncan Sands25cf2272008-11-24 14:53:14 +00003364 if (!LegalOperations || TLI.isCondCodeLegal(NotCC, LHS.getValueType())) {
Bill Wendlingae89bb12008-11-11 08:25:46 +00003365 switch (N0.getOpcode()) {
3366 default:
Torok Edwinc23197a2009-07-14 16:55:14 +00003367 llvm_unreachable("Unhandled SetCC Equivalent!");
Bill Wendlingae89bb12008-11-11 08:25:46 +00003368 case ISD::SETCC:
Bill Wendling317bd702009-01-30 21:14:50 +00003369 return DAG.getSetCC(N->getDebugLoc(), VT, LHS, RHS, NotCC);
Bill Wendlingae89bb12008-11-11 08:25:46 +00003370 case ISD::SELECT_CC:
Bill Wendling317bd702009-01-30 21:14:50 +00003371 return DAG.getSelectCC(N->getDebugLoc(), LHS, RHS, N0.getOperand(2),
Bill Wendlingae89bb12008-11-11 08:25:46 +00003372 N0.getOperand(3), NotCC);
3373 }
3374 }
Nate Begeman1d4d4142005-09-01 00:19:25 +00003375 }
Bill Wendlingae89bb12008-11-11 08:25:46 +00003376
Chris Lattner61c5ff42007-09-10 21:39:07 +00003377 // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
Dan Gohman002e5d02008-03-13 22:13:53 +00003378 if (N1C && N1C->getAPIntValue() == 1 && N0.getOpcode() == ISD::ZERO_EXTEND &&
Gabor Greif12632d22008-08-30 19:29:20 +00003379 N0.getNode()->hasOneUse() &&
3380 isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
Dan Gohman475871a2008-07-27 21:46:04 +00003381 SDValue V = N0.getOperand(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003382 V = DAG.getNode(ISD::XOR, N0.getDebugLoc(), V.getValueType(), V,
Duncan Sands272dce02007-10-10 09:54:50 +00003383 DAG.getConstant(1, V.getValueType()));
Gabor Greifba36cb52008-08-28 21:40:38 +00003384 AddToWorkList(V.getNode());
Bill Wendling317bd702009-01-30 21:14:50 +00003385 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, V);
Chris Lattner61c5ff42007-09-10 21:39:07 +00003386 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003387
Sylvestre Ledru94c22712012-09-27 10:14:43 +00003388 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
Owen Anderson825b72b2009-08-11 20:47:22 +00003389 if (N1C && N1C->getAPIntValue() == 1 && VT == MVT::i1 &&
Nate Begeman99801192005-09-07 23:25:52 +00003390 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
Dan Gohman475871a2008-07-27 21:46:04 +00003391 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
Nate Begeman99801192005-09-07 23:25:52 +00003392 if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
3393 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
Bill Wendling317bd702009-01-30 21:14:50 +00003394 LHS = DAG.getNode(ISD::XOR, LHS.getDebugLoc(), VT, LHS, N1); // LHS = ~LHS
3395 RHS = DAG.getNode(ISD::XOR, RHS.getDebugLoc(), VT, RHS, N1); // RHS = ~RHS
Gabor Greifba36cb52008-08-28 21:40:38 +00003396 AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode());
Bill Wendling317bd702009-01-30 21:14:50 +00003397 return DAG.getNode(NewOpcode, N->getDebugLoc(), VT, LHS, RHS);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003398 }
3399 }
Sylvestre Ledru94c22712012-09-27 10:14:43 +00003400 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
Scott Michelfdc40a02009-02-17 22:15:04 +00003401 if (N1C && N1C->isAllOnesValue() &&
Nate Begeman99801192005-09-07 23:25:52 +00003402 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
Dan Gohman475871a2008-07-27 21:46:04 +00003403 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
Nate Begeman99801192005-09-07 23:25:52 +00003404 if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
3405 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
Bill Wendling317bd702009-01-30 21:14:50 +00003406 LHS = DAG.getNode(ISD::XOR, LHS.getDebugLoc(), VT, LHS, N1); // LHS = ~LHS
3407 RHS = DAG.getNode(ISD::XOR, RHS.getDebugLoc(), VT, RHS, N1); // RHS = ~RHS
Gabor Greifba36cb52008-08-28 21:40:38 +00003408 AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode());
Bill Wendling317bd702009-01-30 21:14:50 +00003409 return DAG.getNode(NewOpcode, N->getDebugLoc(), VT, LHS, RHS);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003410 }
3411 }
Bill Wendling317bd702009-01-30 21:14:50 +00003412 // fold (xor (xor x, c1), c2) -> (xor x, (xor c1, c2))
Nate Begeman223df222005-09-08 20:18:10 +00003413 if (N1C && N0.getOpcode() == ISD::XOR) {
3414 ConstantSDNode *N00C = dyn_cast<ConstantSDNode>(N0.getOperand(0));
3415 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3416 if (N00C)
Bill Wendling317bd702009-01-30 21:14:50 +00003417 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N0.getOperand(1),
3418 DAG.getConstant(N1C->getAPIntValue() ^
Dan Gohman002e5d02008-03-13 22:13:53 +00003419 N00C->getAPIntValue(), VT));
Nate Begeman223df222005-09-08 20:18:10 +00003420 if (N01C)
Bill Wendling317bd702009-01-30 21:14:50 +00003421 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N0.getOperand(0),
3422 DAG.getConstant(N1C->getAPIntValue() ^
Dan Gohman002e5d02008-03-13 22:13:53 +00003423 N01C->getAPIntValue(), VT));
Nate Begeman223df222005-09-08 20:18:10 +00003424 }
3425 // fold (xor x, x) -> 0
Eric Christopher7bccf6a2011-02-16 04:50:12 +00003426 if (N0 == N1)
3427 return tryFoldToZero(N->getDebugLoc(), TLI, VT, DAG, LegalOperations);
Scott Michelfdc40a02009-02-17 22:15:04 +00003428
Chris Lattner35e5c142006-05-05 05:51:50 +00003429 // Simplify: xor (op x...), (op y...) -> (op (xor x, y))
3430 if (N0.getOpcode() == N1.getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003431 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00003432 if (Tmp.getNode()) return Tmp;
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003433 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003434
Chris Lattner3e104b12006-04-08 04:15:24 +00003435 // Simplify the expression using non-local knowledge.
Duncan Sands83ec4b62008-06-06 12:08:01 +00003436 if (!VT.isVector() &&
Dan Gohman475871a2008-07-27 21:46:04 +00003437 SimplifyDemandedBits(SDValue(N, 0)))
3438 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003439
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003440 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003441}
3442
Chris Lattnere70da202007-12-06 07:33:36 +00003443/// visitShiftByConstant - Handle transforms common to the three shifts, when
3444/// the shift amount is a constant.
Dan Gohman475871a2008-07-27 21:46:04 +00003445SDValue DAGCombiner::visitShiftByConstant(SDNode *N, unsigned Amt) {
Gabor Greifba36cb52008-08-28 21:40:38 +00003446 SDNode *LHS = N->getOperand(0).getNode();
Dan Gohman475871a2008-07-27 21:46:04 +00003447 if (!LHS->hasOneUse()) return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00003448
Chris Lattnere70da202007-12-06 07:33:36 +00003449 // We want to pull some binops through shifts, so that we have (and (shift))
3450 // instead of (shift (and)), likewise for add, or, xor, etc. This sort of
3451 // thing happens with address calculations, so it's important to canonicalize
3452 // it.
3453 bool HighBitSet = false; // Can we transform this if the high bit is set?
Scott Michelfdc40a02009-02-17 22:15:04 +00003454
Chris Lattnere70da202007-12-06 07:33:36 +00003455 switch (LHS->getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003456 default: return SDValue();
Chris Lattnere70da202007-12-06 07:33:36 +00003457 case ISD::OR:
3458 case ISD::XOR:
3459 HighBitSet = false; // We can only transform sra if the high bit is clear.
3460 break;
3461 case ISD::AND:
3462 HighBitSet = true; // We can only transform sra if the high bit is set.
3463 break;
3464 case ISD::ADD:
Scott Michelfdc40a02009-02-17 22:15:04 +00003465 if (N->getOpcode() != ISD::SHL)
Dan Gohman475871a2008-07-27 21:46:04 +00003466 return SDValue(); // only shl(add) not sr[al](add).
Chris Lattnere70da202007-12-06 07:33:36 +00003467 HighBitSet = false; // We can only transform sra if the high bit is clear.
3468 break;
3469 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003470
Chris Lattnere70da202007-12-06 07:33:36 +00003471 // We require the RHS of the binop to be a constant as well.
3472 ConstantSDNode *BinOpCst = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
Dan Gohman475871a2008-07-27 21:46:04 +00003473 if (!BinOpCst) return SDValue();
Bill Wendling88103372009-01-30 21:37:17 +00003474
3475 // FIXME: disable this unless the input to the binop is a shift by a constant.
3476 // If it is not a shift, it pessimizes some common cases like:
Chris Lattnerd3fd6d22007-12-06 07:47:55 +00003477 //
Bill Wendling88103372009-01-30 21:37:17 +00003478 // void foo(int *X, int i) { X[i & 1235] = 1; }
3479 // int bar(int *X, int i) { return X[i & 255]; }
Gabor Greifba36cb52008-08-28 21:40:38 +00003480 SDNode *BinOpLHSVal = LHS->getOperand(0).getNode();
Scott Michelfdc40a02009-02-17 22:15:04 +00003481 if ((BinOpLHSVal->getOpcode() != ISD::SHL &&
Chris Lattnerd3fd6d22007-12-06 07:47:55 +00003482 BinOpLHSVal->getOpcode() != ISD::SRA &&
3483 BinOpLHSVal->getOpcode() != ISD::SRL) ||
3484 !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1)))
Dan Gohman475871a2008-07-27 21:46:04 +00003485 return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00003486
Owen Andersone50ed302009-08-10 22:56:29 +00003487 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003488
Bill Wendling88103372009-01-30 21:37:17 +00003489 // If this is a signed shift right, and the high bit is modified by the
3490 // logical operation, do not perform the transformation. The highBitSet
3491 // boolean indicates the value of the high bit of the constant which would
3492 // cause it to be modified for this operation.
Chris Lattnere70da202007-12-06 07:33:36 +00003493 if (N->getOpcode() == ISD::SRA) {
Dan Gohman220a8232008-03-03 23:51:38 +00003494 bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative();
3495 if (BinOpRHSSignSet != HighBitSet)
Dan Gohman475871a2008-07-27 21:46:04 +00003496 return SDValue();
Chris Lattnere70da202007-12-06 07:33:36 +00003497 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003498
Chris Lattnere70da202007-12-06 07:33:36 +00003499 // Fold the constants, shifting the binop RHS by the shift amount.
Bill Wendling88103372009-01-30 21:37:17 +00003500 SDValue NewRHS = DAG.getNode(N->getOpcode(), LHS->getOperand(1).getDebugLoc(),
3501 N->getValueType(0),
3502 LHS->getOperand(1), N->getOperand(1));
Chris Lattnere70da202007-12-06 07:33:36 +00003503
3504 // Create the new shift.
Eric Christopher503a64d2010-12-09 04:48:06 +00003505 SDValue NewShift = DAG.getNode(N->getOpcode(),
3506 LHS->getOperand(0).getDebugLoc(),
Bill Wendling88103372009-01-30 21:37:17 +00003507 VT, LHS->getOperand(0), N->getOperand(1));
Chris Lattnere70da202007-12-06 07:33:36 +00003508
3509 // Create the new binop.
Bill Wendling88103372009-01-30 21:37:17 +00003510 return DAG.getNode(LHS->getOpcode(), N->getDebugLoc(), VT, NewShift, NewRHS);
Chris Lattnere70da202007-12-06 07:33:36 +00003511}
3512
Dan Gohman475871a2008-07-27 21:46:04 +00003513SDValue DAGCombiner::visitSHL(SDNode *N) {
3514 SDValue N0 = N->getOperand(0);
3515 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00003516 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3517 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003518 EVT VT = N0.getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00003519 unsigned OpSizeInBits = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00003520
Nate Begeman1d4d4142005-09-01 00:19:25 +00003521 // fold (shl c1, c2) -> c1<<c2
Nate Begeman646d7e22005-09-02 21:18:40 +00003522 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003523 return DAG.FoldConstantArithmetic(ISD::SHL, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003524 // fold (shl 0, x) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00003525 if (N0C && N0C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003526 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003527 // fold (shl x, c >= size(x)) -> undef
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003528 if (N1C && N1C->getZExtValue() >= OpSizeInBits)
Dale Johannesene8d72302009-02-06 23:05:02 +00003529 return DAG.getUNDEF(VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003530 // fold (shl x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003531 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003532 return N0;
Chad Rosier92bcd962011-06-14 22:29:10 +00003533 // fold (shl undef, x) -> 0
3534 if (N0.getOpcode() == ISD::UNDEF)
3535 return DAG.getConstant(0, VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003536 // if (shl x, c) is known to be zero, return 0
Dan Gohman475871a2008-07-27 21:46:04 +00003537 if (DAG.MaskedValueIsZero(SDValue(N, 0),
Dan Gohman87862e72009-12-11 21:31:27 +00003538 APInt::getAllOnesValue(OpSizeInBits)))
Nate Begeman83e75ec2005-09-06 04:43:02 +00003539 return DAG.getConstant(0, VT);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003540 // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
Evan Chengeb9f8922008-08-30 02:03:58 +00003541 if (N1.getOpcode() == ISD::TRUNCATE &&
Evan Cheng242ebd12008-09-22 18:19:24 +00003542 N1.getOperand(0).getOpcode() == ISD::AND &&
3543 N1.hasOneUse() && N1.getOperand(0).hasOneUse()) {
Evan Chengeb9f8922008-08-30 02:03:58 +00003544 SDValue N101 = N1.getOperand(0).getOperand(1);
Evan Cheng242ebd12008-09-22 18:19:24 +00003545 if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) {
Owen Andersone50ed302009-08-10 22:56:29 +00003546 EVT TruncVT = N1.getValueType();
Evan Cheng242ebd12008-09-22 18:19:24 +00003547 SDValue N100 = N1.getOperand(0).getOperand(0);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003548 APInt TruncC = N101C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00003549 TruncC = TruncC.trunc(TruncVT.getSizeInBits());
Bill Wendling88103372009-01-30 21:37:17 +00003550 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0,
Bill Wendlingfc4b6772009-02-01 11:19:36 +00003551 DAG.getNode(ISD::AND, N->getDebugLoc(), TruncVT,
3552 DAG.getNode(ISD::TRUNCATE,
3553 N->getDebugLoc(),
3554 TruncVT, N100),
Dan Gohmance9bc122009-01-27 20:39:34 +00003555 DAG.getConstant(TruncC, TruncVT)));
Evan Chengeb9f8922008-08-30 02:03:58 +00003556 }
3557 }
3558
Dan Gohman475871a2008-07-27 21:46:04 +00003559 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
3560 return SDValue(N, 0);
Bill Wendling88103372009-01-30 21:37:17 +00003561
3562 // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
Scott Michelfdc40a02009-02-17 22:15:04 +00003563 if (N1C && N0.getOpcode() == ISD::SHL &&
Nate Begeman1d4d4142005-09-01 00:19:25 +00003564 N0.getOperand(1).getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003565 uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
3566 uint64_t c2 = N1C->getZExtValue();
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003567 if (c1 + c2 >= OpSizeInBits)
Nate Begeman83e75ec2005-09-06 04:43:02 +00003568 return DAG.getConstant(0, VT);
Bill Wendling88103372009-01-30 21:37:17 +00003569 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0.getOperand(0),
Nate Begeman83e75ec2005-09-06 04:43:02 +00003570 DAG.getConstant(c1 + c2, N1.getValueType()));
Nate Begeman1d4d4142005-09-01 00:19:25 +00003571 }
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003572
3573 // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2)))
3574 // For this to be valid, the second form must not preserve any of the bits
3575 // that are shifted out by the inner shift in the first form. This means
3576 // the outer shift size must be >= the number of bits added by the ext.
3577 // As a corollary, we don't care what kind of ext it is.
3578 if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
3579 N0.getOpcode() == ISD::ANY_EXTEND ||
3580 N0.getOpcode() == ISD::SIGN_EXTEND) &&
3581 N0.getOperand(0).getOpcode() == ISD::SHL &&
3582 isa<ConstantSDNode>(N0.getOperand(0)->getOperand(1))) {
Owen Anderson95771af2011-02-25 21:41:48 +00003583 uint64_t c1 =
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003584 cast<ConstantSDNode>(N0.getOperand(0)->getOperand(1))->getZExtValue();
3585 uint64_t c2 = N1C->getZExtValue();
3586 EVT InnerShiftVT = N0.getOperand(0).getValueType();
3587 uint64_t InnerShiftSize = InnerShiftVT.getScalarType().getSizeInBits();
3588 if (c2 >= OpSizeInBits - InnerShiftSize) {
3589 if (c1 + c2 >= OpSizeInBits)
3590 return DAG.getConstant(0, VT);
3591 return DAG.getNode(ISD::SHL, N0->getDebugLoc(), VT,
3592 DAG.getNode(N0.getOpcode(), N0->getDebugLoc(), VT,
3593 N0.getOperand(0)->getOperand(0)),
3594 DAG.getConstant(c1 + c2, N1.getValueType()));
3595 }
3596 }
3597
Eli Friedman2a6d9eb2011-06-09 22:14:44 +00003598 // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
3599 // (and (srl x, (sub c1, c2), MASK)
Chandler Carruth62dfc512012-01-05 11:05:55 +00003600 // Only fold this if the inner shift has no other uses -- if it does, folding
3601 // this will increase the total number of instructions.
3602 if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse() &&
Nate Begeman1d4d4142005-09-01 00:19:25 +00003603 N0.getOperand(1).getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003604 uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
Evan Chengd101a722009-07-21 05:40:15 +00003605 if (c1 < VT.getSizeInBits()) {
3606 uint64_t c2 = N1C->getZExtValue();
Eli Friedman2a6d9eb2011-06-09 22:14:44 +00003607 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
3608 VT.getSizeInBits() - c1);
3609 SDValue Shift;
3610 if (c2 > c1) {
3611 Mask = Mask.shl(c2-c1);
3612 Shift = DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0.getOperand(0),
3613 DAG.getConstant(c2-c1, N1.getValueType()));
3614 } else {
3615 Mask = Mask.lshr(c1-c2);
3616 Shift = DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0.getOperand(0),
3617 DAG.getConstant(c1-c2, N1.getValueType()));
3618 }
3619 return DAG.getNode(ISD::AND, N0.getDebugLoc(), VT, Shift,
3620 DAG.getConstant(Mask, VT));
Evan Chengd101a722009-07-21 05:40:15 +00003621 }
Nate Begeman1d4d4142005-09-01 00:19:25 +00003622 }
Bill Wendling88103372009-01-30 21:37:17 +00003623 // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
Dan Gohman5cbd37e2009-08-06 09:18:59 +00003624 if (N1C && N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1)) {
3625 SDValue HiBitsMask =
3626 DAG.getConstant(APInt::getHighBitsSet(VT.getSizeInBits(),
3627 VT.getSizeInBits() -
3628 N1C->getZExtValue()),
3629 VT);
Bill Wendling88103372009-01-30 21:37:17 +00003630 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0.getOperand(0),
Dan Gohman5cbd37e2009-08-06 09:18:59 +00003631 HiBitsMask);
3632 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003633
Evan Chenge5b51ac2010-04-17 06:13:15 +00003634 if (N1C) {
3635 SDValue NewSHL = visitShiftByConstant(N, N1C->getZExtValue());
3636 if (NewSHL.getNode())
3637 return NewSHL;
3638 }
3639
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003640 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003641}
3642
Dan Gohman475871a2008-07-27 21:46:04 +00003643SDValue DAGCombiner::visitSRA(SDNode *N) {
3644 SDValue N0 = N->getOperand(0);
3645 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00003646 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3647 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003648 EVT VT = N0.getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00003649 unsigned OpSizeInBits = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00003650
Bill Wendling88103372009-01-30 21:37:17 +00003651 // fold (sra c1, c2) -> (sra c1, c2)
Nate Begeman646d7e22005-09-02 21:18:40 +00003652 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003653 return DAG.FoldConstantArithmetic(ISD::SRA, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003654 // fold (sra 0, x) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00003655 if (N0C && N0C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003656 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003657 // fold (sra -1, x) -> -1
Nate Begeman646d7e22005-09-02 21:18:40 +00003658 if (N0C && N0C->isAllOnesValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003659 return N0;
Bill Wendling88103372009-01-30 21:37:17 +00003660 // fold (sra x, (setge c, size(x))) -> undef
Dan Gohman87862e72009-12-11 21:31:27 +00003661 if (N1C && N1C->getZExtValue() >= OpSizeInBits)
Dale Johannesene8d72302009-02-06 23:05:02 +00003662 return DAG.getUNDEF(VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003663 // fold (sra x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003664 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003665 return N0;
Nate Begemanfb7217b2006-02-17 19:54:08 +00003666 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
3667 // sext_inreg.
3668 if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
Dan Gohman87862e72009-12-11 21:31:27 +00003669 unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
Dan Gohmand1996362010-01-09 02:13:55 +00003670 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
3671 if (VT.isVector())
3672 ExtVT = EVT::getVectorVT(*DAG.getContext(),
3673 ExtVT, VT.getVectorNumElements());
3674 if ((!LegalOperations ||
3675 TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT)))
Bill Wendling88103372009-01-30 21:37:17 +00003676 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT,
Dan Gohmand1996362010-01-09 02:13:55 +00003677 N0.getOperand(0), DAG.getValueType(ExtVT));
Nate Begemanfb7217b2006-02-17 19:54:08 +00003678 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00003679
Bill Wendling88103372009-01-30 21:37:17 +00003680 // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
Chris Lattner71d9ebc2006-02-28 06:23:04 +00003681 if (N1C && N0.getOpcode() == ISD::SRA) {
3682 if (ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003683 unsigned Sum = N1C->getZExtValue() + C1->getZExtValue();
Dan Gohman87862e72009-12-11 21:31:27 +00003684 if (Sum >= OpSizeInBits) Sum = OpSizeInBits-1;
Bill Wendling88103372009-01-30 21:37:17 +00003685 return DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, N0.getOperand(0),
Chris Lattner71d9ebc2006-02-28 06:23:04 +00003686 DAG.getConstant(Sum, N1C->getValueType(0)));
3687 }
3688 }
Christopher Lamb15cbde32008-03-19 08:30:06 +00003689
Bill Wendling88103372009-01-30 21:37:17 +00003690 // fold (sra (shl X, m), (sub result_size, n))
3691 // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for
Scott Michelfdc40a02009-02-17 22:15:04 +00003692 // result_size - n != m.
3693 // If truncate is free for the target sext(shl) is likely to result in better
Christopher Lambb9b04282008-03-20 04:31:39 +00003694 // code.
Christopher Lamb15cbde32008-03-19 08:30:06 +00003695 if (N0.getOpcode() == ISD::SHL) {
3696 // Get the two constanst of the shifts, CN0 = m, CN = n.
3697 const ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3698 if (N01C && N1C) {
Christopher Lambb9b04282008-03-20 04:31:39 +00003699 // Determine what the truncate's result bitsize and type would be.
Owen Andersone50ed302009-08-10 22:56:29 +00003700 EVT TruncVT =
Eric Christopher503a64d2010-12-09 04:48:06 +00003701 EVT::getIntegerVT(*DAG.getContext(),
3702 OpSizeInBits - N1C->getZExtValue());
Christopher Lambb9b04282008-03-20 04:31:39 +00003703 // Determine the residual right-shift amount.
Torok Edwin6bb49582009-05-23 17:29:48 +00003704 signed ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue();
Duncan Sandsd4b9c172008-06-13 19:07:40 +00003705
Scott Michelfdc40a02009-02-17 22:15:04 +00003706 // If the shift is not a no-op (in which case this should be just a sign
3707 // extend already), the truncated to type is legal, sign_extend is legal
Dan Gohmanf451cb82010-02-10 16:03:48 +00003708 // on that type, and the truncate to that type is both legal and free,
Christopher Lambb9b04282008-03-20 04:31:39 +00003709 // perform the transform.
Torok Edwin6bb49582009-05-23 17:29:48 +00003710 if ((ShiftAmt > 0) &&
Dan Gohmanf560ffa2009-01-28 17:46:25 +00003711 TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) &&
3712 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) &&
Evan Cheng260e07e2008-03-20 02:18:41 +00003713 TLI.isTruncateFree(VT, TruncVT)) {
Christopher Lambb9b04282008-03-20 04:31:39 +00003714
Owen Anderson95771af2011-02-25 21:41:48 +00003715 SDValue Amt = DAG.getConstant(ShiftAmt,
3716 getShiftAmountTy(N0.getOperand(0).getValueType()));
Bill Wendling88103372009-01-30 21:37:17 +00003717 SDValue Shift = DAG.getNode(ISD::SRL, N0.getDebugLoc(), VT,
3718 N0.getOperand(0), Amt);
3719 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(), TruncVT,
3720 Shift);
3721 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(),
3722 N->getValueType(0), Trunc);
Christopher Lamb15cbde32008-03-19 08:30:06 +00003723 }
3724 }
3725 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003726
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003727 // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))).
Evan Chengeb9f8922008-08-30 02:03:58 +00003728 if (N1.getOpcode() == ISD::TRUNCATE &&
Evan Cheng242ebd12008-09-22 18:19:24 +00003729 N1.getOperand(0).getOpcode() == ISD::AND &&
3730 N1.hasOneUse() && N1.getOperand(0).hasOneUse()) {
Evan Chengeb9f8922008-08-30 02:03:58 +00003731 SDValue N101 = N1.getOperand(0).getOperand(1);
Evan Cheng242ebd12008-09-22 18:19:24 +00003732 if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) {
Owen Andersone50ed302009-08-10 22:56:29 +00003733 EVT TruncVT = N1.getValueType();
Evan Cheng242ebd12008-09-22 18:19:24 +00003734 SDValue N100 = N1.getOperand(0).getOperand(0);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003735 APInt TruncC = N101C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00003736 TruncC = TruncC.trunc(TruncVT.getScalarType().getSizeInBits());
Bill Wendling88103372009-01-30 21:37:17 +00003737 return DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, N0,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003738 DAG.getNode(ISD::AND, N->getDebugLoc(),
Bill Wendling88103372009-01-30 21:37:17 +00003739 TruncVT,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003740 DAG.getNode(ISD::TRUNCATE,
3741 N->getDebugLoc(),
3742 TruncVT, N100),
Dan Gohmance9bc122009-01-27 20:39:34 +00003743 DAG.getConstant(TruncC, TruncVT)));
Evan Chengeb9f8922008-08-30 02:03:58 +00003744 }
3745 }
3746
Benjamin Kramer9b108a32011-01-30 16:38:43 +00003747 // fold (sra (trunc (sr x, c1)), c2) -> (trunc (sra x, c1+c2))
3748 // if c1 is equal to the number of bits the trunc removes
3749 if (N0.getOpcode() == ISD::TRUNCATE &&
3750 (N0.getOperand(0).getOpcode() == ISD::SRL ||
3751 N0.getOperand(0).getOpcode() == ISD::SRA) &&
3752 N0.getOperand(0).hasOneUse() &&
3753 N0.getOperand(0).getOperand(1).hasOneUse() &&
3754 N1C && isa<ConstantSDNode>(N0.getOperand(0).getOperand(1))) {
3755 EVT LargeVT = N0.getOperand(0).getValueType();
3756 ConstantSDNode *LargeShiftAmt =
3757 cast<ConstantSDNode>(N0.getOperand(0).getOperand(1));
3758
3759 if (LargeVT.getScalarType().getSizeInBits() - OpSizeInBits ==
3760 LargeShiftAmt->getZExtValue()) {
3761 SDValue Amt =
3762 DAG.getConstant(LargeShiftAmt->getZExtValue() + N1C->getZExtValue(),
Owen Anderson95771af2011-02-25 21:41:48 +00003763 getShiftAmountTy(N0.getOperand(0).getOperand(0).getValueType()));
Benjamin Kramer9b108a32011-01-30 16:38:43 +00003764 SDValue SRA = DAG.getNode(ISD::SRA, N->getDebugLoc(), LargeVT,
3765 N0.getOperand(0).getOperand(0), Amt);
3766 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, SRA);
3767 }
3768 }
3769
Scott Michelfdc40a02009-02-17 22:15:04 +00003770 // Simplify, based on bits shifted out of the LHS.
Dan Gohman475871a2008-07-27 21:46:04 +00003771 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
3772 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003773
3774
Nate Begeman1d4d4142005-09-01 00:19:25 +00003775 // If the sign bit is known to be zero, switch this to a SRL.
Dan Gohman2e68b6f2008-02-25 21:11:39 +00003776 if (DAG.SignBitIsZero(N0))
Bill Wendling88103372009-01-30 21:37:17 +00003777 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0, N1);
Chris Lattnere70da202007-12-06 07:33:36 +00003778
Evan Chenge5b51ac2010-04-17 06:13:15 +00003779 if (N1C) {
3780 SDValue NewSRA = visitShiftByConstant(N, N1C->getZExtValue());
3781 if (NewSRA.getNode())
3782 return NewSRA;
3783 }
3784
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003785 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003786}
3787
Dan Gohman475871a2008-07-27 21:46:04 +00003788SDValue DAGCombiner::visitSRL(SDNode *N) {
3789 SDValue N0 = N->getOperand(0);
3790 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00003791 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3792 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003793 EVT VT = N0.getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00003794 unsigned OpSizeInBits = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00003795
Nate Begeman1d4d4142005-09-01 00:19:25 +00003796 // fold (srl c1, c2) -> c1 >>u c2
Nate Begeman646d7e22005-09-02 21:18:40 +00003797 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003798 return DAG.FoldConstantArithmetic(ISD::SRL, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003799 // fold (srl 0, x) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00003800 if (N0C && N0C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003801 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003802 // fold (srl x, c >= size(x)) -> undef
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003803 if (N1C && N1C->getZExtValue() >= OpSizeInBits)
Dale Johannesene8d72302009-02-06 23:05:02 +00003804 return DAG.getUNDEF(VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003805 // fold (srl x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003806 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003807 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003808 // if (srl x, c) is known to be zero, return 0
Dan Gohman475871a2008-07-27 21:46:04 +00003809 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
Dan Gohman2e68b6f2008-02-25 21:11:39 +00003810 APInt::getAllOnesValue(OpSizeInBits)))
Nate Begeman83e75ec2005-09-06 04:43:02 +00003811 return DAG.getConstant(0, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00003812
Bill Wendling88103372009-01-30 21:37:17 +00003813 // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
Scott Michelfdc40a02009-02-17 22:15:04 +00003814 if (N1C && N0.getOpcode() == ISD::SRL &&
Nate Begeman1d4d4142005-09-01 00:19:25 +00003815 N0.getOperand(1).getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003816 uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
3817 uint64_t c2 = N1C->getZExtValue();
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003818 if (c1 + c2 >= OpSizeInBits)
Nate Begeman83e75ec2005-09-06 04:43:02 +00003819 return DAG.getConstant(0, VT);
Bill Wendling88103372009-01-30 21:37:17 +00003820 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0.getOperand(0),
Nate Begeman83e75ec2005-09-06 04:43:02 +00003821 DAG.getConstant(c1 + c2, N1.getValueType()));
Nate Begeman1d4d4142005-09-01 00:19:25 +00003822 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003823
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003824 // fold (srl (trunc (srl x, c1)), c2) -> 0 or (trunc (srl x, (add c1, c2)))
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003825 if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
3826 N0.getOperand(0).getOpcode() == ISD::SRL &&
Dale Johannesen025cc6e2010-12-20 20:10:50 +00003827 isa<ConstantSDNode>(N0.getOperand(0)->getOperand(1))) {
Owen Anderson95771af2011-02-25 21:41:48 +00003828 uint64_t c1 =
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003829 cast<ConstantSDNode>(N0.getOperand(0)->getOperand(1))->getZExtValue();
3830 uint64_t c2 = N1C->getZExtValue();
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003831 EVT InnerShiftVT = N0.getOperand(0).getValueType();
3832 EVT ShiftCountVT = N0.getOperand(0)->getOperand(1).getValueType();
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003833 uint64_t InnerShiftSize = InnerShiftVT.getScalarType().getSizeInBits();
Dale Johannesen025cc6e2010-12-20 20:10:50 +00003834 // This is only valid if the OpSizeInBits + c1 = size of inner shift.
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003835 if (c1 + OpSizeInBits == InnerShiftSize) {
3836 if (c1 + c2 >= InnerShiftSize)
3837 return DAG.getConstant(0, VT);
3838 return DAG.getNode(ISD::TRUNCATE, N0->getDebugLoc(), VT,
Owen Anderson95771af2011-02-25 21:41:48 +00003839 DAG.getNode(ISD::SRL, N0->getDebugLoc(), InnerShiftVT,
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003840 N0.getOperand(0)->getOperand(0),
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003841 DAG.getConstant(c1 + c2, ShiftCountVT)));
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003842 }
3843 }
3844
Chris Lattnerefcddc32010-04-15 05:28:43 +00003845 // fold (srl (shl x, c), c) -> (and x, cst2)
3846 if (N1C && N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1 &&
3847 N0.getValueSizeInBits() <= 64) {
3848 uint64_t ShAmt = N1C->getZExtValue()+64-N0.getValueSizeInBits();
3849 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0.getOperand(0),
3850 DAG.getConstant(~0ULL >> ShAmt, VT));
3851 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003852
Scott Michelfdc40a02009-02-17 22:15:04 +00003853
Chris Lattner06afe072006-05-05 22:53:17 +00003854 // fold (srl (anyextend x), c) -> (anyextend (srl x, c))
3855 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
3856 // Shifting in all undef bits?
Owen Andersone50ed302009-08-10 22:56:29 +00003857 EVT SmallVT = N0.getOperand(0).getValueType();
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003858 if (N1C->getZExtValue() >= SmallVT.getSizeInBits())
Dale Johannesene8d72302009-02-06 23:05:02 +00003859 return DAG.getUNDEF(VT);
Chris Lattner06afe072006-05-05 22:53:17 +00003860
Evan Chenge5b51ac2010-04-17 06:13:15 +00003861 if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
Owen Andersona34d9362011-04-14 17:30:49 +00003862 uint64_t ShiftAmt = N1C->getZExtValue();
Evan Chenge5b51ac2010-04-17 06:13:15 +00003863 SDValue SmallShift = DAG.getNode(ISD::SRL, N0.getDebugLoc(), SmallVT,
Owen Andersona34d9362011-04-14 17:30:49 +00003864 N0.getOperand(0),
3865 DAG.getConstant(ShiftAmt, getShiftAmountTy(SmallVT)));
Evan Chenge5b51ac2010-04-17 06:13:15 +00003866 AddToWorkList(SmallShift.getNode());
3867 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, SmallShift);
3868 }
Chris Lattner06afe072006-05-05 22:53:17 +00003869 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003870
Chris Lattner3657ffe2006-10-12 20:23:19 +00003871 // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign
3872 // bit, which is unmodified by sra.
Bill Wendling88103372009-01-30 21:37:17 +00003873 if (N1C && N1C->getZExtValue() + 1 == VT.getSizeInBits()) {
Chris Lattner3657ffe2006-10-12 20:23:19 +00003874 if (N0.getOpcode() == ISD::SRA)
Bill Wendling88103372009-01-30 21:37:17 +00003875 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0.getOperand(0), N1);
Chris Lattner3657ffe2006-10-12 20:23:19 +00003876 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003877
Sylvestre Ledru94c22712012-09-27 10:14:43 +00003878 // fold (srl (ctlz x), "5") -> x iff x has one bit set (the low bit).
Scott Michelfdc40a02009-02-17 22:15:04 +00003879 if (N1C && N0.getOpcode() == ISD::CTLZ &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00003880 N1C->getAPIntValue() == Log2_32(VT.getSizeInBits())) {
Dan Gohman948d8ea2008-02-20 16:33:30 +00003881 APInt KnownZero, KnownOne;
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00003882 DAG.ComputeMaskedBits(N0.getOperand(0), KnownZero, KnownOne);
Scott Michelfdc40a02009-02-17 22:15:04 +00003883
Chris Lattner350bec02006-04-02 06:11:11 +00003884 // If any of the input bits are KnownOne, then the input couldn't be all
3885 // zeros, thus the result of the srl will always be zero.
Dan Gohman948d8ea2008-02-20 16:33:30 +00003886 if (KnownOne.getBoolValue()) return DAG.getConstant(0, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00003887
Chris Lattner350bec02006-04-02 06:11:11 +00003888 // If all of the bits input the to ctlz node are known to be zero, then
3889 // the result of the ctlz is "32" and the result of the shift is one.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00003890 APInt UnknownBits = ~KnownZero;
Chris Lattner350bec02006-04-02 06:11:11 +00003891 if (UnknownBits == 0) return DAG.getConstant(1, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00003892
Chris Lattner350bec02006-04-02 06:11:11 +00003893 // Otherwise, check to see if there is exactly one bit input to the ctlz.
Bill Wendling88103372009-01-30 21:37:17 +00003894 if ((UnknownBits & (UnknownBits - 1)) == 0) {
Chris Lattner350bec02006-04-02 06:11:11 +00003895 // Okay, we know that only that the single bit specified by UnknownBits
Bill Wendling88103372009-01-30 21:37:17 +00003896 // could be set on input to the CTLZ node. If this bit is set, the SRL
3897 // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
3898 // to an SRL/XOR pair, which is likely to simplify more.
Dan Gohman948d8ea2008-02-20 16:33:30 +00003899 unsigned ShAmt = UnknownBits.countTrailingZeros();
Dan Gohman475871a2008-07-27 21:46:04 +00003900 SDValue Op = N0.getOperand(0);
Bill Wendling88103372009-01-30 21:37:17 +00003901
Chris Lattner350bec02006-04-02 06:11:11 +00003902 if (ShAmt) {
Bill Wendling88103372009-01-30 21:37:17 +00003903 Op = DAG.getNode(ISD::SRL, N0.getDebugLoc(), VT, Op,
Owen Anderson95771af2011-02-25 21:41:48 +00003904 DAG.getConstant(ShAmt, getShiftAmountTy(Op.getValueType())));
Gabor Greifba36cb52008-08-28 21:40:38 +00003905 AddToWorkList(Op.getNode());
Chris Lattner350bec02006-04-02 06:11:11 +00003906 }
Bill Wendling88103372009-01-30 21:37:17 +00003907
3908 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT,
3909 Op, DAG.getConstant(1, VT));
Chris Lattner350bec02006-04-02 06:11:11 +00003910 }
3911 }
Evan Chengeb9f8922008-08-30 02:03:58 +00003912
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003913 // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))).
Evan Chengeb9f8922008-08-30 02:03:58 +00003914 if (N1.getOpcode() == ISD::TRUNCATE &&
Evan Cheng242ebd12008-09-22 18:19:24 +00003915 N1.getOperand(0).getOpcode() == ISD::AND &&
3916 N1.hasOneUse() && N1.getOperand(0).hasOneUse()) {
Evan Chengeb9f8922008-08-30 02:03:58 +00003917 SDValue N101 = N1.getOperand(0).getOperand(1);
Evan Cheng242ebd12008-09-22 18:19:24 +00003918 if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) {
Owen Andersone50ed302009-08-10 22:56:29 +00003919 EVT TruncVT = N1.getValueType();
Evan Cheng242ebd12008-09-22 18:19:24 +00003920 SDValue N100 = N1.getOperand(0).getOperand(0);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003921 APInt TruncC = N101C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00003922 TruncC = TruncC.trunc(TruncVT.getSizeInBits());
Bill Wendling88103372009-01-30 21:37:17 +00003923 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003924 DAG.getNode(ISD::AND, N->getDebugLoc(),
Bill Wendling88103372009-01-30 21:37:17 +00003925 TruncVT,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003926 DAG.getNode(ISD::TRUNCATE,
3927 N->getDebugLoc(),
3928 TruncVT, N100),
Dan Gohmance9bc122009-01-27 20:39:34 +00003929 DAG.getConstant(TruncC, TruncVT)));
Evan Chengeb9f8922008-08-30 02:03:58 +00003930 }
3931 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003932
Chris Lattner61a4c072007-04-18 03:06:49 +00003933 // fold operands of srl based on knowledge that the low bits are not
3934 // demanded.
Dan Gohman475871a2008-07-27 21:46:04 +00003935 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
3936 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003937
Evan Cheng9ab2b982009-12-18 21:31:31 +00003938 if (N1C) {
3939 SDValue NewSRL = visitShiftByConstant(N, N1C->getZExtValue());
3940 if (NewSRL.getNode())
3941 return NewSRL;
3942 }
3943
Dan Gohman4e39e9d2010-06-24 14:30:44 +00003944 // Attempt to convert a srl of a load into a narrower zero-extending load.
3945 SDValue NarrowLoad = ReduceLoadWidth(N);
3946 if (NarrowLoad.getNode())
3947 return NarrowLoad;
3948
Evan Cheng9ab2b982009-12-18 21:31:31 +00003949 // Here is a common situation. We want to optimize:
3950 //
3951 // %a = ...
3952 // %b = and i32 %a, 2
3953 // %c = srl i32 %b, 1
3954 // brcond i32 %c ...
3955 //
3956 // into
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003957 //
Evan Cheng9ab2b982009-12-18 21:31:31 +00003958 // %a = ...
3959 // %b = and %a, 2
3960 // %c = setcc eq %b, 0
3961 // brcond %c ...
3962 //
3963 // However when after the source operand of SRL is optimized into AND, the SRL
3964 // itself may not be optimized further. Look for it and add the BRCOND into
3965 // the worklist.
Evan Chengd40d03e2010-01-06 19:38:29 +00003966 if (N->hasOneUse()) {
3967 SDNode *Use = *N->use_begin();
3968 if (Use->getOpcode() == ISD::BRCOND)
3969 AddToWorkList(Use);
3970 else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) {
3971 // Also look pass the truncate.
3972 Use = *Use->use_begin();
3973 if (Use->getOpcode() == ISD::BRCOND)
3974 AddToWorkList(Use);
3975 }
3976 }
Evan Cheng9ab2b982009-12-18 21:31:31 +00003977
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003978 return SDValue();
Evan Cheng4c26e932010-04-19 19:29:22 +00003979}
3980
Dan Gohman475871a2008-07-27 21:46:04 +00003981SDValue DAGCombiner::visitCTLZ(SDNode *N) {
3982 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00003983 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003984
3985 // fold (ctlz c1) -> c2
Chris Lattner310b5782006-05-06 23:06:26 +00003986 if (isa<ConstantSDNode>(N0))
Bill Wendling34584e62009-01-30 22:02:18 +00003987 return DAG.getNode(ISD::CTLZ, N->getDebugLoc(), VT, N0);
Dan Gohman475871a2008-07-27 21:46:04 +00003988 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003989}
3990
Chandler Carruth63974b22011-12-13 01:56:10 +00003991SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
3992 SDValue N0 = N->getOperand(0);
3993 EVT VT = N->getValueType(0);
3994
3995 // fold (ctlz_zero_undef c1) -> c2
3996 if (isa<ConstantSDNode>(N0))
3997 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, N->getDebugLoc(), VT, N0);
3998 return SDValue();
3999}
4000
Dan Gohman475871a2008-07-27 21:46:04 +00004001SDValue DAGCombiner::visitCTTZ(SDNode *N) {
4002 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004003 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00004004
Nate Begeman1d4d4142005-09-01 00:19:25 +00004005 // fold (cttz c1) -> c2
Chris Lattner310b5782006-05-06 23:06:26 +00004006 if (isa<ConstantSDNode>(N0))
Bill Wendling34584e62009-01-30 22:02:18 +00004007 return DAG.getNode(ISD::CTTZ, N->getDebugLoc(), VT, N0);
Dan Gohman475871a2008-07-27 21:46:04 +00004008 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00004009}
4010
Chandler Carruth63974b22011-12-13 01:56:10 +00004011SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
4012 SDValue N0 = N->getOperand(0);
4013 EVT VT = N->getValueType(0);
4014
4015 // fold (cttz_zero_undef c1) -> c2
4016 if (isa<ConstantSDNode>(N0))
4017 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, N->getDebugLoc(), VT, N0);
4018 return SDValue();
4019}
4020
Dan Gohman475871a2008-07-27 21:46:04 +00004021SDValue DAGCombiner::visitCTPOP(SDNode *N) {
4022 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004023 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00004024
Nate Begeman1d4d4142005-09-01 00:19:25 +00004025 // fold (ctpop c1) -> c2
Chris Lattner310b5782006-05-06 23:06:26 +00004026 if (isa<ConstantSDNode>(N0))
Bill Wendling34584e62009-01-30 22:02:18 +00004027 return DAG.getNode(ISD::CTPOP, N->getDebugLoc(), VT, N0);
Dan Gohman475871a2008-07-27 21:46:04 +00004028 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00004029}
4030
Dan Gohman475871a2008-07-27 21:46:04 +00004031SDValue DAGCombiner::visitSELECT(SDNode *N) {
4032 SDValue N0 = N->getOperand(0);
4033 SDValue N1 = N->getOperand(1);
4034 SDValue N2 = N->getOperand(2);
Nate Begeman452d7beb2005-09-16 00:54:12 +00004035 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
4036 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
4037 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
Owen Andersone50ed302009-08-10 22:56:29 +00004038 EVT VT = N->getValueType(0);
4039 EVT VT0 = N0.getValueType();
Nate Begeman44728a72005-09-19 22:34:01 +00004040
Bill Wendling34584e62009-01-30 22:02:18 +00004041 // fold (select C, X, X) -> X
Nate Begeman452d7beb2005-09-16 00:54:12 +00004042 if (N1 == N2)
4043 return N1;
Bill Wendling34584e62009-01-30 22:02:18 +00004044 // fold (select true, X, Y) -> X
Nate Begeman452d7beb2005-09-16 00:54:12 +00004045 if (N0C && !N0C->isNullValue())
4046 return N1;
Bill Wendling34584e62009-01-30 22:02:18 +00004047 // fold (select false, X, Y) -> Y
Nate Begeman452d7beb2005-09-16 00:54:12 +00004048 if (N0C && N0C->isNullValue())
4049 return N2;
Bill Wendling34584e62009-01-30 22:02:18 +00004050 // fold (select C, 1, X) -> (or C, X)
Owen Anderson825b72b2009-08-11 20:47:22 +00004051 if (VT == MVT::i1 && N1C && N1C->getAPIntValue() == 1)
Bill Wendling34584e62009-01-30 22:02:18 +00004052 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N2);
4053 // fold (select C, 0, 1) -> (xor C, 1)
Bob Wilson67ba2232009-01-22 22:05:48 +00004054 if (VT.isInteger() &&
Owen Anderson825b72b2009-08-11 20:47:22 +00004055 (VT0 == MVT::i1 ||
Bob Wilson67ba2232009-01-22 22:05:48 +00004056 (VT0.isInteger() &&
Nadav Rotem6dfabb62012-09-20 08:53:31 +00004057 TLI.getBooleanContents(false) ==
4058 TargetLowering::ZeroOrOneBooleanContent)) &&
Dan Gohman002e5d02008-03-13 22:13:53 +00004059 N1C && N2C && N1C->isNullValue() && N2C->getAPIntValue() == 1) {
Bill Wendling34584e62009-01-30 22:02:18 +00004060 SDValue XORNode;
Evan Cheng571c4782007-08-18 05:57:05 +00004061 if (VT == VT0)
Bill Wendling34584e62009-01-30 22:02:18 +00004062 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT0,
4063 N0, DAG.getConstant(1, VT0));
4064 XORNode = DAG.getNode(ISD::XOR, N0.getDebugLoc(), VT0,
4065 N0, DAG.getConstant(1, VT0));
Gabor Greifba36cb52008-08-28 21:40:38 +00004066 AddToWorkList(XORNode.getNode());
Duncan Sands8e4eb092008-06-08 20:54:56 +00004067 if (VT.bitsGT(VT0))
Bill Wendling34584e62009-01-30 22:02:18 +00004068 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, XORNode);
4069 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, XORNode);
Evan Cheng571c4782007-08-18 05:57:05 +00004070 }
Bill Wendling34584e62009-01-30 22:02:18 +00004071 // fold (select C, 0, X) -> (and (not C), X)
Owen Anderson825b72b2009-08-11 20:47:22 +00004072 if (VT == VT0 && VT == MVT::i1 && N1C && N1C->isNullValue()) {
Bill Wendling7581bfa2009-01-30 23:03:19 +00004073 SDValue NOTNode = DAG.getNOT(N0.getDebugLoc(), N0, VT);
Bob Wilson4c245462009-01-22 17:39:32 +00004074 AddToWorkList(NOTNode.getNode());
Bill Wendling7581bfa2009-01-30 23:03:19 +00004075 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, NOTNode, N2);
Nate Begeman452d7beb2005-09-16 00:54:12 +00004076 }
Bill Wendling34584e62009-01-30 22:02:18 +00004077 // fold (select C, X, 1) -> (or (not C), X)
Owen Anderson825b72b2009-08-11 20:47:22 +00004078 if (VT == VT0 && VT == MVT::i1 && N2C && N2C->getAPIntValue() == 1) {
Bill Wendling34584e62009-01-30 22:02:18 +00004079 SDValue NOTNode = DAG.getNOT(N0.getDebugLoc(), N0, VT);
Bob Wilson4c245462009-01-22 17:39:32 +00004080 AddToWorkList(NOTNode.getNode());
Bill Wendling7581bfa2009-01-30 23:03:19 +00004081 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, NOTNode, N1);
Nate Begeman452d7beb2005-09-16 00:54:12 +00004082 }
Bill Wendling34584e62009-01-30 22:02:18 +00004083 // fold (select C, X, 0) -> (and C, X)
Owen Anderson825b72b2009-08-11 20:47:22 +00004084 if (VT == MVT::i1 && N2C && N2C->isNullValue())
Bill Wendling34584e62009-01-30 22:02:18 +00004085 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0, N1);
4086 // fold (select X, X, Y) -> (or X, Y)
4087 // fold (select X, 1, Y) -> (or X, Y)
Owen Anderson825b72b2009-08-11 20:47:22 +00004088 if (VT == MVT::i1 && (N0 == N1 || (N1C && N1C->getAPIntValue() == 1)))
Bill Wendling34584e62009-01-30 22:02:18 +00004089 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N2);
4090 // fold (select X, Y, X) -> (and X, Y)
4091 // fold (select X, Y, 0) -> (and X, Y)
Owen Anderson825b72b2009-08-11 20:47:22 +00004092 if (VT == MVT::i1 && (N0 == N2 || (N2C && N2C->getAPIntValue() == 0)))
Bill Wendling34584e62009-01-30 22:02:18 +00004093 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00004094
Chris Lattner40c62d52005-10-18 06:04:22 +00004095 // If we can fold this based on the true/false value, do so.
4096 if (SimplifySelectOps(N, N1, N2))
Dan Gohman475871a2008-07-27 21:46:04 +00004097 return SDValue(N, 0); // Don't revisit N.
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004098
Nate Begeman44728a72005-09-19 22:34:01 +00004099 // fold selects based on a setcc into other things, such as min/max/abs
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00004100 if (N0.getOpcode() == ISD::SETCC) {
Nate Begeman750ac1b2006-02-01 07:19:44 +00004101 // FIXME:
Owen Anderson825b72b2009-08-11 20:47:22 +00004102 // Check against MVT::Other for SELECT_CC, which is a workaround for targets
Nate Begeman750ac1b2006-02-01 07:19:44 +00004103 // having to say they don't support SELECT_CC on every type the DAG knows
4104 // about, since there is no way to mark an opcode illegal at all value types
Owen Anderson825b72b2009-08-11 20:47:22 +00004105 if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, MVT::Other) &&
Dan Gohman4ea48042009-08-02 16:19:38 +00004106 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))
Bill Wendling34584e62009-01-30 22:02:18 +00004107 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), VT,
4108 N0.getOperand(0), N0.getOperand(1),
Nate Begeman750ac1b2006-02-01 07:19:44 +00004109 N1, N2, N0.getOperand(2));
Chris Lattner600fec32009-03-11 05:08:08 +00004110 return SimplifySelect(N->getDebugLoc(), N0, N1, N2);
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00004111 }
Bill Wendling34584e62009-01-30 22:02:18 +00004112
Dan Gohman475871a2008-07-27 21:46:04 +00004113 return SDValue();
Nate Begeman452d7beb2005-09-16 00:54:12 +00004114}
4115
Dan Gohman475871a2008-07-27 21:46:04 +00004116SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
4117 SDValue N0 = N->getOperand(0);
4118 SDValue N1 = N->getOperand(1);
4119 SDValue N2 = N->getOperand(2);
4120 SDValue N3 = N->getOperand(3);
4121 SDValue N4 = N->getOperand(4);
Nate Begeman44728a72005-09-19 22:34:01 +00004122 ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get();
Scott Michelfdc40a02009-02-17 22:15:04 +00004123
Nate Begeman44728a72005-09-19 22:34:01 +00004124 // fold select_cc lhs, rhs, x, x, cc -> x
4125 if (N2 == N3)
4126 return N2;
Scott Michelfdc40a02009-02-17 22:15:04 +00004127
Chris Lattner5f42a242006-09-20 06:19:26 +00004128 // Determine if the condition we're dealing with is constant
Duncan Sands5480c042009-01-01 15:52:00 +00004129 SDValue SCC = SimplifySetCC(TLI.getSetCCResultType(N0.getValueType()),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00004130 N0, N1, CC, N->getDebugLoc(), false);
Gabor Greifba36cb52008-08-28 21:40:38 +00004131 if (SCC.getNode()) AddToWorkList(SCC.getNode());
Chris Lattner5f42a242006-09-20 06:19:26 +00004132
Gabor Greifba36cb52008-08-28 21:40:38 +00004133 if (ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.getNode())) {
Dan Gohman002e5d02008-03-13 22:13:53 +00004134 if (!SCCC->isNullValue())
Chris Lattner5f42a242006-09-20 06:19:26 +00004135 return N2; // cond always true -> true val
4136 else
4137 return N3; // cond always false -> false val
4138 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004139
Chris Lattner5f42a242006-09-20 06:19:26 +00004140 // Fold to a simpler select_cc
Gabor Greifba36cb52008-08-28 21:40:38 +00004141 if (SCC.getNode() && SCC.getOpcode() == ISD::SETCC)
Scott Michelfdc40a02009-02-17 22:15:04 +00004142 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), N2.getValueType(),
4143 SCC.getOperand(0), SCC.getOperand(1), N2, N3,
Chris Lattner5f42a242006-09-20 06:19:26 +00004144 SCC.getOperand(2));
Scott Michelfdc40a02009-02-17 22:15:04 +00004145
Chris Lattner40c62d52005-10-18 06:04:22 +00004146 // If we can fold this based on the true/false value, do so.
4147 if (SimplifySelectOps(N, N2, N3))
Dan Gohman475871a2008-07-27 21:46:04 +00004148 return SDValue(N, 0); // Don't revisit N.
Scott Michelfdc40a02009-02-17 22:15:04 +00004149
Nate Begeman44728a72005-09-19 22:34:01 +00004150 // fold select_cc into other things, such as min/max/abs
Bill Wendling836ca7d2009-01-30 23:59:18 +00004151 return SimplifySelectCC(N->getDebugLoc(), N0, N1, N2, N3, CC);
Nate Begeman452d7beb2005-09-16 00:54:12 +00004152}
4153
Dan Gohman475871a2008-07-27 21:46:04 +00004154SDValue DAGCombiner::visitSETCC(SDNode *N) {
Nate Begeman452d7beb2005-09-16 00:54:12 +00004155 return SimplifySetCC(N->getValueType(0), N->getOperand(0), N->getOperand(1),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00004156 cast<CondCodeSDNode>(N->getOperand(2))->get(),
4157 N->getDebugLoc());
Nate Begeman452d7beb2005-09-16 00:54:12 +00004158}
4159
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004160// ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
Dan Gohman57fc82d2009-04-09 03:51:29 +00004161// "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004162// transformation. Returns true if extension are possible and the above
Scott Michelfdc40a02009-02-17 22:15:04 +00004163// mentioned transformation is profitable.
Dan Gohman475871a2008-07-27 21:46:04 +00004164static bool ExtendUsesToFormExtLoad(SDNode *N, SDValue N0,
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004165 unsigned ExtOpc,
4166 SmallVector<SDNode*, 4> &ExtendNodes,
Dan Gohman79ce2762009-01-15 19:20:50 +00004167 const TargetLowering &TLI) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004168 bool HasCopyToRegUses = false;
4169 bool isTruncFree = TLI.isTruncateFree(N->getValueType(0), N0.getValueType());
Gabor Greif12632d22008-08-30 19:29:20 +00004170 for (SDNode::use_iterator UI = N0.getNode()->use_begin(),
4171 UE = N0.getNode()->use_end();
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004172 UI != UE; ++UI) {
Dan Gohman89684502008-07-27 20:43:25 +00004173 SDNode *User = *UI;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004174 if (User == N)
4175 continue;
Dan Gohman57fc82d2009-04-09 03:51:29 +00004176 if (UI.getUse().getResNo() != N0.getResNo())
4177 continue;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004178 // FIXME: Only extend SETCC N, N and SETCC N, c for now.
Dan Gohman57fc82d2009-04-09 03:51:29 +00004179 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004180 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get();
4181 if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC))
4182 // Sign bits will be lost after a zext.
4183 return false;
4184 bool Add = false;
4185 for (unsigned i = 0; i != 2; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00004186 SDValue UseOp = User->getOperand(i);
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004187 if (UseOp == N0)
4188 continue;
4189 if (!isa<ConstantSDNode>(UseOp))
4190 return false;
4191 Add = true;
4192 }
4193 if (Add)
4194 ExtendNodes.push_back(User);
Dan Gohman57fc82d2009-04-09 03:51:29 +00004195 continue;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004196 }
Dan Gohman57fc82d2009-04-09 03:51:29 +00004197 // If truncates aren't free and there are users we can't
4198 // extend, it isn't worthwhile.
4199 if (!isTruncFree)
4200 return false;
4201 // Remember if this value is live-out.
4202 if (User->getOpcode() == ISD::CopyToReg)
4203 HasCopyToRegUses = true;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004204 }
4205
4206 if (HasCopyToRegUses) {
4207 bool BothLiveOut = false;
4208 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
4209 UI != UE; ++UI) {
Dan Gohman57fc82d2009-04-09 03:51:29 +00004210 SDUse &Use = UI.getUse();
4211 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) {
4212 BothLiveOut = true;
4213 break;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004214 }
4215 }
4216 if (BothLiveOut)
4217 // Both unextended and extended values are live out. There had better be
Bob Wilsonbebfbc52010-11-28 06:51:19 +00004218 // a good reason for the transformation.
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004219 return ExtendNodes.size();
4220 }
4221 return true;
4222}
4223
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004224void DAGCombiner::ExtendSetCCUses(SmallVector<SDNode*, 4> SetCCs,
4225 SDValue Trunc, SDValue ExtLoad, DebugLoc DL,
4226 ISD::NodeType ExtType) {
4227 // Extend SetCC uses if necessary.
4228 for (unsigned i = 0, e = SetCCs.size(); i != e; ++i) {
4229 SDNode *SetCC = SetCCs[i];
4230 SmallVector<SDValue, 4> Ops;
4231
4232 for (unsigned j = 0; j != 2; ++j) {
4233 SDValue SOp = SetCC->getOperand(j);
4234 if (SOp == Trunc)
4235 Ops.push_back(ExtLoad);
4236 else
4237 Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp));
4238 }
4239
4240 Ops.push_back(SetCC->getOperand(2));
4241 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0),
4242 &Ops[0], Ops.size()));
4243 }
4244}
4245
Dan Gohman475871a2008-07-27 21:46:04 +00004246SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
4247 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004248 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00004249
Nate Begeman1d4d4142005-09-01 00:19:25 +00004250 // fold (sext c1) -> c1
Reid Spencer3ed469c2006-11-02 20:25:50 +00004251 if (isa<ConstantSDNode>(N0))
Bill Wendling6ce610f2009-01-30 22:23:15 +00004252 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00004253
Nate Begeman1d4d4142005-09-01 00:19:25 +00004254 // fold (sext (sext x)) -> (sext x)
Chris Lattner310b5782006-05-06 23:06:26 +00004255 // fold (sext (aext x)) -> (sext x)
4256 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004257 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT,
4258 N0.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00004259
Chris Lattner22558872007-02-26 03:13:59 +00004260 if (N0.getOpcode() == ISD::TRUNCATE) {
Dan Gohman1fdfa6a2008-05-20 20:56:33 +00004261 // fold (sext (truncate (load x))) -> (sext (smaller load x))
4262 // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
Gabor Greifba36cb52008-08-28 21:40:38 +00004263 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
4264 if (NarrowLoad.getNode()) {
Dale Johannesen61734eb2010-05-25 17:50:03 +00004265 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4266 if (NarrowLoad.getNode() != N0.getNode()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004267 CombineTo(N0.getNode(), NarrowLoad);
Dale Johannesen61734eb2010-05-25 17:50:03 +00004268 // CombineTo deleted the truncate, if needed, but not what's under it.
4269 AddToWorkList(oye);
4270 }
Dan Gohmanc7b34442009-04-27 02:00:55 +00004271 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng0b063de2007-03-23 02:16:52 +00004272 }
Evan Chengc88138f2007-03-22 01:54:19 +00004273
Dan Gohman1fdfa6a2008-05-20 20:56:33 +00004274 // See if the value being truncated is already sign extended. If so, just
4275 // eliminate the trunc/sext pair.
Dan Gohman475871a2008-07-27 21:46:04 +00004276 SDValue Op = N0.getOperand(0);
Dan Gohmand1996362010-01-09 02:13:55 +00004277 unsigned OpBits = Op.getValueType().getScalarType().getSizeInBits();
4278 unsigned MidBits = N0.getValueType().getScalarType().getSizeInBits();
4279 unsigned DestBits = VT.getScalarType().getSizeInBits();
Dan Gohmanea859be2007-06-22 14:59:07 +00004280 unsigned NumSignBits = DAG.ComputeNumSignBits(Op);
Scott Michelfdc40a02009-02-17 22:15:04 +00004281
Chris Lattner22558872007-02-26 03:13:59 +00004282 if (OpBits == DestBits) {
4283 // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign
4284 // bits, it is already ready.
4285 if (NumSignBits > DestBits-MidBits)
4286 return Op;
4287 } else if (OpBits < DestBits) {
4288 // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign
4289 // bits, just sext from i32.
4290 if (NumSignBits > OpBits-MidBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004291 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, Op);
Chris Lattner22558872007-02-26 03:13:59 +00004292 } else {
4293 // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign
4294 // bits, just truncate to i32.
4295 if (NumSignBits > OpBits-MidBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004296 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, Op);
Chris Lattner6007b842006-09-21 06:00:20 +00004297 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004298
Chris Lattner22558872007-02-26 03:13:59 +00004299 // fold (sext (truncate x)) -> (sextinreg x).
Duncan Sands25cf2272008-11-24 14:53:14 +00004300 if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG,
4301 N0.getValueType())) {
Dan Gohmand1996362010-01-09 02:13:55 +00004302 if (OpBits < DestBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004303 Op = DAG.getNode(ISD::ANY_EXTEND, N0.getDebugLoc(), VT, Op);
Dan Gohmand1996362010-01-09 02:13:55 +00004304 else if (OpBits > DestBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004305 Op = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(), VT, Op);
4306 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT, Op,
Dan Gohmand1996362010-01-09 02:13:55 +00004307 DAG.getValueType(N0.getValueType()));
Chris Lattner22558872007-02-26 03:13:59 +00004308 }
Chris Lattner6007b842006-09-21 06:00:20 +00004309 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004310
Evan Cheng110dec22005-12-14 02:19:23 +00004311 // fold (sext (load x)) -> (sext (truncate (sextload x)))
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004312 // None of the supported targets knows how to perform load and sign extend
Nadav Rotemfcd96192011-02-27 07:40:43 +00004313 // on vectors in one instruction. We only perform this transformation on
4314 // scalars.
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004315 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004316 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00004317 TLI.isLoadExtLegal(ISD::SEXTLOAD, N0.getValueType()))) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004318 bool DoXform = true;
4319 SmallVector<SDNode*, 4> SetCCs;
4320 if (!N0.hasOneUse())
4321 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::SIGN_EXTEND, SetCCs, TLI);
4322 if (DoXform) {
4323 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00004324 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Dan Gohman57fc82d2009-04-09 03:51:29 +00004325 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004326 LN0->getBasePtr(), LN0->getPointerInfo(),
Duncan Sands25cf2272008-11-24 14:53:14 +00004327 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00004328 LN0->isVolatile(), LN0->isNonTemporal(),
4329 LN0->getAlignment());
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004330 CombineTo(N, ExtLoad);
Bill Wendling6ce610f2009-01-30 22:23:15 +00004331 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4332 N0.getValueType(), ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00004333 CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1));
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004334 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4335 ISD::SIGN_EXTEND);
Dan Gohman475871a2008-07-27 21:46:04 +00004336 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004337 }
Nate Begeman3df4d522005-10-12 20:40:40 +00004338 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004339
4340 // fold (sext (sextload x)) -> (sext (truncate (sextload x)))
4341 // fold (sext ( extload x)) -> (sext (truncate (sextload x)))
Gabor Greifba36cb52008-08-28 21:40:38 +00004342 if ((ISD::isSEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) &&
4343 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
Evan Cheng466685d2006-10-09 20:57:25 +00004344 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00004345 EVT MemVT = LN0->getMemoryVT();
Duncan Sands25cf2272008-11-24 14:53:14 +00004346 if ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00004347 TLI.isLoadExtLegal(ISD::SEXTLOAD, MemVT)) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00004348 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling6ce610f2009-01-30 22:23:15 +00004349 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004350 LN0->getBasePtr(), LN0->getPointerInfo(),
4351 MemVT,
David Greene1e559442010-02-15 17:00:31 +00004352 LN0->isVolatile(), LN0->isNonTemporal(),
4353 LN0->getAlignment());
Jim Laskeyf6c4ccf2006-12-15 21:38:30 +00004354 CombineTo(N, ExtLoad);
Gabor Greif12632d22008-08-30 19:29:20 +00004355 CombineTo(N0.getNode(),
Bill Wendling6ce610f2009-01-30 22:23:15 +00004356 DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4357 N0.getValueType(), ExtLoad),
Jim Laskeyf6c4ccf2006-12-15 21:38:30 +00004358 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00004359 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Jim Laskeyf6c4ccf2006-12-15 21:38:30 +00004360 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004361 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004362
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004363 // fold (sext (and/or/xor (load x), cst)) ->
4364 // (and/or/xor (sextload x), (sext cst))
4365 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
4366 N0.getOpcode() == ISD::XOR) &&
4367 isa<LoadSDNode>(N0.getOperand(0)) &&
4368 N0.getOperand(1).getOpcode() == ISD::Constant &&
4369 TLI.isLoadExtLegal(ISD::SEXTLOAD, N0.getValueType()) &&
4370 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
4371 LoadSDNode *LN0 = cast<LoadSDNode>(N0.getOperand(0));
4372 if (LN0->getExtensionType() != ISD::ZEXTLOAD) {
4373 bool DoXform = true;
4374 SmallVector<SDNode*, 4> SetCCs;
4375 if (!N0.hasOneUse())
4376 DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::SIGN_EXTEND,
4377 SetCCs, TLI);
4378 if (DoXform) {
4379 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, LN0->getDebugLoc(), VT,
4380 LN0->getChain(), LN0->getBasePtr(),
4381 LN0->getPointerInfo(),
4382 LN0->getMemoryVT(),
4383 LN0->isVolatile(),
4384 LN0->isNonTemporal(),
4385 LN0->getAlignment());
4386 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
4387 Mask = Mask.sext(VT.getSizeInBits());
4388 SDValue And = DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
4389 ExtLoad, DAG.getConstant(Mask, VT));
4390 SDValue Trunc = DAG.getNode(ISD::TRUNCATE,
4391 N0.getOperand(0).getDebugLoc(),
4392 N0.getOperand(0).getValueType(), ExtLoad);
4393 CombineTo(N, And);
4394 CombineTo(N0.getOperand(0).getNode(), Trunc, ExtLoad.getValue(1));
4395 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4396 ISD::SIGN_EXTEND);
4397 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4398 }
4399 }
4400 }
4401
Chris Lattner20a35c32007-04-11 05:32:27 +00004402 if (N0.getOpcode() == ISD::SETCC) {
Chris Lattner2b7a2712009-07-08 00:31:33 +00004403 // sext(setcc) -> sext_in_reg(vsetcc) for vectors.
Dan Gohman3ce89f42010-04-30 17:19:19 +00004404 // Only do this before legalize for now.
4405 if (VT.isVector() && !LegalOperations) {
4406 EVT N0VT = N0.getOperand(0).getValueType();
Nadav Rotem2e506192012-04-11 08:26:11 +00004407 // On some architectures (such as SSE/NEON/etc) the SETCC result type is
4408 // of the same size as the compared operands. Only optimize sext(setcc())
4409 // if this is the case.
4410 EVT SVT = TLI.getSetCCResultType(N0VT);
4411
4412 // We know that the # elements of the results is the same as the
4413 // # elements of the compare (and the # elements of the compare result
4414 // for that matter). Check to see that they are the same size. If so,
4415 // we know that the element size of the sext'd result matches the
4416 // element size of the compare operands.
4417 if (VT.getSizeInBits() == SVT.getSizeInBits())
Duncan Sands28b77e92011-09-06 19:07:46 +00004418 return DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
Duncan Sands34727662010-07-12 08:16:59 +00004419 N0.getOperand(1),
4420 cast<CondCodeSDNode>(N0.getOperand(2))->get());
Dan Gohman3ce89f42010-04-30 17:19:19 +00004421 // If the desired elements are smaller or larger than the source
4422 // elements we can use a matching integer vector type and then
4423 // truncate/sign extend
4424 else {
Duncan Sands34727662010-07-12 08:16:59 +00004425 EVT MatchingElementType =
4426 EVT::getIntegerVT(*DAG.getContext(),
4427 N0VT.getScalarType().getSizeInBits());
4428 EVT MatchingVectorType =
4429 EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
4430 N0VT.getVectorNumElements());
Nadav Rotem2e506192012-04-11 08:26:11 +00004431
4432 if (SVT == MatchingVectorType) {
4433 SDValue VsetCC = DAG.getSetCC(N->getDebugLoc(), MatchingVectorType,
4434 N0.getOperand(0), N0.getOperand(1),
4435 cast<CondCodeSDNode>(N0.getOperand(2))->get());
4436 return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT);
4437 }
Dan Gohman3ce89f42010-04-30 17:19:19 +00004438 }
Chris Lattner2b7a2712009-07-08 00:31:33 +00004439 }
Dan Gohman3ce89f42010-04-30 17:19:19 +00004440
Chris Lattner2b7a2712009-07-08 00:31:33 +00004441 // sext(setcc x, y, cc) -> (select_cc x, y, -1, 0, cc)
Dan Gohmana7bcef12010-04-24 01:17:30 +00004442 unsigned ElementWidth = VT.getScalarType().getSizeInBits();
Dan Gohman5cbd37e2009-08-06 09:18:59 +00004443 SDValue NegOne =
Dan Gohmana7bcef12010-04-24 01:17:30 +00004444 DAG.getConstant(APInt::getAllOnesValue(ElementWidth), VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00004445 SDValue SCC =
Bill Wendling836ca7d2009-01-30 23:59:18 +00004446 SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1),
Dan Gohman5cbd37e2009-08-06 09:18:59 +00004447 NegOne, DAG.getConstant(0, VT),
Chris Lattner1eba01e2007-04-11 06:50:51 +00004448 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
Gabor Greifba36cb52008-08-28 21:40:38 +00004449 if (SCC.getNode()) return SCC;
Evan Cheng8c7ecaf2010-01-26 02:00:44 +00004450 if (!LegalOperations ||
4451 TLI.isOperationLegal(ISD::SETCC, TLI.getSetCCResultType(VT)))
4452 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), VT,
4453 DAG.getSetCC(N->getDebugLoc(),
4454 TLI.getSetCCResultType(VT),
4455 N0.getOperand(0), N0.getOperand(1),
4456 cast<CondCodeSDNode>(N0.getOperand(2))->get()),
4457 NegOne, DAG.getConstant(0, VT));
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004458 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004459
Dan Gohman8f0ad582008-04-28 16:58:24 +00004460 // fold (sext x) -> (zext x) if the sign bit is known zero.
Duncan Sands25cf2272008-11-24 14:53:14 +00004461 if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) &&
Dan Gohman187db7b2008-04-28 18:47:17 +00004462 DAG.SignBitIsZero(N0))
Bill Wendling6ce610f2009-01-30 22:23:15 +00004463 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00004464
Evan Chengb3a3d5e2010-04-28 07:10:39 +00004465 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00004466}
4467
Rafael Espindoladecbc432012-04-09 16:06:03 +00004468// isTruncateOf - If N is a truncate of some other value, return true, record
4469// the value being truncated in Op and which of Op's bits are zero in KnownZero.
4470// This function computes KnownZero to avoid a duplicated call to
4471// ComputeMaskedBits in the caller.
4472static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
4473 APInt &KnownZero) {
4474 APInt KnownOne;
4475 if (N->getOpcode() == ISD::TRUNCATE) {
4476 Op = N->getOperand(0);
4477 DAG.ComputeMaskedBits(Op, KnownZero, KnownOne);
4478 return true;
4479 }
4480
4481 if (N->getOpcode() != ISD::SETCC || N->getValueType(0) != MVT::i1 ||
4482 cast<CondCodeSDNode>(N->getOperand(2))->get() != ISD::SETNE)
4483 return false;
4484
4485 SDValue Op0 = N->getOperand(0);
4486 SDValue Op1 = N->getOperand(1);
4487 assert(Op0.getValueType() == Op1.getValueType());
4488
4489 ConstantSDNode *COp0 = dyn_cast<ConstantSDNode>(Op0);
4490 ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1);
Rafael Espindolafdb230a2012-04-10 00:16:22 +00004491 if (COp0 && COp0->isNullValue())
Rafael Espindoladecbc432012-04-09 16:06:03 +00004492 Op = Op1;
Rafael Espindolafdb230a2012-04-10 00:16:22 +00004493 else if (COp1 && COp1->isNullValue())
Rafael Espindoladecbc432012-04-09 16:06:03 +00004494 Op = Op0;
4495 else
4496 return false;
4497
4498 DAG.ComputeMaskedBits(Op, KnownZero, KnownOne);
4499
4500 if (!(KnownZero | APInt(Op.getValueSizeInBits(), 1)).isAllOnesValue())
4501 return false;
4502
4503 return true;
4504}
4505
Dan Gohman475871a2008-07-27 21:46:04 +00004506SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
4507 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004508 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00004509
Nate Begeman1d4d4142005-09-01 00:19:25 +00004510 // fold (zext c1) -> c1
Reid Spencer3ed469c2006-11-02 20:25:50 +00004511 if (isa<ConstantSDNode>(N0))
Bill Wendling6ce610f2009-01-30 22:23:15 +00004512 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00004513 // fold (zext (zext x)) -> (zext x)
Chris Lattner310b5782006-05-06 23:06:26 +00004514 // fold (zext (aext x)) -> (zext x)
4515 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004516 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT,
4517 N0.getOperand(0));
Chris Lattner6007b842006-09-21 06:00:20 +00004518
Chandler Carruthf103b3d2012-01-11 08:41:08 +00004519 // fold (zext (truncate x)) -> (zext x) or
4520 // (zext (truncate x)) -> (truncate x)
4521 // This is valid when the truncated bits of x are already zero.
4522 // FIXME: We should extend this to work for vectors too.
Rafael Espindoladecbc432012-04-09 16:06:03 +00004523 SDValue Op;
4524 APInt KnownZero;
4525 if (!VT.isVector() && isTruncateOf(DAG, N0, Op, KnownZero)) {
4526 APInt TruncatedBits =
4527 (Op.getValueSizeInBits() == N0.getValueSizeInBits()) ?
4528 APInt(Op.getValueSizeInBits(), 0) :
4529 APInt::getBitsSet(Op.getValueSizeInBits(),
4530 N0.getValueSizeInBits(),
4531 std::min(Op.getValueSizeInBits(),
4532 VT.getSizeInBits()));
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00004533 if (TruncatedBits == (KnownZero & TruncatedBits)) {
Chandler Carruthf103b3d2012-01-11 08:41:08 +00004534 if (VT.bitsGT(Op.getValueType()))
4535 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, Op);
4536 if (VT.bitsLT(Op.getValueType()))
4537 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, Op);
4538
4539 return Op;
4540 }
4541 }
4542
Evan Chengc88138f2007-03-22 01:54:19 +00004543 // fold (zext (truncate (load x))) -> (zext (smaller load x))
4544 // fold (zext (truncate (srl (load x), c))) -> (zext (small load (x+c/n)))
Dale Johannesen2041a0e2007-03-30 21:38:07 +00004545 if (N0.getOpcode() == ISD::TRUNCATE) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004546 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
4547 if (NarrowLoad.getNode()) {
Dale Johannesen61734eb2010-05-25 17:50:03 +00004548 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4549 if (NarrowLoad.getNode() != N0.getNode()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004550 CombineTo(N0.getNode(), NarrowLoad);
Dale Johannesen61734eb2010-05-25 17:50:03 +00004551 // CombineTo deleted the truncate, if needed, but not what's under it.
4552 AddToWorkList(oye);
4553 }
Eli Friedmane545d382011-04-16 23:25:34 +00004554 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng0b063de2007-03-23 02:16:52 +00004555 }
Evan Chengc88138f2007-03-22 01:54:19 +00004556 }
4557
Chris Lattner6007b842006-09-21 06:00:20 +00004558 // fold (zext (truncate x)) -> (and x, mask)
4559 if (N0.getOpcode() == ISD::TRUNCATE &&
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004560 (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT))) {
Dan Gohman394d6292010-11-03 01:47:46 +00004561
4562 // fold (zext (truncate (load x))) -> (zext (smaller load x))
4563 // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
4564 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
4565 if (NarrowLoad.getNode()) {
4566 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4567 if (NarrowLoad.getNode() != N0.getNode()) {
4568 CombineTo(N0.getNode(), NarrowLoad);
4569 // CombineTo deleted the truncate, if needed, but not what's under it.
4570 AddToWorkList(oye);
4571 }
4572 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4573 }
4574
Dan Gohman475871a2008-07-27 21:46:04 +00004575 SDValue Op = N0.getOperand(0);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004576 if (Op.getValueType().bitsLT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004577 Op = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, Op);
Elena Demikhovsky1da58672012-04-22 09:39:03 +00004578 AddToWorkList(Op.getNode());
Duncan Sands8e4eb092008-06-08 20:54:56 +00004579 } else if (Op.getValueType().bitsGT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004580 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, Op);
Elena Demikhovsky1da58672012-04-22 09:39:03 +00004581 AddToWorkList(Op.getNode());
Chris Lattner6007b842006-09-21 06:00:20 +00004582 }
Dan Gohman87862e72009-12-11 21:31:27 +00004583 return DAG.getZeroExtendInReg(Op, N->getDebugLoc(),
4584 N0.getValueType().getScalarType());
Chris Lattner6007b842006-09-21 06:00:20 +00004585 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004586
Dan Gohman97121ba2009-04-08 00:15:30 +00004587 // Fold (zext (and (trunc x), cst)) -> (and x, cst),
4588 // if either of the casts is not free.
Chris Lattner111c2282006-09-21 06:14:31 +00004589 if (N0.getOpcode() == ISD::AND &&
4590 N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
Dan Gohman97121ba2009-04-08 00:15:30 +00004591 N0.getOperand(1).getOpcode() == ISD::Constant &&
4592 (!TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
4593 N0.getValueType()) ||
4594 !TLI.isZExtFree(N0.getValueType(), VT))) {
Dan Gohman475871a2008-07-27 21:46:04 +00004595 SDValue X = N0.getOperand(0).getOperand(0);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004596 if (X.getValueType().bitsLT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004597 X = DAG.getNode(ISD::ANY_EXTEND, X.getDebugLoc(), VT, X);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004598 } else if (X.getValueType().bitsGT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004599 X = DAG.getNode(ISD::TRUNCATE, X.getDebugLoc(), VT, X);
Chris Lattner111c2282006-09-21 06:14:31 +00004600 }
Dan Gohman220a8232008-03-03 23:51:38 +00004601 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00004602 Mask = Mask.zext(VT.getSizeInBits());
Bill Wendling6ce610f2009-01-30 22:23:15 +00004603 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
4604 X, DAG.getConstant(Mask, VT));
Chris Lattner111c2282006-09-21 06:14:31 +00004605 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004606
Evan Cheng110dec22005-12-14 02:19:23 +00004607 // fold (zext (load x)) -> (zext (truncate (zextload x)))
Nadav Rotemed9b9342011-02-20 12:37:50 +00004608 // None of the supported targets knows how to perform load and vector_zext
Nadav Rotemfcd96192011-02-27 07:40:43 +00004609 // on vectors in one instruction. We only perform this transformation on
4610 // scalars.
Nadav Rotemed9b9342011-02-20 12:37:50 +00004611 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004612 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00004613 TLI.isLoadExtLegal(ISD::ZEXTLOAD, N0.getValueType()))) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004614 bool DoXform = true;
4615 SmallVector<SDNode*, 4> SetCCs;
4616 if (!N0.hasOneUse())
4617 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ZERO_EXTEND, SetCCs, TLI);
4618 if (DoXform) {
4619 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00004620 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling6ce610f2009-01-30 22:23:15 +00004621 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004622 LN0->getBasePtr(), LN0->getPointerInfo(),
Duncan Sands25cf2272008-11-24 14:53:14 +00004623 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00004624 LN0->isVolatile(), LN0->isNonTemporal(),
4625 LN0->getAlignment());
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004626 CombineTo(N, ExtLoad);
Bill Wendling6ce610f2009-01-30 22:23:15 +00004627 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4628 N0.getValueType(), ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00004629 CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1));
Bill Wendling6ce610f2009-01-30 22:23:15 +00004630
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004631 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4632 ISD::ZERO_EXTEND);
Dan Gohman475871a2008-07-27 21:46:04 +00004633 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004634 }
Evan Cheng110dec22005-12-14 02:19:23 +00004635 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004636
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004637 // fold (zext (and/or/xor (load x), cst)) ->
4638 // (and/or/xor (zextload x), (zext cst))
4639 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
4640 N0.getOpcode() == ISD::XOR) &&
4641 isa<LoadSDNode>(N0.getOperand(0)) &&
4642 N0.getOperand(1).getOpcode() == ISD::Constant &&
4643 TLI.isLoadExtLegal(ISD::ZEXTLOAD, N0.getValueType()) &&
4644 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
4645 LoadSDNode *LN0 = cast<LoadSDNode>(N0.getOperand(0));
4646 if (LN0->getExtensionType() != ISD::SEXTLOAD) {
4647 bool DoXform = true;
4648 SmallVector<SDNode*, 4> SetCCs;
4649 if (!N0.hasOneUse())
4650 DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::ZERO_EXTEND,
4651 SetCCs, TLI);
4652 if (DoXform) {
4653 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, LN0->getDebugLoc(), VT,
4654 LN0->getChain(), LN0->getBasePtr(),
4655 LN0->getPointerInfo(),
4656 LN0->getMemoryVT(),
4657 LN0->isVolatile(),
4658 LN0->isNonTemporal(),
4659 LN0->getAlignment());
4660 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
4661 Mask = Mask.zext(VT.getSizeInBits());
4662 SDValue And = DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
4663 ExtLoad, DAG.getConstant(Mask, VT));
4664 SDValue Trunc = DAG.getNode(ISD::TRUNCATE,
4665 N0.getOperand(0).getDebugLoc(),
4666 N0.getOperand(0).getValueType(), ExtLoad);
4667 CombineTo(N, And);
4668 CombineTo(N0.getOperand(0).getNode(), Trunc, ExtLoad.getValue(1));
4669 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4670 ISD::ZERO_EXTEND);
4671 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4672 }
4673 }
4674 }
4675
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004676 // fold (zext (zextload x)) -> (zext (truncate (zextload x)))
4677 // fold (zext ( extload x)) -> (zext (truncate (zextload x)))
Gabor Greifba36cb52008-08-28 21:40:38 +00004678 if ((ISD::isZEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) &&
4679 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
Evan Cheng466685d2006-10-09 20:57:25 +00004680 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00004681 EVT MemVT = LN0->getMemoryVT();
Duncan Sands25cf2272008-11-24 14:53:14 +00004682 if ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00004683 TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT)) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00004684 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling6ce610f2009-01-30 22:23:15 +00004685 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004686 LN0->getBasePtr(), LN0->getPointerInfo(),
4687 MemVT,
David Greene1e559442010-02-15 17:00:31 +00004688 LN0->isVolatile(), LN0->isNonTemporal(),
4689 LN0->getAlignment());
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004690 CombineTo(N, ExtLoad);
Gabor Greif12632d22008-08-30 19:29:20 +00004691 CombineTo(N0.getNode(),
Bill Wendling6ce610f2009-01-30 22:23:15 +00004692 DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(), N0.getValueType(),
4693 ExtLoad),
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004694 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00004695 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004696 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004697 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004698
Chris Lattner20a35c32007-04-11 05:32:27 +00004699 if (N0.getOpcode() == ISD::SETCC) {
Evan Cheng0a942db2010-05-19 01:08:17 +00004700 if (!LegalOperations && VT.isVector()) {
4701 // zext(setcc) -> (and (vsetcc), (1, 1, ...) for vectors.
4702 // Only do this before legalize for now.
4703 EVT N0VT = N0.getOperand(0).getValueType();
4704 EVT EltVT = VT.getVectorElementType();
4705 SmallVector<SDValue,8> OneOps(VT.getVectorNumElements(),
4706 DAG.getConstant(1, EltVT));
Dan Gohman71dc7c92011-05-17 22:20:36 +00004707 if (VT.getSizeInBits() == N0VT.getSizeInBits())
Evan Cheng0a942db2010-05-19 01:08:17 +00004708 // We know that the # elements of the results is the same as the
4709 // # elements of the compare (and the # elements of the compare result
4710 // for that matter). Check to see that they are the same size. If so,
4711 // we know that the element size of the sext'd result matches the
4712 // element size of the compare operands.
4713 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
Duncan Sands28b77e92011-09-06 19:07:46 +00004714 DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
Evan Cheng0a942db2010-05-19 01:08:17 +00004715 N0.getOperand(1),
4716 cast<CondCodeSDNode>(N0.getOperand(2))->get()),
4717 DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT,
4718 &OneOps[0], OneOps.size()));
Dan Gohman71dc7c92011-05-17 22:20:36 +00004719
4720 // If the desired elements are smaller or larger than the source
4721 // elements we can use a matching integer vector type and then
4722 // truncate/sign extend
4723 EVT MatchingElementType =
4724 EVT::getIntegerVT(*DAG.getContext(),
4725 N0VT.getScalarType().getSizeInBits());
4726 EVT MatchingVectorType =
4727 EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
4728 N0VT.getVectorNumElements());
4729 SDValue VsetCC =
Duncan Sands28b77e92011-09-06 19:07:46 +00004730 DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
Dan Gohman71dc7c92011-05-17 22:20:36 +00004731 N0.getOperand(1),
4732 cast<CondCodeSDNode>(N0.getOperand(2))->get());
4733 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
4734 DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT),
4735 DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT,
4736 &OneOps[0], OneOps.size()));
Evan Cheng0a942db2010-05-19 01:08:17 +00004737 }
4738
4739 // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
Scott Michelfdc40a02009-02-17 22:15:04 +00004740 SDValue SCC =
Bill Wendling836ca7d2009-01-30 23:59:18 +00004741 SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1),
Chris Lattner20a35c32007-04-11 05:32:27 +00004742 DAG.getConstant(1, VT), DAG.getConstant(0, VT),
Chris Lattner1eba01e2007-04-11 06:50:51 +00004743 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
Gabor Greifba36cb52008-08-28 21:40:38 +00004744 if (SCC.getNode()) return SCC;
Chris Lattner20a35c32007-04-11 05:32:27 +00004745 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004746
Evan Cheng9818c042009-12-15 03:00:32 +00004747 // (zext (shl (zext x), cst)) -> (shl (zext x), cst)
Evan Cheng99b653c2009-12-15 00:41:36 +00004748 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
Evan Cheng9818c042009-12-15 03:00:32 +00004749 isa<ConstantSDNode>(N0.getOperand(1)) &&
Evan Cheng99b653c2009-12-15 00:41:36 +00004750 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND &&
4751 N0.hasOneUse()) {
Chris Lattnere0751182011-02-13 19:09:16 +00004752 SDValue ShAmt = N0.getOperand(1);
4753 unsigned ShAmtVal = cast<ConstantSDNode>(ShAmt)->getZExtValue();
Evan Cheng9818c042009-12-15 03:00:32 +00004754 if (N0.getOpcode() == ISD::SHL) {
Chris Lattnere0751182011-02-13 19:09:16 +00004755 SDValue InnerZExt = N0.getOperand(0);
Evan Cheng9818c042009-12-15 03:00:32 +00004756 // If the original shl may be shifting out bits, do not perform this
4757 // transformation.
Chris Lattnere0751182011-02-13 19:09:16 +00004758 unsigned KnownZeroBits = InnerZExt.getValueType().getSizeInBits() -
4759 InnerZExt.getOperand(0).getValueType().getSizeInBits();
4760 if (ShAmtVal > KnownZeroBits)
Evan Cheng9818c042009-12-15 03:00:32 +00004761 return SDValue();
4762 }
Chris Lattnere0751182011-02-13 19:09:16 +00004763
4764 DebugLoc DL = N->getDebugLoc();
Owen Anderson95771af2011-02-25 21:41:48 +00004765
4766 // Ensure that the shift amount is wide enough for the shifted value.
Chris Lattnere0751182011-02-13 19:09:16 +00004767 if (VT.getSizeInBits() >= 256)
4768 ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt);
Owen Anderson95771af2011-02-25 21:41:48 +00004769
Chris Lattnere0751182011-02-13 19:09:16 +00004770 return DAG.getNode(N0.getOpcode(), DL, VT,
4771 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)),
4772 ShAmt);
Evan Cheng99b653c2009-12-15 00:41:36 +00004773 }
4774
Evan Chengb3a3d5e2010-04-28 07:10:39 +00004775 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00004776}
4777
Dan Gohman475871a2008-07-27 21:46:04 +00004778SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
4779 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004780 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00004781
Chris Lattner5ffc0662006-05-05 05:58:59 +00004782 // fold (aext c1) -> c1
Chris Lattner310b5782006-05-06 23:06:26 +00004783 if (isa<ConstantSDNode>(N0))
Bill Wendlingfc4b6772009-02-01 11:19:36 +00004784 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, N0);
Chris Lattner5ffc0662006-05-05 05:58:59 +00004785 // fold (aext (aext x)) -> (aext x)
4786 // fold (aext (zext x)) -> (zext x)
4787 // fold (aext (sext x)) -> (sext x)
4788 if (N0.getOpcode() == ISD::ANY_EXTEND ||
4789 N0.getOpcode() == ISD::ZERO_EXTEND ||
4790 N0.getOpcode() == ISD::SIGN_EXTEND)
Bill Wendling683c9572009-01-30 22:27:33 +00004791 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT, N0.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00004792
Evan Chengc88138f2007-03-22 01:54:19 +00004793 // fold (aext (truncate (load x))) -> (aext (smaller load x))
4794 // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n)))
4795 if (N0.getOpcode() == ISD::TRUNCATE) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004796 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
4797 if (NarrowLoad.getNode()) {
Dale Johannesen86234c32010-05-25 18:47:23 +00004798 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4799 if (NarrowLoad.getNode() != N0.getNode()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004800 CombineTo(N0.getNode(), NarrowLoad);
Dale Johannesen86234c32010-05-25 18:47:23 +00004801 // CombineTo deleted the truncate, if needed, but not what's under it.
4802 AddToWorkList(oye);
4803 }
Eli Friedmane545d382011-04-16 23:25:34 +00004804 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng0b063de2007-03-23 02:16:52 +00004805 }
Evan Chengc88138f2007-03-22 01:54:19 +00004806 }
4807
Chris Lattner84750582006-09-20 06:29:17 +00004808 // fold (aext (truncate x))
4809 if (N0.getOpcode() == ISD::TRUNCATE) {
Dan Gohman475871a2008-07-27 21:46:04 +00004810 SDValue TruncOp = N0.getOperand(0);
Chris Lattner84750582006-09-20 06:29:17 +00004811 if (TruncOp.getValueType() == VT)
Sylvestre Ledru94c22712012-09-27 10:14:43 +00004812 return TruncOp; // x iff x size == zext size.
Duncan Sands8e4eb092008-06-08 20:54:56 +00004813 if (TruncOp.getValueType().bitsGT(VT))
Bill Wendling683c9572009-01-30 22:27:33 +00004814 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, TruncOp);
4815 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, TruncOp);
Chris Lattner84750582006-09-20 06:29:17 +00004816 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004817
Dan Gohman97121ba2009-04-08 00:15:30 +00004818 // Fold (aext (and (trunc x), cst)) -> (and x, cst)
4819 // if the trunc is not free.
Chris Lattner0e4b9222006-09-21 06:40:43 +00004820 if (N0.getOpcode() == ISD::AND &&
4821 N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
Dan Gohman97121ba2009-04-08 00:15:30 +00004822 N0.getOperand(1).getOpcode() == ISD::Constant &&
4823 !TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
4824 N0.getValueType())) {
Dan Gohman475871a2008-07-27 21:46:04 +00004825 SDValue X = N0.getOperand(0).getOperand(0);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004826 if (X.getValueType().bitsLT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004827 X = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, X);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004828 } else if (X.getValueType().bitsGT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004829 X = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, X);
Chris Lattner0e4b9222006-09-21 06:40:43 +00004830 }
Dan Gohman220a8232008-03-03 23:51:38 +00004831 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00004832 Mask = Mask.zext(VT.getSizeInBits());
Bill Wendling683c9572009-01-30 22:27:33 +00004833 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
4834 X, DAG.getConstant(Mask, VT));
Chris Lattner0e4b9222006-09-21 06:40:43 +00004835 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004836
Chris Lattner5ffc0662006-05-05 05:58:59 +00004837 // fold (aext (load x)) -> (aext (truncate (extload x)))
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004838 // None of the supported targets knows how to perform load and any_ext
Nadav Rotemfcd96192011-02-27 07:40:43 +00004839 // on vectors in one instruction. We only perform this transformation on
4840 // scalars.
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004841 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004842 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00004843 TLI.isLoadExtLegal(ISD::EXTLOAD, N0.getValueType()))) {
Dan Gohman57fc82d2009-04-09 03:51:29 +00004844 bool DoXform = true;
4845 SmallVector<SDNode*, 4> SetCCs;
4846 if (!N0.hasOneUse())
4847 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ANY_EXTEND, SetCCs, TLI);
4848 if (DoXform) {
4849 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00004850 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, N->getDebugLoc(), VT,
Dan Gohman57fc82d2009-04-09 03:51:29 +00004851 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004852 LN0->getBasePtr(), LN0->getPointerInfo(),
Dan Gohman57fc82d2009-04-09 03:51:29 +00004853 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00004854 LN0->isVolatile(), LN0->isNonTemporal(),
4855 LN0->getAlignment());
Dan Gohman57fc82d2009-04-09 03:51:29 +00004856 CombineTo(N, ExtLoad);
4857 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4858 N0.getValueType(), ExtLoad);
4859 CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1));
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004860 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4861 ISD::ANY_EXTEND);
Dan Gohman57fc82d2009-04-09 03:51:29 +00004862 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4863 }
Chris Lattner5ffc0662006-05-05 05:58:59 +00004864 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004865
Chris Lattner5ffc0662006-05-05 05:58:59 +00004866 // fold (aext (zextload x)) -> (aext (truncate (zextload x)))
4867 // fold (aext (sextload x)) -> (aext (truncate (sextload x)))
4868 // fold (aext ( extload x)) -> (aext (truncate (extload x)))
Evan Cheng83060c52007-03-07 08:07:03 +00004869 if (N0.getOpcode() == ISD::LOAD &&
Gabor Greifba36cb52008-08-28 21:40:38 +00004870 !ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
Evan Cheng466685d2006-10-09 20:57:25 +00004871 N0.hasOneUse()) {
4872 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00004873 EVT MemVT = LN0->getMemoryVT();
Stuart Hastingsa9011292011-02-16 16:23:55 +00004874 SDValue ExtLoad = DAG.getExtLoad(LN0->getExtensionType(), N->getDebugLoc(),
4875 VT, LN0->getChain(), LN0->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004876 LN0->getPointerInfo(), MemVT,
David Greene1e559442010-02-15 17:00:31 +00004877 LN0->isVolatile(), LN0->isNonTemporal(),
4878 LN0->getAlignment());
Chris Lattner5ffc0662006-05-05 05:58:59 +00004879 CombineTo(N, ExtLoad);
Evan Cheng45299662008-08-29 23:20:46 +00004880 CombineTo(N0.getNode(),
Bill Wendling683c9572009-01-30 22:27:33 +00004881 DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4882 N0.getValueType(), ExtLoad),
Chris Lattner5ffc0662006-05-05 05:58:59 +00004883 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00004884 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattner5ffc0662006-05-05 05:58:59 +00004885 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004886
Chris Lattner20a35c32007-04-11 05:32:27 +00004887 if (N0.getOpcode() == ISD::SETCC) {
Evan Cheng0a942db2010-05-19 01:08:17 +00004888 // aext(setcc) -> sext_in_reg(vsetcc) for vectors.
4889 // Only do this before legalize for now.
4890 if (VT.isVector() && !LegalOperations) {
4891 EVT N0VT = N0.getOperand(0).getValueType();
4892 // We know that the # elements of the results is the same as the
4893 // # elements of the compare (and the # elements of the compare result
4894 // for that matter). Check to see that they are the same size. If so,
4895 // we know that the element size of the sext'd result matches the
4896 // element size of the compare operands.
4897 if (VT.getSizeInBits() == N0VT.getSizeInBits())
Duncan Sands28b77e92011-09-06 19:07:46 +00004898 return DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
Duncan Sands34727662010-07-12 08:16:59 +00004899 N0.getOperand(1),
4900 cast<CondCodeSDNode>(N0.getOperand(2))->get());
Evan Cheng0a942db2010-05-19 01:08:17 +00004901 // If the desired elements are smaller or larger than the source
4902 // elements we can use a matching integer vector type and then
4903 // truncate/sign extend
4904 else {
Duncan Sands34727662010-07-12 08:16:59 +00004905 EVT MatchingElementType =
4906 EVT::getIntegerVT(*DAG.getContext(),
4907 N0VT.getScalarType().getSizeInBits());
4908 EVT MatchingVectorType =
4909 EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
4910 N0VT.getVectorNumElements());
4911 SDValue VsetCC =
Duncan Sands28b77e92011-09-06 19:07:46 +00004912 DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
Duncan Sands34727662010-07-12 08:16:59 +00004913 N0.getOperand(1),
4914 cast<CondCodeSDNode>(N0.getOperand(2))->get());
4915 return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT);
Evan Cheng0a942db2010-05-19 01:08:17 +00004916 }
4917 }
4918
4919 // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
Scott Michelfdc40a02009-02-17 22:15:04 +00004920 SDValue SCC =
Bill Wendling836ca7d2009-01-30 23:59:18 +00004921 SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1),
Chris Lattner1eba01e2007-04-11 06:50:51 +00004922 DAG.getConstant(1, VT), DAG.getConstant(0, VT),
Chris Lattnerc24bbad2007-04-11 16:51:53 +00004923 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
Gabor Greifba36cb52008-08-28 21:40:38 +00004924 if (SCC.getNode())
Chris Lattnerc56a81d2007-04-11 06:43:25 +00004925 return SCC;
Chris Lattner20a35c32007-04-11 05:32:27 +00004926 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004927
Evan Chengb3a3d5e2010-04-28 07:10:39 +00004928 return SDValue();
Chris Lattner5ffc0662006-05-05 05:58:59 +00004929}
4930
Chris Lattner2b4c2792007-10-13 06:35:54 +00004931/// GetDemandedBits - See if the specified operand can be simplified with the
4932/// knowledge that only the bits specified by Mask are used. If so, return the
Dan Gohman475871a2008-07-27 21:46:04 +00004933/// simpler operand, otherwise return a null SDValue.
4934SDValue DAGCombiner::GetDemandedBits(SDValue V, const APInt &Mask) {
Chris Lattner2b4c2792007-10-13 06:35:54 +00004935 switch (V.getOpcode()) {
4936 default: break;
Lang Hames5207bf22011-11-08 18:56:23 +00004937 case ISD::Constant: {
4938 const ConstantSDNode *CV = cast<ConstantSDNode>(V.getNode());
4939 assert(CV != 0 && "Const value should be ConstSDNode.");
4940 const APInt &CVal = CV->getAPIntValue();
4941 APInt NewVal = CVal & Mask;
4942 if (NewVal != CVal) {
4943 return DAG.getConstant(NewVal, V.getValueType());
4944 }
4945 break;
4946 }
Chris Lattner2b4c2792007-10-13 06:35:54 +00004947 case ISD::OR:
4948 case ISD::XOR:
4949 // If the LHS or RHS don't contribute bits to the or, drop them.
4950 if (DAG.MaskedValueIsZero(V.getOperand(0), Mask))
4951 return V.getOperand(1);
4952 if (DAG.MaskedValueIsZero(V.getOperand(1), Mask))
4953 return V.getOperand(0);
4954 break;
Chris Lattnere33544c2007-10-13 06:58:48 +00004955 case ISD::SRL:
4956 // Only look at single-use SRLs.
Gabor Greifba36cb52008-08-28 21:40:38 +00004957 if (!V.getNode()->hasOneUse())
Chris Lattnere33544c2007-10-13 06:58:48 +00004958 break;
4959 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
4960 // See if we can recursively simplify the LHS.
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00004961 unsigned Amt = RHSC->getZExtValue();
Bill Wendling8509c902009-01-30 22:33:24 +00004962
Dan Gohmancc91d632009-01-03 19:22:06 +00004963 // Watch out for shift count overflow though.
4964 if (Amt >= Mask.getBitWidth()) break;
Dan Gohman2e68b6f2008-02-25 21:11:39 +00004965 APInt NewMask = Mask << Amt;
Dan Gohman475871a2008-07-27 21:46:04 +00004966 SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask);
Bill Wendling8509c902009-01-30 22:33:24 +00004967 if (SimplifyLHS.getNode())
Scott Michelfdc40a02009-02-17 22:15:04 +00004968 return DAG.getNode(ISD::SRL, V.getDebugLoc(), V.getValueType(),
Chris Lattnere33544c2007-10-13 06:58:48 +00004969 SimplifyLHS, V.getOperand(1));
Chris Lattnere33544c2007-10-13 06:58:48 +00004970 }
Chris Lattner2b4c2792007-10-13 06:35:54 +00004971 }
Dan Gohman475871a2008-07-27 21:46:04 +00004972 return SDValue();
Chris Lattner2b4c2792007-10-13 06:35:54 +00004973}
4974
Evan Chengc88138f2007-03-22 01:54:19 +00004975/// ReduceLoadWidth - If the result of a wider load is shifted to right of N
4976/// bits and then truncated to a narrower type and where N is a multiple
4977/// of number of bits of the narrower type, transform it to a narrower load
4978/// from address + N / num of bits of new type. If the result is to be
4979/// extended, also fold the extension to form a extending load.
Dan Gohman475871a2008-07-27 21:46:04 +00004980SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
Evan Chengc88138f2007-03-22 01:54:19 +00004981 unsigned Opc = N->getOpcode();
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004982
Evan Chengc88138f2007-03-22 01:54:19 +00004983 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
Dan Gohman475871a2008-07-27 21:46:04 +00004984 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004985 EVT VT = N->getValueType(0);
4986 EVT ExtVT = VT;
Evan Chengc88138f2007-03-22 01:54:19 +00004987
Dan Gohman7f8613e2008-08-14 20:04:46 +00004988 // This transformation isn't valid for vector loads.
4989 if (VT.isVector())
4990 return SDValue();
4991
Dan Gohmand1996362010-01-09 02:13:55 +00004992 // Special case: SIGN_EXTEND_INREG is basically truncating to ExtVT then
Evan Chenge177e302007-03-23 22:13:36 +00004993 // extended to VT.
Evan Chengc88138f2007-03-22 01:54:19 +00004994 if (Opc == ISD::SIGN_EXTEND_INREG) {
4995 ExtType = ISD::SEXTLOAD;
Owen Andersone50ed302009-08-10 22:56:29 +00004996 ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT();
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004997 } else if (Opc == ISD::SRL) {
Chris Lattner90b03642010-12-21 18:05:22 +00004998 // Another special-case: SRL is basically zero-extending a narrower value.
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004999 ExtType = ISD::ZEXTLOAD;
5000 N0 = SDValue(N, 0);
5001 ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5002 if (!N01) return SDValue();
5003 ExtVT = EVT::getIntegerVT(*DAG.getContext(),
5004 VT.getSizeInBits() - N01->getZExtValue());
Evan Chengc88138f2007-03-22 01:54:19 +00005005 }
Richard Osborne4e3740e2011-01-31 17:41:44 +00005006 if (LegalOperations && !TLI.isLoadExtLegal(ExtType, ExtVT))
5007 return SDValue();
Evan Chengc88138f2007-03-22 01:54:19 +00005008
Owen Andersone50ed302009-08-10 22:56:29 +00005009 unsigned EVTBits = ExtVT.getSizeInBits();
Owen Anderson95771af2011-02-25 21:41:48 +00005010
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00005011 // Do not generate loads of non-round integer types since these can
5012 // be expensive (and would be wrong if the type is not byte sized).
5013 if (!ExtVT.isRound())
5014 return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00005015
Evan Chengc88138f2007-03-22 01:54:19 +00005016 unsigned ShAmt = 0;
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00005017 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
Evan Chengc88138f2007-03-22 01:54:19 +00005018 if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00005019 ShAmt = N01->getZExtValue();
Evan Chengc88138f2007-03-22 01:54:19 +00005020 // Is the shift amount a multiple of size of VT?
5021 if ((ShAmt & (EVTBits-1)) == 0) {
5022 N0 = N0.getOperand(0);
Eli Friedmand68eea22009-08-19 08:46:10 +00005023 // Is the load width a multiple of size of VT?
5024 if ((N0.getValueType().getSizeInBits() & (EVTBits-1)) != 0)
Dan Gohman475871a2008-07-27 21:46:04 +00005025 return SDValue();
Evan Chengc88138f2007-03-22 01:54:19 +00005026 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005027
Chris Lattnercbf68df2010-12-22 08:02:57 +00005028 // At this point, we must have a load or else we can't do the transform.
5029 if (!isa<LoadSDNode>(N0)) return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00005030
Chris Lattner2831a192010-10-01 05:36:09 +00005031 // If the shift amount is larger than the input type then we're not
5032 // accessing any of the loaded bytes. If the load was a zextload/extload
5033 // then the result of the shift+trunc is zero/undef (handled elsewhere).
5034 // If the load was a sextload then the result is a splat of the sign bit
5035 // of the extended byte. This is not worth optimizing for.
Chris Lattnercbf68df2010-12-22 08:02:57 +00005036 if (ShAmt >= cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits())
Chris Lattner2831a192010-10-01 05:36:09 +00005037 return SDValue();
Evan Chengc88138f2007-03-22 01:54:19 +00005038 }
5039 }
5040
Dan Gohman394d6292010-11-03 01:47:46 +00005041 // If the load is shifted left (and the result isn't shifted back right),
5042 // we can fold the truncate through the shift.
5043 unsigned ShLeftAmt = 0;
5044 if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
Chris Lattner4c32bc22010-12-22 07:36:50 +00005045 ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) {
Dan Gohman394d6292010-11-03 01:47:46 +00005046 if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
5047 ShLeftAmt = N01->getZExtValue();
5048 N0 = N0.getOperand(0);
5049 }
5050 }
Owen Anderson95771af2011-02-25 21:41:48 +00005051
Chris Lattner4c32bc22010-12-22 07:36:50 +00005052 // If we haven't found a load, we can't narrow it. Don't transform one with
5053 // multiple uses, this would require adding a new load.
5054 if (!isa<LoadSDNode>(N0) || !N0.hasOneUse() ||
5055 // Don't change the width of a volatile load.
5056 cast<LoadSDNode>(N0)->isVolatile())
5057 return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00005058
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00005059 // Verify that we are actually reducing a load width here.
5060 if (cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits() < EVTBits)
Chris Lattner4c32bc22010-12-22 07:36:50 +00005061 return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00005062
Chris Lattner4c32bc22010-12-22 07:36:50 +00005063 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
5064 EVT PtrType = N0.getOperand(1).getValueType();
Bill Wendling8509c902009-01-30 22:33:24 +00005065
Evan Cheng16436df2012-06-26 01:19:33 +00005066 if (PtrType == MVT::Untyped || PtrType.isExtended())
5067 // It's not possible to generate a constant of extended or untyped type.
5068 return SDValue();
5069
Chris Lattner4c32bc22010-12-22 07:36:50 +00005070 // For big endian targets, we need to adjust the offset to the pointer to
5071 // load the correct bytes.
5072 if (TLI.isBigEndian()) {
5073 unsigned LVTStoreBits = LN0->getMemoryVT().getStoreSizeInBits();
5074 unsigned EVTStoreBits = ExtVT.getStoreSizeInBits();
5075 ShAmt = LVTStoreBits - EVTStoreBits - ShAmt;
Evan Chengc88138f2007-03-22 01:54:19 +00005076 }
5077
Chris Lattner4c32bc22010-12-22 07:36:50 +00005078 uint64_t PtrOff = ShAmt / 8;
5079 unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff);
5080 SDValue NewPtr = DAG.getNode(ISD::ADD, LN0->getDebugLoc(),
5081 PtrType, LN0->getBasePtr(),
5082 DAG.getConstant(PtrOff, PtrType));
5083 AddToWorkList(NewPtr.getNode());
5084
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00005085 SDValue Load;
5086 if (ExtType == ISD::NON_EXTLOAD)
5087 Load = DAG.getLoad(VT, N0.getDebugLoc(), LN0->getChain(), NewPtr,
5088 LN0->getPointerInfo().getWithOffset(PtrOff),
Pete Cooperd752e0f2011-11-08 18:42:53 +00005089 LN0->isVolatile(), LN0->isNonTemporal(),
5090 LN0->isInvariant(), NewAlign);
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00005091 else
Stuart Hastingsa9011292011-02-16 16:23:55 +00005092 Load = DAG.getExtLoad(ExtType, N0.getDebugLoc(), VT, LN0->getChain(),NewPtr,
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00005093 LN0->getPointerInfo().getWithOffset(PtrOff),
5094 ExtVT, LN0->isVolatile(), LN0->isNonTemporal(),
5095 NewAlign);
Chris Lattner4c32bc22010-12-22 07:36:50 +00005096
5097 // Replace the old load's chain with the new load's chain.
5098 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005099 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
Chris Lattner4c32bc22010-12-22 07:36:50 +00005100
5101 // Shift the result left, if we've swallowed a left shift.
5102 SDValue Result = Load;
5103 if (ShLeftAmt != 0) {
Owen Anderson95771af2011-02-25 21:41:48 +00005104 EVT ShImmTy = getShiftAmountTy(Result.getValueType());
Chris Lattner4c32bc22010-12-22 07:36:50 +00005105 if (!isUIntN(ShImmTy.getSizeInBits(), ShLeftAmt))
5106 ShImmTy = VT;
5107 Result = DAG.getNode(ISD::SHL, N0.getDebugLoc(), VT,
5108 Result, DAG.getConstant(ShLeftAmt, ShImmTy));
5109 }
5110
5111 // Return the new loaded value.
5112 return Result;
Evan Chengc88138f2007-03-22 01:54:19 +00005113}
5114
Dan Gohman475871a2008-07-27 21:46:04 +00005115SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
5116 SDValue N0 = N->getOperand(0);
5117 SDValue N1 = N->getOperand(1);
Owen Andersone50ed302009-08-10 22:56:29 +00005118 EVT VT = N->getValueType(0);
5119 EVT EVT = cast<VTSDNode>(N1)->getVT();
Dan Gohman87862e72009-12-11 21:31:27 +00005120 unsigned VTBits = VT.getScalarType().getSizeInBits();
Dan Gohmand1996362010-01-09 02:13:55 +00005121 unsigned EVTBits = EVT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00005122
Nate Begeman1d4d4142005-09-01 00:19:25 +00005123 // fold (sext_in_reg c1) -> c1
Chris Lattnereaeda562006-05-08 20:59:41 +00005124 if (isa<ConstantSDNode>(N0) || N0.getOpcode() == ISD::UNDEF)
Bill Wendling8509c902009-01-30 22:33:24 +00005125 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00005126
Chris Lattner541a24f2006-05-06 22:43:44 +00005127 // If the input is already sign extended, just drop the extension.
Dan Gohman87862e72009-12-11 21:31:27 +00005128 if (DAG.ComputeNumSignBits(N0) >= VTBits-EVTBits+1)
Chris Lattneree4ea922006-05-06 09:30:03 +00005129 return N0;
Scott Michelfdc40a02009-02-17 22:15:04 +00005130
Nate Begeman646d7e22005-09-02 21:18:40 +00005131 // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
5132 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
Duncan Sands8e4eb092008-06-08 20:54:56 +00005133 EVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT())) {
Bill Wendling8509c902009-01-30 22:33:24 +00005134 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT,
5135 N0.getOperand(0), N1);
Nate Begeman646d7e22005-09-02 21:18:40 +00005136 }
Chris Lattner4b37e872006-05-08 21:18:59 +00005137
Dan Gohman75dcf082008-07-31 00:50:31 +00005138 // fold (sext_in_reg (sext x)) -> (sext x)
5139 // fold (sext_in_reg (aext x)) -> (sext x)
5140 // if x is small enough.
5141 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
5142 SDValue N00 = N0.getOperand(0);
Evan Cheng003d7c42010-04-16 22:26:19 +00005143 if (N00.getValueType().getScalarType().getSizeInBits() <= EVTBits &&
5144 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
Bill Wendling8509c902009-01-30 22:33:24 +00005145 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, N00, N1);
Dan Gohman75dcf082008-07-31 00:50:31 +00005146 }
5147
Chris Lattner95a5e052007-04-17 19:03:21 +00005148 // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero.
Dan Gohman2e68b6f2008-02-25 21:11:39 +00005149 if (DAG.MaskedValueIsZero(N0, APInt::getBitsSet(VTBits, EVTBits-1, EVTBits)))
Bill Wendlingfc4b6772009-02-01 11:19:36 +00005150 return DAG.getZeroExtendInReg(N0, N->getDebugLoc(), EVT);
Scott Michelfdc40a02009-02-17 22:15:04 +00005151
Chris Lattner95a5e052007-04-17 19:03:21 +00005152 // fold operands of sext_in_reg based on knowledge that the top bits are not
5153 // demanded.
Dan Gohman475871a2008-07-27 21:46:04 +00005154 if (SimplifyDemandedBits(SDValue(N, 0)))
5155 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005156
Evan Chengc88138f2007-03-22 01:54:19 +00005157 // fold (sext_in_reg (load x)) -> (smaller sextload x)
5158 // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
Dan Gohman475871a2008-07-27 21:46:04 +00005159 SDValue NarrowLoad = ReduceLoadWidth(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005160 if (NarrowLoad.getNode())
Evan Chengc88138f2007-03-22 01:54:19 +00005161 return NarrowLoad;
5162
Bill Wendling8509c902009-01-30 22:33:24 +00005163 // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24)
Sylvestre Ledru94c22712012-09-27 10:14:43 +00005164 // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible.
Chris Lattner4b37e872006-05-08 21:18:59 +00005165 // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above.
5166 if (N0.getOpcode() == ISD::SRL) {
5167 if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
Dan Gohman87862e72009-12-11 21:31:27 +00005168 if (ShAmt->getZExtValue()+EVTBits <= VTBits) {
Sylvestre Ledru94c22712012-09-27 10:14:43 +00005169 // We can turn this into an SRA iff the input to the SRL is already sign
Chris Lattner4b37e872006-05-08 21:18:59 +00005170 // extended enough.
Dan Gohmanea859be2007-06-22 14:59:07 +00005171 unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
Dan Gohman87862e72009-12-11 21:31:27 +00005172 if (VTBits-(ShAmt->getZExtValue()+EVTBits) < InSignBits)
Bill Wendling8509c902009-01-30 22:33:24 +00005173 return DAG.getNode(ISD::SRA, N->getDebugLoc(), VT,
5174 N0.getOperand(0), N0.getOperand(1));
Chris Lattner4b37e872006-05-08 21:18:59 +00005175 }
5176 }
Evan Chengc88138f2007-03-22 01:54:19 +00005177
Nate Begemanded49632005-10-13 03:11:28 +00005178 // fold (sext_inreg (extload x)) -> (sextload x)
Scott Michelfdc40a02009-02-17 22:15:04 +00005179 if (ISD::isEXTLoad(N0.getNode()) &&
Gabor Greifba36cb52008-08-28 21:40:38 +00005180 ISD::isUNINDEXEDLoad(N0.getNode()) &&
Dan Gohmanb625f2f2008-01-30 00:15:11 +00005181 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00005182 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00005183 TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT))) {
Evan Cheng466685d2006-10-09 20:57:25 +00005184 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00005185 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling8509c902009-01-30 22:33:24 +00005186 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00005187 LN0->getBasePtr(), LN0->getPointerInfo(),
5188 EVT,
David Greene1e559442010-02-15 17:00:31 +00005189 LN0->isVolatile(), LN0->isNonTemporal(),
5190 LN0->getAlignment());
Chris Lattnerd4771842005-12-14 19:25:30 +00005191 CombineTo(N, ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00005192 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00005193 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00005194 }
Sylvestre Ledru94c22712012-09-27 10:14:43 +00005195 // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
Gabor Greifba36cb52008-08-28 21:40:38 +00005196 if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
Evan Cheng83060c52007-03-07 08:07:03 +00005197 N0.hasOneUse() &&
Dan Gohmanb625f2f2008-01-30 00:15:11 +00005198 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00005199 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00005200 TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT))) {
Evan Cheng466685d2006-10-09 20:57:25 +00005201 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00005202 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling8509c902009-01-30 22:33:24 +00005203 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00005204 LN0->getBasePtr(), LN0->getPointerInfo(),
5205 EVT,
David Greene1e559442010-02-15 17:00:31 +00005206 LN0->isVolatile(), LN0->isNonTemporal(),
5207 LN0->getAlignment());
Chris Lattnerd4771842005-12-14 19:25:30 +00005208 CombineTo(N, ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00005209 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00005210 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00005211 }
Evan Cheng9568e5c2011-06-21 06:01:08 +00005212
5213 // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
5214 if (EVTBits <= 16 && N0.getOpcode() == ISD::OR) {
5215 SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
5216 N0.getOperand(1), false);
5217 if (BSwap.getNode() != 0)
5218 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT,
5219 BSwap, N1);
5220 }
5221
Dan Gohman475871a2008-07-27 21:46:04 +00005222 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005223}
5224
Dan Gohman475871a2008-07-27 21:46:04 +00005225SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
5226 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00005227 EVT VT = N->getValueType(0);
Nadav Rotem7e413e9c2012-02-03 13:18:25 +00005228 bool isLE = TLI.isLittleEndian();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005229
5230 // noop truncate
5231 if (N0.getValueType() == N->getValueType(0))
Nate Begeman83e75ec2005-09-06 04:43:02 +00005232 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00005233 // fold (truncate c1) -> c1
Chris Lattner310b5782006-05-06 23:06:26 +00005234 if (isa<ConstantSDNode>(N0))
Bill Wendling67a67682009-01-30 22:44:24 +00005235 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00005236 // fold (truncate (truncate x)) -> (truncate x)
5237 if (N0.getOpcode() == ISD::TRUNCATE)
Bill Wendling67a67682009-01-30 22:44:24 +00005238 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, N0.getOperand(0));
Nate Begeman1d4d4142005-09-01 00:19:25 +00005239 // fold (truncate (ext x)) -> (ext x) or (truncate x) or x
Chris Lattner7f893c02010-04-07 18:13:33 +00005240 if (N0.getOpcode() == ISD::ZERO_EXTEND ||
5241 N0.getOpcode() == ISD::SIGN_EXTEND ||
Chris Lattnerb72773b2006-05-05 22:56:26 +00005242 N0.getOpcode() == ISD::ANY_EXTEND) {
Duncan Sands8e4eb092008-06-08 20:54:56 +00005243 if (N0.getOperand(0).getValueType().bitsLT(VT))
Nate Begeman1d4d4142005-09-01 00:19:25 +00005244 // if the source is smaller than the dest, we still need an extend
Bill Wendling67a67682009-01-30 22:44:24 +00005245 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
5246 N0.getOperand(0));
Duncan Sands8e4eb092008-06-08 20:54:56 +00005247 else if (N0.getOperand(0).getValueType().bitsGT(VT))
Nate Begeman1d4d4142005-09-01 00:19:25 +00005248 // if the source is larger than the dest, than we just need the truncate
Bill Wendling67a67682009-01-30 22:44:24 +00005249 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, N0.getOperand(0));
Nate Begeman1d4d4142005-09-01 00:19:25 +00005250 else
5251 // if the source and dest are the same type, we can drop both the extend
Evan Chengd40d03e2010-01-06 19:38:29 +00005252 // and the truncate.
Nate Begeman83e75ec2005-09-06 04:43:02 +00005253 return N0.getOperand(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00005254 }
Evan Cheng007b69e2007-03-21 20:14:05 +00005255
Nadav Rotemcc870a82012-02-05 11:39:23 +00005256 // Fold extract-and-trunc into a narrow extract. For example:
5257 // i64 x = EXTRACT_VECTOR_ELT(v2i64 val, i32 1)
5258 // i32 y = TRUNCATE(i64 x)
5259 // -- becomes --
5260 // v16i8 b = BITCAST (v2i64 val)
5261 // i8 x = EXTRACT_VECTOR_ELT(v16i8 b, i32 8)
5262 //
5263 // Note: We only run this optimization after type legalization (which often
Nadav Rotem7e413e9c2012-02-03 13:18:25 +00005264 // creates this pattern) and before operation legalization after which
5265 // we need to be more careful about the vector instructions that we generate.
5266 if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5267 LegalTypes && !LegalOperations && N0->hasOneUse()) {
5268
5269 EVT VecTy = N0.getOperand(0).getValueType();
5270 EVT ExTy = N0.getValueType();
5271 EVT TrTy = N->getValueType(0);
5272
5273 unsigned NumElem = VecTy.getVectorNumElements();
5274 unsigned SizeRatio = ExTy.getSizeInBits()/TrTy.getSizeInBits();
5275
5276 EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, SizeRatio * NumElem);
5277 assert(NVT.getSizeInBits() == VecTy.getSizeInBits() && "Invalid Size");
5278
5279 SDValue EltNo = N0->getOperand(1);
5280 if (isa<ConstantSDNode>(EltNo) && isTypeLegal(NVT)) {
5281 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
Jim Grosbacha249f7d2012-05-08 20:56:07 +00005282 EVT IndexTy = N0->getOperand(1).getValueType();
Nadav Rotem7e413e9c2012-02-03 13:18:25 +00005283 int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
5284
5285 SDValue V = DAG.getNode(ISD::BITCAST, N->getDebugLoc(),
5286 NVT, N0.getOperand(0));
5287
5288 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT,
5289 N->getDebugLoc(), TrTy, V,
Jim Grosbacha249f7d2012-05-08 20:56:07 +00005290 DAG.getConstant(Index, IndexTy));
Nadav Rotem7e413e9c2012-02-03 13:18:25 +00005291 }
5292 }
5293
Chris Lattner2b4c2792007-10-13 06:35:54 +00005294 // See if we can simplify the input to this truncate through knowledge that
Nadav Rotem8c20ec52011-02-24 21:01:34 +00005295 // only the low bits are being used.
5296 // For example "trunc (or (shl x, 8), y)" // -> trunc y
Nadav Rotemfcd96192011-02-27 07:40:43 +00005297 // Currently we only perform this optimization on scalars because vectors
Nadav Rotem8c20ec52011-02-24 21:01:34 +00005298 // may have different active low bits.
5299 if (!VT.isVector()) {
5300 SDValue Shorter =
5301 GetDemandedBits(N0, APInt::getLowBitsSet(N0.getValueSizeInBits(),
5302 VT.getSizeInBits()));
5303 if (Shorter.getNode())
5304 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, Shorter);
5305 }
Nate Begeman3df4d522005-10-12 20:40:40 +00005306 // fold (truncate (load x)) -> (smaller load x)
Evan Cheng007b69e2007-03-21 20:14:05 +00005307 // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
Dan Gohman4e39e9d2010-06-24 14:30:44 +00005308 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) {
5309 SDValue Reduced = ReduceLoadWidth(N);
5310 if (Reduced.getNode())
5311 return Reduced;
5312 }
5313
5314 // Simplify the operands using demanded-bits information.
5315 if (!VT.isVector() &&
5316 SimplifyDemandedBits(SDValue(N, 0)))
5317 return SDValue(N, 0);
5318
Evan Chenge5b51ac2010-04-17 06:13:15 +00005319 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005320}
5321
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005322static SDNode *getBuildPairElt(SDNode *N, unsigned i) {
Dan Gohman475871a2008-07-27 21:46:04 +00005323 SDValue Elt = N->getOperand(i);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005324 if (Elt.getOpcode() != ISD::MERGE_VALUES)
Gabor Greifba36cb52008-08-28 21:40:38 +00005325 return Elt.getNode();
5326 return Elt.getOperand(Elt.getResNo()).getNode();
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005327}
5328
5329/// CombineConsecutiveLoads - build_pair (load, load) -> load
Scott Michelfdc40a02009-02-17 22:15:04 +00005330/// if load locations are consecutive.
Owen Andersone50ed302009-08-10 22:56:29 +00005331SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005332 assert(N->getOpcode() == ISD::BUILD_PAIR);
5333
Nate Begemanabc01992009-06-05 21:37:30 +00005334 LoadSDNode *LD1 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 0));
5335 LoadSDNode *LD2 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 1));
Chris Lattnerfa459012010-09-21 16:08:50 +00005336 if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !LD1->hasOneUse() ||
5337 LD1->getPointerInfo().getAddrSpace() !=
5338 LD2->getPointerInfo().getAddrSpace())
Dan Gohman475871a2008-07-27 21:46:04 +00005339 return SDValue();
Owen Andersone50ed302009-08-10 22:56:29 +00005340 EVT LD1VT = LD1->getValueType(0);
Bill Wendling67a67682009-01-30 22:44:24 +00005341
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005342 if (ISD::isNON_EXTLoad(LD2) &&
5343 LD2->hasOneUse() &&
Duncan Sandsd4b9c172008-06-13 19:07:40 +00005344 // If both are volatile this would reduce the number of volatile loads.
5345 // If one is volatile it might be ok, but play conservative and bail out.
Nate Begemanabc01992009-06-05 21:37:30 +00005346 !LD1->isVolatile() &&
5347 !LD2->isVolatile() &&
Evan Cheng64fa4a92009-12-09 01:36:00 +00005348 DAG.isConsecutiveLoad(LD2, LD1, LD1VT.getSizeInBits()/8, 1)) {
Nate Begemanabc01992009-06-05 21:37:30 +00005349 unsigned Align = LD1->getAlignment();
Dan Gohman6448d912008-09-04 15:39:15 +00005350 unsigned NewAlign = TLI.getTargetData()->
Owen Anderson23b9b192009-08-12 00:36:31 +00005351 getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
Bill Wendling67a67682009-01-30 22:44:24 +00005352
Duncan Sandsd4b9c172008-06-13 19:07:40 +00005353 if (NewAlign <= Align &&
Duncan Sands25cf2272008-11-24 14:53:14 +00005354 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)))
Nate Begemanabc01992009-06-05 21:37:30 +00005355 return DAG.getLoad(VT, N->getDebugLoc(), LD1->getChain(),
Chris Lattnerfa459012010-09-21 16:08:50 +00005356 LD1->getBasePtr(), LD1->getPointerInfo(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00005357 false, false, false, Align);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005358 }
Bill Wendling67a67682009-01-30 22:44:24 +00005359
Dan Gohman475871a2008-07-27 21:46:04 +00005360 return SDValue();
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005361}
5362
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005363SDValue DAGCombiner::visitBITCAST(SDNode *N) {
Dan Gohman475871a2008-07-27 21:46:04 +00005364 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00005365 EVT VT = N->getValueType(0);
Chris Lattner94683772005-12-23 05:30:37 +00005366
Dan Gohman7f321562007-06-25 16:23:39 +00005367 // If the input is a BUILD_VECTOR with all constant elements, fold this now.
5368 // Only do this before legalize, since afterward the target may be depending
5369 // on the bitconvert.
5370 // First check to see if this is all constant.
Duncan Sands25cf2272008-11-24 14:53:14 +00005371 if (!LegalTypes &&
Gabor Greifba36cb52008-08-28 21:40:38 +00005372 N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNode()->hasOneUse() &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00005373 VT.isVector()) {
Dan Gohman7f321562007-06-25 16:23:39 +00005374 bool isSimple = true;
5375 for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i)
5376 if (N0.getOperand(i).getOpcode() != ISD::UNDEF &&
5377 N0.getOperand(i).getOpcode() != ISD::Constant &&
5378 N0.getOperand(i).getOpcode() != ISD::ConstantFP) {
Scott Michelfdc40a02009-02-17 22:15:04 +00005379 isSimple = false;
Dan Gohman7f321562007-06-25 16:23:39 +00005380 break;
5381 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005382
Owen Andersone50ed302009-08-10 22:56:29 +00005383 EVT DestEltVT = N->getValueType(0).getVectorElementType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00005384 assert(!DestEltVT.isVector() &&
Dan Gohman7f321562007-06-25 16:23:39 +00005385 "Element type of vector ValueType must not be vector!");
Bill Wendling67a67682009-01-30 22:44:24 +00005386 if (isSimple)
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005387 return ConstantFoldBITCASTofBUILD_VECTOR(N0.getNode(), DestEltVT);
Dan Gohman7f321562007-06-25 16:23:39 +00005388 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005389
Dan Gohman3dd168d2008-09-05 01:58:21 +00005390 // If the input is a constant, let getNode fold it.
Chris Lattner94683772005-12-23 05:30:37 +00005391 if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005392 SDValue Res = DAG.getNode(ISD::BITCAST, N->getDebugLoc(), VT, N0);
Dan Gohmana407ca12009-08-10 23:15:10 +00005393 if (Res.getNode() != N) {
5394 if (!LegalOperations ||
5395 TLI.isOperationLegal(Res.getNode()->getOpcode(), VT))
5396 return Res;
5397
5398 // Folding it resulted in an illegal node, and it's too late to
5399 // do that. Clean up the old node and forego the transformation.
5400 // Ideally this won't happen very often, because instcombine
5401 // and the earlier dagcombine runs (where illegal nodes are
5402 // permitted) should have folded most of them already.
5403 DAG.DeleteNode(Res.getNode());
5404 }
Chris Lattner94683772005-12-23 05:30:37 +00005405 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005406
Bill Wendling67a67682009-01-30 22:44:24 +00005407 // (conv (conv x, t1), t2) -> (conv x, t2)
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005408 if (N0.getOpcode() == ISD::BITCAST)
5409 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005410 N0.getOperand(0));
Chris Lattner6258fb22006-04-02 02:53:43 +00005411
Chris Lattner57104102005-12-23 05:44:41 +00005412 // fold (conv (load x)) -> (load (conv*)x)
Evan Cheng513da432007-10-06 08:19:55 +00005413 // If the resultant load doesn't need a higher alignment than the original!
Gabor Greifba36cb52008-08-28 21:40:38 +00005414 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
Duncan Sandsd4b9c172008-06-13 19:07:40 +00005415 // Do not change the width of a volatile load.
5416 !cast<LoadSDNode>(N0)->isVolatile() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00005417 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT))) {
Evan Cheng466685d2006-10-09 20:57:25 +00005418 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman6448d912008-09-04 15:39:15 +00005419 unsigned Align = TLI.getTargetData()->
Owen Anderson23b9b192009-08-12 00:36:31 +00005420 getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
Evan Cheng59d5b682007-05-07 21:27:48 +00005421 unsigned OrigAlign = LN0->getAlignment();
Bill Wendling67a67682009-01-30 22:44:24 +00005422
Evan Cheng59d5b682007-05-07 21:27:48 +00005423 if (Align <= OrigAlign) {
Bill Wendling67a67682009-01-30 22:44:24 +00005424 SDValue Load = DAG.getLoad(VT, N->getDebugLoc(), LN0->getChain(),
Chris Lattnerfa459012010-09-21 16:08:50 +00005425 LN0->getBasePtr(), LN0->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00005426 LN0->isVolatile(), LN0->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00005427 LN0->isInvariant(), OrigAlign);
Evan Cheng59d5b682007-05-07 21:27:48 +00005428 AddToWorkList(N);
Gabor Greif12632d22008-08-30 19:29:20 +00005429 CombineTo(N0.getNode(),
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005430 DAG.getNode(ISD::BITCAST, N0.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005431 N0.getValueType(), Load),
Evan Cheng59d5b682007-05-07 21:27:48 +00005432 Load.getValue(1));
5433 return Load;
5434 }
Chris Lattner57104102005-12-23 05:44:41 +00005435 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00005436
Bill Wendling67a67682009-01-30 22:44:24 +00005437 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
5438 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
Chris Lattner3bd39d42008-01-27 17:42:27 +00005439 // This often reduces constant pool loads.
Owen Anderson29f60f32012-04-02 22:10:29 +00005440 if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(VT)) ||
5441 (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(VT))) &&
Nadav Rotem91a7e012012-09-13 14:54:28 +00005442 N0.getNode()->hasOneUse() && VT.isInteger() &&
5443 !VT.isVector() && !N0.getValueType().isVector()) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005444 SDValue NewConv = DAG.getNode(ISD::BITCAST, N0.getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005445 N0.getOperand(0));
Gabor Greifba36cb52008-08-28 21:40:38 +00005446 AddToWorkList(NewConv.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00005447
Duncan Sands83ec4b62008-06-06 12:08:01 +00005448 APInt SignBit = APInt::getSignBit(VT.getSizeInBits());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005449 if (N0.getOpcode() == ISD::FNEG)
Bill Wendling67a67682009-01-30 22:44:24 +00005450 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT,
5451 NewConv, DAG.getConstant(SignBit, VT));
Chris Lattner3bd39d42008-01-27 17:42:27 +00005452 assert(N0.getOpcode() == ISD::FABS);
Bill Wendling67a67682009-01-30 22:44:24 +00005453 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
5454 NewConv, DAG.getConstant(~SignBit, VT));
Chris Lattner3bd39d42008-01-27 17:42:27 +00005455 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005456
Bill Wendling67a67682009-01-30 22:44:24 +00005457 // fold (bitconvert (fcopysign cst, x)) ->
5458 // (or (and (bitconvert x), sign), (and cst, (not sign)))
5459 // Note that we don't handle (copysign x, cst) because this can always be
5460 // folded to an fneg or fabs.
Gabor Greifba36cb52008-08-28 21:40:38 +00005461 if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse() &&
Chris Lattnerf32aac32008-01-27 23:32:17 +00005462 isa<ConstantFPSDNode>(N0.getOperand(0)) &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00005463 VT.isInteger() && !VT.isVector()) {
5464 unsigned OrigXWidth = N0.getOperand(1).getValueType().getSizeInBits();
Owen Anderson23b9b192009-08-12 00:36:31 +00005465 EVT IntXVT = EVT::getIntegerVT(*DAG.getContext(), OrigXWidth);
Chris Lattner2392ae72010-04-15 04:48:01 +00005466 if (isTypeLegal(IntXVT)) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005467 SDValue X = DAG.getNode(ISD::BITCAST, N0.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005468 IntXVT, N0.getOperand(1));
Duncan Sands25cf2272008-11-24 14:53:14 +00005469 AddToWorkList(X.getNode());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005470
Duncan Sands25cf2272008-11-24 14:53:14 +00005471 // If X has a different width than the result/lhs, sext it or truncate it.
5472 unsigned VTWidth = VT.getSizeInBits();
5473 if (OrigXWidth < VTWidth) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00005474 X = DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, X);
Duncan Sands25cf2272008-11-24 14:53:14 +00005475 AddToWorkList(X.getNode());
5476 } else if (OrigXWidth > VTWidth) {
5477 // To get the sign bit in the right place, we have to shift it right
5478 // before truncating.
Bill Wendling9729c5a2009-01-31 03:12:48 +00005479 X = DAG.getNode(ISD::SRL, X.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005480 X.getValueType(), X,
Duncan Sands25cf2272008-11-24 14:53:14 +00005481 DAG.getConstant(OrigXWidth-VTWidth, X.getValueType()));
5482 AddToWorkList(X.getNode());
Bill Wendling9729c5a2009-01-31 03:12:48 +00005483 X = DAG.getNode(ISD::TRUNCATE, X.getDebugLoc(), VT, X);
Duncan Sands25cf2272008-11-24 14:53:14 +00005484 AddToWorkList(X.getNode());
5485 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005486
Duncan Sands25cf2272008-11-24 14:53:14 +00005487 APInt SignBit = APInt::getSignBit(VT.getSizeInBits());
Bill Wendling9729c5a2009-01-31 03:12:48 +00005488 X = DAG.getNode(ISD::AND, X.getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005489 X, DAG.getConstant(SignBit, VT));
Duncan Sands25cf2272008-11-24 14:53:14 +00005490 AddToWorkList(X.getNode());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005491
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005492 SDValue Cst = DAG.getNode(ISD::BITCAST, N0.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005493 VT, N0.getOperand(0));
Bill Wendling9729c5a2009-01-31 03:12:48 +00005494 Cst = DAG.getNode(ISD::AND, Cst.getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005495 Cst, DAG.getConstant(~SignBit, VT));
Duncan Sands25cf2272008-11-24 14:53:14 +00005496 AddToWorkList(Cst.getNode());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005497
Bill Wendling67a67682009-01-30 22:44:24 +00005498 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, X, Cst);
Duncan Sands25cf2272008-11-24 14:53:14 +00005499 }
Chris Lattner3bd39d42008-01-27 17:42:27 +00005500 }
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005501
Sylvestre Ledru94c22712012-09-27 10:14:43 +00005502 // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive.
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005503 if (N0.getOpcode() == ISD::BUILD_PAIR) {
Gabor Greifba36cb52008-08-28 21:40:38 +00005504 SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT);
5505 if (CombineLD.getNode())
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005506 return CombineLD;
5507 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005508
Dan Gohman475871a2008-07-27 21:46:04 +00005509 return SDValue();
Chris Lattner94683772005-12-23 05:30:37 +00005510}
5511
Dan Gohman475871a2008-07-27 21:46:04 +00005512SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
Owen Andersone50ed302009-08-10 22:56:29 +00005513 EVT VT = N->getValueType(0);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005514 return CombineConsecutiveLoads(N, VT);
5515}
5516
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005517/// ConstantFoldBITCASTofBUILD_VECTOR - We know that BV is a build_vector
Scott Michelfdc40a02009-02-17 22:15:04 +00005518/// node with Constant, ConstantFP or Undef operands. DstEltVT indicates the
Chris Lattner6258fb22006-04-02 02:53:43 +00005519/// destination element value type.
Dan Gohman475871a2008-07-27 21:46:04 +00005520SDValue DAGCombiner::
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005521ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
Owen Andersone50ed302009-08-10 22:56:29 +00005522 EVT SrcEltVT = BV->getValueType(0).getVectorElementType();
Scott Michelfdc40a02009-02-17 22:15:04 +00005523
Chris Lattner6258fb22006-04-02 02:53:43 +00005524 // If this is already the right type, we're done.
Dan Gohman475871a2008-07-27 21:46:04 +00005525 if (SrcEltVT == DstEltVT) return SDValue(BV, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005526
Duncan Sands83ec4b62008-06-06 12:08:01 +00005527 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
5528 unsigned DstBitSize = DstEltVT.getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00005529
Chris Lattner6258fb22006-04-02 02:53:43 +00005530 // If this is a conversion of N elements of one type to N elements of another
5531 // type, convert each element. This handles FP<->INT cases.
5532 if (SrcBitSize == DstBitSize) {
Nate Begemane0efc212010-07-27 18:02:18 +00005533 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
5534 BV->getValueType(0).getVectorNumElements());
5535
5536 // Due to the FP element handling below calling this routine recursively,
5537 // we can end up with a scalar-to-vector node here.
5538 if (BV->getOpcode() == ISD::SCALAR_TO_VECTOR)
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005539 return DAG.getNode(ISD::SCALAR_TO_VECTOR, BV->getDebugLoc(), VT,
5540 DAG.getNode(ISD::BITCAST, BV->getDebugLoc(),
Nate Begemane0efc212010-07-27 18:02:18 +00005541 DstEltVT, BV->getOperand(0)));
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005542
Dan Gohman475871a2008-07-27 21:46:04 +00005543 SmallVector<SDValue, 8> Ops;
Dan Gohman7f321562007-06-25 16:23:39 +00005544 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
Bob Wilsonb1303d02009-04-13 22:05:19 +00005545 SDValue Op = BV->getOperand(i);
5546 // If the vector element type is not legal, the BUILD_VECTOR operands
5547 // are promoted and implicitly truncated. Make that explicit here.
Bob Wilsonc8851652009-04-20 17:27:09 +00005548 if (Op.getValueType() != SrcEltVT)
5549 Op = DAG.getNode(ISD::TRUNCATE, BV->getDebugLoc(), SrcEltVT, Op);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005550 Ops.push_back(DAG.getNode(ISD::BITCAST, BV->getDebugLoc(),
Bob Wilsonb1303d02009-04-13 22:05:19 +00005551 DstEltVT, Op));
Gabor Greifba36cb52008-08-28 21:40:38 +00005552 AddToWorkList(Ops.back().getNode());
Chris Lattner3e104b12006-04-08 04:15:24 +00005553 }
Evan Chenga87008d2009-02-25 22:49:59 +00005554 return DAG.getNode(ISD::BUILD_VECTOR, BV->getDebugLoc(), VT,
5555 &Ops[0], Ops.size());
Chris Lattner6258fb22006-04-02 02:53:43 +00005556 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005557
Chris Lattner6258fb22006-04-02 02:53:43 +00005558 // Otherwise, we're growing or shrinking the elements. To avoid having to
5559 // handle annoying details of growing/shrinking FP values, we convert them to
5560 // int first.
Duncan Sands83ec4b62008-06-06 12:08:01 +00005561 if (SrcEltVT.isFloatingPoint()) {
Chris Lattner6258fb22006-04-02 02:53:43 +00005562 // Convert the input float vector to a int vector where the elements are the
5563 // same sizes.
Owen Anderson825b72b2009-08-11 20:47:22 +00005564 assert((SrcEltVT == MVT::f32 || SrcEltVT == MVT::f64) && "Unknown FP VT!");
Owen Anderson23b9b192009-08-12 00:36:31 +00005565 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltVT.getSizeInBits());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005566 BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
Chris Lattner6258fb22006-04-02 02:53:43 +00005567 SrcEltVT = IntVT;
5568 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005569
Chris Lattner6258fb22006-04-02 02:53:43 +00005570 // Now we know the input is an integer vector. If the output is a FP type,
5571 // convert to integer first, then to FP of the right size.
Duncan Sands83ec4b62008-06-06 12:08:01 +00005572 if (DstEltVT.isFloatingPoint()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00005573 assert((DstEltVT == MVT::f32 || DstEltVT == MVT::f64) && "Unknown FP VT!");
Owen Anderson23b9b192009-08-12 00:36:31 +00005574 EVT TmpVT = EVT::getIntegerVT(*DAG.getContext(), DstEltVT.getSizeInBits());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005575 SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
Scott Michelfdc40a02009-02-17 22:15:04 +00005576
Chris Lattner6258fb22006-04-02 02:53:43 +00005577 // Next, convert to FP elements of the same size.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005578 return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
Chris Lattner6258fb22006-04-02 02:53:43 +00005579 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005580
Chris Lattner6258fb22006-04-02 02:53:43 +00005581 // Okay, we know the src/dst types are both integers of differing types.
5582 // Handling growing first.
Duncan Sands83ec4b62008-06-06 12:08:01 +00005583 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
Chris Lattner6258fb22006-04-02 02:53:43 +00005584 if (SrcBitSize < DstBitSize) {
5585 unsigned NumInputsPerOutput = DstBitSize/SrcBitSize;
Scott Michelfdc40a02009-02-17 22:15:04 +00005586
Dan Gohman475871a2008-07-27 21:46:04 +00005587 SmallVector<SDValue, 8> Ops;
Dan Gohman7f321562007-06-25 16:23:39 +00005588 for (unsigned i = 0, e = BV->getNumOperands(); i != e;
Chris Lattner6258fb22006-04-02 02:53:43 +00005589 i += NumInputsPerOutput) {
5590 bool isLE = TLI.isLittleEndian();
Dan Gohman220a8232008-03-03 23:51:38 +00005591 APInt NewBits = APInt(DstBitSize, 0);
Chris Lattner6258fb22006-04-02 02:53:43 +00005592 bool EltIsUndef = true;
5593 for (unsigned j = 0; j != NumInputsPerOutput; ++j) {
5594 // Shift the previously computed bits over.
5595 NewBits <<= SrcBitSize;
Dan Gohman475871a2008-07-27 21:46:04 +00005596 SDValue Op = BV->getOperand(i+ (isLE ? (NumInputsPerOutput-j-1) : j));
Chris Lattner6258fb22006-04-02 02:53:43 +00005597 if (Op.getOpcode() == ISD::UNDEF) continue;
5598 EltIsUndef = false;
Scott Michelfdc40a02009-02-17 22:15:04 +00005599
Jay Foad40f8f622010-12-07 08:25:19 +00005600 NewBits |= cast<ConstantSDNode>(Op)->getAPIntValue().
Dan Gohman58c25872010-04-12 02:24:01 +00005601 zextOrTrunc(SrcBitSize).zext(DstBitSize);
Chris Lattner6258fb22006-04-02 02:53:43 +00005602 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005603
Chris Lattner6258fb22006-04-02 02:53:43 +00005604 if (EltIsUndef)
Dale Johannesene8d72302009-02-06 23:05:02 +00005605 Ops.push_back(DAG.getUNDEF(DstEltVT));
Chris Lattner6258fb22006-04-02 02:53:43 +00005606 else
5607 Ops.push_back(DAG.getConstant(NewBits, DstEltVT));
5608 }
5609
Owen Anderson23b9b192009-08-12 00:36:31 +00005610 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size());
Evan Chenga87008d2009-02-25 22:49:59 +00005611 return DAG.getNode(ISD::BUILD_VECTOR, BV->getDebugLoc(), VT,
5612 &Ops[0], Ops.size());
Chris Lattner6258fb22006-04-02 02:53:43 +00005613 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005614
Chris Lattner6258fb22006-04-02 02:53:43 +00005615 // Finally, this must be the case where we are shrinking elements: each input
5616 // turns into multiple outputs.
Evan Chengefec7512008-02-18 23:04:32 +00005617 bool isS2V = ISD::isScalarToVector(BV);
Chris Lattner6258fb22006-04-02 02:53:43 +00005618 unsigned NumOutputsPerInput = SrcBitSize/DstBitSize;
Owen Anderson23b9b192009-08-12 00:36:31 +00005619 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
5620 NumOutputsPerInput*BV->getNumOperands());
Dan Gohman475871a2008-07-27 21:46:04 +00005621 SmallVector<SDValue, 8> Ops;
Bill Wendlingb0162f52009-01-30 22:53:48 +00005622
Dan Gohman7f321562007-06-25 16:23:39 +00005623 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
Chris Lattner6258fb22006-04-02 02:53:43 +00005624 if (BV->getOperand(i).getOpcode() == ISD::UNDEF) {
5625 for (unsigned j = 0; j != NumOutputsPerInput; ++j)
Dale Johannesene8d72302009-02-06 23:05:02 +00005626 Ops.push_back(DAG.getUNDEF(DstEltVT));
Chris Lattner6258fb22006-04-02 02:53:43 +00005627 continue;
5628 }
Bill Wendlingb0162f52009-01-30 22:53:48 +00005629
Jay Foad40f8f622010-12-07 08:25:19 +00005630 APInt OpVal = cast<ConstantSDNode>(BV->getOperand(i))->
5631 getAPIntValue().zextOrTrunc(SrcBitSize);
Bill Wendlingb0162f52009-01-30 22:53:48 +00005632
Chris Lattner6258fb22006-04-02 02:53:43 +00005633 for (unsigned j = 0; j != NumOutputsPerInput; ++j) {
Jay Foad40f8f622010-12-07 08:25:19 +00005634 APInt ThisVal = OpVal.trunc(DstBitSize);
Chris Lattner6258fb22006-04-02 02:53:43 +00005635 Ops.push_back(DAG.getConstant(ThisVal, DstEltVT));
Jay Foad40f8f622010-12-07 08:25:19 +00005636 if (isS2V && i == 0 && j == 0 && ThisVal.zext(SrcBitSize) == OpVal)
Evan Chengefec7512008-02-18 23:04:32 +00005637 // Simply turn this into a SCALAR_TO_VECTOR of the new type.
Bill Wendlingb0162f52009-01-30 22:53:48 +00005638 return DAG.getNode(ISD::SCALAR_TO_VECTOR, BV->getDebugLoc(), VT,
5639 Ops[0]);
Dan Gohman220a8232008-03-03 23:51:38 +00005640 OpVal = OpVal.lshr(DstBitSize);
Chris Lattner6258fb22006-04-02 02:53:43 +00005641 }
5642
5643 // For big endian targets, swap the order of the pieces of each element.
Duncan Sands0753fc12008-02-11 10:37:04 +00005644 if (TLI.isBigEndian())
Chris Lattner6258fb22006-04-02 02:53:43 +00005645 std::reverse(Ops.end()-NumOutputsPerInput, Ops.end());
5646 }
Bill Wendlingb0162f52009-01-30 22:53:48 +00005647
Evan Chenga87008d2009-02-25 22:49:59 +00005648 return DAG.getNode(ISD::BUILD_VECTOR, BV->getDebugLoc(), VT,
5649 &Ops[0], Ops.size());
Chris Lattner6258fb22006-04-02 02:53:43 +00005650}
5651
Dan Gohman475871a2008-07-27 21:46:04 +00005652SDValue DAGCombiner::visitFADD(SDNode *N) {
5653 SDValue N0 = N->getOperand(0);
5654 SDValue N1 = N->getOperand(1);
Nate Begemana0e221d2005-10-18 00:28:13 +00005655 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5656 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005657 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005658
Dan Gohman7f321562007-06-25 16:23:39 +00005659 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00005660 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005661 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005662 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00005663 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005664
Lang Hames01806942012-06-14 20:37:15 +00005665 // fold (fadd c1, c2) -> c1 + c2
Owen Anderson825b72b2009-08-11 20:47:22 +00005666 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005667 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0, N1);
Nate Begemana0e221d2005-10-18 00:28:13 +00005668 // canonicalize constant to RHS
5669 if (N0CFP && !N1CFP)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005670 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N1, N0);
5671 // fold (fadd A, 0) -> A
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005672 if (DAG.getTarget().Options.UnsafeFPMath && N1CFP &&
5673 N1CFP->getValueAPF().isZero())
Dan Gohman760f86f2009-01-22 21:58:43 +00005674 return N0;
Bill Wendlingb0162f52009-01-30 22:53:48 +00005675 // fold (fadd A, (fneg B)) -> (fsub A, B)
Owen Andersonafd3d562012-03-06 00:29:31 +00005676 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
Nadav Rotem6dfabb62012-09-20 08:53:31 +00005677 isNegatibleForFree(N1, LegalOperations, TLI, &DAG.getTarget().Options) == 2)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005678 return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N0,
Duncan Sands25cf2272008-11-24 14:53:14 +00005679 GetNegatedExpression(N1, DAG, LegalOperations));
Bill Wendlingb0162f52009-01-30 22:53:48 +00005680 // fold (fadd (fneg A), B) -> (fsub B, A)
Owen Andersonafd3d562012-03-06 00:29:31 +00005681 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
Nadav Rotem6dfabb62012-09-20 08:53:31 +00005682 isNegatibleForFree(N0, LegalOperations, TLI, &DAG.getTarget().Options) == 2)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005683 return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N1,
Duncan Sands25cf2272008-11-24 14:53:14 +00005684 GetNegatedExpression(N0, DAG, LegalOperations));
Scott Michelfdc40a02009-02-17 22:15:04 +00005685
Chris Lattnerddae4bd2007-01-08 23:04:05 +00005686 // If allowed, fold (fadd (fadd x, c1), c2) -> (fadd x, (fadd c1, c2))
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005687 if (DAG.getTarget().Options.UnsafeFPMath && N1CFP &&
5688 N0.getOpcode() == ISD::FADD && N0.getNode()->hasOneUse() &&
5689 isa<ConstantFPSDNode>(N0.getOperand(1)))
Bill Wendlingb0162f52009-01-30 22:53:48 +00005690 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0.getOperand(0),
Bill Wendlingfc4b6772009-02-01 11:19:36 +00005691 DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
5692 N0.getOperand(1), N1));
Scott Michelfdc40a02009-02-17 22:15:04 +00005693
Owen Anderson43da6c72012-08-30 23:35:16 +00005694 // In unsafe math mode, we can fold chains of FADD's of the same value
5695 // into multiplications. This transform is not safe in general because
5696 // we are reducing the number of rounding steps.
5697 if (DAG.getTarget().Options.UnsafeFPMath &&
5698 TLI.isOperationLegalOrCustom(ISD::FMUL, VT) &&
5699 !N0CFP && !N1CFP) {
5700 if (N0.getOpcode() == ISD::FMUL) {
5701 ConstantFPSDNode *CFP00 = dyn_cast<ConstantFPSDNode>(N0.getOperand(0));
5702 ConstantFPSDNode *CFP01 = dyn_cast<ConstantFPSDNode>(N0.getOperand(1));
5703
5704 // (fadd (fmul c, x), x) -> (fmul c+1, x)
5705 if (CFP00 && !CFP01 && N0.getOperand(1) == N1) {
5706 SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
5707 SDValue(CFP00, 0),
5708 DAG.getConstantFP(1.0, VT));
5709 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
5710 N1, NewCFP);
5711 }
5712
5713 // (fadd (fmul x, c), x) -> (fmul c+1, x)
5714 if (CFP01 && !CFP00 && N0.getOperand(0) == N1) {
5715 SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
5716 SDValue(CFP01, 0),
5717 DAG.getConstantFP(1.0, VT));
5718 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
5719 N1, NewCFP);
5720 }
5721
5722 // (fadd (fadd x, x), x) -> (fmul 3.0, x)
5723 if (!CFP00 && !CFP01 && N0.getOperand(0) == N0.getOperand(1) &&
5724 N0.getOperand(0) == N1) {
5725 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
5726 N1, DAG.getConstantFP(3.0, VT));
5727 }
5728
5729 // (fadd (fmul c, x), (fadd x, x)) -> (fmul c+2, x)
5730 if (CFP00 && !CFP01 && N1.getOpcode() == ISD::FADD &&
5731 N1.getOperand(0) == N1.getOperand(1) &&
5732 N0.getOperand(1) == N1.getOperand(0)) {
5733 SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
5734 SDValue(CFP00, 0),
5735 DAG.getConstantFP(2.0, VT));
5736 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
5737 N0.getOperand(1), NewCFP);
5738 }
5739
5740 // (fadd (fmul x, c), (fadd x, x)) -> (fmul c+2, x)
5741 if (CFP01 && !CFP00 && N1.getOpcode() == ISD::FADD &&
5742 N1.getOperand(0) == N1.getOperand(1) &&
5743 N0.getOperand(0) == N1.getOperand(0)) {
5744 SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
5745 SDValue(CFP01, 0),
5746 DAG.getConstantFP(2.0, VT));
5747 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
5748 N0.getOperand(0), NewCFP);
5749 }
5750 }
5751
5752 if (N1.getOpcode() == ISD::FMUL) {
5753 ConstantFPSDNode *CFP10 = dyn_cast<ConstantFPSDNode>(N1.getOperand(0));
5754 ConstantFPSDNode *CFP11 = dyn_cast<ConstantFPSDNode>(N1.getOperand(1));
5755
5756 // (fadd x, (fmul c, x)) -> (fmul c+1, x)
5757 if (CFP10 && !CFP11 && N1.getOperand(1) == N0) {
5758 SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
5759 SDValue(CFP10, 0),
5760 DAG.getConstantFP(1.0, VT));
5761 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
5762 N0, NewCFP);
5763 }
5764
5765 // (fadd x, (fmul x, c)) -> (fmul c+1, x)
5766 if (CFP11 && !CFP10 && N1.getOperand(0) == N0) {
5767 SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
5768 SDValue(CFP11, 0),
5769 DAG.getConstantFP(1.0, VT));
5770 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
5771 N0, NewCFP);
5772 }
5773
5774 // (fadd x, (fadd x, x)) -> (fmul 3.0, x)
5775 if (!CFP10 && !CFP11 && N1.getOperand(0) == N1.getOperand(1) &&
5776 N1.getOperand(0) == N0) {
5777 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
5778 N0, DAG.getConstantFP(3.0, VT));
5779 }
5780
5781 // (fadd (fadd x, x), (fmul c, x)) -> (fmul c+2, x)
5782 if (CFP10 && !CFP11 && N1.getOpcode() == ISD::FADD &&
5783 N1.getOperand(0) == N1.getOperand(1) &&
5784 N0.getOperand(1) == N1.getOperand(0)) {
5785 SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
5786 SDValue(CFP10, 0),
5787 DAG.getConstantFP(2.0, VT));
5788 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
5789 N0.getOperand(1), NewCFP);
5790 }
5791
5792 // (fadd (fadd x, x), (fmul x, c)) -> (fmul c+2, x)
5793 if (CFP11 && !CFP10 && N1.getOpcode() == ISD::FADD &&
5794 N1.getOperand(0) == N1.getOperand(1) &&
5795 N0.getOperand(0) == N1.getOperand(0)) {
5796 SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
5797 SDValue(CFP11, 0),
5798 DAG.getConstantFP(2.0, VT));
5799 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
5800 N0.getOperand(0), NewCFP);
5801 }
5802 }
5803
5804 // (fadd (fadd x, x), (fadd x, x)) -> (fmul 4.0, x)
5805 if (N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD &&
5806 N0.getOperand(0) == N0.getOperand(1) &&
5807 N1.getOperand(0) == N1.getOperand(1) &&
5808 N0.getOperand(0) == N1.getOperand(0)) {
5809 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
5810 N0.getOperand(0),
5811 DAG.getConstantFP(4.0, VT));
5812 }
5813 }
5814
Lang Hamesd693caf2012-06-19 22:51:23 +00005815 // FADD -> FMA combines:
Lang Hamese0231412012-06-22 01:09:09 +00005816 if ((DAG.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast ||
Lang Hamesd693caf2012-06-19 22:51:23 +00005817 DAG.getTarget().Options.UnsafeFPMath) &&
5818 DAG.getTarget().getTargetLowering()->isFMAFasterThanMulAndAdd(VT) &&
Elena Demikhovsky1503aba2012-08-01 12:06:00 +00005819 TLI.isOperationLegalOrCustom(ISD::FMA, VT)) {
Lang Hamesd693caf2012-06-19 22:51:23 +00005820
5821 // fold (fadd (fmul x, y), z) -> (fma x, y, z)
5822 if (N0.getOpcode() == ISD::FMUL && N0->hasOneUse()) {
5823 return DAG.getNode(ISD::FMA, N->getDebugLoc(), VT,
5824 N0.getOperand(0), N0.getOperand(1), N1);
5825 }
Owen Anderson43da6c72012-08-30 23:35:16 +00005826
Michael Liaob79bff52012-09-01 04:09:16 +00005827 // fold (fadd x, (fmul y, z)) -> (fma y, z, x)
Lang Hamesd693caf2012-06-19 22:51:23 +00005828 // Note: Commutes FADD operands.
5829 if (N1.getOpcode() == ISD::FMUL && N1->hasOneUse()) {
5830 return DAG.getNode(ISD::FMA, N->getDebugLoc(), VT,
5831 N1.getOperand(0), N1.getOperand(1), N0);
5832 }
5833 }
5834
Dan Gohman475871a2008-07-27 21:46:04 +00005835 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005836}
5837
Dan Gohman475871a2008-07-27 21:46:04 +00005838SDValue DAGCombiner::visitFSUB(SDNode *N) {
5839 SDValue N0 = N->getOperand(0);
5840 SDValue N1 = N->getOperand(1);
Nate Begemana0e221d2005-10-18 00:28:13 +00005841 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5842 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005843 EVT VT = N->getValueType(0);
Elena Demikhovsky1503aba2012-08-01 12:06:00 +00005844 DebugLoc dl = N->getDebugLoc();
Scott Michelfdc40a02009-02-17 22:15:04 +00005845
Dan Gohman7f321562007-06-25 16:23:39 +00005846 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00005847 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005848 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005849 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00005850 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005851
Nate Begemana0e221d2005-10-18 00:28:13 +00005852 // fold (fsub c1, c2) -> c1-c2
Owen Anderson825b72b2009-08-11 20:47:22 +00005853 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlingfc4b6772009-02-01 11:19:36 +00005854 return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N0, N1);
Bill Wendlingb0162f52009-01-30 22:53:48 +00005855 // fold (fsub A, 0) -> A
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005856 if (DAG.getTarget().Options.UnsafeFPMath &&
5857 N1CFP && N1CFP->getValueAPF().isZero())
Dan Gohmana90c8e62009-01-23 19:10:37 +00005858 return N0;
Bill Wendlingb0162f52009-01-30 22:53:48 +00005859 // fold (fsub 0, B) -> -B
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005860 if (DAG.getTarget().Options.UnsafeFPMath &&
5861 N0CFP && N0CFP->getValueAPF().isZero()) {
Owen Andersonafd3d562012-03-06 00:29:31 +00005862 if (isNegatibleForFree(N1, LegalOperations, TLI, &DAG.getTarget().Options))
Duncan Sands25cf2272008-11-24 14:53:14 +00005863 return GetNegatedExpression(N1, DAG, LegalOperations);
Dan Gohman760f86f2009-01-22 21:58:43 +00005864 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
Elena Demikhovsky1503aba2012-08-01 12:06:00 +00005865 return DAG.getNode(ISD::FNEG, dl, VT, N1);
Dan Gohman23ff1822007-07-02 15:48:56 +00005866 }
Bill Wendlingb0162f52009-01-30 22:53:48 +00005867 // fold (fsub A, (fneg B)) -> (fadd A, B)
Owen Andersonafd3d562012-03-06 00:29:31 +00005868 if (isNegatibleForFree(N1, LegalOperations, TLI, &DAG.getTarget().Options))
Elena Demikhovsky1503aba2012-08-01 12:06:00 +00005869 return DAG.getNode(ISD::FADD, dl, VT, N0,
Duncan Sands25cf2272008-11-24 14:53:14 +00005870 GetNegatedExpression(N1, DAG, LegalOperations));
Scott Michelfdc40a02009-02-17 22:15:04 +00005871
Bill Wendling5a894342012-03-15 05:12:00 +00005872 // If 'unsafe math' is enabled, fold
Owen Anderson713e9532012-05-07 20:51:25 +00005873 // (fsub x, x) -> 0.0 &
Bill Wendling5a894342012-03-15 05:12:00 +00005874 // (fsub x, (fadd x, y)) -> (fneg y) &
5875 // (fsub x, (fadd y, x)) -> (fneg y)
5876 if (DAG.getTarget().Options.UnsafeFPMath) {
Owen Anderson713e9532012-05-07 20:51:25 +00005877 if (N0 == N1)
5878 return DAG.getConstantFP(0.0f, VT);
5879
Bill Wendling5a894342012-03-15 05:12:00 +00005880 if (N1.getOpcode() == ISD::FADD) {
5881 SDValue N10 = N1->getOperand(0);
5882 SDValue N11 = N1->getOperand(1);
5883
5884 if (N10 == N0 && isNegatibleForFree(N11, LegalOperations, TLI,
5885 &DAG.getTarget().Options))
5886 return GetNegatedExpression(N11, DAG, LegalOperations);
5887 else if (N11 == N0 && isNegatibleForFree(N10, LegalOperations, TLI,
5888 &DAG.getTarget().Options))
5889 return GetNegatedExpression(N10, DAG, LegalOperations);
5890 }
5891 }
5892
Lang Hamesd693caf2012-06-19 22:51:23 +00005893 // FSUB -> FMA combines:
Lang Hamese0231412012-06-22 01:09:09 +00005894 if ((DAG.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast ||
Lang Hamesd693caf2012-06-19 22:51:23 +00005895 DAG.getTarget().Options.UnsafeFPMath) &&
5896 DAG.getTarget().getTargetLowering()->isFMAFasterThanMulAndAdd(VT) &&
Elena Demikhovsky1503aba2012-08-01 12:06:00 +00005897 TLI.isOperationLegalOrCustom(ISD::FMA, VT)) {
Lang Hamesd693caf2012-06-19 22:51:23 +00005898
5899 // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
5900 if (N0.getOpcode() == ISD::FMUL && N0->hasOneUse()) {
Elena Demikhovsky1503aba2012-08-01 12:06:00 +00005901 return DAG.getNode(ISD::FMA, dl, VT,
Lang Hamesd693caf2012-06-19 22:51:23 +00005902 N0.getOperand(0), N0.getOperand(1),
Elena Demikhovsky1503aba2012-08-01 12:06:00 +00005903 DAG.getNode(ISD::FNEG, dl, VT, N1));
Lang Hamesd693caf2012-06-19 22:51:23 +00005904 }
5905
5906 // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
5907 // Note: Commutes FSUB operands.
5908 if (N1.getOpcode() == ISD::FMUL && N1->hasOneUse()) {
Elena Demikhovsky1503aba2012-08-01 12:06:00 +00005909 return DAG.getNode(ISD::FMA, dl, VT,
5910 DAG.getNode(ISD::FNEG, dl, VT,
Lang Hamesd693caf2012-06-19 22:51:23 +00005911 N1.getOperand(0)),
5912 N1.getOperand(1), N0);
5913 }
Elena Demikhovsky1503aba2012-08-01 12:06:00 +00005914
5915 // fold (fsub (-(fmul, x, y)), z) -> (fma (fneg x), y, (fneg z))
5916 if (N0.getOpcode() == ISD::FNEG &&
5917 N0.getOperand(0).getOpcode() == ISD::FMUL &&
5918 N0->hasOneUse() && N0.getOperand(0).hasOneUse()) {
5919 SDValue N00 = N0.getOperand(0).getOperand(0);
5920 SDValue N01 = N0.getOperand(0).getOperand(1);
5921 return DAG.getNode(ISD::FMA, dl, VT,
5922 DAG.getNode(ISD::FNEG, dl, VT, N00), N01,
5923 DAG.getNode(ISD::FNEG, dl, VT, N1));
5924 }
Lang Hamesd693caf2012-06-19 22:51:23 +00005925 }
5926
Dan Gohman475871a2008-07-27 21:46:04 +00005927 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005928}
5929
Dan Gohman475871a2008-07-27 21:46:04 +00005930SDValue DAGCombiner::visitFMUL(SDNode *N) {
5931 SDValue N0 = N->getOperand(0);
5932 SDValue N1 = N->getOperand(1);
Nate Begeman11af4ea2005-10-17 20:40:11 +00005933 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5934 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005935 EVT VT = N->getValueType(0);
Owen Andersonafd3d562012-03-06 00:29:31 +00005936 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Chris Lattner01b3d732005-09-28 22:28:18 +00005937
Dan Gohman7f321562007-06-25 16:23:39 +00005938 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00005939 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005940 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005941 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00005942 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005943
Nate Begeman11af4ea2005-10-17 20:40:11 +00005944 // fold (fmul c1, c2) -> c1*c2
Owen Anderson825b72b2009-08-11 20:47:22 +00005945 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005946 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N0, N1);
Nate Begeman11af4ea2005-10-17 20:40:11 +00005947 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00005948 if (N0CFP && !N1CFP)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005949 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N1, N0);
5950 // fold (fmul A, 0) -> 0
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005951 if (DAG.getTarget().Options.UnsafeFPMath &&
5952 N1CFP && N1CFP->getValueAPF().isZero())
Dan Gohman760f86f2009-01-22 21:58:43 +00005953 return N1;
Dan Gohman77b81fe2009-06-04 17:12:12 +00005954 // fold (fmul A, 0) -> 0, vector edition.
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005955 if (DAG.getTarget().Options.UnsafeFPMath &&
5956 ISD::isBuildVectorAllZeros(N1.getNode()))
Dan Gohman77b81fe2009-06-04 17:12:12 +00005957 return N1;
Owen Anderson363e4b92012-05-02 21:32:35 +00005958 // fold (fmul A, 1.0) -> A
5959 if (N1CFP && N1CFP->isExactlyValue(1.0))
5960 return N0;
Nate Begeman11af4ea2005-10-17 20:40:11 +00005961 // fold (fmul X, 2.0) -> (fadd X, X)
5962 if (N1CFP && N1CFP->isExactlyValue(+2.0))
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005963 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0, N0);
Dan Gohmaneb1fedc2009-08-10 16:50:32 +00005964 // fold (fmul X, -1.0) -> (fneg X)
Chris Lattner29446522007-05-14 22:04:50 +00005965 if (N1CFP && N1CFP->isExactlyValue(-1.0))
Dan Gohman760f86f2009-01-22 21:58:43 +00005966 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005967 return DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005968
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005969 // fold (fmul (fneg X), (fneg Y)) -> (fmul X, Y)
Owen Andersonafd3d562012-03-06 00:29:31 +00005970 if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, TLI,
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005971 &DAG.getTarget().Options)) {
Owen Andersonafd3d562012-03-06 00:29:31 +00005972 if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, TLI,
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005973 &DAG.getTarget().Options)) {
Chris Lattner29446522007-05-14 22:04:50 +00005974 // Both can be negated for free, check to see if at least one is cheaper
5975 // negated.
5976 if (LHSNeg == 2 || RHSNeg == 2)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005977 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
Duncan Sands25cf2272008-11-24 14:53:14 +00005978 GetNegatedExpression(N0, DAG, LegalOperations),
5979 GetNegatedExpression(N1, DAG, LegalOperations));
Chris Lattner29446522007-05-14 22:04:50 +00005980 }
5981 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005982
Chris Lattnerddae4bd2007-01-08 23:04:05 +00005983 // If allowed, fold (fmul (fmul x, c1), c2) -> (fmul x, (fmul c1, c2))
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005984 if (DAG.getTarget().Options.UnsafeFPMath &&
5985 N1CFP && N0.getOpcode() == ISD::FMUL &&
Gabor Greifba36cb52008-08-28 21:40:38 +00005986 N0.getNode()->hasOneUse() && isa<ConstantFPSDNode>(N0.getOperand(1)))
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005987 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N0.getOperand(0),
Scott Michelfdc40a02009-02-17 22:15:04 +00005988 DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
Dale Johannesende064702009-02-06 21:50:26 +00005989 N0.getOperand(1), N1));
Scott Michelfdc40a02009-02-17 22:15:04 +00005990
Dan Gohman475871a2008-07-27 21:46:04 +00005991 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005992}
5993
Owen Anderson062c0a52012-05-02 22:17:40 +00005994SDValue DAGCombiner::visitFMA(SDNode *N) {
5995 SDValue N0 = N->getOperand(0);
5996 SDValue N1 = N->getOperand(1);
5997 SDValue N2 = N->getOperand(2);
5998 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5999 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
6000 EVT VT = N->getValueType(0);
Owen Anderson58d57292012-09-01 06:04:27 +00006001 DebugLoc dl = N->getDebugLoc();
Owen Anderson062c0a52012-05-02 22:17:40 +00006002
6003 if (N0CFP && N0CFP->isExactlyValue(1.0))
6004 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N1, N2);
6005 if (N1CFP && N1CFP->isExactlyValue(1.0))
6006 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0, N2);
6007
Owen Anderson85ef6f42012-05-30 18:50:39 +00006008 // Canonicalize (fma c, x, y) -> (fma x, c, y)
Owen Andersonf917d202012-05-30 18:54:50 +00006009 if (N0CFP && !N1CFP)
Owen Anderson85ef6f42012-05-30 18:50:39 +00006010 return DAG.getNode(ISD::FMA, N->getDebugLoc(), VT, N1, N0, N2);
6011
Owen Anderson58d57292012-09-01 06:04:27 +00006012 // (fma x, c1, (fmul x, c2)) -> (fmul x, c1+c2)
6013 if (DAG.getTarget().Options.UnsafeFPMath && N1CFP &&
6014 N2.getOpcode() == ISD::FMUL &&
6015 N0 == N2.getOperand(0) &&
6016 N2.getOperand(1).getOpcode() == ISD::ConstantFP) {
6017 return DAG.getNode(ISD::FMUL, dl, VT, N0,
6018 DAG.getNode(ISD::FADD, dl, VT, N1, N2.getOperand(1)));
6019 }
6020
6021
6022 // (fma (fmul x, c1), c2, y) -> (fma x, c1*c2, y)
6023 if (DAG.getTarget().Options.UnsafeFPMath &&
6024 N0.getOpcode() == ISD::FMUL && N1CFP &&
6025 N0.getOperand(1).getOpcode() == ISD::ConstantFP) {
6026 return DAG.getNode(ISD::FMA, dl, VT,
6027 N0.getOperand(0),
6028 DAG.getNode(ISD::FMUL, dl, VT, N1, N0.getOperand(1)),
6029 N2);
6030 }
6031
6032 // (fma x, 1, y) -> (fadd x, y)
6033 // (fma x, -1, y) -> (fadd (fneg x), y)
6034 if (N1CFP) {
6035 if (N1CFP->isExactlyValue(1.0))
6036 return DAG.getNode(ISD::FADD, dl, VT, N0, N2);
6037
6038 if (N1CFP->isExactlyValue(-1.0) &&
6039 (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) {
6040 SDValue RHSNeg = DAG.getNode(ISD::FNEG, dl, VT, N0);
6041 AddToWorkList(RHSNeg.getNode());
6042 return DAG.getNode(ISD::FADD, dl, VT, N2, RHSNeg);
6043 }
6044 }
6045
6046 // (fma x, c, x) -> (fmul x, (c+1))
6047 if (DAG.getTarget().Options.UnsafeFPMath && N1CFP && N0 == N2) {
6048 return DAG.getNode(ISD::FMUL, dl, VT,
6049 N0,
6050 DAG.getNode(ISD::FADD, dl, VT,
6051 N1, DAG.getConstantFP(1.0, VT)));
6052 }
6053
6054 // (fma x, c, (fneg x)) -> (fmul x, (c-1))
6055 if (DAG.getTarget().Options.UnsafeFPMath && N1CFP &&
6056 N2.getOpcode() == ISD::FNEG && N2.getOperand(0) == N0) {
6057 return DAG.getNode(ISD::FMUL, dl, VT,
6058 N0,
6059 DAG.getNode(ISD::FADD, dl, VT,
6060 N1, DAG.getConstantFP(-1.0, VT)));
6061 }
6062
6063
Owen Anderson062c0a52012-05-02 22:17:40 +00006064 return SDValue();
6065}
6066
Dan Gohman475871a2008-07-27 21:46:04 +00006067SDValue DAGCombiner::visitFDIV(SDNode *N) {
6068 SDValue N0 = N->getOperand(0);
6069 SDValue N1 = N->getOperand(1);
Nate Begemana148d982006-01-18 22:35:16 +00006070 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
6071 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00006072 EVT VT = N->getValueType(0);
Owen Andersonafd3d562012-03-06 00:29:31 +00006073 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Chris Lattner01b3d732005-09-28 22:28:18 +00006074
Dan Gohman7f321562007-06-25 16:23:39 +00006075 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00006076 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00006077 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00006078 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00006079 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006080
Nate Begemana148d982006-01-18 22:35:16 +00006081 // fold (fdiv c1, c2) -> c1/c2
Owen Anderson825b72b2009-08-11 20:47:22 +00006082 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00006083 return DAG.getNode(ISD::FDIV, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00006084
Duncan Sands3ef3fcf2012-04-08 18:08:12 +00006085 // fold (fdiv X, c2) -> fmul X, 1/c2 if losing precision is acceptable.
6086 if (N1CFP && VT != MVT::ppcf128 && DAG.getTarget().Options.UnsafeFPMath) {
Duncan Sands961d6662012-04-07 20:04:00 +00006087 // Compute the reciprocal 1.0 / c2.
6088 APFloat N1APF = N1CFP->getValueAPF();
6089 APFloat Recip(N1APF.getSemantics(), 1); // 1.0
6090 APFloat::opStatus st = Recip.divide(N1APF, APFloat::rmNearestTiesToEven);
Duncan Sands507bb7a2012-04-10 20:35:27 +00006091 // Only do the transform if the reciprocal is a legal fp immediate that
6092 // isn't too nasty (eg NaN, denormal, ...).
6093 if ((st == APFloat::opOK || st == APFloat::opInexact) && // Not too nasty
Anton Korobeynikov999821c2012-04-10 13:22:49 +00006094 (!LegalOperations ||
6095 // FIXME: custom lowering of ConstantFP might fail (see e.g. ARM
6096 // backend)... we should handle this gracefully after Legalize.
6097 // TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT) ||
6098 TLI.isOperationLegal(llvm::ISD::ConstantFP, VT) ||
6099 TLI.isFPImmLegal(Recip, VT)))
Duncan Sands961d6662012-04-07 20:04:00 +00006100 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N0,
6101 DAG.getConstantFP(Recip, VT));
6102 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006103
Bill Wendlinga03e74b2009-01-30 22:57:07 +00006104 // (fdiv (fneg X), (fneg Y)) -> (fdiv X, Y)
Owen Andersonafd3d562012-03-06 00:29:31 +00006105 if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, TLI,
Nick Lewycky8a8d4792011-12-02 22:16:29 +00006106 &DAG.getTarget().Options)) {
Owen Andersonafd3d562012-03-06 00:29:31 +00006107 if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, TLI,
Nick Lewycky8a8d4792011-12-02 22:16:29 +00006108 &DAG.getTarget().Options)) {
Chris Lattner29446522007-05-14 22:04:50 +00006109 // Both can be negated for free, check to see if at least one is cheaper
6110 // negated.
6111 if (LHSNeg == 2 || RHSNeg == 2)
Scott Michelfdc40a02009-02-17 22:15:04 +00006112 return DAG.getNode(ISD::FDIV, N->getDebugLoc(), VT,
Duncan Sands25cf2272008-11-24 14:53:14 +00006113 GetNegatedExpression(N0, DAG, LegalOperations),
6114 GetNegatedExpression(N1, DAG, LegalOperations));
Chris Lattner29446522007-05-14 22:04:50 +00006115 }
6116 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006117
Dan Gohman475871a2008-07-27 21:46:04 +00006118 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00006119}
6120
Dan Gohman475871a2008-07-27 21:46:04 +00006121SDValue DAGCombiner::visitFREM(SDNode *N) {
6122 SDValue N0 = N->getOperand(0);
6123 SDValue N1 = N->getOperand(1);
Nate Begemana148d982006-01-18 22:35:16 +00006124 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
6125 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00006126 EVT VT = N->getValueType(0);
Chris Lattner01b3d732005-09-28 22:28:18 +00006127
Nate Begemana148d982006-01-18 22:35:16 +00006128 // fold (frem c1, c2) -> fmod(c1,c2)
Owen Anderson825b72b2009-08-11 20:47:22 +00006129 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00006130 return DAG.getNode(ISD::FREM, N->getDebugLoc(), VT, N0, N1);
Dan Gohman7f321562007-06-25 16:23:39 +00006131
Dan Gohman475871a2008-07-27 21:46:04 +00006132 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00006133}
6134
Dan Gohman475871a2008-07-27 21:46:04 +00006135SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
6136 SDValue N0 = N->getOperand(0);
6137 SDValue N1 = N->getOperand(1);
Chris Lattner12d83032006-03-05 05:30:57 +00006138 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
6139 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00006140 EVT VT = N->getValueType(0);
Chris Lattner12d83032006-03-05 05:30:57 +00006141
Owen Anderson825b72b2009-08-11 20:47:22 +00006142 if (N0CFP && N1CFP && VT != MVT::ppcf128) // Constant fold
Bill Wendlingfc4b6772009-02-01 11:19:36 +00006143 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00006144
Chris Lattner12d83032006-03-05 05:30:57 +00006145 if (N1CFP) {
Dale Johannesene6c17422007-08-26 01:18:27 +00006146 const APFloat& V = N1CFP->getValueAPF();
Sylvestre Ledru94c22712012-09-27 10:14:43 +00006147 // copysign(x, c1) -> fabs(x) iff ispos(c1)
6148 // copysign(x, c1) -> fneg(fabs(x)) iff isneg(c1)
Dan Gohman760f86f2009-01-22 21:58:43 +00006149 if (!V.isNegative()) {
6150 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT))
Bill Wendling0225a1d2009-01-30 23:15:49 +00006151 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0);
Dan Gohman760f86f2009-01-22 21:58:43 +00006152 } else {
6153 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
Bill Wendling0225a1d2009-01-30 23:15:49 +00006154 return DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT,
Bill Wendling9729c5a2009-01-31 03:12:48 +00006155 DAG.getNode(ISD::FABS, N0.getDebugLoc(), VT, N0));
Dan Gohman760f86f2009-01-22 21:58:43 +00006156 }
Chris Lattner12d83032006-03-05 05:30:57 +00006157 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006158
Chris Lattner12d83032006-03-05 05:30:57 +00006159 // copysign(fabs(x), y) -> copysign(x, y)
6160 // copysign(fneg(x), y) -> copysign(x, y)
6161 // copysign(copysign(x,z), y) -> copysign(x, y)
6162 if (N0.getOpcode() == ISD::FABS || N0.getOpcode() == ISD::FNEG ||
6163 N0.getOpcode() == ISD::FCOPYSIGN)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006164 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
6165 N0.getOperand(0), N1);
Chris Lattner12d83032006-03-05 05:30:57 +00006166
6167 // copysign(x, abs(y)) -> abs(x)
6168 if (N1.getOpcode() == ISD::FABS)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006169 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006170
Chris Lattner12d83032006-03-05 05:30:57 +00006171 // copysign(x, copysign(y,z)) -> copysign(x, z)
6172 if (N1.getOpcode() == ISD::FCOPYSIGN)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006173 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
6174 N0, N1.getOperand(1));
Scott Michelfdc40a02009-02-17 22:15:04 +00006175
Chris Lattner12d83032006-03-05 05:30:57 +00006176 // copysign(x, fp_extend(y)) -> copysign(x, y)
6177 // copysign(x, fp_round(y)) -> copysign(x, y)
6178 if (N1.getOpcode() == ISD::FP_EXTEND || N1.getOpcode() == ISD::FP_ROUND)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006179 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
6180 N0, N1.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00006181
Dan Gohman475871a2008-07-27 21:46:04 +00006182 return SDValue();
Chris Lattner12d83032006-03-05 05:30:57 +00006183}
6184
Dan Gohman475871a2008-07-27 21:46:04 +00006185SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
6186 SDValue N0 = N->getOperand(0);
Nate Begeman646d7e22005-09-02 21:18:40 +00006187 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006188 EVT VT = N->getValueType(0);
6189 EVT OpVT = N0.getValueType();
Chris Lattnercda88752008-06-26 00:16:49 +00006190
Nate Begeman1d4d4142005-09-01 00:19:25 +00006191 // fold (sint_to_fp c1) -> c1fp
Stuart Hastings7e334182011-03-02 19:36:30 +00006192 if (N0C && OpVT != MVT::ppcf128 &&
6193 // ...but only if the target supports immediate floating-point values
Eli Friedman50185242011-11-12 00:35:34 +00006194 (!LegalOperations ||
Evan Cheng9568e5c2011-06-21 06:01:08 +00006195 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT)))
Bill Wendling0225a1d2009-01-30 23:15:49 +00006196 return DAG.getNode(ISD::SINT_TO_FP, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006197
Chris Lattnercda88752008-06-26 00:16:49 +00006198 // If the input is a legal type, and SINT_TO_FP is not legal on this target,
6199 // but UINT_TO_FP is legal on this target, try to convert.
Dan Gohmanf560ffa2009-01-28 17:46:25 +00006200 if (!TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT) &&
6201 TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00006202 // If the sign bit is known to be zero, we can change this to UINT_TO_FP.
Chris Lattnercda88752008-06-26 00:16:49 +00006203 if (DAG.SignBitIsZero(N0))
Bill Wendling0225a1d2009-01-30 23:15:49 +00006204 return DAG.getNode(ISD::UINT_TO_FP, N->getDebugLoc(), VT, N0);
Chris Lattnercda88752008-06-26 00:16:49 +00006205 }
Bill Wendling0225a1d2009-01-30 23:15:49 +00006206
Nadav Rotemed1a3352012-07-23 07:59:50 +00006207 // The next optimizations are desireable only if SELECT_CC can be lowered.
6208 // Check against MVT::Other for SELECT_CC, which is a workaround for targets
6209 // having to say they don't support SELECT_CC on every type the DAG knows
6210 // about, since there is no way to mark an opcode illegal at all value types
6211 // (See also visitSELECT)
6212 if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, MVT::Other)) {
6213 // fold (sint_to_fp (setcc x, y, cc)) -> (select_cc x, y, -1.0, 0.0,, cc)
6214 if (N0.getOpcode() == ISD::SETCC && N0.getValueType() == MVT::i1 &&
6215 !VT.isVector() &&
6216 (!LegalOperations ||
6217 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) {
6218 SDValue Ops[] =
6219 { N0.getOperand(0), N0.getOperand(1),
6220 DAG.getConstantFP(-1.0, VT) , DAG.getConstantFP(0.0, VT),
6221 N0.getOperand(2) };
6222 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), VT, Ops, 5);
6223 }
Owen Andersond9bf71f2012-07-09 20:31:12 +00006224
Nadav Rotemed1a3352012-07-23 07:59:50 +00006225 // fold (sint_to_fp (zext (setcc x, y, cc))) ->
6226 // (select_cc x, y, 1.0, 0.0,, cc)
6227 if (N0.getOpcode() == ISD::ZERO_EXTEND &&
6228 N0.getOperand(0).getOpcode() == ISD::SETCC &&!VT.isVector() &&
6229 (!LegalOperations ||
6230 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) {
6231 SDValue Ops[] =
6232 { N0.getOperand(0).getOperand(0), N0.getOperand(0).getOperand(1),
6233 DAG.getConstantFP(1.0, VT) , DAG.getConstantFP(0.0, VT),
6234 N0.getOperand(0).getOperand(2) };
6235 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), VT, Ops, 5);
6236 }
Owen Andersond9bf71f2012-07-09 20:31:12 +00006237 }
6238
Dan Gohman475871a2008-07-27 21:46:04 +00006239 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006240}
6241
Dan Gohman475871a2008-07-27 21:46:04 +00006242SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
6243 SDValue N0 = N->getOperand(0);
Nate Begeman646d7e22005-09-02 21:18:40 +00006244 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006245 EVT VT = N->getValueType(0);
6246 EVT OpVT = N0.getValueType();
Nate Begemana148d982006-01-18 22:35:16 +00006247
Nate Begeman1d4d4142005-09-01 00:19:25 +00006248 // fold (uint_to_fp c1) -> c1fp
Stuart Hastings7e334182011-03-02 19:36:30 +00006249 if (N0C && OpVT != MVT::ppcf128 &&
6250 // ...but only if the target supports immediate floating-point values
Eli Friedman50185242011-11-12 00:35:34 +00006251 (!LegalOperations ||
Evan Cheng9568e5c2011-06-21 06:01:08 +00006252 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT)))
Bill Wendling0225a1d2009-01-30 23:15:49 +00006253 return DAG.getNode(ISD::UINT_TO_FP, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006254
Chris Lattnercda88752008-06-26 00:16:49 +00006255 // If the input is a legal type, and UINT_TO_FP is not legal on this target,
6256 // but SINT_TO_FP is legal on this target, try to convert.
Dan Gohmanf560ffa2009-01-28 17:46:25 +00006257 if (!TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT) &&
6258 TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00006259 // If the sign bit is known to be zero, we can change this to SINT_TO_FP.
Chris Lattnercda88752008-06-26 00:16:49 +00006260 if (DAG.SignBitIsZero(N0))
Bill Wendling0225a1d2009-01-30 23:15:49 +00006261 return DAG.getNode(ISD::SINT_TO_FP, N->getDebugLoc(), VT, N0);
Chris Lattnercda88752008-06-26 00:16:49 +00006262 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006263
Nadav Rotemed1a3352012-07-23 07:59:50 +00006264 // The next optimizations are desireable only if SELECT_CC can be lowered.
6265 // Check against MVT::Other for SELECT_CC, which is a workaround for targets
6266 // having to say they don't support SELECT_CC on every type the DAG knows
6267 // about, since there is no way to mark an opcode illegal at all value types
6268 // (See also visitSELECT)
6269 if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, MVT::Other)) {
6270 // fold (uint_to_fp (setcc x, y, cc)) -> (select_cc x, y, -1.0, 0.0,, cc)
Owen Andersond9bf71f2012-07-09 20:31:12 +00006271
Nadav Rotemed1a3352012-07-23 07:59:50 +00006272 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() &&
6273 (!LegalOperations ||
6274 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) {
6275 SDValue Ops[] =
6276 { N0.getOperand(0), N0.getOperand(1),
6277 DAG.getConstantFP(1.0, VT), DAG.getConstantFP(0.0, VT),
6278 N0.getOperand(2) };
6279 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), VT, Ops, 5);
6280 }
6281 }
Owen Andersond9bf71f2012-07-09 20:31:12 +00006282
Dan Gohman475871a2008-07-27 21:46:04 +00006283 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006284}
6285
Dan Gohman475871a2008-07-27 21:46:04 +00006286SDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) {
6287 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00006288 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006289 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006290
Nate Begeman1d4d4142005-09-01 00:19:25 +00006291 // fold (fp_to_sint c1fp) -> c1
Nate Begeman646d7e22005-09-02 21:18:40 +00006292 if (N0CFP)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006293 return DAG.getNode(ISD::FP_TO_SINT, N->getDebugLoc(), VT, N0);
6294
Dan Gohman475871a2008-07-27 21:46:04 +00006295 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006296}
6297
Dan Gohman475871a2008-07-27 21:46:04 +00006298SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) {
6299 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00006300 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006301 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006302
Nate Begeman1d4d4142005-09-01 00:19:25 +00006303 // fold (fp_to_uint c1fp) -> c1
Owen Anderson825b72b2009-08-11 20:47:22 +00006304 if (N0CFP && VT != MVT::ppcf128)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006305 return DAG.getNode(ISD::FP_TO_UINT, N->getDebugLoc(), VT, N0);
6306
Dan Gohman475871a2008-07-27 21:46:04 +00006307 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006308}
6309
Dan Gohman475871a2008-07-27 21:46:04 +00006310SDValue DAGCombiner::visitFP_ROUND(SDNode *N) {
6311 SDValue N0 = N->getOperand(0);
6312 SDValue N1 = N->getOperand(1);
Nate Begemana148d982006-01-18 22:35:16 +00006313 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006314 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006315
Nate Begeman1d4d4142005-09-01 00:19:25 +00006316 // fold (fp_round c1fp) -> c1fp
Owen Anderson825b72b2009-08-11 20:47:22 +00006317 if (N0CFP && N0.getValueType() != MVT::ppcf128)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006318 return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00006319
Chris Lattner79dbea52006-03-13 06:26:26 +00006320 // fold (fp_round (fp_extend x)) -> x
6321 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType())
6322 return N0.getOperand(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006323
Chris Lattner0aa5e6f2008-01-24 06:45:35 +00006324 // fold (fp_round (fp_round x)) -> (fp_round x)
6325 if (N0.getOpcode() == ISD::FP_ROUND) {
6326 // This is a value preserving truncation if both round's are.
6327 bool IsTrunc = N->getConstantOperandVal(1) == 1 &&
Gabor Greifba36cb52008-08-28 21:40:38 +00006328 N0.getNode()->getConstantOperandVal(1) == 1;
Bill Wendling0225a1d2009-01-30 23:15:49 +00006329 return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), VT, N0.getOperand(0),
Chris Lattner0aa5e6f2008-01-24 06:45:35 +00006330 DAG.getIntPtrConstant(IsTrunc));
6331 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006332
Chris Lattner79dbea52006-03-13 06:26:26 +00006333 // fold (fp_round (copysign X, Y)) -> (copysign (fp_round X), Y)
Gabor Greifba36cb52008-08-28 21:40:38 +00006334 if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse()) {
Bill Wendling0225a1d2009-01-30 23:15:49 +00006335 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, N0.getDebugLoc(), VT,
6336 N0.getOperand(0), N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00006337 AddToWorkList(Tmp.getNode());
Bill Wendling0225a1d2009-01-30 23:15:49 +00006338 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
6339 Tmp, N0.getOperand(1));
Chris Lattner79dbea52006-03-13 06:26:26 +00006340 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006341
Dan Gohman475871a2008-07-27 21:46:04 +00006342 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006343}
6344
Dan Gohman475871a2008-07-27 21:46:04 +00006345SDValue DAGCombiner::visitFP_ROUND_INREG(SDNode *N) {
6346 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00006347 EVT VT = N->getValueType(0);
6348 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
Nate Begeman646d7e22005-09-02 21:18:40 +00006349 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006350
Nate Begeman1d4d4142005-09-01 00:19:25 +00006351 // fold (fp_round_inreg c1fp) -> c1fp
Chris Lattner2392ae72010-04-15 04:48:01 +00006352 if (N0CFP && isTypeLegal(EVT)) {
Dan Gohman4fbd7962008-09-12 18:08:03 +00006353 SDValue Round = DAG.getConstantFP(*N0CFP->getConstantFPValue(), EVT);
Bill Wendling0225a1d2009-01-30 23:15:49 +00006354 return DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), VT, Round);
Nate Begeman1d4d4142005-09-01 00:19:25 +00006355 }
Bill Wendling0225a1d2009-01-30 23:15:49 +00006356
Dan Gohman475871a2008-07-27 21:46:04 +00006357 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006358}
6359
Dan Gohman475871a2008-07-27 21:46:04 +00006360SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
6361 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00006362 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006363 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006364
Chris Lattner5938bef2007-12-29 06:55:23 +00006365 // If this is fp_round(fpextend), don't fold it, allow ourselves to be folded.
Scott Michelfdc40a02009-02-17 22:15:04 +00006366 if (N->hasOneUse() &&
Dan Gohmane7852d02009-01-26 04:35:06 +00006367 N->use_begin()->getOpcode() == ISD::FP_ROUND)
Dan Gohman475871a2008-07-27 21:46:04 +00006368 return SDValue();
Chris Lattner0bd48932008-01-17 07:00:52 +00006369
Nate Begeman1d4d4142005-09-01 00:19:25 +00006370 // fold (fp_extend c1fp) -> c1fp
Owen Anderson825b72b2009-08-11 20:47:22 +00006371 if (N0CFP && VT != MVT::ppcf128)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006372 return DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), VT, N0);
Chris Lattner0bd48932008-01-17 07:00:52 +00006373
6374 // Turn fp_extend(fp_round(X, 1)) -> x since the fp_round doesn't affect the
6375 // value of X.
Gabor Greif12632d22008-08-30 19:29:20 +00006376 if (N0.getOpcode() == ISD::FP_ROUND
6377 && N0.getNode()->getConstantOperandVal(1) == 1) {
Dan Gohman475871a2008-07-27 21:46:04 +00006378 SDValue In = N0.getOperand(0);
Chris Lattner0bd48932008-01-17 07:00:52 +00006379 if (In.getValueType() == VT) return In;
Duncan Sands8e4eb092008-06-08 20:54:56 +00006380 if (VT.bitsLT(In.getValueType()))
Bill Wendling0225a1d2009-01-30 23:15:49 +00006381 return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), VT,
6382 In, N0.getOperand(1));
6383 return DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), VT, In);
Chris Lattner0bd48932008-01-17 07:00:52 +00006384 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006385
Chris Lattner0bd48932008-01-17 07:00:52 +00006386 // fold (fpext (load x)) -> (fpext (fptrunc (extload x)))
Gabor Greifba36cb52008-08-28 21:40:38 +00006387 if (ISD::isNON_EXTLoad(N0.getNode()) && N0.hasOneUse() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00006388 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00006389 TLI.isLoadExtLegal(ISD::EXTLOAD, N0.getValueType()))) {
Evan Cheng466685d2006-10-09 20:57:25 +00006390 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00006391 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, N->getDebugLoc(), VT,
Bill Wendling0225a1d2009-01-30 23:15:49 +00006392 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00006393 LN0->getBasePtr(), LN0->getPointerInfo(),
Duncan Sands25cf2272008-11-24 14:53:14 +00006394 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00006395 LN0->isVolatile(), LN0->isNonTemporal(),
6396 LN0->getAlignment());
Chris Lattnere564dbb2006-05-05 21:34:35 +00006397 CombineTo(N, ExtLoad);
Bill Wendling0225a1d2009-01-30 23:15:49 +00006398 CombineTo(N0.getNode(),
6399 DAG.getNode(ISD::FP_ROUND, N0.getDebugLoc(),
6400 N0.getValueType(), ExtLoad, DAG.getIntPtrConstant(1)),
Chris Lattnere564dbb2006-05-05 21:34:35 +00006401 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00006402 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattnere564dbb2006-05-05 21:34:35 +00006403 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00006404
Dan Gohman475871a2008-07-27 21:46:04 +00006405 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006406}
6407
Dan Gohman475871a2008-07-27 21:46:04 +00006408SDValue DAGCombiner::visitFNEG(SDNode *N) {
6409 SDValue N0 = N->getOperand(0);
Anton Korobeynikov2bcf60a2009-10-20 21:37:45 +00006410 EVT VT = N->getValueType(0);
Nate Begemana148d982006-01-18 22:35:16 +00006411
Craig Topperdd201ff2012-09-11 01:45:21 +00006412 if (VT.isVector()) {
6413 SDValue FoldedVOp = SimplifyVUnaryOp(N);
6414 if (FoldedVOp.getNode()) return FoldedVOp;
Craig Topper956342b2012-09-09 22:58:45 +00006415 }
6416
Owen Andersonafd3d562012-03-06 00:29:31 +00006417 if (isNegatibleForFree(N0, LegalOperations, DAG.getTargetLoweringInfo(),
6418 &DAG.getTarget().Options))
Duncan Sands25cf2272008-11-24 14:53:14 +00006419 return GetNegatedExpression(N0, DAG, LegalOperations);
Dan Gohman23ff1822007-07-02 15:48:56 +00006420
Chris Lattner3bd39d42008-01-27 17:42:27 +00006421 // Transform fneg(bitconvert(x)) -> bitconvert(x^sign) to avoid loading
6422 // constant pool values.
Owen Anderson29f60f32012-04-02 22:10:29 +00006423 if (!TLI.isFNegFree(VT) && N0.getOpcode() == ISD::BITCAST &&
Anton Korobeynikov2bcf60a2009-10-20 21:37:45 +00006424 !VT.isVector() &&
6425 N0.getNode()->hasOneUse() &&
6426 N0.getOperand(0).getValueType().isInteger()) {
Dan Gohman475871a2008-07-27 21:46:04 +00006427 SDValue Int = N0.getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00006428 EVT IntVT = Int.getValueType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00006429 if (IntVT.isInteger() && !IntVT.isVector()) {
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00006430 Int = DAG.getNode(ISD::XOR, N0.getDebugLoc(), IntVT, Int,
6431 DAG.getConstant(APInt::getSignBit(IntVT.getSizeInBits()), IntVT));
Gabor Greifba36cb52008-08-28 21:40:38 +00006432 AddToWorkList(Int.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006433 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(),
Anton Korobeynikov2bcf60a2009-10-20 21:37:45 +00006434 VT, Int);
Chris Lattner3bd39d42008-01-27 17:42:27 +00006435 }
6436 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006437
Owen Anderson58d57292012-09-01 06:04:27 +00006438 // (fneg (fmul c, x)) -> (fmul -c, x)
6439 if (N0.getOpcode() == ISD::FMUL) {
6440 ConstantFPSDNode *CFP1 = dyn_cast<ConstantFPSDNode>(N0.getOperand(1));
6441 if (CFP1) {
6442 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
6443 N0.getOperand(0),
6444 DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT,
6445 N0.getOperand(1)));
6446 }
6447 }
6448
Dan Gohman475871a2008-07-27 21:46:04 +00006449 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006450}
6451
Owen Anderson7c626d32012-08-13 23:32:49 +00006452SDValue DAGCombiner::visitFCEIL(SDNode *N) {
6453 SDValue N0 = N->getOperand(0);
6454 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
6455 EVT VT = N->getValueType(0);
6456
6457 // fold (fceil c1) -> fceil(c1)
6458 if (N0CFP && VT != MVT::ppcf128)
6459 return DAG.getNode(ISD::FCEIL, N->getDebugLoc(), VT, N0);
6460
6461 return SDValue();
6462}
6463
6464SDValue DAGCombiner::visitFTRUNC(SDNode *N) {
6465 SDValue N0 = N->getOperand(0);
6466 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
6467 EVT VT = N->getValueType(0);
6468
6469 // fold (ftrunc c1) -> ftrunc(c1)
6470 if (N0CFP && VT != MVT::ppcf128)
6471 return DAG.getNode(ISD::FTRUNC, N->getDebugLoc(), VT, N0);
6472
6473 return SDValue();
6474}
6475
6476SDValue DAGCombiner::visitFFLOOR(SDNode *N) {
6477 SDValue N0 = N->getOperand(0);
6478 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
6479 EVT VT = N->getValueType(0);
6480
6481 // fold (ffloor c1) -> ffloor(c1)
6482 if (N0CFP && VT != MVT::ppcf128)
6483 return DAG.getNode(ISD::FFLOOR, N->getDebugLoc(), VT, N0);
6484
6485 return SDValue();
6486}
6487
Dan Gohman475871a2008-07-27 21:46:04 +00006488SDValue DAGCombiner::visitFABS(SDNode *N) {
6489 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00006490 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006491 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006492
Craig Topperdd201ff2012-09-11 01:45:21 +00006493 if (VT.isVector()) {
6494 SDValue FoldedVOp = SimplifyVUnaryOp(N);
6495 if (FoldedVOp.getNode()) return FoldedVOp;
6496 }
6497
Nate Begeman1d4d4142005-09-01 00:19:25 +00006498 // fold (fabs c1) -> fabs(c1)
Owen Anderson825b72b2009-08-11 20:47:22 +00006499 if (N0CFP && VT != MVT::ppcf128)
Bill Wendlingc0debad2009-01-30 23:27:35 +00006500 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00006501 // fold (fabs (fabs x)) -> (fabs x)
Chris Lattner12d83032006-03-05 05:30:57 +00006502 if (N0.getOpcode() == ISD::FABS)
Nate Begeman83e75ec2005-09-06 04:43:02 +00006503 return N->getOperand(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00006504 // fold (fabs (fneg x)) -> (fabs x)
Chris Lattner12d83032006-03-05 05:30:57 +00006505 // fold (fabs (fcopysign x, y)) -> (fabs x)
6506 if (N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FCOPYSIGN)
Bill Wendlingc0debad2009-01-30 23:27:35 +00006507 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00006508
Chris Lattner3bd39d42008-01-27 17:42:27 +00006509 // Transform fabs(bitconvert(x)) -> bitconvert(x&~sign) to avoid loading
6510 // constant pool values.
Owen Anderson29f60f32012-04-02 22:10:29 +00006511 if (!TLI.isFAbsFree(VT) &&
6512 N0.getOpcode() == ISD::BITCAST && N0.getNode()->hasOneUse() &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00006513 N0.getOperand(0).getValueType().isInteger() &&
6514 !N0.getOperand(0).getValueType().isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00006515 SDValue Int = N0.getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00006516 EVT IntVT = Int.getValueType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00006517 if (IntVT.isInteger() && !IntVT.isVector()) {
Scott Michelfdc40a02009-02-17 22:15:04 +00006518 Int = DAG.getNode(ISD::AND, N0.getDebugLoc(), IntVT, Int,
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00006519 DAG.getConstant(~APInt::getSignBit(IntVT.getSizeInBits()), IntVT));
Gabor Greifba36cb52008-08-28 21:40:38 +00006520 AddToWorkList(Int.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006521 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(),
Bill Wendlingc0debad2009-01-30 23:27:35 +00006522 N->getValueType(0), Int);
Chris Lattner3bd39d42008-01-27 17:42:27 +00006523 }
6524 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006525
Dan Gohman475871a2008-07-27 21:46:04 +00006526 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006527}
6528
Dan Gohman475871a2008-07-27 21:46:04 +00006529SDValue DAGCombiner::visitBRCOND(SDNode *N) {
6530 SDValue Chain = N->getOperand(0);
6531 SDValue N1 = N->getOperand(1);
6532 SDValue N2 = N->getOperand(2);
Scott Michelfdc40a02009-02-17 22:15:04 +00006533
Dan Gohmane0f06c72009-11-17 00:47:23 +00006534 // If N is a constant we could fold this into a fallthrough or unconditional
6535 // branch. However that doesn't happen very often in normal code, because
6536 // Instcombine/SimplifyCFG should have handled the available opportunities.
6537 // If we did this folding here, it would be necessary to update the
6538 // MachineBasicBlock CFG, which is awkward.
6539
Nate Begeman750ac1b2006-02-01 07:19:44 +00006540 // fold a brcond with a setcc condition into a BR_CC node if BR_CC is legal
6541 // on the target.
Scott Michelfdc40a02009-02-17 22:15:04 +00006542 if (N1.getOpcode() == ISD::SETCC &&
Owen Anderson825b72b2009-08-11 20:47:22 +00006543 TLI.isOperationLegalOrCustom(ISD::BR_CC, MVT::Other)) {
6544 return DAG.getNode(ISD::BR_CC, N->getDebugLoc(), MVT::Other,
Bill Wendlingc0debad2009-01-30 23:27:35 +00006545 Chain, N1.getOperand(2),
Nate Begeman750ac1b2006-02-01 07:19:44 +00006546 N1.getOperand(0), N1.getOperand(1), N2);
6547 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006548
Evan Cheng2a135ae2010-10-04 22:41:01 +00006549 if ((N1.hasOneUse() && N1.getOpcode() == ISD::SRL) ||
6550 ((N1.getOpcode() == ISD::TRUNCATE && N1.hasOneUse()) &&
6551 (N1.getOperand(0).hasOneUse() &&
6552 N1.getOperand(0).getOpcode() == ISD::SRL))) {
6553 SDNode *Trunc = 0;
6554 if (N1.getOpcode() == ISD::TRUNCATE) {
6555 // Look pass the truncate.
6556 Trunc = N1.getNode();
6557 N1 = N1.getOperand(0);
6558 }
Evan Chengd40d03e2010-01-06 19:38:29 +00006559
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006560 // Match this pattern so that we can generate simpler code:
6561 //
6562 // %a = ...
6563 // %b = and i32 %a, 2
6564 // %c = srl i32 %b, 1
6565 // brcond i32 %c ...
6566 //
6567 // into
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006568 //
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006569 // %a = ...
Evan Chengd40d03e2010-01-06 19:38:29 +00006570 // %b = and i32 %a, 2
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006571 // %c = setcc eq %b, 0
6572 // brcond %c ...
6573 //
6574 // This applies only when the AND constant value has one bit set and the
6575 // SRL constant is equal to the log2 of the AND constant. The back-end is
6576 // smart enough to convert the result into a TEST/JMP sequence.
6577 SDValue Op0 = N1.getOperand(0);
6578 SDValue Op1 = N1.getOperand(1);
6579
6580 if (Op0.getOpcode() == ISD::AND &&
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006581 Op1.getOpcode() == ISD::Constant) {
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006582 SDValue AndOp1 = Op0.getOperand(1);
6583
6584 if (AndOp1.getOpcode() == ISD::Constant) {
6585 const APInt &AndConst = cast<ConstantSDNode>(AndOp1)->getAPIntValue();
6586
6587 if (AndConst.isPowerOf2() &&
6588 cast<ConstantSDNode>(Op1)->getAPIntValue()==AndConst.logBase2()) {
6589 SDValue SetCC =
6590 DAG.getSetCC(N->getDebugLoc(),
6591 TLI.getSetCCResultType(Op0.getValueType()),
6592 Op0, DAG.getConstant(0, Op0.getValueType()),
6593 ISD::SETNE);
6594
Evan Chengd40d03e2010-01-06 19:38:29 +00006595 SDValue NewBRCond = DAG.getNode(ISD::BRCOND, N->getDebugLoc(),
6596 MVT::Other, Chain, SetCC, N2);
6597 // Don't add the new BRCond into the worklist or else SimplifySelectCC
6598 // will convert it back to (X & C1) >> C2.
6599 CombineTo(N, NewBRCond, false);
6600 // Truncate is dead.
6601 if (Trunc) {
6602 removeFromWorkList(Trunc);
6603 DAG.DeleteNode(Trunc);
6604 }
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006605 // Replace the uses of SRL with SETCC
Evan Cheng2c755ba2010-02-27 07:36:59 +00006606 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006607 DAG.ReplaceAllUsesOfValueWith(N1, SetCC);
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006608 removeFromWorkList(N1.getNode());
6609 DAG.DeleteNode(N1.getNode());
Evan Chengd40d03e2010-01-06 19:38:29 +00006610 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006611 }
6612 }
6613 }
Evan Cheng2a135ae2010-10-04 22:41:01 +00006614
6615 if (Trunc)
6616 // Restore N1 if the above transformation doesn't match.
6617 N1 = N->getOperand(1);
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006618 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006619
Evan Cheng2c755ba2010-02-27 07:36:59 +00006620 // Transform br(xor(x, y)) -> br(x != y)
6621 // Transform br(xor(xor(x,y), 1)) -> br (x == y)
6622 if (N1.hasOneUse() && N1.getOpcode() == ISD::XOR) {
6623 SDNode *TheXor = N1.getNode();
6624 SDValue Op0 = TheXor->getOperand(0);
6625 SDValue Op1 = TheXor->getOperand(1);
6626 if (Op0.getOpcode() == Op1.getOpcode()) {
6627 // Avoid missing important xor optimizations.
6628 SDValue Tmp = visitXOR(TheXor);
Bill Wendling86c5abb2010-04-20 01:25:01 +00006629 if (Tmp.getNode() && Tmp.getNode() != TheXor) {
Evan Cheng2c755ba2010-02-27 07:36:59 +00006630 DEBUG(dbgs() << "\nReplacing.8 ";
6631 TheXor->dump(&DAG);
6632 dbgs() << "\nWith: ";
6633 Tmp.getNode()->dump(&DAG);
6634 dbgs() << '\n');
6635 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006636 DAG.ReplaceAllUsesOfValueWith(N1, Tmp);
Evan Cheng2c755ba2010-02-27 07:36:59 +00006637 removeFromWorkList(TheXor);
6638 DAG.DeleteNode(TheXor);
6639 return DAG.getNode(ISD::BRCOND, N->getDebugLoc(),
6640 MVT::Other, Chain, Tmp, N2);
6641 }
6642 }
6643
6644 if (Op0.getOpcode() != ISD::SETCC && Op1.getOpcode() != ISD::SETCC) {
6645 bool Equal = false;
6646 if (ConstantSDNode *RHSCI = dyn_cast<ConstantSDNode>(Op0))
6647 if (RHSCI->getAPIntValue() == 1 && Op0.hasOneUse() &&
6648 Op0.getOpcode() == ISD::XOR) {
6649 TheXor = Op0.getNode();
6650 Equal = true;
6651 }
6652
Evan Cheng2a135ae2010-10-04 22:41:01 +00006653 EVT SetCCVT = N1.getValueType();
Evan Cheng2c755ba2010-02-27 07:36:59 +00006654 if (LegalTypes)
6655 SetCCVT = TLI.getSetCCResultType(SetCCVT);
6656 SDValue SetCC = DAG.getSetCC(TheXor->getDebugLoc(),
6657 SetCCVT,
6658 Op0, Op1,
6659 Equal ? ISD::SETEQ : ISD::SETNE);
6660 // Replace the uses of XOR with SETCC
6661 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006662 DAG.ReplaceAllUsesOfValueWith(N1, SetCC);
Evan Cheng2a135ae2010-10-04 22:41:01 +00006663 removeFromWorkList(N1.getNode());
6664 DAG.DeleteNode(N1.getNode());
Evan Cheng2c755ba2010-02-27 07:36:59 +00006665 return DAG.getNode(ISD::BRCOND, N->getDebugLoc(),
6666 MVT::Other, Chain, SetCC, N2);
6667 }
6668 }
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006669
Dan Gohman475871a2008-07-27 21:46:04 +00006670 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00006671}
6672
Chris Lattner3ea0b472005-10-05 06:47:48 +00006673// Operand List for BR_CC: Chain, CondCC, CondLHS, CondRHS, DestBB.
6674//
Dan Gohman475871a2008-07-27 21:46:04 +00006675SDValue DAGCombiner::visitBR_CC(SDNode *N) {
Chris Lattner3ea0b472005-10-05 06:47:48 +00006676 CondCodeSDNode *CC = cast<CondCodeSDNode>(N->getOperand(1));
Dan Gohman475871a2008-07-27 21:46:04 +00006677 SDValue CondLHS = N->getOperand(2), CondRHS = N->getOperand(3);
Scott Michelfdc40a02009-02-17 22:15:04 +00006678
Dan Gohmane0f06c72009-11-17 00:47:23 +00006679 // If N is a constant we could fold this into a fallthrough or unconditional
6680 // branch. However that doesn't happen very often in normal code, because
6681 // Instcombine/SimplifyCFG should have handled the available opportunities.
6682 // If we did this folding here, it would be necessary to update the
6683 // MachineBasicBlock CFG, which is awkward.
6684
Duncan Sands8eab8a22008-06-09 11:32:28 +00006685 // Use SimplifySetCC to simplify SETCC's.
Duncan Sands5480c042009-01-01 15:52:00 +00006686 SDValue Simp = SimplifySetCC(TLI.getSetCCResultType(CondLHS.getValueType()),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00006687 CondLHS, CondRHS, CC->get(), N->getDebugLoc(),
6688 false);
Gabor Greifba36cb52008-08-28 21:40:38 +00006689 if (Simp.getNode()) AddToWorkList(Simp.getNode());
Chris Lattner30f73e72006-10-14 03:52:46 +00006690
Nate Begemane17daeb2005-10-05 21:43:42 +00006691 // fold to a simpler setcc
Gabor Greifba36cb52008-08-28 21:40:38 +00006692 if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC)
Owen Anderson825b72b2009-08-11 20:47:22 +00006693 return DAG.getNode(ISD::BR_CC, N->getDebugLoc(), MVT::Other,
Bill Wendlingc0debad2009-01-30 23:27:35 +00006694 N->getOperand(0), Simp.getOperand(2),
6695 Simp.getOperand(0), Simp.getOperand(1),
6696 N->getOperand(4));
6697
Dan Gohman475871a2008-07-27 21:46:04 +00006698 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00006699}
6700
Evan Chengc4b527a2012-01-13 01:37:24 +00006701/// canFoldInAddressingMode - Return true if 'Use' is a load or a store that
6702/// uses N as its base pointer and that N may be folded in the load / store
Evan Cheng03be3622012-03-06 23:33:32 +00006703/// addressing mode.
Evan Chengc4b527a2012-01-13 01:37:24 +00006704static bool canFoldInAddressingMode(SDNode *N, SDNode *Use,
6705 SelectionDAG &DAG,
6706 const TargetLowering &TLI) {
6707 EVT VT;
6708 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Use)) {
6709 if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
6710 return false;
6711 VT = Use->getValueType(0);
6712 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Use)) {
6713 if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
6714 return false;
6715 VT = ST->getValue().getValueType();
6716 } else
6717 return false;
6718
6719 TargetLowering::AddrMode AM;
6720 if (N->getOpcode() == ISD::ADD) {
6721 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
6722 if (Offset)
Evan Cheng03be3622012-03-06 23:33:32 +00006723 // [reg +/- imm]
Evan Chengc4b527a2012-01-13 01:37:24 +00006724 AM.BaseOffs = Offset->getSExtValue();
6725 else
Evan Cheng03be3622012-03-06 23:33:32 +00006726 // [reg +/- reg]
6727 AM.Scale = 1;
Evan Chengc4b527a2012-01-13 01:37:24 +00006728 } else if (N->getOpcode() == ISD::SUB) {
6729 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
6730 if (Offset)
Evan Cheng03be3622012-03-06 23:33:32 +00006731 // [reg +/- imm]
Evan Chengc4b527a2012-01-13 01:37:24 +00006732 AM.BaseOffs = -Offset->getSExtValue();
6733 else
Evan Cheng03be3622012-03-06 23:33:32 +00006734 // [reg +/- reg]
6735 AM.Scale = 1;
Evan Chengc4b527a2012-01-13 01:37:24 +00006736 } else
6737 return false;
6738
6739 return TLI.isLegalAddressingMode(AM, VT.getTypeForEVT(*DAG.getContext()));
6740}
6741
Duncan Sandsec87aa82008-06-15 20:12:31 +00006742/// CombineToPreIndexedLoadStore - Try turning a load / store into a
6743/// pre-indexed load / store when the base pointer is an add or subtract
Chris Lattner448f2192006-11-11 00:39:41 +00006744/// and it has other uses besides the load / store. After the
6745/// transformation, the new indexed load / store has effectively folded
6746/// the add / subtract in and all of its other uses are redirected to the
6747/// new load / store.
6748bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) {
Eli Friedman50185242011-11-12 00:35:34 +00006749 if (Level < AfterLegalizeDAG)
Chris Lattner448f2192006-11-11 00:39:41 +00006750 return false;
6751
6752 bool isLoad = true;
Dan Gohman475871a2008-07-27 21:46:04 +00006753 SDValue Ptr;
Owen Andersone50ed302009-08-10 22:56:29 +00006754 EVT VT;
Chris Lattner448f2192006-11-11 00:39:41 +00006755 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00006756 if (LD->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00006757 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006758 VT = LD->getMemoryVT();
Evan Cheng83060c52007-03-07 08:07:03 +00006759 if (!TLI.isIndexedLoadLegal(ISD::PRE_INC, VT) &&
Chris Lattner448f2192006-11-11 00:39:41 +00006760 !TLI.isIndexedLoadLegal(ISD::PRE_DEC, VT))
6761 return false;
6762 Ptr = LD->getBasePtr();
6763 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00006764 if (ST->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00006765 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006766 VT = ST->getMemoryVT();
Chris Lattner448f2192006-11-11 00:39:41 +00006767 if (!TLI.isIndexedStoreLegal(ISD::PRE_INC, VT) &&
6768 !TLI.isIndexedStoreLegal(ISD::PRE_DEC, VT))
6769 return false;
6770 Ptr = ST->getBasePtr();
6771 isLoad = false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00006772 } else {
Chris Lattner448f2192006-11-11 00:39:41 +00006773 return false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00006774 }
Chris Lattner448f2192006-11-11 00:39:41 +00006775
Chris Lattner9f1794e2006-11-11 00:56:29 +00006776 // If the pointer is not an add/sub, or if it doesn't have multiple uses, bail
6777 // out. There is no reason to make this a preinc/predec.
6778 if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) ||
Gabor Greifba36cb52008-08-28 21:40:38 +00006779 Ptr.getNode()->hasOneUse())
Chris Lattner9f1794e2006-11-11 00:56:29 +00006780 return false;
Chris Lattner448f2192006-11-11 00:39:41 +00006781
Chris Lattner9f1794e2006-11-11 00:56:29 +00006782 // Ask the target to do addressing mode selection.
Dan Gohman475871a2008-07-27 21:46:04 +00006783 SDValue BasePtr;
6784 SDValue Offset;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006785 ISD::MemIndexedMode AM = ISD::UNINDEXED;
6786 if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG))
6787 return false;
Evan Chenga7d4a042007-05-03 23:52:19 +00006788 // Don't create a indexed load / store with zero offset.
6789 if (isa<ConstantSDNode>(Offset) &&
Dan Gohman002e5d02008-03-13 22:13:53 +00006790 cast<ConstantSDNode>(Offset)->isNullValue())
Evan Chenga7d4a042007-05-03 23:52:19 +00006791 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +00006792
Chris Lattner41e53fd2006-11-11 01:00:15 +00006793 // Try turning it into a pre-indexed load / store except when:
Evan Chengc843abe2007-05-24 02:35:39 +00006794 // 1) The new base ptr is a frame index.
6795 // 2) If N is a store and the new base ptr is either the same as or is a
Chris Lattner9f1794e2006-11-11 00:56:29 +00006796 // predecessor of the value being stored.
Evan Chengc843abe2007-05-24 02:35:39 +00006797 // 3) Another use of old base ptr is a predecessor of N. If ptr is folded
Chris Lattner9f1794e2006-11-11 00:56:29 +00006798 // that would create a cycle.
Evan Chengc843abe2007-05-24 02:35:39 +00006799 // 4) All uses are load / store ops that use it as old base ptr.
Chris Lattner448f2192006-11-11 00:39:41 +00006800
Chris Lattner41e53fd2006-11-11 01:00:15 +00006801 // Check #1. Preinc'ing a frame index would require copying the stack pointer
6802 // (plus the implicit offset) to a register to preinc anyway.
Evan Chengcaab1292009-05-06 18:25:01 +00006803 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
Chris Lattner41e53fd2006-11-11 01:00:15 +00006804 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +00006805
Chris Lattner41e53fd2006-11-11 01:00:15 +00006806 // Check #2.
Chris Lattner9f1794e2006-11-11 00:56:29 +00006807 if (!isLoad) {
Dan Gohman475871a2008-07-27 21:46:04 +00006808 SDValue Val = cast<StoreSDNode>(N)->getValue();
Gabor Greifba36cb52008-08-28 21:40:38 +00006809 if (Val == BasePtr || BasePtr.getNode()->isPredecessorOf(Val.getNode()))
Chris Lattner9f1794e2006-11-11 00:56:29 +00006810 return false;
Chris Lattner448f2192006-11-11 00:39:41 +00006811 }
Chris Lattner9f1794e2006-11-11 00:56:29 +00006812
Evan Chengc843abe2007-05-24 02:35:39 +00006813 // Now check for #3 and #4.
Chris Lattner9f1794e2006-11-11 00:56:29 +00006814 bool RealUse = false;
Lang Hames944520f2011-07-07 04:31:51 +00006815
6816 // Caches for hasPredecessorHelper
6817 SmallPtrSet<const SDNode *, 32> Visited;
6818 SmallVector<const SDNode *, 16> Worklist;
6819
Gabor Greifba36cb52008-08-28 21:40:38 +00006820 for (SDNode::use_iterator I = Ptr.getNode()->use_begin(),
6821 E = Ptr.getNode()->use_end(); I != E; ++I) {
Dan Gohman89684502008-07-27 20:43:25 +00006822 SDNode *Use = *I;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006823 if (Use == N)
6824 continue;
Lang Hames944520f2011-07-07 04:31:51 +00006825 if (N->hasPredecessorHelper(Use, Visited, Worklist))
Chris Lattner9f1794e2006-11-11 00:56:29 +00006826 return false;
6827
Evan Chengc4b527a2012-01-13 01:37:24 +00006828 // If Ptr may be folded in addressing mode of other use, then it's
6829 // not profitable to do this transformation.
6830 if (!canFoldInAddressingMode(Ptr.getNode(), Use, DAG, TLI))
Chris Lattner9f1794e2006-11-11 00:56:29 +00006831 RealUse = true;
6832 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006833
Chris Lattner9f1794e2006-11-11 00:56:29 +00006834 if (!RealUse)
6835 return false;
6836
Dan Gohman475871a2008-07-27 21:46:04 +00006837 SDValue Result;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006838 if (isLoad)
Bill Wendlingc0debad2009-01-30 23:27:35 +00006839 Result = DAG.getIndexedLoad(SDValue(N,0), N->getDebugLoc(),
6840 BasePtr, Offset, AM);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006841 else
Bill Wendlingc0debad2009-01-30 23:27:35 +00006842 Result = DAG.getIndexedStore(SDValue(N,0), N->getDebugLoc(),
6843 BasePtr, Offset, AM);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006844 ++PreIndexedNodes;
6845 ++NodesCombined;
David Greenef1090292010-01-05 01:25:00 +00006846 DEBUG(dbgs() << "\nReplacing.4 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006847 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006848 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006849 Result.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006850 dbgs() << '\n');
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006851 WorkListRemover DeadNodes(*this);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006852 if (isLoad) {
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006853 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0));
6854 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2));
Chris Lattner9f1794e2006-11-11 00:56:29 +00006855 } else {
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006856 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1));
Chris Lattner9f1794e2006-11-11 00:56:29 +00006857 }
6858
Chris Lattner9f1794e2006-11-11 00:56:29 +00006859 // Finally, since the node is now dead, remove it from the graph.
6860 DAG.DeleteNode(N);
6861
6862 // Replace the uses of Ptr with uses of the updated base value.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006863 DAG.ReplaceAllUsesOfValueWith(Ptr, Result.getValue(isLoad ? 1 : 0));
Gabor Greifba36cb52008-08-28 21:40:38 +00006864 removeFromWorkList(Ptr.getNode());
6865 DAG.DeleteNode(Ptr.getNode());
Chris Lattner9f1794e2006-11-11 00:56:29 +00006866
6867 return true;
Chris Lattner448f2192006-11-11 00:39:41 +00006868}
6869
Duncan Sandsec87aa82008-06-15 20:12:31 +00006870/// CombineToPostIndexedLoadStore - Try to combine a load / store with a
Chris Lattner448f2192006-11-11 00:39:41 +00006871/// add / sub of the base pointer node into a post-indexed load / store.
6872/// The transformation folded the add / subtract into the new indexed
6873/// load / store effectively and all of its uses are redirected to the
6874/// new load / store.
6875bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) {
Eli Friedman50185242011-11-12 00:35:34 +00006876 if (Level < AfterLegalizeDAG)
Chris Lattner448f2192006-11-11 00:39:41 +00006877 return false;
6878
6879 bool isLoad = true;
Dan Gohman475871a2008-07-27 21:46:04 +00006880 SDValue Ptr;
Owen Andersone50ed302009-08-10 22:56:29 +00006881 EVT VT;
Chris Lattner448f2192006-11-11 00:39:41 +00006882 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00006883 if (LD->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00006884 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006885 VT = LD->getMemoryVT();
Chris Lattner448f2192006-11-11 00:39:41 +00006886 if (!TLI.isIndexedLoadLegal(ISD::POST_INC, VT) &&
6887 !TLI.isIndexedLoadLegal(ISD::POST_DEC, VT))
6888 return false;
6889 Ptr = LD->getBasePtr();
6890 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00006891 if (ST->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00006892 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006893 VT = ST->getMemoryVT();
Chris Lattner448f2192006-11-11 00:39:41 +00006894 if (!TLI.isIndexedStoreLegal(ISD::POST_INC, VT) &&
6895 !TLI.isIndexedStoreLegal(ISD::POST_DEC, VT))
6896 return false;
6897 Ptr = ST->getBasePtr();
6898 isLoad = false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00006899 } else {
Chris Lattner448f2192006-11-11 00:39:41 +00006900 return false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00006901 }
Chris Lattner448f2192006-11-11 00:39:41 +00006902
Gabor Greifba36cb52008-08-28 21:40:38 +00006903 if (Ptr.getNode()->hasOneUse())
Chris Lattner9f1794e2006-11-11 00:56:29 +00006904 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +00006905
Gabor Greifba36cb52008-08-28 21:40:38 +00006906 for (SDNode::use_iterator I = Ptr.getNode()->use_begin(),
6907 E = Ptr.getNode()->use_end(); I != E; ++I) {
Dan Gohman89684502008-07-27 20:43:25 +00006908 SDNode *Op = *I;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006909 if (Op == N ||
6910 (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB))
6911 continue;
6912
Dan Gohman475871a2008-07-27 21:46:04 +00006913 SDValue BasePtr;
6914 SDValue Offset;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006915 ISD::MemIndexedMode AM = ISD::UNINDEXED;
6916 if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) {
Evan Chenga7d4a042007-05-03 23:52:19 +00006917 // Don't create a indexed load / store with zero offset.
6918 if (isa<ConstantSDNode>(Offset) &&
Dan Gohman002e5d02008-03-13 22:13:53 +00006919 cast<ConstantSDNode>(Offset)->isNullValue())
Evan Chenga7d4a042007-05-03 23:52:19 +00006920 continue;
Chris Lattner448f2192006-11-11 00:39:41 +00006921
Chris Lattner9f1794e2006-11-11 00:56:29 +00006922 // Try turning it into a post-indexed load / store except when
Evan Chengc4b527a2012-01-13 01:37:24 +00006923 // 1) All uses are load / store ops that use it as base ptr (and
6924 // it may be folded as addressing mmode).
Chris Lattner9f1794e2006-11-11 00:56:29 +00006925 // 2) Op must be independent of N, i.e. Op is neither a predecessor
6926 // nor a successor of N. Otherwise, if Op is folded that would
6927 // create a cycle.
6928
Evan Chengcaab1292009-05-06 18:25:01 +00006929 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
6930 continue;
6931
Chris Lattner9f1794e2006-11-11 00:56:29 +00006932 // Check for #1.
6933 bool TryNext = false;
Gabor Greifba36cb52008-08-28 21:40:38 +00006934 for (SDNode::use_iterator II = BasePtr.getNode()->use_begin(),
6935 EE = BasePtr.getNode()->use_end(); II != EE; ++II) {
Dan Gohman89684502008-07-27 20:43:25 +00006936 SDNode *Use = *II;
Gabor Greifba36cb52008-08-28 21:40:38 +00006937 if (Use == Ptr.getNode())
Chris Lattner448f2192006-11-11 00:39:41 +00006938 continue;
6939
Chris Lattner9f1794e2006-11-11 00:56:29 +00006940 // If all the uses are load / store addresses, then don't do the
6941 // transformation.
6942 if (Use->getOpcode() == ISD::ADD || Use->getOpcode() == ISD::SUB){
6943 bool RealUse = false;
6944 for (SDNode::use_iterator III = Use->use_begin(),
6945 EEE = Use->use_end(); III != EEE; ++III) {
Dan Gohman89684502008-07-27 20:43:25 +00006946 SDNode *UseUse = *III;
Evan Chengc4b527a2012-01-13 01:37:24 +00006947 if (!canFoldInAddressingMode(Use, UseUse, DAG, TLI))
Chris Lattner9f1794e2006-11-11 00:56:29 +00006948 RealUse = true;
6949 }
Chris Lattner448f2192006-11-11 00:39:41 +00006950
Chris Lattner9f1794e2006-11-11 00:56:29 +00006951 if (!RealUse) {
6952 TryNext = true;
6953 break;
Chris Lattner448f2192006-11-11 00:39:41 +00006954 }
6955 }
Chris Lattner9f1794e2006-11-11 00:56:29 +00006956 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006957
Chris Lattner9f1794e2006-11-11 00:56:29 +00006958 if (TryNext)
6959 continue;
Chris Lattner448f2192006-11-11 00:39:41 +00006960
Chris Lattner9f1794e2006-11-11 00:56:29 +00006961 // Check for #2
Evan Cheng917be682008-03-04 00:41:45 +00006962 if (!Op->isPredecessorOf(N) && !N->isPredecessorOf(Op)) {
Dan Gohman475871a2008-07-27 21:46:04 +00006963 SDValue Result = isLoad
Bill Wendlingc0debad2009-01-30 23:27:35 +00006964 ? DAG.getIndexedLoad(SDValue(N,0), N->getDebugLoc(),
6965 BasePtr, Offset, AM)
6966 : DAG.getIndexedStore(SDValue(N,0), N->getDebugLoc(),
6967 BasePtr, Offset, AM);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006968 ++PostIndexedNodes;
6969 ++NodesCombined;
David Greenef1090292010-01-05 01:25:00 +00006970 DEBUG(dbgs() << "\nReplacing.5 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006971 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006972 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006973 Result.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006974 dbgs() << '\n');
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006975 WorkListRemover DeadNodes(*this);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006976 if (isLoad) {
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006977 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0));
6978 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2));
Chris Lattner9f1794e2006-11-11 00:56:29 +00006979 } else {
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006980 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1));
Chris Lattner448f2192006-11-11 00:39:41 +00006981 }
Chris Lattner9f1794e2006-11-11 00:56:29 +00006982
Chris Lattner9f1794e2006-11-11 00:56:29 +00006983 // Finally, since the node is now dead, remove it from the graph.
6984 DAG.DeleteNode(N);
6985
6986 // Replace the uses of Use with uses of the updated base value.
Dan Gohman475871a2008-07-27 21:46:04 +00006987 DAG.ReplaceAllUsesOfValueWith(SDValue(Op, 0),
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006988 Result.getValue(isLoad ? 1 : 0));
Chris Lattner9f1794e2006-11-11 00:56:29 +00006989 removeFromWorkList(Op);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006990 DAG.DeleteNode(Op);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006991 return true;
Chris Lattner448f2192006-11-11 00:39:41 +00006992 }
6993 }
6994 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006995
Chris Lattner448f2192006-11-11 00:39:41 +00006996 return false;
6997}
6998
Dan Gohman475871a2008-07-27 21:46:04 +00006999SDValue DAGCombiner::visitLOAD(SDNode *N) {
Evan Cheng466685d2006-10-09 20:57:25 +00007000 LoadSDNode *LD = cast<LoadSDNode>(N);
Dan Gohman475871a2008-07-27 21:46:04 +00007001 SDValue Chain = LD->getChain();
7002 SDValue Ptr = LD->getBasePtr();
Scott Michelfdc40a02009-02-17 22:15:04 +00007003
Evan Cheng45a7ca92007-05-01 00:38:21 +00007004 // If load is not volatile and there are no uses of the loaded value (and
7005 // the updated indexed value in case of indexed loads), change uses of the
7006 // chain value into uses of the chain input (i.e. delete the dead load).
7007 if (!LD->isVolatile()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00007008 if (N->getValueType(1) == MVT::Other) {
Evan Cheng498f5592007-05-01 08:53:39 +00007009 // Unindexed loads.
Craig Topper704e1a02012-01-07 18:31:09 +00007010 if (!N->hasAnyUseOfValue(0)) {
Evan Cheng02c42852008-01-16 23:11:54 +00007011 // It's not safe to use the two value CombineTo variant here. e.g.
7012 // v1, chain2 = load chain1, loc
7013 // v2, chain3 = load chain2, loc
7014 // v3 = add v2, c
Chris Lattner125991a2008-01-24 07:57:06 +00007015 // Now we replace use of chain2 with chain1. This makes the second load
7016 // isomorphic to the one we are deleting, and thus makes this load live.
David Greenef1090292010-01-05 01:25:00 +00007017 DEBUG(dbgs() << "\nReplacing.6 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00007018 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00007019 dbgs() << "\nWith chain: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00007020 Chain.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00007021 dbgs() << "\n");
Chris Lattnerf8dc0612008-02-03 06:49:24 +00007022 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00007023 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain);
Bill Wendlingc0debad2009-01-30 23:27:35 +00007024
Chris Lattner125991a2008-01-24 07:57:06 +00007025 if (N->use_empty()) {
7026 removeFromWorkList(N);
7027 DAG.DeleteNode(N);
7028 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00007029
Dan Gohman475871a2008-07-27 21:46:04 +00007030 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng02c42852008-01-16 23:11:54 +00007031 }
Evan Cheng498f5592007-05-01 08:53:39 +00007032 } else {
7033 // Indexed loads.
Owen Anderson825b72b2009-08-11 20:47:22 +00007034 assert(N->getValueType(2) == MVT::Other && "Malformed indexed loads?");
Craig Topper704e1a02012-01-07 18:31:09 +00007035 if (!N->hasAnyUseOfValue(0) && !N->hasAnyUseOfValue(1)) {
Dale Johannesene8d72302009-02-06 23:05:02 +00007036 SDValue Undef = DAG.getUNDEF(N->getValueType(0));
Evan Cheng2c755ba2010-02-27 07:36:59 +00007037 DEBUG(dbgs() << "\nReplacing.7 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00007038 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00007039 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00007040 Undef.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00007041 dbgs() << " and 2 other values\n");
Chris Lattnerf8dc0612008-02-03 06:49:24 +00007042 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00007043 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Undef);
Dan Gohman475871a2008-07-27 21:46:04 +00007044 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1),
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00007045 DAG.getUNDEF(N->getValueType(1)));
7046 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 2), Chain);
Evan Cheng02c42852008-01-16 23:11:54 +00007047 removeFromWorkList(N);
Evan Cheng02c42852008-01-16 23:11:54 +00007048 DAG.DeleteNode(N);
Dan Gohman475871a2008-07-27 21:46:04 +00007049 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng45a7ca92007-05-01 00:38:21 +00007050 }
Evan Cheng45a7ca92007-05-01 00:38:21 +00007051 }
7052 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007053
Chris Lattner01a22022005-10-10 22:04:48 +00007054 // If this load is directly stored, replace the load value with the stored
7055 // value.
7056 // TODO: Handle store large -> read small portion.
Jim Laskeyc2b19f32006-10-11 17:47:52 +00007057 // TODO: Handle TRUNCSTORE/LOADEXT
Evan Cheng9ef82ce2011-03-11 00:48:56 +00007058 if (ISD::isNormalLoad(N) && !LD->isVolatile()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00007059 if (ISD::isNON_TRUNCStore(Chain.getNode())) {
Evan Cheng8b2794a2006-10-13 21:14:26 +00007060 StoreSDNode *PrevST = cast<StoreSDNode>(Chain);
7061 if (PrevST->getBasePtr() == Ptr &&
7062 PrevST->getValue().getValueType() == N->getValueType(0))
Jim Laskeyc2b19f32006-10-11 17:47:52 +00007063 return CombineTo(N, Chain.getOperand(1), Chain);
Evan Cheng8b2794a2006-10-13 21:14:26 +00007064 }
Jim Laskeyc2b19f32006-10-11 17:47:52 +00007065 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007066
Evan Cheng255f20f2010-04-01 06:04:33 +00007067 // Try to infer better alignment information than the load already has.
7068 if (OptLevel != CodeGenOpt::None && LD->isUnindexed()) {
Evan Chenged1c0c72011-11-28 22:37:34 +00007069 if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
7070 if (Align > LD->getAlignment())
7071 return DAG.getExtLoad(LD->getExtensionType(), N->getDebugLoc(),
7072 LD->getValueType(0),
7073 Chain, Ptr, LD->getPointerInfo(),
7074 LD->getMemoryVT(),
7075 LD->isVolatile(), LD->isNonTemporal(), Align);
Evan Cheng255f20f2010-04-01 06:04:33 +00007076 }
7077 }
7078
Jim Laskey7ca56af2006-10-11 13:47:09 +00007079 if (CombinerAA) {
Jim Laskey279f0532006-09-25 16:29:54 +00007080 // Walk up chain skipping non-aliasing memory nodes.
Dan Gohman475871a2008-07-27 21:46:04 +00007081 SDValue BetterChain = FindBetterChain(N, Chain);
Scott Michelfdc40a02009-02-17 22:15:04 +00007082
Jim Laskey6ff23e52006-10-04 16:53:27 +00007083 // If there is a better chain.
Jim Laskey279f0532006-09-25 16:29:54 +00007084 if (Chain != BetterChain) {
Dan Gohman475871a2008-07-27 21:46:04 +00007085 SDValue ReplLoad;
Jim Laskeyc2b19f32006-10-11 17:47:52 +00007086
Jim Laskey279f0532006-09-25 16:29:54 +00007087 // Replace the chain to void dependency.
Jim Laskeyc2b19f32006-10-11 17:47:52 +00007088 if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
Bill Wendlingc0debad2009-01-30 23:27:35 +00007089 ReplLoad = DAG.getLoad(N->getValueType(0), LD->getDebugLoc(),
Chris Lattnerfa459012010-09-21 16:08:50 +00007090 BetterChain, Ptr, LD->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00007091 LD->isVolatile(), LD->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00007092 LD->isInvariant(), LD->getAlignment());
Jim Laskeyc2b19f32006-10-11 17:47:52 +00007093 } else {
Stuart Hastingsa9011292011-02-16 16:23:55 +00007094 ReplLoad = DAG.getExtLoad(LD->getExtensionType(), LD->getDebugLoc(),
7095 LD->getValueType(0),
Chris Lattnerfa459012010-09-21 16:08:50 +00007096 BetterChain, Ptr, LD->getPointerInfo(),
Dan Gohmanb625f2f2008-01-30 00:15:11 +00007097 LD->getMemoryVT(),
Scott Michelfdc40a02009-02-17 22:15:04 +00007098 LD->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00007099 LD->isNonTemporal(),
Christopher Lamb95c218a2007-04-22 23:15:30 +00007100 LD->getAlignment());
Jim Laskeyc2b19f32006-10-11 17:47:52 +00007101 }
Jim Laskey279f0532006-09-25 16:29:54 +00007102
Jim Laskey6ff23e52006-10-04 16:53:27 +00007103 // Create token factor to keep old chain connected.
Bill Wendlingc0debad2009-01-30 23:27:35 +00007104 SDValue Token = DAG.getNode(ISD::TokenFactor, N->getDebugLoc(),
Owen Anderson825b72b2009-08-11 20:47:22 +00007105 MVT::Other, Chain, ReplLoad.getValue(1));
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007106
Nate Begemanb6aef5c2009-09-15 00:18:30 +00007107 // Make sure the new and old chains are cleaned up.
7108 AddToWorkList(Token.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007109
Jim Laskey274062c2006-10-13 23:32:28 +00007110 // Replace uses with load result and token factor. Don't add users
7111 // to work list.
7112 return CombineTo(N, ReplLoad.getValue(0), Token, false);
Jim Laskey279f0532006-09-25 16:29:54 +00007113 }
7114 }
7115
Evan Cheng7fc033a2006-11-03 03:06:21 +00007116 // Try transforming N to an indexed load.
Evan Chengbbd6f6e2006-11-07 09:03:05 +00007117 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
Dan Gohman475871a2008-07-27 21:46:04 +00007118 return SDValue(N, 0);
Evan Cheng7fc033a2006-11-03 03:06:21 +00007119
Dan Gohman475871a2008-07-27 21:46:04 +00007120 return SDValue();
Chris Lattner01a22022005-10-10 22:04:48 +00007121}
7122
Chris Lattner2392ae72010-04-15 04:48:01 +00007123/// CheckForMaskedLoad - Check to see if V is (and load (ptr), imm), where the
7124/// load is having specific bytes cleared out. If so, return the byte size
7125/// being masked out and the shift amount.
7126static std::pair<unsigned, unsigned>
7127CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) {
7128 std::pair<unsigned, unsigned> Result(0, 0);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007129
Chris Lattner2392ae72010-04-15 04:48:01 +00007130 // Check for the structure we're looking for.
7131 if (V->getOpcode() != ISD::AND ||
7132 !isa<ConstantSDNode>(V->getOperand(1)) ||
7133 !ISD::isNormalLoad(V->getOperand(0).getNode()))
7134 return Result;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007135
Chris Lattnere6987582010-04-15 06:10:49 +00007136 // Check the chain and pointer.
Chris Lattner2392ae72010-04-15 04:48:01 +00007137 LoadSDNode *LD = cast<LoadSDNode>(V->getOperand(0));
Chris Lattnere6987582010-04-15 06:10:49 +00007138 if (LD->getBasePtr() != Ptr) return Result; // Not from same pointer.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007139
Chris Lattnere6987582010-04-15 06:10:49 +00007140 // The store should be chained directly to the load or be an operand of a
7141 // tokenfactor.
7142 if (LD == Chain.getNode())
7143 ; // ok.
7144 else if (Chain->getOpcode() != ISD::TokenFactor)
7145 return Result; // Fail.
7146 else {
7147 bool isOk = false;
7148 for (unsigned i = 0, e = Chain->getNumOperands(); i != e; ++i)
7149 if (Chain->getOperand(i).getNode() == LD) {
7150 isOk = true;
7151 break;
7152 }
7153 if (!isOk) return Result;
7154 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007155
Chris Lattner2392ae72010-04-15 04:48:01 +00007156 // This only handles simple types.
7157 if (V.getValueType() != MVT::i16 &&
7158 V.getValueType() != MVT::i32 &&
7159 V.getValueType() != MVT::i64)
7160 return Result;
7161
7162 // Check the constant mask. Invert it so that the bits being masked out are
7163 // 0 and the bits being kept are 1. Use getSExtValue so that leading bits
7164 // follow the sign bit for uniformity.
7165 uint64_t NotMask = ~cast<ConstantSDNode>(V->getOperand(1))->getSExtValue();
7166 unsigned NotMaskLZ = CountLeadingZeros_64(NotMask);
7167 if (NotMaskLZ & 7) return Result; // Must be multiple of a byte.
7168 unsigned NotMaskTZ = CountTrailingZeros_64(NotMask);
7169 if (NotMaskTZ & 7) return Result; // Must be multiple of a byte.
7170 if (NotMaskLZ == 64) return Result; // All zero mask.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007171
Chris Lattner2392ae72010-04-15 04:48:01 +00007172 // See if we have a continuous run of bits. If so, we have 0*1+0*
7173 if (CountTrailingOnes_64(NotMask >> NotMaskTZ)+NotMaskTZ+NotMaskLZ != 64)
7174 return Result;
7175
7176 // Adjust NotMaskLZ down to be from the actual size of the int instead of i64.
7177 if (V.getValueType() != MVT::i64 && NotMaskLZ)
7178 NotMaskLZ -= 64-V.getValueSizeInBits();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007179
Chris Lattner2392ae72010-04-15 04:48:01 +00007180 unsigned MaskedBytes = (V.getValueSizeInBits()-NotMaskLZ-NotMaskTZ)/8;
7181 switch (MaskedBytes) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007182 case 1:
7183 case 2:
Chris Lattner2392ae72010-04-15 04:48:01 +00007184 case 4: break;
7185 default: return Result; // All one mask, or 5-byte mask.
7186 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007187
Chris Lattner2392ae72010-04-15 04:48:01 +00007188 // Verify that the first bit starts at a multiple of mask so that the access
7189 // is aligned the same as the access width.
7190 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes) return Result;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007191
Chris Lattner2392ae72010-04-15 04:48:01 +00007192 Result.first = MaskedBytes;
7193 Result.second = NotMaskTZ/8;
7194 return Result;
7195}
7196
7197
7198/// ShrinkLoadReplaceStoreWithStore - Check to see if IVal is something that
7199/// provides a value as specified by MaskInfo. If so, replace the specified
7200/// store with a narrower store of truncated IVal.
7201static SDNode *
7202ShrinkLoadReplaceStoreWithStore(const std::pair<unsigned, unsigned> &MaskInfo,
7203 SDValue IVal, StoreSDNode *St,
7204 DAGCombiner *DC) {
7205 unsigned NumBytes = MaskInfo.first;
7206 unsigned ByteShift = MaskInfo.second;
7207 SelectionDAG &DAG = DC->getDAG();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007208
Chris Lattner2392ae72010-04-15 04:48:01 +00007209 // Check to see if IVal is all zeros in the part being masked in by the 'or'
7210 // that uses this. If not, this is not a replacement.
7211 APInt Mask = ~APInt::getBitsSet(IVal.getValueSizeInBits(),
7212 ByteShift*8, (ByteShift+NumBytes)*8);
7213 if (!DAG.MaskedValueIsZero(IVal, Mask)) return 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007214
Chris Lattner2392ae72010-04-15 04:48:01 +00007215 // Check that it is legal on the target to do this. It is legal if the new
7216 // VT we're shrinking to (i8/i16/i32) is legal or we're still before type
7217 // legalization.
7218 MVT VT = MVT::getIntegerVT(NumBytes*8);
7219 if (!DC->isTypeLegal(VT))
7220 return 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007221
Chris Lattner2392ae72010-04-15 04:48:01 +00007222 // Okay, we can do this! Replace the 'St' store with a store of IVal that is
7223 // shifted by ByteShift and truncated down to NumBytes.
7224 if (ByteShift)
7225 IVal = DAG.getNode(ISD::SRL, IVal->getDebugLoc(), IVal.getValueType(), IVal,
Owen Anderson95771af2011-02-25 21:41:48 +00007226 DAG.getConstant(ByteShift*8,
7227 DC->getShiftAmountTy(IVal.getValueType())));
Chris Lattner2392ae72010-04-15 04:48:01 +00007228
7229 // Figure out the offset for the store and the alignment of the access.
7230 unsigned StOffset;
7231 unsigned NewAlign = St->getAlignment();
7232
7233 if (DAG.getTargetLoweringInfo().isLittleEndian())
7234 StOffset = ByteShift;
7235 else
7236 StOffset = IVal.getValueType().getStoreSize() - ByteShift - NumBytes;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007237
Chris Lattner2392ae72010-04-15 04:48:01 +00007238 SDValue Ptr = St->getBasePtr();
7239 if (StOffset) {
7240 Ptr = DAG.getNode(ISD::ADD, IVal->getDebugLoc(), Ptr.getValueType(),
7241 Ptr, DAG.getConstant(StOffset, Ptr.getValueType()));
7242 NewAlign = MinAlign(NewAlign, StOffset);
7243 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007244
Chris Lattner2392ae72010-04-15 04:48:01 +00007245 // Truncate down to the new size.
7246 IVal = DAG.getNode(ISD::TRUNCATE, IVal->getDebugLoc(), VT, IVal);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007247
Chris Lattner2392ae72010-04-15 04:48:01 +00007248 ++OpsNarrowed;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007249 return DAG.getStore(St->getChain(), St->getDebugLoc(), IVal, Ptr,
Chris Lattner6229d0a2010-09-21 18:41:36 +00007250 St->getPointerInfo().getWithOffset(StOffset),
Chris Lattner2392ae72010-04-15 04:48:01 +00007251 false, false, NewAlign).getNode();
7252}
7253
Evan Cheng8b944d32009-05-28 00:35:15 +00007254
7255/// ReduceLoadOpStoreWidth - Look for sequence of load / op / store where op is
7256/// one of 'or', 'xor', and 'and' of immediates. If 'op' is only touching some
7257/// of the loaded bits, try narrowing the load and store if it would end up
7258/// being a win for performance or code size.
7259SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
7260 StoreSDNode *ST = cast<StoreSDNode>(N);
Evan Chengcdcecc02009-05-28 18:41:02 +00007261 if (ST->isVolatile())
7262 return SDValue();
7263
Evan Cheng8b944d32009-05-28 00:35:15 +00007264 SDValue Chain = ST->getChain();
7265 SDValue Value = ST->getValue();
7266 SDValue Ptr = ST->getBasePtr();
Owen Andersone50ed302009-08-10 22:56:29 +00007267 EVT VT = Value.getValueType();
Evan Cheng8b944d32009-05-28 00:35:15 +00007268
7269 if (ST->isTruncatingStore() || VT.isVector() || !Value.hasOneUse())
Evan Chengcdcecc02009-05-28 18:41:02 +00007270 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00007271
7272 unsigned Opc = Value.getOpcode();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007273
Chris Lattner2392ae72010-04-15 04:48:01 +00007274 // If this is "store (or X, Y), P" and X is "(and (load P), cst)", where cst
7275 // is a byte mask indicating a consecutive number of bytes, check to see if
7276 // Y is known to provide just those bytes. If so, we try to replace the
7277 // load + replace + store sequence with a single (narrower) store, which makes
7278 // the load dead.
7279 if (Opc == ISD::OR) {
7280 std::pair<unsigned, unsigned> MaskedLoad;
7281 MaskedLoad = CheckForMaskedLoad(Value.getOperand(0), Ptr, Chain);
7282 if (MaskedLoad.first)
7283 if (SDNode *NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad,
7284 Value.getOperand(1), ST,this))
7285 return SDValue(NewST, 0);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007286
Chris Lattner2392ae72010-04-15 04:48:01 +00007287 // Or is commutative, so try swapping X and Y.
7288 MaskedLoad = CheckForMaskedLoad(Value.getOperand(1), Ptr, Chain);
7289 if (MaskedLoad.first)
7290 if (SDNode *NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad,
7291 Value.getOperand(0), ST,this))
7292 return SDValue(NewST, 0);
7293 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007294
Evan Cheng8b944d32009-05-28 00:35:15 +00007295 if ((Opc != ISD::OR && Opc != ISD::XOR && Opc != ISD::AND) ||
7296 Value.getOperand(1).getOpcode() != ISD::Constant)
Evan Chengcdcecc02009-05-28 18:41:02 +00007297 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00007298
7299 SDValue N0 = Value.getOperand(0);
Dan Gohman24bde5b2010-09-02 21:18:42 +00007300 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
7301 Chain == SDValue(N0.getNode(), 1)) {
Evan Cheng8b944d32009-05-28 00:35:15 +00007302 LoadSDNode *LD = cast<LoadSDNode>(N0);
Chris Lattnerfa459012010-09-21 16:08:50 +00007303 if (LD->getBasePtr() != Ptr ||
7304 LD->getPointerInfo().getAddrSpace() !=
7305 ST->getPointerInfo().getAddrSpace())
Evan Chengcdcecc02009-05-28 18:41:02 +00007306 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00007307
7308 // Find the type to narrow it the load / op / store to.
7309 SDValue N1 = Value.getOperand(1);
7310 unsigned BitWidth = N1.getValueSizeInBits();
7311 APInt Imm = cast<ConstantSDNode>(N1)->getAPIntValue();
7312 if (Opc == ISD::AND)
7313 Imm ^= APInt::getAllOnesValue(BitWidth);
Evan Chengd3c76bb2009-05-28 23:52:18 +00007314 if (Imm == 0 || Imm.isAllOnesValue())
7315 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00007316 unsigned ShAmt = Imm.countTrailingZeros();
7317 unsigned MSB = BitWidth - Imm.countLeadingZeros() - 1;
7318 unsigned NewBW = NextPowerOf2(MSB - ShAmt);
Owen Anderson23b9b192009-08-12 00:36:31 +00007319 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);
Evan Cheng8b944d32009-05-28 00:35:15 +00007320 while (NewBW < BitWidth &&
Evan Chengcdcecc02009-05-28 18:41:02 +00007321 !(TLI.isOperationLegalOrCustom(Opc, NewVT) &&
Evan Cheng8b944d32009-05-28 00:35:15 +00007322 TLI.isNarrowingProfitable(VT, NewVT))) {
7323 NewBW = NextPowerOf2(NewBW);
Owen Anderson23b9b192009-08-12 00:36:31 +00007324 NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);
Evan Cheng8b944d32009-05-28 00:35:15 +00007325 }
Evan Chengcdcecc02009-05-28 18:41:02 +00007326 if (NewBW >= BitWidth)
7327 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00007328
7329 // If the lsb changed does not start at the type bitwidth boundary,
7330 // start at the previous one.
7331 if (ShAmt % NewBW)
7332 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW;
7333 APInt Mask = APInt::getBitsSet(BitWidth, ShAmt, ShAmt + NewBW);
7334 if ((Imm & Mask) == Imm) {
7335 APInt NewImm = (Imm & Mask).lshr(ShAmt).trunc(NewBW);
7336 if (Opc == ISD::AND)
7337 NewImm ^= APInt::getAllOnesValue(NewBW);
7338 uint64_t PtrOff = ShAmt / 8;
7339 // For big endian targets, we need to adjust the offset to the pointer to
7340 // load the correct bytes.
7341 if (TLI.isBigEndian())
Evan Chengcdcecc02009-05-28 18:41:02 +00007342 PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff;
Evan Cheng8b944d32009-05-28 00:35:15 +00007343
7344 unsigned NewAlign = MinAlign(LD->getAlignment(), PtrOff);
Chris Lattnerdb125cf2011-07-18 04:54:35 +00007345 Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext());
Chris Lattner2392ae72010-04-15 04:48:01 +00007346 if (NewAlign < TLI.getTargetData()->getABITypeAlignment(NewVTTy))
Evan Chengcdcecc02009-05-28 18:41:02 +00007347 return SDValue();
7348
Evan Cheng8b944d32009-05-28 00:35:15 +00007349 SDValue NewPtr = DAG.getNode(ISD::ADD, LD->getDebugLoc(),
7350 Ptr.getValueType(), Ptr,
7351 DAG.getConstant(PtrOff, Ptr.getValueType()));
7352 SDValue NewLD = DAG.getLoad(NewVT, N0.getDebugLoc(),
7353 LD->getChain(), NewPtr,
Chris Lattnerfa459012010-09-21 16:08:50 +00007354 LD->getPointerInfo().getWithOffset(PtrOff),
David Greene1e559442010-02-15 17:00:31 +00007355 LD->isVolatile(), LD->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00007356 LD->isInvariant(), NewAlign);
Evan Cheng8b944d32009-05-28 00:35:15 +00007357 SDValue NewVal = DAG.getNode(Opc, Value.getDebugLoc(), NewVT, NewLD,
7358 DAG.getConstant(NewImm, NewVT));
7359 SDValue NewST = DAG.getStore(Chain, N->getDebugLoc(),
7360 NewVal, NewPtr,
Chris Lattnerfa459012010-09-21 16:08:50 +00007361 ST->getPointerInfo().getWithOffset(PtrOff),
David Greene1e559442010-02-15 17:00:31 +00007362 false, false, NewAlign);
Evan Cheng8b944d32009-05-28 00:35:15 +00007363
7364 AddToWorkList(NewPtr.getNode());
7365 AddToWorkList(NewLD.getNode());
7366 AddToWorkList(NewVal.getNode());
7367 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00007368 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLD.getValue(1));
Evan Cheng8b944d32009-05-28 00:35:15 +00007369 ++OpsNarrowed;
7370 return NewST;
7371 }
7372 }
7373
Evan Chengcdcecc02009-05-28 18:41:02 +00007374 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00007375}
7376
Evan Cheng31959b12011-02-02 01:06:55 +00007377/// TransformFPLoadStorePair - For a given floating point load / store pair,
7378/// if the load value isn't used by any other operations, then consider
7379/// transforming the pair to integer load / store operations if the target
7380/// deems the transformation profitable.
7381SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
7382 StoreSDNode *ST = cast<StoreSDNode>(N);
7383 SDValue Chain = ST->getChain();
7384 SDValue Value = ST->getValue();
7385 if (ISD::isNormalStore(ST) && ISD::isNormalLoad(Value.getNode()) &&
7386 Value.hasOneUse() &&
7387 Chain == SDValue(Value.getNode(), 1)) {
7388 LoadSDNode *LD = cast<LoadSDNode>(Value);
7389 EVT VT = LD->getMemoryVT();
7390 if (!VT.isFloatingPoint() ||
7391 VT != ST->getMemoryVT() ||
7392 LD->isNonTemporal() ||
7393 ST->isNonTemporal() ||
7394 LD->getPointerInfo().getAddrSpace() != 0 ||
7395 ST->getPointerInfo().getAddrSpace() != 0)
7396 return SDValue();
7397
7398 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
7399 if (!TLI.isOperationLegal(ISD::LOAD, IntVT) ||
7400 !TLI.isOperationLegal(ISD::STORE, IntVT) ||
7401 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) ||
7402 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT))
7403 return SDValue();
7404
7405 unsigned LDAlign = LD->getAlignment();
7406 unsigned STAlign = ST->getAlignment();
Chris Lattnerdb125cf2011-07-18 04:54:35 +00007407 Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext());
Evan Cheng31959b12011-02-02 01:06:55 +00007408 unsigned ABIAlign = TLI.getTargetData()->getABITypeAlignment(IntVTTy);
7409 if (LDAlign < ABIAlign || STAlign < ABIAlign)
7410 return SDValue();
7411
7412 SDValue NewLD = DAG.getLoad(IntVT, Value.getDebugLoc(),
7413 LD->getChain(), LD->getBasePtr(),
7414 LD->getPointerInfo(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00007415 false, false, false, LDAlign);
Evan Cheng31959b12011-02-02 01:06:55 +00007416
7417 SDValue NewST = DAG.getStore(NewLD.getValue(1), N->getDebugLoc(),
7418 NewLD, ST->getBasePtr(),
7419 ST->getPointerInfo(),
7420 false, false, STAlign);
7421
7422 AddToWorkList(NewLD.getNode());
7423 AddToWorkList(NewST.getNode());
7424 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00007425 DAG.ReplaceAllUsesOfValueWith(Value.getValue(1), NewLD.getValue(1));
Evan Cheng31959b12011-02-02 01:06:55 +00007426 ++LdStFP2Int;
7427 return NewST;
7428 }
7429
7430 return SDValue();
7431}
7432
Nadav Rotem72f7b082012-09-29 06:33:25 +00007433/// Returns the base pointer and an integer offset from that object.
7434static std::pair<SDValue, int64_t> GetPointerBaseAndOffset(SDValue Ptr) {
7435 if (Ptr->getOpcode() == ISD::ADD && isa<ConstantSDNode>(Ptr->getOperand(1))) {
7436 int64_t Offset = cast<ConstantSDNode>(Ptr->getOperand(1))->getSExtValue();
7437 SDValue Base = Ptr->getOperand(0);
7438 return std::make_pair(Base, Offset);
7439 }
7440
7441 return std::make_pair(Ptr, 0);
7442}
7443
7444struct ConsecutiveMemoryChainSorter {
7445 typedef std::pair<LSBaseSDNode*, int64_t> MemLink;
7446 bool operator()(MemLink LHS, MemLink RHS) {
7447 return LHS.second < RHS.second;
7448 }
7449};
7450
7451bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
7452 EVT MemVT = St->getMemoryVT();
7453 int64_t ElementSizeBytes = MemVT.getSizeInBits()/8;
7454
7455 // Don't handle vectors.
7456 if (MemVT.isVector() || !MemVT.isSimple())
7457 return false;
7458
7459 // Perform an early exit check. Do not bother looking at stored values that
7460 // are not constants or loads.
7461 SDValue StoredVal = St->getValue();
7462 if (!isa<ConstantSDNode>(StoredVal) && !isa<ConstantFPSDNode>(StoredVal) &&
7463 !isa<LoadSDNode>(StoredVal))
7464 return false;
7465
7466 // Is this a load-to-store or a const-store.
7467 bool IsLoadSrc = isa<LoadSDNode>(StoredVal);
7468
7469 // Only look at ends of store chains.
7470 SDValue Chain = SDValue(St, 1);
7471 if (Chain->hasOneUse() && Chain->use_begin()->getOpcode() == ISD::STORE)
7472 return false;
7473
7474 // This holds the base pointer and the offset in bytes from the base pointer.
7475 std::pair<SDValue, int64_t> BasePtr =
7476 GetPointerBaseAndOffset(St->getBasePtr());
7477
7478 // We must have a base and an offset.
7479 if (!BasePtr.first.getNode())
7480 return false;
7481
7482 // Do not handle stores to undef base pointers.
7483 if (BasePtr.first.getOpcode() == ISD::UNDEF)
7484 return false;
7485
7486 SmallVector<std::pair<StoreSDNode*, int64_t>, 8> StoreNodes;
7487 // Walk up the chain and look for nodes with offsets from the same
7488 // base pointer. Stop when reaching an instruction with a different kind
7489 // or instruction which has a different base pointer.
7490 StoreSDNode *Index = St;
7491 while (Index) {
7492 // If the chain has more than one use, then we can't reorder the mem ops.
7493 if (Index != St && !SDValue(Index, 1)->hasOneUse())
7494 break;
7495
7496 // Find the base pointer and offset for this memory node.
7497 std::pair<SDValue, int64_t> Ptr =
7498 GetPointerBaseAndOffset(Index->getBasePtr());
7499
7500 // Check that the base pointer is the same as the original one.
7501 if (Ptr.first.getNode() != BasePtr.first.getNode())
7502 break;
7503
7504 // Check that the alignment is the same.
7505 if (Index->getAlignment() != St->getAlignment())
7506 break;
7507
7508 // The memory operands must not be volatile.
7509 if (Index->isVolatile() || Index->isIndexed())
7510 break;
7511
7512 // No truncation.
7513 if (StoreSDNode *St = dyn_cast<StoreSDNode>(Index))
7514 if (St->isTruncatingStore())
7515 break;
7516
7517 // The stored memory type must be the same.
7518 if (Index->getMemoryVT() != MemVT)
7519 break;
7520
7521 // We found a potential memory operand to merge.
7522 StoreNodes.push_back(std::make_pair(Index,Ptr.second));
7523
7524 // Move up the chain to the next memory operation.
7525 Index = dyn_cast<StoreSDNode>(Index->getChain().getNode());
7526 }
7527
7528 // Check if there is anything to merge.
7529 if (StoreNodes.size() < 2)
7530 return false;
7531
7532 // Remember which node is the earliest node in the chain.
7533 LSBaseSDNode *EarliestOp = StoreNodes.back().first;
7534
7535 // Sort the memory operands according to their distance from the base pointer.
7536 std::sort(StoreNodes.begin(), StoreNodes.end(),
7537 ConsecutiveMemoryChainSorter());
7538
7539 // Scan the memory operations on the chain and find the first non-consecutive
7540 // store memory address.
7541 unsigned LastConsecutiveStore = 0;
7542 int64_t StartAddress = StoreNodes[0].second;
7543 for (unsigned i=1; i<StoreNodes.size(); ++i) {
7544 int64_t CurrAddress = StoreNodes[i].second;
7545 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
7546 break;
7547 LastConsecutiveStore = i;
7548 }
7549
7550 // Store the constants into memory as one consecutive store.
7551 if (!IsLoadSrc) {
7552 unsigned LastConst = 0;
7553 for (unsigned i=0; i<LastConsecutiveStore+1; ++i) {
7554 SDValue StoredVal = StoreNodes[i].first->getValue();
7555 bool IsConst = (isa<ConstantSDNode>(StoredVal) || isa<ConstantFPSDNode>(StoredVal));
7556 if (!IsConst)
7557 break;
7558 LastConst = i;
7559 }
7560 unsigned NumElem = std::min(LastConsecutiveStore + 1, LastConst + 1);
7561 if (NumElem < 2)
7562 return false;
7563
7564 EVT JointMemOpVT = EVT::getVectorVT(*DAG.getContext(), MemVT, NumElem);
7565 DebugLoc DL = StoreNodes[0].first->getDebugLoc();
7566 SmallVector<SDValue, 8> Ops;
7567
7568 for (unsigned i = 0; i < NumElem ; ++i) {
7569 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].first);
7570 Ops.push_back(St->getValue());
7571 }
7572
7573 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, DL,
7574 JointMemOpVT, &Ops[0], Ops.size());
7575
7576 SDValue NewStore = DAG.getStore(EarliestOp->getChain(), DL, BV,
7577 EarliestOp->getBasePtr(),
7578 EarliestOp->getPointerInfo(), false, false,
7579 EarliestOp->getAlignment());
7580
7581 for (unsigned i = 0; i < NumElem ; ++i) {
7582 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].first);
7583 CombineTo(St, NewStore);
7584 }
7585 return true;
7586 }
7587
7588 // Look for load nodes wich are used by the stored values.
7589 SmallVector<std::pair<LoadSDNode*, int64_t>, 8> LoadNodes;
7590
7591 // Find acceptible loads. Loads need to have the same chain (token factor),
7592 // must not be zext, volatile, indexed, and they must be consecutive.
7593 SDValue LdBasePtr;
7594 for (unsigned i=0; i<LastConsecutiveStore+1; ++i) {
7595 LoadSDNode *Ld = dyn_cast<LoadSDNode>(StoreNodes[i].first->getValue());
7596 if (!Ld) break;
7597
7598 // Loads must only have one use.
7599 if (!Ld->hasNUsesOfValue(1, 0))
7600 break;
7601
7602 // Check that the alignment is the same as the stores.
7603 if (Ld->getAlignment() != St->getAlignment())
7604 break;
7605
7606 // The memory operands must not be volatile.
7607 if (Ld->isVolatile() || Ld->isIndexed())
7608 break;
7609
7610 if (Ld->getExtensionType() != ISD::NON_EXTLOAD)
7611 break;
7612
7613 // The stored memory type must be the same.
7614 if (Ld->getMemoryVT() != MemVT)
7615 break;
7616
7617 std::pair<SDValue, int64_t> LdPtr =
7618 GetPointerBaseAndOffset(Ld->getBasePtr());
7619
7620 // If this is not the first ptr that we check.
7621 if (LdBasePtr.getNode()) {
7622 // The base ptr must be the same,
7623 if (LdPtr.first != LdBasePtr)
7624 break;
7625 } else {
7626 LdBasePtr = LdPtr.first;
7627 }
7628
7629 // We found a potential memory operand to merge.
7630 LoadNodes.push_back(std::make_pair(Ld, LdPtr.second));
7631 }
7632
7633 if (LoadNodes.size() < 2)
7634 return false;
7635
7636 // Scan the memory operations on the chain and find the first non-consecutive
7637 // load memory address.
7638 unsigned LastConsecutiveLoad = 0;
7639 StartAddress = LoadNodes[0].second;
7640 for (unsigned i=1; i<LoadNodes.size(); ++i) {
7641 int64_t CurrAddress = LoadNodes[i].second;
7642 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
7643 break;
7644 LastConsecutiveLoad = i;
7645 }
7646
7647 unsigned NumElem =
7648 std::min(LastConsecutiveStore + 1, LastConsecutiveLoad + 1);
7649
7650 EVT JointMemOpVT = EVT::getVectorVT(*DAG.getContext(), MemVT, NumElem);
7651 DebugLoc LoadDL = LoadNodes[0].first->getDebugLoc();
7652 DebugLoc StoreDL = StoreNodes[0].first->getDebugLoc();
7653
7654 LoadSDNode *FirstLoad = LoadNodes[0].first;
7655 SDValue NewLoad = DAG.getLoad(JointMemOpVT, LoadDL,
7656 FirstLoad->getChain(),
7657 FirstLoad->getBasePtr(),
7658 FirstLoad->getPointerInfo(),
7659 false, false, false,
7660 FirstLoad->getAlignment());
7661
7662 SDValue NewStore = DAG.getStore(EarliestOp->getChain(), StoreDL, NewLoad,
7663 EarliestOp->getBasePtr(),
7664 EarliestOp->getPointerInfo(), false, false,
7665 EarliestOp->getAlignment());
7666
7667 for (unsigned i = 0; i < NumElem ; ++i) {
7668 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].first);
7669 CombineTo(St, NewStore);
7670 }
7671
7672 return true;
7673}
7674
Dan Gohman475871a2008-07-27 21:46:04 +00007675SDValue DAGCombiner::visitSTORE(SDNode *N) {
Evan Cheng8b2794a2006-10-13 21:14:26 +00007676 StoreSDNode *ST = cast<StoreSDNode>(N);
Dan Gohman475871a2008-07-27 21:46:04 +00007677 SDValue Chain = ST->getChain();
7678 SDValue Value = ST->getValue();
7679 SDValue Ptr = ST->getBasePtr();
Scott Michelfdc40a02009-02-17 22:15:04 +00007680
Evan Cheng59d5b682007-05-07 21:27:48 +00007681 // If this is a store of a bit convert, store the input value if the
Evan Cheng2c4f9432007-05-09 21:49:47 +00007682 // resultant store does not need a higher alignment than the original.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007683 if (Value.getOpcode() == ISD::BITCAST && !ST->isTruncatingStore() &&
Chris Lattnerddf89562008-01-17 19:59:44 +00007684 ST->isUnindexed()) {
Dan Gohman1ba519b2009-02-20 23:29:13 +00007685 unsigned OrigAlign = ST->getAlignment();
Owen Andersone50ed302009-08-10 22:56:29 +00007686 EVT SVT = Value.getOperand(0).getValueType();
Dan Gohman1ba519b2009-02-20 23:29:13 +00007687 unsigned Align = TLI.getTargetData()->
Owen Anderson23b9b192009-08-12 00:36:31 +00007688 getABITypeAlignment(SVT.getTypeForEVT(*DAG.getContext()));
Duncan Sandsd4b9c172008-06-13 19:07:40 +00007689 if (Align <= OrigAlign &&
Duncan Sands25cf2272008-11-24 14:53:14 +00007690 ((!LegalOperations && !ST->isVolatile()) ||
Dan Gohmanf560ffa2009-01-28 17:46:25 +00007691 TLI.isOperationLegalOrCustom(ISD::STORE, SVT)))
Bill Wendlingc144a572009-01-30 23:36:47 +00007692 return DAG.getStore(Chain, N->getDebugLoc(), Value.getOperand(0),
Chris Lattner6229d0a2010-09-21 18:41:36 +00007693 Ptr, ST->getPointerInfo(), ST->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00007694 ST->isNonTemporal(), OrigAlign);
Jim Laskey279f0532006-09-25 16:29:54 +00007695 }
Owen Andersona34d9362011-04-14 17:30:49 +00007696
Chris Lattnerb3452ea2011-04-09 02:32:02 +00007697 // Turn 'store undef, Ptr' -> nothing.
7698 if (Value.getOpcode() == ISD::UNDEF && ST->isUnindexed())
7699 return Chain;
Duncan Sandsd4b9c172008-06-13 19:07:40 +00007700
Nate Begeman2cbba892006-12-11 02:23:46 +00007701 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
Nate Begeman2cbba892006-12-11 02:23:46 +00007702 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
Duncan Sandsd4b9c172008-06-13 19:07:40 +00007703 // NOTE: If the original store is volatile, this transform must not increase
7704 // the number of stores. For example, on x86-32 an f64 can be stored in one
7705 // processor operation but an i64 (which is not legal) requires two. So the
7706 // transform should not be done in this case.
Evan Cheng25ece662006-12-11 17:25:19 +00007707 if (Value.getOpcode() != ISD::TargetConstantFP) {
Dan Gohman475871a2008-07-27 21:46:04 +00007708 SDValue Tmp;
Owen Anderson825b72b2009-08-11 20:47:22 +00007709 switch (CFP->getValueType(0).getSimpleVT().SimpleTy) {
Torok Edwinc23197a2009-07-14 16:55:14 +00007710 default: llvm_unreachable("Unknown FP type");
Pete Cooper438c0402012-06-21 18:00:39 +00007711 case MVT::f16: // We don't do this for these yet.
7712 case MVT::f80:
Owen Anderson825b72b2009-08-11 20:47:22 +00007713 case MVT::f128:
7714 case MVT::ppcf128:
Dale Johannesenc7b21d52007-09-18 18:36:59 +00007715 break;
Owen Anderson825b72b2009-08-11 20:47:22 +00007716 case MVT::f32:
Chris Lattner2392ae72010-04-15 04:48:01 +00007717 if ((isTypeLegal(MVT::i32) && !LegalOperations && !ST->isVolatile()) ||
Owen Anderson825b72b2009-08-11 20:47:22 +00007718 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) {
Dale Johannesen9d5f4562007-09-12 03:30:33 +00007719 Tmp = DAG.getConstant((uint32_t)CFP->getValueAPF().
Owen Anderson825b72b2009-08-11 20:47:22 +00007720 bitcastToAPInt().getZExtValue(), MVT::i32);
Bill Wendlingc144a572009-01-30 23:36:47 +00007721 return DAG.getStore(Chain, N->getDebugLoc(), Tmp,
Chris Lattner6229d0a2010-09-21 18:41:36 +00007722 Ptr, ST->getPointerInfo(), ST->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00007723 ST->isNonTemporal(), ST->getAlignment());
Chris Lattner62be1a72006-12-12 04:16:14 +00007724 }
7725 break;
Owen Anderson825b72b2009-08-11 20:47:22 +00007726 case MVT::f64:
Chris Lattner2392ae72010-04-15 04:48:01 +00007727 if ((TLI.isTypeLegal(MVT::i64) && !LegalOperations &&
Dan Gohmanf560ffa2009-01-28 17:46:25 +00007728 !ST->isVolatile()) ||
Owen Anderson825b72b2009-08-11 20:47:22 +00007729 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) {
Dale Johannesen7111b022008-10-09 18:53:47 +00007730 Tmp = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
Owen Anderson825b72b2009-08-11 20:47:22 +00007731 getZExtValue(), MVT::i64);
Bill Wendlingc144a572009-01-30 23:36:47 +00007732 return DAG.getStore(Chain, N->getDebugLoc(), Tmp,
Chris Lattner6229d0a2010-09-21 18:41:36 +00007733 Ptr, ST->getPointerInfo(), ST->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00007734 ST->isNonTemporal(), ST->getAlignment());
Chris Lattnerb3452ea2011-04-09 02:32:02 +00007735 }
Owen Andersona34d9362011-04-14 17:30:49 +00007736
Chris Lattnerb3452ea2011-04-09 02:32:02 +00007737 if (!ST->isVolatile() &&
7738 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) {
Duncan Sandsdc846502007-10-28 12:59:45 +00007739 // Many FP stores are not made apparent until after legalize, e.g. for
Chris Lattner62be1a72006-12-12 04:16:14 +00007740 // argument passing. Since this is so common, custom legalize the
7741 // 64-bit integer store into two 32-bit stores.
Dale Johannesen7111b022008-10-09 18:53:47 +00007742 uint64_t Val = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
Owen Anderson825b72b2009-08-11 20:47:22 +00007743 SDValue Lo = DAG.getConstant(Val & 0xFFFFFFFF, MVT::i32);
7744 SDValue Hi = DAG.getConstant(Val >> 32, MVT::i32);
Duncan Sands0753fc12008-02-11 10:37:04 +00007745 if (TLI.isBigEndian()) std::swap(Lo, Hi);
Chris Lattner62be1a72006-12-12 04:16:14 +00007746
Dan Gohmand6fd1bc2007-07-09 22:18:38 +00007747 unsigned Alignment = ST->getAlignment();
7748 bool isVolatile = ST->isVolatile();
David Greene1e559442010-02-15 17:00:31 +00007749 bool isNonTemporal = ST->isNonTemporal();
Dan Gohmand6fd1bc2007-07-09 22:18:38 +00007750
Bill Wendlingc144a572009-01-30 23:36:47 +00007751 SDValue St0 = DAG.getStore(Chain, ST->getDebugLoc(), Lo,
Chris Lattner6229d0a2010-09-21 18:41:36 +00007752 Ptr, ST->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00007753 isVolatile, isNonTemporal,
7754 ST->getAlignment());
Bill Wendlingc144a572009-01-30 23:36:47 +00007755 Ptr = DAG.getNode(ISD::ADD, N->getDebugLoc(), Ptr.getValueType(), Ptr,
Chris Lattner62be1a72006-12-12 04:16:14 +00007756 DAG.getConstant(4, Ptr.getValueType()));
Duncan Sandsdc846502007-10-28 12:59:45 +00007757 Alignment = MinAlign(Alignment, 4U);
Bill Wendlingc144a572009-01-30 23:36:47 +00007758 SDValue St1 = DAG.getStore(Chain, ST->getDebugLoc(), Hi,
Chris Lattner6229d0a2010-09-21 18:41:36 +00007759 Ptr, ST->getPointerInfo().getWithOffset(4),
7760 isVolatile, isNonTemporal,
David Greene1e559442010-02-15 17:00:31 +00007761 Alignment);
Owen Anderson825b72b2009-08-11 20:47:22 +00007762 return DAG.getNode(ISD::TokenFactor, N->getDebugLoc(), MVT::Other,
Bill Wendlingc144a572009-01-30 23:36:47 +00007763 St0, St1);
Chris Lattner62be1a72006-12-12 04:16:14 +00007764 }
Bill Wendlingc144a572009-01-30 23:36:47 +00007765
Chris Lattner62be1a72006-12-12 04:16:14 +00007766 break;
Evan Cheng25ece662006-12-11 17:25:19 +00007767 }
Nate Begeman2cbba892006-12-11 02:23:46 +00007768 }
Nate Begeman2cbba892006-12-11 02:23:46 +00007769 }
7770
Evan Cheng255f20f2010-04-01 06:04:33 +00007771 // Try to infer better alignment information than the store already has.
7772 if (OptLevel != CodeGenOpt::None && ST->isUnindexed()) {
Evan Chenged1c0c72011-11-28 22:37:34 +00007773 if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
7774 if (Align > ST->getAlignment())
7775 return DAG.getTruncStore(Chain, N->getDebugLoc(), Value,
7776 Ptr, ST->getPointerInfo(), ST->getMemoryVT(),
7777 ST->isVolatile(), ST->isNonTemporal(), Align);
Evan Cheng255f20f2010-04-01 06:04:33 +00007778 }
7779 }
7780
Evan Cheng31959b12011-02-02 01:06:55 +00007781 // Try transforming a pair floating point load / store ops to integer
7782 // load / store ops.
7783 SDValue NewST = TransformFPLoadStorePair(N);
7784 if (NewST.getNode())
7785 return NewST;
7786
Scott Michelfdc40a02009-02-17 22:15:04 +00007787 if (CombinerAA) {
Jim Laskey279f0532006-09-25 16:29:54 +00007788 // Walk up chain skipping non-aliasing memory nodes.
Dan Gohman475871a2008-07-27 21:46:04 +00007789 SDValue BetterChain = FindBetterChain(N, Chain);
Scott Michelfdc40a02009-02-17 22:15:04 +00007790
Jim Laskey6ff23e52006-10-04 16:53:27 +00007791 // If there is a better chain.
Jim Laskey279f0532006-09-25 16:29:54 +00007792 if (Chain != BetterChain) {
Dan Gohman475871a2008-07-27 21:46:04 +00007793 SDValue ReplStore;
Nate Begemanb6aef5c2009-09-15 00:18:30 +00007794
7795 // Replace the chain to avoid dependency.
Jim Laskeyd4edf2c2006-10-14 12:14:27 +00007796 if (ST->isTruncatingStore()) {
Bill Wendlingc144a572009-01-30 23:36:47 +00007797 ReplStore = DAG.getTruncStore(BetterChain, N->getDebugLoc(), Value, Ptr,
Chris Lattnerda2d8e12010-09-21 17:42:31 +00007798 ST->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00007799 ST->getMemoryVT(), ST->isVolatile(),
7800 ST->isNonTemporal(), ST->getAlignment());
Jim Laskeyd4edf2c2006-10-14 12:14:27 +00007801 } else {
Bill Wendlingc144a572009-01-30 23:36:47 +00007802 ReplStore = DAG.getStore(BetterChain, N->getDebugLoc(), Value, Ptr,
Chris Lattner6229d0a2010-09-21 18:41:36 +00007803 ST->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00007804 ST->isVolatile(), ST->isNonTemporal(),
7805 ST->getAlignment());
Jim Laskeyd4edf2c2006-10-14 12:14:27 +00007806 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007807
Jim Laskey279f0532006-09-25 16:29:54 +00007808 // Create token to keep both nodes around.
Bill Wendlingc144a572009-01-30 23:36:47 +00007809 SDValue Token = DAG.getNode(ISD::TokenFactor, N->getDebugLoc(),
Owen Anderson825b72b2009-08-11 20:47:22 +00007810 MVT::Other, Chain, ReplStore);
Bill Wendlingc144a572009-01-30 23:36:47 +00007811
Nate Begemanb6aef5c2009-09-15 00:18:30 +00007812 // Make sure the new and old chains are cleaned up.
7813 AddToWorkList(Token.getNode());
7814
Jim Laskey274062c2006-10-13 23:32:28 +00007815 // Don't add users to work list.
7816 return CombineTo(N, Token, false);
Jim Laskey279f0532006-09-25 16:29:54 +00007817 }
Jim Laskeyd1aed7a2006-09-21 16:28:59 +00007818 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007819
Evan Cheng33dbedc2006-11-05 09:31:14 +00007820 // Try transforming N to an indexed store.
Evan Chengbbd6f6e2006-11-07 09:03:05 +00007821 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
Dan Gohman475871a2008-07-27 21:46:04 +00007822 return SDValue(N, 0);
Evan Cheng33dbedc2006-11-05 09:31:14 +00007823
Chris Lattner3c872852007-12-29 06:26:16 +00007824 // FIXME: is there such a thing as a truncating indexed store?
Chris Lattnerddf89562008-01-17 19:59:44 +00007825 if (ST->isTruncatingStore() && ST->isUnindexed() &&
Nadav Rotembaff46f2011-06-15 11:19:12 +00007826 Value.getValueType().isInteger()) {
Chris Lattner2b4c2792007-10-13 06:35:54 +00007827 // See if we can simplify the input to this truncstore with knowledge that
7828 // only the low bits are being used. For example:
7829 // "truncstore (or (shl x, 8), y), i8" -> "truncstore y, i8"
Scott Michelfdc40a02009-02-17 22:15:04 +00007830 SDValue Shorter =
Dan Gohman2e68b6f2008-02-25 21:11:39 +00007831 GetDemandedBits(Value,
Nadav Rotembaff46f2011-06-15 11:19:12 +00007832 APInt::getLowBitsSet(
7833 Value.getValueType().getScalarType().getSizeInBits(),
7834 ST->getMemoryVT().getScalarType().getSizeInBits()));
Gabor Greifba36cb52008-08-28 21:40:38 +00007835 AddToWorkList(Value.getNode());
7836 if (Shorter.getNode())
Bill Wendlingc144a572009-01-30 23:36:47 +00007837 return DAG.getTruncStore(Chain, N->getDebugLoc(), Shorter,
Chris Lattnerda2d8e12010-09-21 17:42:31 +00007838 Ptr, ST->getPointerInfo(), ST->getMemoryVT(),
David Greene1e559442010-02-15 17:00:31 +00007839 ST->isVolatile(), ST->isNonTemporal(),
7840 ST->getAlignment());
Scott Michelfdc40a02009-02-17 22:15:04 +00007841
Chris Lattnere33544c2007-10-13 06:58:48 +00007842 // Otherwise, see if we can simplify the operation with
7843 // SimplifyDemandedBits, which only works if the value has a single use.
Dan Gohman7b8d4a92008-02-27 00:25:32 +00007844 if (SimplifyDemandedBits(Value,
Eric Christopher503a64d2010-12-09 04:48:06 +00007845 APInt::getLowBitsSet(
7846 Value.getValueType().getScalarType().getSizeInBits(),
7847 ST->getMemoryVT().getScalarType().getSizeInBits())))
Dan Gohman475871a2008-07-27 21:46:04 +00007848 return SDValue(N, 0);
Chris Lattner2b4c2792007-10-13 06:35:54 +00007849 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007850
Chris Lattner3c872852007-12-29 06:26:16 +00007851 // If this is a load followed by a store to the same location, then the store
7852 // is dead/noop.
7853 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Value)) {
Dan Gohmanb625f2f2008-01-30 00:15:11 +00007854 if (Ld->getBasePtr() == Ptr && ST->getMemoryVT() == Ld->getMemoryVT() &&
Chris Lattnerddf89562008-01-17 19:59:44 +00007855 ST->isUnindexed() && !ST->isVolatile() &&
Chris Lattner07649d92008-01-08 23:08:06 +00007856 // There can't be any side effects between the load and store, such as
7857 // a call or store.
Dan Gohman475871a2008-07-27 21:46:04 +00007858 Chain.reachesChainWithoutSideEffects(SDValue(Ld, 1))) {
Chris Lattner3c872852007-12-29 06:26:16 +00007859 // The store is dead, remove it.
7860 return Chain;
7861 }
7862 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00007863
Chris Lattnerddf89562008-01-17 19:59:44 +00007864 // If this is an FP_ROUND or TRUNC followed by a store, fold this into a
7865 // truncating store. We can do this even if this is already a truncstore.
7866 if ((Value.getOpcode() == ISD::FP_ROUND || Value.getOpcode() == ISD::TRUNCATE)
Gabor Greifba36cb52008-08-28 21:40:38 +00007867 && Value.getNode()->hasOneUse() && ST->isUnindexed() &&
Chris Lattnerddf89562008-01-17 19:59:44 +00007868 TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(),
Dan Gohmanb625f2f2008-01-30 00:15:11 +00007869 ST->getMemoryVT())) {
Bill Wendlingc144a572009-01-30 23:36:47 +00007870 return DAG.getTruncStore(Chain, N->getDebugLoc(), Value.getOperand(0),
Chris Lattnerda2d8e12010-09-21 17:42:31 +00007871 Ptr, ST->getPointerInfo(), ST->getMemoryVT(),
David Greene1e559442010-02-15 17:00:31 +00007872 ST->isVolatile(), ST->isNonTemporal(),
7873 ST->getAlignment());
Chris Lattnerddf89562008-01-17 19:59:44 +00007874 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00007875
Nadav Rotem72f7b082012-09-29 06:33:25 +00007876
7877 // Only perform this optimization before the types are legal, because we
7878 // don't want to generate illegal types in this optimization.
7879 if (!LegalTypes && MergeConsecutiveStores(ST))
7880 return SDValue(N, 0);
7881
Evan Cheng8b944d32009-05-28 00:35:15 +00007882 return ReduceLoadOpStoreWidth(N);
Chris Lattner87514ca2005-10-10 22:31:19 +00007883}
7884
Dan Gohman475871a2008-07-27 21:46:04 +00007885SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
7886 SDValue InVec = N->getOperand(0);
7887 SDValue InVal = N->getOperand(1);
7888 SDValue EltNo = N->getOperand(2);
Eli Friedman9db817f2011-09-09 21:04:06 +00007889 DebugLoc dl = N->getDebugLoc();
Scott Michelfdc40a02009-02-17 22:15:04 +00007890
Bob Wilson492fd452010-05-19 23:42:58 +00007891 // If the inserted element is an UNDEF, just use the input vector.
7892 if (InVal.getOpcode() == ISD::UNDEF)
7893 return InVec;
7894
Nadav Rotem609d54e2011-02-12 14:40:33 +00007895 EVT VT = InVec.getValueType();
7896
Owen Anderson95771af2011-02-25 21:41:48 +00007897 // If we can't generate a legal BUILD_VECTOR, exit
Nadav Rotem609d54e2011-02-12 14:40:33 +00007898 if (LegalOperations && !TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
7899 return SDValue();
7900
Eli Friedman9db817f2011-09-09 21:04:06 +00007901 // Check that we know which element is being inserted
7902 if (!isa<ConstantSDNode>(EltNo))
7903 return SDValue();
7904 unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00007905
Eli Friedman9db817f2011-09-09 21:04:06 +00007906 // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially
7907 // be converted to a BUILD_VECTOR). Fill in the Ops vector with the
7908 // vector elements.
7909 SmallVector<SDValue, 8> Ops;
7910 if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
7911 Ops.append(InVec.getNode()->op_begin(),
7912 InVec.getNode()->op_end());
7913 } else if (InVec.getOpcode() == ISD::UNDEF) {
7914 unsigned NElts = VT.getVectorNumElements();
7915 Ops.append(NElts, DAG.getUNDEF(InVal.getValueType()));
7916 } else {
7917 return SDValue();
Nate Begeman9008ca62009-04-27 18:41:29 +00007918 }
Eli Friedman9db817f2011-09-09 21:04:06 +00007919
7920 // Insert the element
7921 if (Elt < Ops.size()) {
7922 // All the operands of BUILD_VECTOR must have the same type;
7923 // we enforce that here.
7924 EVT OpVT = Ops[0].getValueType();
7925 if (InVal.getValueType() != OpVT)
7926 InVal = OpVT.bitsGT(InVal.getValueType()) ?
7927 DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) :
7928 DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal);
7929 Ops[Elt] = InVal;
7930 }
7931
7932 // Return the new vector
7933 return DAG.getNode(ISD::BUILD_VECTOR, dl,
7934 VT, &Ops[0], Ops.size());
Chris Lattnerca242442006-03-19 01:27:56 +00007935}
7936
Dan Gohman475871a2008-07-27 21:46:04 +00007937SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
Mon P Wang7ac9cdf2009-01-17 00:07:25 +00007938 // (vextract (scalar_to_vector val, 0) -> val
7939 SDValue InVec = N->getOperand(0);
Nadav Rotemba05c912012-01-17 21:44:01 +00007940 EVT VT = InVec.getValueType();
7941 EVT NVT = N->getValueType(0);
Mon P Wang7ac9cdf2009-01-17 00:07:25 +00007942
Duncan Sandsc356f332011-05-09 08:03:33 +00007943 if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR) {
7944 // Check if the result type doesn't match the inserted element type. A
7945 // SCALAR_TO_VECTOR may truncate the inserted element and the
7946 // EXTRACT_VECTOR_ELT may widen the extracted vector.
7947 SDValue InOp = InVec.getOperand(0);
Duncan Sandsc356f332011-05-09 08:03:33 +00007948 if (InOp.getValueType() != NVT) {
7949 assert(InOp.getValueType().isInteger() && NVT.isInteger());
7950 return DAG.getSExtOrTrunc(InOp, InVec.getDebugLoc(), NVT);
7951 }
7952 return InOp;
7953 }
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007954
Nadav Rotemba05c912012-01-17 21:44:01 +00007955 SDValue EltNo = N->getOperand(1);
7956 bool ConstEltNo = isa<ConstantSDNode>(EltNo);
7957
7958 // Transform: (EXTRACT_VECTOR_ELT( VECTOR_SHUFFLE )) -> EXTRACT_VECTOR_ELT.
7959 // We only perform this optimization before the op legalization phase because
Nadav Rotem6dfabb62012-09-20 08:53:31 +00007960 // we may introduce new vector instructions which are not backed by TD
7961 // patterns. For example on AVX, extracting elements from a wide vector
7962 // without using extract_subvector.
Nadav Rotemba05c912012-01-17 21:44:01 +00007963 if (InVec.getOpcode() == ISD::VECTOR_SHUFFLE
7964 && ConstEltNo && !LegalOperations) {
7965 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
7966 int NumElem = VT.getVectorNumElements();
7967 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(InVec);
7968 // Find the new index to extract from.
7969 int OrigElt = SVOp->getMaskElt(Elt);
7970
7971 // Extracting an undef index is undef.
7972 if (OrigElt == -1)
7973 return DAG.getUNDEF(NVT);
7974
7975 // Select the right vector half to extract from.
7976 if (OrigElt < NumElem) {
7977 InVec = InVec->getOperand(0);
7978 } else {
7979 InVec = InVec->getOperand(1);
7980 OrigElt -= NumElem;
7981 }
7982
Jim Grosbacha249f7d2012-05-08 20:56:07 +00007983 EVT IndexTy = N->getOperand(1).getValueType();
Nadav Rotemba05c912012-01-17 21:44:01 +00007984 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), NVT,
Jim Grosbacha249f7d2012-05-08 20:56:07 +00007985 InVec, DAG.getConstant(OrigElt, IndexTy));
Nadav Rotemba05c912012-01-17 21:44:01 +00007986 }
7987
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007988 // Perform only after legalization to ensure build_vector / vector_shuffle
7989 // optimizations have already been done.
Duncan Sands25cf2272008-11-24 14:53:14 +00007990 if (!LegalOperations) return SDValue();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007991
Mon P Wang7ac9cdf2009-01-17 00:07:25 +00007992 // (vextract (v4f32 load $addr), c) -> (f32 load $addr+c*size)
7993 // (vextract (v4f32 s2v (f32 load $addr)), c) -> (f32 load $addr+c*size)
7994 // (vextract (v4f32 shuffle (load $addr), <1,u,u,u>), 0) -> (f32 load $addr)
Evan Cheng513da432007-10-06 08:19:55 +00007995
Nadav Rotemba05c912012-01-17 21:44:01 +00007996 if (ConstEltNo) {
Eric Christophercaebdd42010-11-03 09:36:40 +00007997 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
Evan Cheng513da432007-10-06 08:19:55 +00007998 bool NewLoad = false;
Mon P Wanga60b5232008-12-11 00:26:16 +00007999 bool BCNumEltsChanged = false;
Owen Andersone50ed302009-08-10 22:56:29 +00008000 EVT ExtVT = VT.getVectorElementType();
8001 EVT LVT = ExtVT;
Bill Wendlingc144a572009-01-30 23:36:47 +00008002
Evan Cheng84387ea2012-03-13 22:00:52 +00008003 // If the result of load has to be truncated, then it's not necessarily
8004 // profitable.
Evan Chenga03d3662012-03-13 22:16:11 +00008005 if (NVT.bitsLT(LVT) && !TLI.isTruncateFree(LVT, NVT))
Evan Cheng84387ea2012-03-13 22:00:52 +00008006 return SDValue();
8007
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008008 if (InVec.getOpcode() == ISD::BITCAST) {
Eli Friedmand6e25602011-12-26 22:49:32 +00008009 // Don't duplicate a load with other uses.
8010 if (!InVec.hasOneUse())
8011 return SDValue();
8012
Owen Andersone50ed302009-08-10 22:56:29 +00008013 EVT BCVT = InVec.getOperand(0).getValueType();
8014 if (!BCVT.isVector() || ExtVT.bitsGT(BCVT.getVectorElementType()))
Dan Gohman475871a2008-07-27 21:46:04 +00008015 return SDValue();
Mon P Wanga60b5232008-12-11 00:26:16 +00008016 if (VT.getVectorNumElements() != BCVT.getVectorNumElements())
8017 BCNumEltsChanged = true;
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008018 InVec = InVec.getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00008019 ExtVT = BCVT.getVectorElementType();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008020 NewLoad = true;
8021 }
Evan Cheng513da432007-10-06 08:19:55 +00008022
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008023 LoadSDNode *LN0 = NULL;
Nate Begeman5a5ca152009-04-29 05:20:52 +00008024 const ShuffleVectorSDNode *SVN = NULL;
Bill Wendlingc144a572009-01-30 23:36:47 +00008025 if (ISD::isNormalLoad(InVec.getNode())) {
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008026 LN0 = cast<LoadSDNode>(InVec);
Bill Wendlingc144a572009-01-30 23:36:47 +00008027 } else if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR &&
Owen Andersone50ed302009-08-10 22:56:29 +00008028 InVec.getOperand(0).getValueType() == ExtVT &&
Bill Wendlingc144a572009-01-30 23:36:47 +00008029 ISD::isNormalLoad(InVec.getOperand(0).getNode())) {
Eli Friedmand6e25602011-12-26 22:49:32 +00008030 // Don't duplicate a load with other uses.
8031 if (!InVec.hasOneUse())
8032 return SDValue();
8033
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008034 LN0 = cast<LoadSDNode>(InVec.getOperand(0));
Nate Begeman5a5ca152009-04-29 05:20:52 +00008035 } else if ((SVN = dyn_cast<ShuffleVectorSDNode>(InVec))) {
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008036 // (vextract (vector_shuffle (load $addr), v2, <1, u, u, u>), 1)
8037 // =>
8038 // (load $addr+1*size)
Scott Michelfdc40a02009-02-17 22:15:04 +00008039
Eli Friedmand6e25602011-12-26 22:49:32 +00008040 // Don't duplicate a load with other uses.
8041 if (!InVec.hasOneUse())
8042 return SDValue();
8043
Mon P Wanga60b5232008-12-11 00:26:16 +00008044 // If the bit convert changed the number of elements, it is unsafe
8045 // to examine the mask.
8046 if (BCNumEltsChanged)
8047 return SDValue();
Nate Begeman5a5ca152009-04-29 05:20:52 +00008048
8049 // Select the input vector, guarding against out of range extract vector.
8050 unsigned NumElems = VT.getVectorNumElements();
Eric Christophercaebdd42010-11-03 09:36:40 +00008051 int Idx = (Elt > (int)NumElems) ? -1 : SVN->getMaskElt(Elt);
Nate Begeman5a5ca152009-04-29 05:20:52 +00008052 InVec = (Idx < (int)NumElems) ? InVec.getOperand(0) : InVec.getOperand(1);
8053
Eli Friedmand6e25602011-12-26 22:49:32 +00008054 if (InVec.getOpcode() == ISD::BITCAST) {
8055 // Don't duplicate a load with other uses.
8056 if (!InVec.hasOneUse())
8057 return SDValue();
8058
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008059 InVec = InVec.getOperand(0);
Eli Friedmand6e25602011-12-26 22:49:32 +00008060 }
Gabor Greifba36cb52008-08-28 21:40:38 +00008061 if (ISD::isNormalLoad(InVec.getNode())) {
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008062 LN0 = cast<LoadSDNode>(InVec);
Ted Kremenekd0e88f32010-04-08 18:49:30 +00008063 Elt = (Idx < (int)NumElems) ? Idx : Idx - (int)NumElems;
Evan Cheng513da432007-10-06 08:19:55 +00008064 }
8065 }
Bill Wendlingc144a572009-01-30 23:36:47 +00008066
Eli Friedmand6e25602011-12-26 22:49:32 +00008067 // Make sure we found a non-volatile load and the extractelement is
8068 // the only use.
Nadav Rotem42febc62011-05-11 14:40:50 +00008069 if (!LN0 || !LN0->hasNUsesOfValue(1,0) || LN0->isVolatile())
Dan Gohman475871a2008-07-27 21:46:04 +00008070 return SDValue();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008071
Eric Christopherd81f17a2010-11-03 20:44:42 +00008072 // If Idx was -1 above, Elt is going to be -1, so just return undef.
8073 if (Elt == -1)
Eli Friedmaned4b4272011-07-25 22:25:42 +00008074 return DAG.getUNDEF(LVT);
Eric Christopherd81f17a2010-11-03 20:44:42 +00008075
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008076 unsigned Align = LN0->getAlignment();
8077 if (NewLoad) {
8078 // Check the resultant load doesn't need a higher alignment than the
8079 // original load.
Bill Wendlingc144a572009-01-30 23:36:47 +00008080 unsigned NewAlign =
Eric Christopher503a64d2010-12-09 04:48:06 +00008081 TLI.getTargetData()
8082 ->getABITypeAlignment(LVT.getTypeForEVT(*DAG.getContext()));
Bill Wendlingc144a572009-01-30 23:36:47 +00008083
Dan Gohmanf560ffa2009-01-28 17:46:25 +00008084 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, LVT))
Dan Gohman475871a2008-07-27 21:46:04 +00008085 return SDValue();
Bill Wendlingc144a572009-01-30 23:36:47 +00008086
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008087 Align = NewAlign;
8088 }
8089
Dan Gohman475871a2008-07-27 21:46:04 +00008090 SDValue NewPtr = LN0->getBasePtr();
Chris Lattnerfa459012010-09-21 16:08:50 +00008091 unsigned PtrOff = 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008092
Eric Christopherd81f17a2010-11-03 20:44:42 +00008093 if (Elt) {
Chris Lattnerfa459012010-09-21 16:08:50 +00008094 PtrOff = LVT.getSizeInBits() * Elt / 8;
Owen Andersone50ed302009-08-10 22:56:29 +00008095 EVT PtrType = NewPtr.getValueType();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008096 if (TLI.isBigEndian())
Duncan Sands83ec4b62008-06-06 12:08:01 +00008097 PtrOff = VT.getSizeInBits() / 8 - PtrOff;
Bill Wendlingc144a572009-01-30 23:36:47 +00008098 NewPtr = DAG.getNode(ISD::ADD, N->getDebugLoc(), PtrType, NewPtr,
Evan Cheng77f0b7a2008-05-13 08:35:03 +00008099 DAG.getConstant(PtrOff, PtrType));
8100 }
Bill Wendlingc144a572009-01-30 23:36:47 +00008101
Eli Friedman4db4add2011-11-16 23:50:22 +00008102 // The replacement we need to do here is a little tricky: we need to
8103 // replace an extractelement of a load with a load.
8104 // Use ReplaceAllUsesOfValuesWith to do the replacement.
Eli Friedmand6e25602011-12-26 22:49:32 +00008105 // Note that this replacement assumes that the extractvalue is the only
8106 // use of the load; that's okay because we don't want to perform this
8107 // transformation in other cases anyway.
Evan Cheng84387ea2012-03-13 22:00:52 +00008108 SDValue Load;
Evan Chenga03d3662012-03-13 22:16:11 +00008109 SDValue Chain;
Evan Cheng84387ea2012-03-13 22:00:52 +00008110 if (NVT.bitsGT(LVT)) {
8111 // If the result type of vextract is wider than the load, then issue an
8112 // extending load instead.
8113 ISD::LoadExtType ExtType = TLI.isLoadExtLegal(ISD::ZEXTLOAD, LVT)
8114 ? ISD::ZEXTLOAD : ISD::EXTLOAD;
8115 Load = DAG.getExtLoad(ExtType, N->getDebugLoc(), NVT, LN0->getChain(),
8116 NewPtr, LN0->getPointerInfo().getWithOffset(PtrOff),
8117 LVT, LN0->isVolatile(), LN0->isNonTemporal(),Align);
Evan Chenga03d3662012-03-13 22:16:11 +00008118 Chain = Load.getValue(1);
8119 } else {
Evan Cheng84387ea2012-03-13 22:00:52 +00008120 Load = DAG.getLoad(LVT, N->getDebugLoc(), LN0->getChain(), NewPtr,
8121 LN0->getPointerInfo().getWithOffset(PtrOff),
8122 LN0->isVolatile(), LN0->isNonTemporal(),
8123 LN0->isInvariant(), Align);
Evan Chenga03d3662012-03-13 22:16:11 +00008124 Chain = Load.getValue(1);
8125 if (NVT.bitsLT(LVT))
8126 Load = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), NVT, Load);
8127 else
8128 Load = DAG.getNode(ISD::BITCAST, N->getDebugLoc(), NVT, Load);
8129 }
Eli Friedman4db4add2011-11-16 23:50:22 +00008130 WorkListRemover DeadNodes(*this);
8131 SDValue From[] = { SDValue(N, 0), SDValue(LN0,1) };
Evan Chenga03d3662012-03-13 22:16:11 +00008132 SDValue To[] = { Load, Chain };
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00008133 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
Eli Friedman4db4add2011-11-16 23:50:22 +00008134 // Since we're explcitly calling ReplaceAllUses, add the new node to the
8135 // worklist explicitly as well.
8136 AddToWorkList(Load.getNode());
Craig Topper0c9da212012-03-20 05:28:39 +00008137 AddUsersToWorkList(Load.getNode()); // Add users too
Eli Friedman4db4add2011-11-16 23:50:22 +00008138 // Make sure to revisit this node to clean it up; it will usually be dead.
8139 AddToWorkList(N);
8140 return SDValue(N, 0);
Evan Cheng513da432007-10-06 08:19:55 +00008141 }
Bill Wendlingc144a572009-01-30 23:36:47 +00008142
Dan Gohman475871a2008-07-27 21:46:04 +00008143 return SDValue();
Evan Cheng513da432007-10-06 08:19:55 +00008144}
Evan Cheng513da432007-10-06 08:19:55 +00008145
Dan Gohman475871a2008-07-27 21:46:04 +00008146SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
Dan Gohman7f321562007-06-25 16:23:39 +00008147 unsigned NumInScalars = N->getNumOperands();
Nadav Rotemb00418a2011-10-29 21:23:04 +00008148 DebugLoc dl = N->getDebugLoc();
Owen Andersone50ed302009-08-10 22:56:29 +00008149 EVT VT = N->getValueType(0);
Nadav Rotemb87bdac2012-07-15 08:38:23 +00008150
8151 // A vector built entirely of undefs is undef.
8152 if (ISD::allOperandsUndef(N))
8153 return DAG.getUNDEF(VT);
8154
Nadav Rotemb00418a2011-10-29 21:23:04 +00008155 // Check to see if this is a BUILD_VECTOR of a bunch of values
8156 // which come from any_extend or zero_extend nodes. If so, we can create
8157 // a new BUILD_VECTOR using bit-casts which may enable other BUILD_VECTOR
Nadav Rotemf47368b2011-10-31 20:08:25 +00008158 // optimizations. We do not handle sign-extend because we can't fill the sign
8159 // using shuffles.
Nadav Rotemb00418a2011-10-29 21:23:04 +00008160 EVT SourceType = MVT::Other;
Craig Topperd3b58892012-01-17 09:09:48 +00008161 bool AllAnyExt = true;
Nadav Rotemb87bdac2012-07-15 08:38:23 +00008162
Craig Topperd3b58892012-01-17 09:09:48 +00008163 for (unsigned i = 0; i != NumInScalars; ++i) {
Nadav Rotemb00418a2011-10-29 21:23:04 +00008164 SDValue In = N->getOperand(i);
8165 // Ignore undef inputs.
8166 if (In.getOpcode() == ISD::UNDEF) continue;
8167
8168 bool AnyExt = In.getOpcode() == ISD::ANY_EXTEND;
8169 bool ZeroExt = In.getOpcode() == ISD::ZERO_EXTEND;
8170
Nadav Rotemf47368b2011-10-31 20:08:25 +00008171 // Abort if the element is not an extension.
Nadav Rotemb00418a2011-10-29 21:23:04 +00008172 if (!ZeroExt && !AnyExt) {
Nadav Rotemf47368b2011-10-31 20:08:25 +00008173 SourceType = MVT::Other;
Nadav Rotemb00418a2011-10-29 21:23:04 +00008174 break;
8175 }
8176
8177 // The input is a ZeroExt or AnyExt. Check the original type.
8178 EVT InTy = In.getOperand(0).getValueType();
8179
8180 // Check that all of the widened source types are the same.
8181 if (SourceType == MVT::Other)
Nadav Rotemf47368b2011-10-31 20:08:25 +00008182 // First time.
Nadav Rotemb00418a2011-10-29 21:23:04 +00008183 SourceType = InTy;
8184 else if (InTy != SourceType) {
8185 // Multiple income types. Abort.
Nadav Rotemf47368b2011-10-31 20:08:25 +00008186 SourceType = MVT::Other;
Nadav Rotemb00418a2011-10-29 21:23:04 +00008187 break;
8188 }
8189
8190 // Check if all of the extends are ANY_EXTENDs.
Craig Topperd3b58892012-01-17 09:09:48 +00008191 AllAnyExt &= AnyExt;
Nadav Rotemb00418a2011-10-29 21:23:04 +00008192 }
8193
Nadav Rotemf47368b2011-10-31 20:08:25 +00008194 // In order to have valid types, all of the inputs must be extended from the
8195 // same source type and all of the inputs must be any or zero extend.
8196 // Scalar sizes must be a power of two.
8197 EVT OutScalarTy = N->getValueType(0).getScalarType();
Nadav Rotem2ee746b2012-02-12 15:05:31 +00008198 bool ValidTypes = SourceType != MVT::Other &&
Nadav Rotemf47368b2011-10-31 20:08:25 +00008199 isPowerOf2_32(OutScalarTy.getSizeInBits()) &&
8200 isPowerOf2_32(SourceType.getSizeInBits());
8201
8202 // We perform this optimization post type-legalization because
8203 // the type-legalizer often scalarizes integer-promoted vectors.
8204 // Performing this optimization before may create bit-casts which
8205 // will be type-legalized to complex code sequences.
8206 // We perform this optimization only before the operation legalizer because we
8207 // may introduce illegal operations.
Nadav Rotem6431ff92012-03-15 08:49:06 +00008208 // Create a new simpler BUILD_VECTOR sequence which other optimizations can
8209 // turn into a single shuffle instruction.
Nadav Rotem2ee746b2012-02-12 15:05:31 +00008210 if ((Level == AfterLegalizeVectorOps || Level == AfterLegalizeTypes) &&
8211 ValidTypes) {
Nadav Rotemb00418a2011-10-29 21:23:04 +00008212 bool isLE = TLI.isLittleEndian();
Nadav Rotemf47368b2011-10-31 20:08:25 +00008213 unsigned ElemRatio = OutScalarTy.getSizeInBits()/SourceType.getSizeInBits();
Nadav Rotemb00418a2011-10-29 21:23:04 +00008214 assert(ElemRatio > 1 && "Invalid element size ratio");
Craig Topperd3b58892012-01-17 09:09:48 +00008215 SDValue Filler = AllAnyExt ? DAG.getUNDEF(SourceType):
Nadav Rotemf47368b2011-10-31 20:08:25 +00008216 DAG.getConstant(0, SourceType);
Nadav Rotemb00418a2011-10-29 21:23:04 +00008217
8218 unsigned NewBVElems = ElemRatio * N->getValueType(0).getVectorNumElements();
Benjamin Kramer50bf86e2011-10-30 08:39:55 +00008219 SmallVector<SDValue, 8> Ops(NewBVElems, Filler);
Nadav Rotemb00418a2011-10-29 21:23:04 +00008220
8221 // Populate the new build_vector
8222 for (unsigned i=0; i < N->getNumOperands(); ++i) {
8223 SDValue Cast = N->getOperand(i);
Benjamin Kramer50bf86e2011-10-30 08:39:55 +00008224 assert((Cast.getOpcode() == ISD::ANY_EXTEND ||
8225 Cast.getOpcode() == ISD::ZERO_EXTEND ||
8226 Cast.getOpcode() == ISD::UNDEF) && "Invalid cast opcode");
Nadav Rotemb00418a2011-10-29 21:23:04 +00008227 SDValue In;
8228 if (Cast.getOpcode() == ISD::UNDEF)
Nadav Rotemf47368b2011-10-31 20:08:25 +00008229 In = DAG.getUNDEF(SourceType);
Nadav Rotemb00418a2011-10-29 21:23:04 +00008230 else
8231 In = Cast->getOperand(0);
8232 unsigned Index = isLE ? (i * ElemRatio) :
8233 (i * ElemRatio + (ElemRatio - 1));
8234
8235 assert(Index < Ops.size() && "Invalid index");
8236 Ops[Index] = In;
8237 }
8238
8239 // The type of the new BUILD_VECTOR node.
Nadav Rotemf47368b2011-10-31 20:08:25 +00008240 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), SourceType, NewBVElems);
Nadav Rotemb00418a2011-10-29 21:23:04 +00008241 assert(VecVT.getSizeInBits() == N->getValueType(0).getSizeInBits() &&
8242 "Invalid vector size");
Nadav Rotemf47368b2011-10-31 20:08:25 +00008243 // Check if the new vector type is legal.
8244 if (!isTypeLegal(VecVT)) return SDValue();
Nadav Rotemb00418a2011-10-29 21:23:04 +00008245
8246 // Make the new BUILD_VECTOR.
8247 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
8248 VecVT, &Ops[0], Ops.size());
8249
Nadav Rotem6431ff92012-03-15 08:49:06 +00008250 // The new BUILD_VECTOR node has the potential to be further optimized.
8251 AddToWorkList(BV.getNode());
Nadav Rotemb00418a2011-10-29 21:23:04 +00008252 // Bitcast to the desired type.
8253 return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), BV);
8254 }
Chris Lattnerca242442006-03-19 01:27:56 +00008255
Dan Gohman7f321562007-06-25 16:23:39 +00008256 // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT
8257 // operations. If so, and if the EXTRACT_VECTOR_ELT vector inputs come from
8258 // at most two distinct vectors, turn this into a shuffle node.
Duncan Sands00294ca2012-03-19 15:35:44 +00008259
8260 // May only combine to shuffle after legalize if shuffle is legal.
8261 if (LegalOperations &&
8262 !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))
8263 return SDValue();
8264
Dan Gohman475871a2008-07-27 21:46:04 +00008265 SDValue VecIn1, VecIn2;
Chris Lattnerd7648c82006-03-28 20:28:38 +00008266 for (unsigned i = 0; i != NumInScalars; ++i) {
8267 // Ignore undef inputs.
8268 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00008269
Dan Gohman7f321562007-06-25 16:23:39 +00008270 // If this input is something other than a EXTRACT_VECTOR_ELT with a
Chris Lattnerd7648c82006-03-28 20:28:38 +00008271 // constant index, bail out.
Dan Gohman7f321562007-06-25 16:23:39 +00008272 if (N->getOperand(i).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
Chris Lattnerd7648c82006-03-28 20:28:38 +00008273 !isa<ConstantSDNode>(N->getOperand(i).getOperand(1))) {
Dan Gohman475871a2008-07-27 21:46:04 +00008274 VecIn1 = VecIn2 = SDValue(0, 0);
Chris Lattnerd7648c82006-03-28 20:28:38 +00008275 break;
8276 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008277
Nadav Rotem2ee746b2012-02-12 15:05:31 +00008278 // We allow up to two distinct input vectors.
Dan Gohman475871a2008-07-27 21:46:04 +00008279 SDValue ExtractedFromVec = N->getOperand(i).getOperand(0);
Chris Lattnerd7648c82006-03-28 20:28:38 +00008280 if (ExtractedFromVec == VecIn1 || ExtractedFromVec == VecIn2)
8281 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00008282
Gabor Greifba36cb52008-08-28 21:40:38 +00008283 if (VecIn1.getNode() == 0) {
Chris Lattnerd7648c82006-03-28 20:28:38 +00008284 VecIn1 = ExtractedFromVec;
Gabor Greifba36cb52008-08-28 21:40:38 +00008285 } else if (VecIn2.getNode() == 0) {
Chris Lattnerd7648c82006-03-28 20:28:38 +00008286 VecIn2 = ExtractedFromVec;
8287 } else {
8288 // Too many inputs.
Dan Gohman475871a2008-07-27 21:46:04 +00008289 VecIn1 = VecIn2 = SDValue(0, 0);
Chris Lattnerd7648c82006-03-28 20:28:38 +00008290 break;
8291 }
8292 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008293
Nadav Rotem2ee746b2012-02-12 15:05:31 +00008294 // If everything is good, we can make a shuffle operation.
Gabor Greifba36cb52008-08-28 21:40:38 +00008295 if (VecIn1.getNode()) {
Nate Begeman9008ca62009-04-27 18:41:29 +00008296 SmallVector<int, 8> Mask;
Chris Lattnerd7648c82006-03-28 20:28:38 +00008297 for (unsigned i = 0; i != NumInScalars; ++i) {
8298 if (N->getOperand(i).getOpcode() == ISD::UNDEF) {
Nate Begeman9008ca62009-04-27 18:41:29 +00008299 Mask.push_back(-1);
Chris Lattnerd7648c82006-03-28 20:28:38 +00008300 continue;
8301 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008302
Rafael Espindola15684b22009-04-24 12:40:33 +00008303 // If extracting from the first vector, just use the index directly.
Nate Begeman9008ca62009-04-27 18:41:29 +00008304 SDValue Extract = N->getOperand(i);
Mon P Wang93b74152009-03-17 06:33:10 +00008305 SDValue ExtVal = Extract.getOperand(1);
Chris Lattnerd7648c82006-03-28 20:28:38 +00008306 if (Extract.getOperand(0) == VecIn1) {
Nate Begeman5a5ca152009-04-29 05:20:52 +00008307 unsigned ExtIndex = cast<ConstantSDNode>(ExtVal)->getZExtValue();
8308 if (ExtIndex > VT.getVectorNumElements())
8309 return SDValue();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008310
Nate Begeman5a5ca152009-04-29 05:20:52 +00008311 Mask.push_back(ExtIndex);
Chris Lattnerd7648c82006-03-28 20:28:38 +00008312 continue;
8313 }
8314
8315 // Otherwise, use InIdx + VecSize
Mon P Wang93b74152009-03-17 06:33:10 +00008316 unsigned Idx = cast<ConstantSDNode>(ExtVal)->getZExtValue();
Nate Begeman9008ca62009-04-27 18:41:29 +00008317 Mask.push_back(Idx+NumInScalars);
Chris Lattnerd7648c82006-03-28 20:28:38 +00008318 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008319
Nadav Rotem2ee746b2012-02-12 15:05:31 +00008320 // We can't generate a shuffle node with mismatched input and output types.
8321 // Attempt to transform a single input vector to the correct type.
8322 if ((VT != VecIn1.getValueType())) {
8323 // We don't support shuffeling between TWO values of different types.
8324 if (VecIn2.getNode() != 0)
8325 return SDValue();
8326
8327 // We only support widening of vectors which are half the size of the
8328 // output registers. For example XMM->YMM widening on X86 with AVX.
8329 if (VecIn1.getValueType().getSizeInBits()*2 != VT.getSizeInBits())
8330 return SDValue();
8331
James Molloy8cd08bf2012-09-10 14:01:21 +00008332 // If the input vector type has a different base type to the output
8333 // vector type, bail out.
8334 if (VecIn1.getValueType().getVectorElementType() !=
8335 VT.getVectorElementType())
8336 return SDValue();
8337
Stepan Dyatkovskiyfdeb9fe2012-08-22 09:33:55 +00008338 // Widen the input vector by adding undef values.
8339 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, N->getDebugLoc(), VT,
8340 VecIn1, DAG.getUNDEF(VecIn1.getValueType()));
Nadav Rotem2ee746b2012-02-12 15:05:31 +00008341 }
8342
8343 // If VecIn2 is unused then change it to undef.
8344 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
8345
Nadav Rotem6dfabb62012-09-20 08:53:31 +00008346 // Check that we were able to transform all incoming values to the same
8347 // type.
Nadav Rotem0877fdf2012-02-13 12:42:26 +00008348 if (VecIn2.getValueType() != VecIn1.getValueType() ||
8349 VecIn1.getValueType() != VT)
8350 return SDValue();
8351
Nadav Rotem2ee746b2012-02-12 15:05:31 +00008352 // Only type-legal BUILD_VECTOR nodes are converted to shuffle nodes.
Nadav Rotem0877fdf2012-02-13 12:42:26 +00008353 if (!isTypeLegal(VT))
Duncan Sands25cf2272008-11-24 14:53:14 +00008354 return SDValue();
8355
Dan Gohman7f321562007-06-25 16:23:39 +00008356 // Return the new VECTOR_SHUFFLE node.
Nate Begeman9008ca62009-04-27 18:41:29 +00008357 SDValue Ops[2];
Chris Lattnerbd564bf2006-08-08 02:23:42 +00008358 Ops[0] = VecIn1;
Nadav Rotem2ee746b2012-02-12 15:05:31 +00008359 Ops[1] = VecIn2;
Nate Begeman9008ca62009-04-27 18:41:29 +00008360 return DAG.getVectorShuffle(VT, N->getDebugLoc(), Ops[0], Ops[1], &Mask[0]);
Chris Lattnerd7648c82006-03-28 20:28:38 +00008361 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008362
Dan Gohman475871a2008-07-27 21:46:04 +00008363 return SDValue();
Chris Lattnerd7648c82006-03-28 20:28:38 +00008364}
8365
Dan Gohman475871a2008-07-27 21:46:04 +00008366SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
Dan Gohman7f321562007-06-25 16:23:39 +00008367 // TODO: Check to see if this is a CONCAT_VECTORS of a bunch of
8368 // EXTRACT_SUBVECTOR operations. If so, and if the EXTRACT_SUBVECTOR vector
8369 // inputs come from at most two distinct vectors, turn this into a shuffle
8370 // node.
8371
8372 // If we only have one input vector, we don't need to do any concatenation.
Bill Wendlingc144a572009-01-30 23:36:47 +00008373 if (N->getNumOperands() == 1)
Dan Gohman7f321562007-06-25 16:23:39 +00008374 return N->getOperand(0);
Dan Gohman7f321562007-06-25 16:23:39 +00008375
Nadav Rotemb7e230d2012-07-14 21:30:27 +00008376 // Check if all of the operands are undefs.
Nadav Rotemb87bdac2012-07-15 08:38:23 +00008377 if (ISD::allOperandsUndef(N))
Nadav Rotemb7e230d2012-07-14 21:30:27 +00008378 return DAG.getUNDEF(N->getValueType(0));
8379
Dan Gohman475871a2008-07-27 21:46:04 +00008380 return SDValue();
Dan Gohman7f321562007-06-25 16:23:39 +00008381}
8382
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00008383SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) {
8384 EVT NVT = N->getValueType(0);
8385 SDValue V = N->getOperand(0);
8386
8387 if (V->getOpcode() == ISD::INSERT_SUBVECTOR) {
8388 // Handle only simple case where vector being inserted and vector
8389 // being extracted are of same type, and are half size of larger vectors.
8390 EVT BigVT = V->getOperand(0).getValueType();
8391 EVT SmallVT = V->getOperand(1).getValueType();
8392 if (NVT != SmallVT || NVT.getSizeInBits()*2 != BigVT.getSizeInBits())
8393 return SDValue();
8394
Eli Friedman26323442011-12-07 00:11:56 +00008395 // Only handle cases where both indexes are constants with the same type.
8396 ConstantSDNode *InsIdx = dyn_cast<ConstantSDNode>(N->getOperand(1));
8397 ConstantSDNode *ExtIdx = dyn_cast<ConstantSDNode>(V->getOperand(2));
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00008398
Eli Friedman26323442011-12-07 00:11:56 +00008399 if (InsIdx && ExtIdx &&
8400 InsIdx->getValueType(0).getSizeInBits() <= 64 &&
8401 ExtIdx->getValueType(0).getSizeInBits() <= 64) {
8402 // Combine:
8403 // (extract_subvec (insert_subvec V1, V2, InsIdx), ExtIdx)
8404 // Into:
8405 // indices are equal => V1
8406 // otherwise => (extract_subvec V1, ExtIdx)
8407 if (InsIdx->getZExtValue() == ExtIdx->getZExtValue())
8408 return V->getOperand(1);
8409 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), NVT,
8410 V->getOperand(0), N->getOperand(1));
8411 }
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00008412 }
8413
8414 return SDValue();
8415}
8416
Dan Gohman475871a2008-07-27 21:46:04 +00008417SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
Owen Andersone50ed302009-08-10 22:56:29 +00008418 EVT VT = N->getValueType(0);
Nate Begeman9008ca62009-04-27 18:41:29 +00008419 unsigned NumElts = VT.getVectorNumElements();
Chris Lattnerf1d0c622006-03-31 22:16:43 +00008420
Mon P Wangaeb06d22008-11-10 04:46:22 +00008421 SDValue N0 = N->getOperand(0);
Craig Topper481b79c2012-01-04 08:07:43 +00008422 SDValue N1 = N->getOperand(1);
Mon P Wangaeb06d22008-11-10 04:46:22 +00008423
Craig Topperae1bec52012-04-09 05:16:56 +00008424 assert(N0.getValueType() == VT && "Vector shuffle must be normalized in DAG");
Mon P Wangaeb06d22008-11-10 04:46:22 +00008425
Craig Topper481b79c2012-01-04 08:07:43 +00008426 // Canonicalize shuffle undef, undef -> undef
8427 if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF)
8428 return DAG.getUNDEF(VT);
8429
8430 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
8431
8432 // Canonicalize shuffle v, v -> v, undef
8433 if (N0 == N1) {
8434 SmallVector<int, 8> NewMask;
8435 for (unsigned i = 0; i != NumElts; ++i) {
8436 int Idx = SVN->getMaskElt(i);
8437 if (Idx >= (int)NumElts) Idx -= NumElts;
8438 NewMask.push_back(Idx);
8439 }
8440 return DAG.getVectorShuffle(VT, N->getDebugLoc(), N0, DAG.getUNDEF(VT),
8441 &NewMask[0]);
8442 }
8443
8444 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
8445 if (N0.getOpcode() == ISD::UNDEF) {
8446 SmallVector<int, 8> NewMask;
8447 for (unsigned i = 0; i != NumElts; ++i) {
8448 int Idx = SVN->getMaskElt(i);
Craig Topper4b206bd2012-04-09 05:55:33 +00008449 if (Idx >= 0) {
8450 if (Idx < (int)NumElts)
8451 Idx += NumElts;
8452 else
8453 Idx -= NumElts;
8454 }
8455 NewMask.push_back(Idx);
Craig Topper481b79c2012-01-04 08:07:43 +00008456 }
8457 return DAG.getVectorShuffle(VT, N->getDebugLoc(), N1, DAG.getUNDEF(VT),
8458 &NewMask[0]);
8459 }
8460
8461 // Remove references to rhs if it is undef
8462 if (N1.getOpcode() == ISD::UNDEF) {
8463 bool Changed = false;
8464 SmallVector<int, 8> NewMask;
8465 for (unsigned i = 0; i != NumElts; ++i) {
8466 int Idx = SVN->getMaskElt(i);
8467 if (Idx >= (int)NumElts) {
8468 Idx = -1;
8469 Changed = true;
8470 }
8471 NewMask.push_back(Idx);
8472 }
8473 if (Changed)
8474 return DAG.getVectorShuffle(VT, N->getDebugLoc(), N0, N1, &NewMask[0]);
8475 }
Evan Chenge7bec0d2006-07-20 22:44:41 +00008476
Bob Wilson0f1db1a2010-10-28 17:06:14 +00008477 // If it is a splat, check if the argument vector is another splat or a
8478 // build_vector with all scalar elements the same.
Bob Wilson0f1db1a2010-10-28 17:06:14 +00008479 if (SVN->isSplat() && SVN->getSplatIndex() < (int)NumElts) {
Gabor Greifba36cb52008-08-28 21:40:38 +00008480 SDNode *V = N0.getNode();
Evan Cheng917ec982006-07-21 08:25:53 +00008481
Dan Gohman7f321562007-06-25 16:23:39 +00008482 // If this is a bit convert that changes the element type of the vector but
Evan Cheng59569222006-10-16 22:49:37 +00008483 // not the number of vector elements, look through it. Be careful not to
8484 // look though conversions that change things like v4f32 to v2f64.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008485 if (V->getOpcode() == ISD::BITCAST) {
Dan Gohman475871a2008-07-27 21:46:04 +00008486 SDValue ConvInput = V->getOperand(0);
Evan Cheng29257862008-07-22 20:42:56 +00008487 if (ConvInput.getValueType().isVector() &&
8488 ConvInput.getValueType().getVectorNumElements() == NumElts)
Gabor Greifba36cb52008-08-28 21:40:38 +00008489 V = ConvInput.getNode();
Evan Cheng59569222006-10-16 22:49:37 +00008490 }
8491
Dan Gohman7f321562007-06-25 16:23:39 +00008492 if (V->getOpcode() == ISD::BUILD_VECTOR) {
Bob Wilson0f1db1a2010-10-28 17:06:14 +00008493 assert(V->getNumOperands() == NumElts &&
8494 "BUILD_VECTOR has wrong number of operands");
8495 SDValue Base;
8496 bool AllSame = true;
8497 for (unsigned i = 0; i != NumElts; ++i) {
8498 if (V->getOperand(i).getOpcode() != ISD::UNDEF) {
8499 Base = V->getOperand(i);
8500 break;
Evan Cheng917ec982006-07-21 08:25:53 +00008501 }
Evan Cheng917ec982006-07-21 08:25:53 +00008502 }
Bob Wilson0f1db1a2010-10-28 17:06:14 +00008503 // Splat of <u, u, u, u>, return <u, u, u, u>
8504 if (!Base.getNode())
8505 return N0;
8506 for (unsigned i = 0; i != NumElts; ++i) {
8507 if (V->getOperand(i) != Base) {
8508 AllSame = false;
8509 break;
8510 }
8511 }
8512 // Splat of <x, x, x, x>, return <x, x, x, x>
8513 if (AllSame)
8514 return N0;
Evan Cheng917ec982006-07-21 08:25:53 +00008515 }
8516 }
Nadav Rotem4ac90812012-04-01 19:31:22 +00008517
8518 // If this shuffle node is simply a swizzle of another shuffle node,
Nadav Rotemd16c8d02012-04-07 21:19:08 +00008519 // and it reverses the swizzle of the previous shuffle then we can
8520 // optimize shuffle(shuffle(x, undef), undef) -> x.
Nadav Rotem4ac90812012-04-01 19:31:22 +00008521 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG &&
8522 N1.getOpcode() == ISD::UNDEF) {
8523
Nadav Rotem4ac90812012-04-01 19:31:22 +00008524 ShuffleVectorSDNode *OtherSV = cast<ShuffleVectorSDNode>(N0);
8525
Nadav Rotemd16c8d02012-04-07 21:19:08 +00008526 // Shuffle nodes can only reverse shuffles with a single non-undef value.
8527 if (N0.getOperand(1).getOpcode() != ISD::UNDEF)
8528 return SDValue();
8529
Craig Topperae1bec52012-04-09 05:16:56 +00008530 // The incoming shuffle must be of the same type as the result of the
8531 // current shuffle.
8532 assert(OtherSV->getOperand(0).getValueType() == VT &&
8533 "Shuffle types don't match");
Nadav Rotem4ac90812012-04-01 19:31:22 +00008534
8535 for (unsigned i = 0; i != NumElts; ++i) {
8536 int Idx = SVN->getMaskElt(i);
Craig Topperae1bec52012-04-09 05:16:56 +00008537 assert(Idx < (int)NumElts && "Index references undef operand");
Nadav Rotem4ac90812012-04-01 19:31:22 +00008538 // Next, this index comes from the first value, which is the incoming
8539 // shuffle. Adopt the incoming index.
8540 if (Idx >= 0)
8541 Idx = OtherSV->getMaskElt(Idx);
8542
Nadav Rotemd16c8d02012-04-07 21:19:08 +00008543 // The combined shuffle must map each index to itself.
Craig Topperae1bec52012-04-09 05:16:56 +00008544 if (Idx >= 0 && (unsigned)Idx != i)
Nadav Rotemd16c8d02012-04-07 21:19:08 +00008545 return SDValue();
Nadav Rotem4ac90812012-04-01 19:31:22 +00008546 }
Nadav Rotemd16c8d02012-04-07 21:19:08 +00008547
8548 return OtherSV->getOperand(0);
Nadav Rotem4ac90812012-04-01 19:31:22 +00008549 }
8550
Dan Gohman475871a2008-07-27 21:46:04 +00008551 return SDValue();
Chris Lattnerf1d0c622006-03-31 22:16:43 +00008552}
8553
Jim Grosbach9a526492010-06-23 16:07:42 +00008554SDValue DAGCombiner::visitMEMBARRIER(SDNode* N) {
8555 if (!TLI.getShouldFoldAtomicFences())
8556 return SDValue();
8557
8558 SDValue atomic = N->getOperand(0);
8559 switch (atomic.getOpcode()) {
8560 case ISD::ATOMIC_CMP_SWAP:
8561 case ISD::ATOMIC_SWAP:
8562 case ISD::ATOMIC_LOAD_ADD:
8563 case ISD::ATOMIC_LOAD_SUB:
8564 case ISD::ATOMIC_LOAD_AND:
8565 case ISD::ATOMIC_LOAD_OR:
8566 case ISD::ATOMIC_LOAD_XOR:
8567 case ISD::ATOMIC_LOAD_NAND:
8568 case ISD::ATOMIC_LOAD_MIN:
8569 case ISD::ATOMIC_LOAD_MAX:
8570 case ISD::ATOMIC_LOAD_UMIN:
8571 case ISD::ATOMIC_LOAD_UMAX:
8572 break;
8573 default:
8574 return SDValue();
8575 }
8576
8577 SDValue fence = atomic.getOperand(0);
8578 if (fence.getOpcode() != ISD::MEMBARRIER)
8579 return SDValue();
8580
8581 switch (atomic.getOpcode()) {
8582 case ISD::ATOMIC_CMP_SWAP:
8583 return SDValue(DAG.UpdateNodeOperands(atomic.getNode(),
8584 fence.getOperand(0),
8585 atomic.getOperand(1), atomic.getOperand(2),
8586 atomic.getOperand(3)), atomic.getResNo());
8587 case ISD::ATOMIC_SWAP:
8588 case ISD::ATOMIC_LOAD_ADD:
8589 case ISD::ATOMIC_LOAD_SUB:
8590 case ISD::ATOMIC_LOAD_AND:
8591 case ISD::ATOMIC_LOAD_OR:
8592 case ISD::ATOMIC_LOAD_XOR:
8593 case ISD::ATOMIC_LOAD_NAND:
8594 case ISD::ATOMIC_LOAD_MIN:
8595 case ISD::ATOMIC_LOAD_MAX:
8596 case ISD::ATOMIC_LOAD_UMIN:
8597 case ISD::ATOMIC_LOAD_UMAX:
8598 return SDValue(DAG.UpdateNodeOperands(atomic.getNode(),
8599 fence.getOperand(0),
8600 atomic.getOperand(1), atomic.getOperand(2)),
8601 atomic.getResNo());
8602 default:
8603 return SDValue();
8604 }
8605}
8606
Evan Cheng44f1f092006-04-20 08:56:16 +00008607/// XformToShuffleWithZero - Returns a vector_shuffle if it able to transform
Dan Gohman7f321562007-06-25 16:23:39 +00008608/// an AND to a vector_shuffle with the destination vector and a zero vector.
8609/// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==>
Evan Cheng44f1f092006-04-20 08:56:16 +00008610/// vector_shuffle V, Zero, <0, 4, 2, 4>
Dan Gohman475871a2008-07-27 21:46:04 +00008611SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) {
Owen Andersone50ed302009-08-10 22:56:29 +00008612 EVT VT = N->getValueType(0);
Nate Begeman9008ca62009-04-27 18:41:29 +00008613 DebugLoc dl = N->getDebugLoc();
Dan Gohman475871a2008-07-27 21:46:04 +00008614 SDValue LHS = N->getOperand(0);
8615 SDValue RHS = N->getOperand(1);
Dan Gohman7f321562007-06-25 16:23:39 +00008616 if (N->getOpcode() == ISD::AND) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008617 if (RHS.getOpcode() == ISD::BITCAST)
Evan Cheng44f1f092006-04-20 08:56:16 +00008618 RHS = RHS.getOperand(0);
Dan Gohman7f321562007-06-25 16:23:39 +00008619 if (RHS.getOpcode() == ISD::BUILD_VECTOR) {
Nate Begeman9008ca62009-04-27 18:41:29 +00008620 SmallVector<int, 8> Indices;
8621 unsigned NumElts = RHS.getNumOperands();
Evan Cheng44f1f092006-04-20 08:56:16 +00008622 for (unsigned i = 0; i != NumElts; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00008623 SDValue Elt = RHS.getOperand(i);
Evan Cheng44f1f092006-04-20 08:56:16 +00008624 if (!isa<ConstantSDNode>(Elt))
Dan Gohman475871a2008-07-27 21:46:04 +00008625 return SDValue();
Craig Topperb7135e52012-04-09 05:59:53 +00008626
8627 if (cast<ConstantSDNode>(Elt)->isAllOnesValue())
Nate Begeman9008ca62009-04-27 18:41:29 +00008628 Indices.push_back(i);
Evan Cheng44f1f092006-04-20 08:56:16 +00008629 else if (cast<ConstantSDNode>(Elt)->isNullValue())
Nate Begeman9008ca62009-04-27 18:41:29 +00008630 Indices.push_back(NumElts);
Evan Cheng44f1f092006-04-20 08:56:16 +00008631 else
Dan Gohman475871a2008-07-27 21:46:04 +00008632 return SDValue();
Evan Cheng44f1f092006-04-20 08:56:16 +00008633 }
8634
8635 // Let's see if the target supports this vector_shuffle.
Owen Andersone50ed302009-08-10 22:56:29 +00008636 EVT RVT = RHS.getValueType();
Nate Begeman9008ca62009-04-27 18:41:29 +00008637 if (!TLI.isVectorClearMaskLegal(Indices, RVT))
Dan Gohman475871a2008-07-27 21:46:04 +00008638 return SDValue();
Evan Cheng44f1f092006-04-20 08:56:16 +00008639
Dan Gohman7f321562007-06-25 16:23:39 +00008640 // Return the new VECTOR_SHUFFLE node.
Dan Gohman8a55ce42009-09-23 21:02:20 +00008641 EVT EltVT = RVT.getVectorElementType();
Nate Begeman9008ca62009-04-27 18:41:29 +00008642 SmallVector<SDValue,8> ZeroOps(RVT.getVectorNumElements(),
Dan Gohman8a55ce42009-09-23 21:02:20 +00008643 DAG.getConstant(0, EltVT));
Nate Begeman9008ca62009-04-27 18:41:29 +00008644 SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
8645 RVT, &ZeroOps[0], ZeroOps.size());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008646 LHS = DAG.getNode(ISD::BITCAST, dl, RVT, LHS);
Nate Begeman9008ca62009-04-27 18:41:29 +00008647 SDValue Shuf = DAG.getVectorShuffle(RVT, dl, LHS, Zero, &Indices[0]);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008648 return DAG.getNode(ISD::BITCAST, dl, VT, Shuf);
Evan Cheng44f1f092006-04-20 08:56:16 +00008649 }
8650 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008651
Dan Gohman475871a2008-07-27 21:46:04 +00008652 return SDValue();
Evan Cheng44f1f092006-04-20 08:56:16 +00008653}
8654
Dan Gohman7f321562007-06-25 16:23:39 +00008655/// SimplifyVBinOp - Visit a binary vector operation, like ADD.
Dan Gohman475871a2008-07-27 21:46:04 +00008656SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) {
Dan Gohman7f321562007-06-25 16:23:39 +00008657 // After legalize, the target may be depending on adds and other
8658 // binary ops to provide legal ways to construct constants or other
8659 // things. Simplifying them may result in a loss of legality.
Duncan Sands25cf2272008-11-24 14:53:14 +00008660 if (LegalOperations) return SDValue();
Dan Gohman7f321562007-06-25 16:23:39 +00008661
Bob Wilsond7273432010-12-17 23:06:49 +00008662 assert(N->getValueType(0).isVector() &&
8663 "SimplifyVBinOp only works on vectors!");
Dan Gohman7f321562007-06-25 16:23:39 +00008664
Dan Gohman475871a2008-07-27 21:46:04 +00008665 SDValue LHS = N->getOperand(0);
8666 SDValue RHS = N->getOperand(1);
8667 SDValue Shuffle = XformToShuffleWithZero(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00008668 if (Shuffle.getNode()) return Shuffle;
Evan Cheng44f1f092006-04-20 08:56:16 +00008669
Dan Gohman7f321562007-06-25 16:23:39 +00008670 // If the LHS and RHS are BUILD_VECTOR nodes, see if we can constant fold
Chris Lattneredab1b92006-04-02 03:25:57 +00008671 // this operation.
Scott Michelfdc40a02009-02-17 22:15:04 +00008672 if (LHS.getOpcode() == ISD::BUILD_VECTOR &&
Dan Gohman7f321562007-06-25 16:23:39 +00008673 RHS.getOpcode() == ISD::BUILD_VECTOR) {
Dan Gohman475871a2008-07-27 21:46:04 +00008674 SmallVector<SDValue, 8> Ops;
Dan Gohman7f321562007-06-25 16:23:39 +00008675 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00008676 SDValue LHSOp = LHS.getOperand(i);
8677 SDValue RHSOp = RHS.getOperand(i);
Chris Lattneredab1b92006-04-02 03:25:57 +00008678 // If these two elements can't be folded, bail out.
8679 if ((LHSOp.getOpcode() != ISD::UNDEF &&
8680 LHSOp.getOpcode() != ISD::Constant &&
8681 LHSOp.getOpcode() != ISD::ConstantFP) ||
8682 (RHSOp.getOpcode() != ISD::UNDEF &&
8683 RHSOp.getOpcode() != ISD::Constant &&
8684 RHSOp.getOpcode() != ISD::ConstantFP))
8685 break;
Bill Wendling836ca7d2009-01-30 23:59:18 +00008686
Evan Cheng7b336a82006-05-31 06:08:35 +00008687 // Can't fold divide by zero.
Dan Gohman7f321562007-06-25 16:23:39 +00008688 if (N->getOpcode() == ISD::SDIV || N->getOpcode() == ISD::UDIV ||
8689 N->getOpcode() == ISD::FDIV) {
Evan Cheng7b336a82006-05-31 06:08:35 +00008690 if ((RHSOp.getOpcode() == ISD::Constant &&
Gabor Greifba36cb52008-08-28 21:40:38 +00008691 cast<ConstantSDNode>(RHSOp.getNode())->isNullValue()) ||
Evan Cheng7b336a82006-05-31 06:08:35 +00008692 (RHSOp.getOpcode() == ISD::ConstantFP &&
Gabor Greifba36cb52008-08-28 21:40:38 +00008693 cast<ConstantFPSDNode>(RHSOp.getNode())->getValueAPF().isZero()))
Evan Cheng7b336a82006-05-31 06:08:35 +00008694 break;
8695 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008696
Bob Wilsond7273432010-12-17 23:06:49 +00008697 EVT VT = LHSOp.getValueType();
Bob Wilsondb2b18f2011-10-18 17:34:47 +00008698 EVT RVT = RHSOp.getValueType();
8699 if (RVT != VT) {
8700 // Integer BUILD_VECTOR operands may have types larger than the element
8701 // size (e.g., when the element type is not legal). Prior to type
8702 // legalization, the types may not match between the two BUILD_VECTORS.
8703 // Truncate one of the operands to make them match.
8704 if (RVT.getSizeInBits() > VT.getSizeInBits()) {
8705 RHSOp = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, RHSOp);
8706 } else {
8707 LHSOp = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), RVT, LHSOp);
8708 VT = RVT;
8709 }
8710 }
Bob Wilsond7273432010-12-17 23:06:49 +00008711 SDValue FoldOp = DAG.getNode(N->getOpcode(), LHS.getDebugLoc(), VT,
Evan Chenga0839882010-05-18 00:03:40 +00008712 LHSOp, RHSOp);
8713 if (FoldOp.getOpcode() != ISD::UNDEF &&
8714 FoldOp.getOpcode() != ISD::Constant &&
8715 FoldOp.getOpcode() != ISD::ConstantFP)
8716 break;
8717 Ops.push_back(FoldOp);
8718 AddToWorkList(FoldOp.getNode());
Chris Lattneredab1b92006-04-02 03:25:57 +00008719 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008720
Bob Wilsond7273432010-12-17 23:06:49 +00008721 if (Ops.size() == LHS.getNumOperands())
8722 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
8723 LHS.getValueType(), &Ops[0], Ops.size());
Chris Lattneredab1b92006-04-02 03:25:57 +00008724 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008725
Dan Gohman475871a2008-07-27 21:46:04 +00008726 return SDValue();
Chris Lattneredab1b92006-04-02 03:25:57 +00008727}
8728
Craig Topperdd201ff2012-09-11 01:45:21 +00008729/// SimplifyVUnaryOp - Visit a binary vector operation, like FABS/FNEG.
8730SDValue DAGCombiner::SimplifyVUnaryOp(SDNode *N) {
8731 // After legalize, the target may be depending on adds and other
8732 // binary ops to provide legal ways to construct constants or other
8733 // things. Simplifying them may result in a loss of legality.
8734 if (LegalOperations) return SDValue();
8735
8736 assert(N->getValueType(0).isVector() &&
8737 "SimplifyVUnaryOp only works on vectors!");
8738
8739 SDValue N0 = N->getOperand(0);
8740
8741 if (N0.getOpcode() != ISD::BUILD_VECTOR)
8742 return SDValue();
8743
8744 // Operand is a BUILD_VECTOR node, see if we can constant fold it.
8745 SmallVector<SDValue, 8> Ops;
8746 for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) {
8747 SDValue Op = N0.getOperand(i);
8748 if (Op.getOpcode() != ISD::UNDEF &&
8749 Op.getOpcode() != ISD::ConstantFP)
8750 break;
8751 EVT EltVT = Op.getValueType();
8752 SDValue FoldOp = DAG.getNode(N->getOpcode(), N0.getDebugLoc(), EltVT, Op);
8753 if (FoldOp.getOpcode() != ISD::UNDEF &&
8754 FoldOp.getOpcode() != ISD::ConstantFP)
8755 break;
8756 Ops.push_back(FoldOp);
8757 AddToWorkList(FoldOp.getNode());
8758 }
8759
8760 if (Ops.size() != N0.getNumOperands())
8761 return SDValue();
8762
8763 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
8764 N0.getValueType(), &Ops[0], Ops.size());
8765}
8766
Bill Wendling836ca7d2009-01-30 23:59:18 +00008767SDValue DAGCombiner::SimplifySelect(DebugLoc DL, SDValue N0,
8768 SDValue N1, SDValue N2){
Nate Begemanf845b452005-10-08 00:29:44 +00008769 assert(N0.getOpcode() ==ISD::SETCC && "First argument must be a SetCC node!");
Scott Michelfdc40a02009-02-17 22:15:04 +00008770
Bill Wendling836ca7d2009-01-30 23:59:18 +00008771 SDValue SCC = SimplifySelectCC(DL, N0.getOperand(0), N0.getOperand(1), N1, N2,
Nate Begemanf845b452005-10-08 00:29:44 +00008772 cast<CondCodeSDNode>(N0.getOperand(2))->get());
Bill Wendling836ca7d2009-01-30 23:59:18 +00008773
Nate Begemanf845b452005-10-08 00:29:44 +00008774 // If we got a simplified select_cc node back from SimplifySelectCC, then
8775 // break it down into a new SETCC node, and a new SELECT node, and then return
8776 // the SELECT node, since we were called with a SELECT node.
Gabor Greifba36cb52008-08-28 21:40:38 +00008777 if (SCC.getNode()) {
Nate Begemanf845b452005-10-08 00:29:44 +00008778 // Check to see if we got a select_cc back (to turn into setcc/select).
8779 // Otherwise, just return whatever node we got back, like fabs.
8780 if (SCC.getOpcode() == ISD::SELECT_CC) {
Bill Wendling836ca7d2009-01-30 23:59:18 +00008781 SDValue SETCC = DAG.getNode(ISD::SETCC, N0.getDebugLoc(),
8782 N0.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +00008783 SCC.getOperand(0), SCC.getOperand(1),
Bill Wendling836ca7d2009-01-30 23:59:18 +00008784 SCC.getOperand(4));
Gabor Greifba36cb52008-08-28 21:40:38 +00008785 AddToWorkList(SETCC.getNode());
Bill Wendling836ca7d2009-01-30 23:59:18 +00008786 return DAG.getNode(ISD::SELECT, SCC.getDebugLoc(), SCC.getValueType(),
8787 SCC.getOperand(2), SCC.getOperand(3), SETCC);
Nate Begemanf845b452005-10-08 00:29:44 +00008788 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008789
Nate Begemanf845b452005-10-08 00:29:44 +00008790 return SCC;
8791 }
Dan Gohman475871a2008-07-27 21:46:04 +00008792 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00008793}
8794
Chris Lattner40c62d52005-10-18 06:04:22 +00008795/// SimplifySelectOps - Given a SELECT or a SELECT_CC node, where LHS and RHS
8796/// are the two values being selected between, see if we can simplify the
Chris Lattner729c6d12006-05-27 00:43:02 +00008797/// select. Callers of this should assume that TheSelect is deleted if this
8798/// returns true. As such, they should return the appropriate thing (e.g. the
8799/// node) back to the top-level of the DAG combiner loop to avoid it being
8800/// looked at.
Scott Michelfdc40a02009-02-17 22:15:04 +00008801bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
Dan Gohman475871a2008-07-27 21:46:04 +00008802 SDValue RHS) {
Scott Michelfdc40a02009-02-17 22:15:04 +00008803
Nadav Rotemf94fdb62011-02-11 19:57:47 +00008804 // Cannot simplify select with vector condition
8805 if (TheSelect->getOperand(0).getValueType().isVector()) return false;
8806
Chris Lattner40c62d52005-10-18 06:04:22 +00008807 // If this is a select from two identical things, try to pull the operation
8808 // through the select.
Chris Lattner18061612010-09-21 15:46:59 +00008809 if (LHS.getOpcode() != RHS.getOpcode() ||
8810 !LHS.hasOneUse() || !RHS.hasOneUse())
8811 return false;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008812
Chris Lattner18061612010-09-21 15:46:59 +00008813 // If this is a load and the token chain is identical, replace the select
8814 // of two loads with a load through a select of the address to load from.
8815 // This triggers in things like "select bool X, 10.0, 123.0" after the FP
8816 // constants have been dropped into the constant pool.
8817 if (LHS.getOpcode() == ISD::LOAD) {
8818 LoadSDNode *LLD = cast<LoadSDNode>(LHS);
8819 LoadSDNode *RLD = cast<LoadSDNode>(RHS);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008820
Chris Lattner18061612010-09-21 15:46:59 +00008821 // Token chains must be identical.
8822 if (LHS.getOperand(0) != RHS.getOperand(0) ||
Duncan Sandsd4b9c172008-06-13 19:07:40 +00008823 // Do not let this transformation reduce the number of volatile loads.
Chris Lattner18061612010-09-21 15:46:59 +00008824 LLD->isVolatile() || RLD->isVolatile() ||
8825 // If this is an EXTLOAD, the VT's must match.
8826 LLD->getMemoryVT() != RLD->getMemoryVT() ||
Duncan Sandsdcfd3a72010-11-18 20:05:18 +00008827 // If this is an EXTLOAD, the kind of extension must match.
8828 (LLD->getExtensionType() != RLD->getExtensionType() &&
8829 // The only exception is if one of the extensions is anyext.
8830 LLD->getExtensionType() != ISD::EXTLOAD &&
8831 RLD->getExtensionType() != ISD::EXTLOAD) ||
Dan Gohman75832d72009-10-31 14:14:04 +00008832 // FIXME: this discards src value information. This is
8833 // over-conservative. It would be beneficial to be able to remember
Mon P Wangfe240b12010-01-11 20:12:49 +00008834 // both potential memory locations. Since we are discarding
8835 // src value info, don't do the transformation if the memory
8836 // locations are not in the default address space.
Chris Lattner18061612010-09-21 15:46:59 +00008837 LLD->getPointerInfo().getAddrSpace() != 0 ||
8838 RLD->getPointerInfo().getAddrSpace() != 0)
8839 return false;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008840
Chris Lattnerf1658062010-09-21 15:58:55 +00008841 // Check that the select condition doesn't reach either load. If so,
8842 // folding this will induce a cycle into the DAG. If not, this is safe to
8843 // xform, so create a select of the addresses.
Chris Lattner18061612010-09-21 15:46:59 +00008844 SDValue Addr;
8845 if (TheSelect->getOpcode() == ISD::SELECT) {
Chris Lattnerf1658062010-09-21 15:58:55 +00008846 SDNode *CondNode = TheSelect->getOperand(0).getNode();
8847 if ((LLD->hasAnyUseOfValue(1) && LLD->isPredecessorOf(CondNode)) ||
8848 (RLD->hasAnyUseOfValue(1) && RLD->isPredecessorOf(CondNode)))
8849 return false;
8850 Addr = DAG.getNode(ISD::SELECT, TheSelect->getDebugLoc(),
8851 LLD->getBasePtr().getValueType(),
8852 TheSelect->getOperand(0), LLD->getBasePtr(),
8853 RLD->getBasePtr());
Chris Lattner18061612010-09-21 15:46:59 +00008854 } else { // Otherwise SELECT_CC
Chris Lattnerf1658062010-09-21 15:58:55 +00008855 SDNode *CondLHS = TheSelect->getOperand(0).getNode();
8856 SDNode *CondRHS = TheSelect->getOperand(1).getNode();
8857
8858 if ((LLD->hasAnyUseOfValue(1) &&
8859 (LLD->isPredecessorOf(CondLHS) || LLD->isPredecessorOf(CondRHS))) ||
Chris Lattner77d95212012-03-27 16:27:21 +00008860 (RLD->hasAnyUseOfValue(1) &&
8861 (RLD->isPredecessorOf(CondLHS) || RLD->isPredecessorOf(CondRHS))))
Chris Lattnerf1658062010-09-21 15:58:55 +00008862 return false;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008863
Chris Lattnerf1658062010-09-21 15:58:55 +00008864 Addr = DAG.getNode(ISD::SELECT_CC, TheSelect->getDebugLoc(),
8865 LLD->getBasePtr().getValueType(),
8866 TheSelect->getOperand(0),
8867 TheSelect->getOperand(1),
8868 LLD->getBasePtr(), RLD->getBasePtr(),
8869 TheSelect->getOperand(4));
Chris Lattner18061612010-09-21 15:46:59 +00008870 }
8871
Chris Lattnerf1658062010-09-21 15:58:55 +00008872 SDValue Load;
8873 if (LLD->getExtensionType() == ISD::NON_EXTLOAD) {
8874 Load = DAG.getLoad(TheSelect->getValueType(0),
8875 TheSelect->getDebugLoc(),
8876 // FIXME: Discards pointer info.
8877 LLD->getChain(), Addr, MachinePointerInfo(),
8878 LLD->isVolatile(), LLD->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00008879 LLD->isInvariant(), LLD->getAlignment());
Chris Lattnerf1658062010-09-21 15:58:55 +00008880 } else {
Duncan Sandsb9064bb2010-11-18 21:16:28 +00008881 Load = DAG.getExtLoad(LLD->getExtensionType() == ISD::EXTLOAD ?
8882 RLD->getExtensionType() : LLD->getExtensionType(),
Chris Lattnerf1658062010-09-21 15:58:55 +00008883 TheSelect->getDebugLoc(),
Stuart Hastingsa9011292011-02-16 16:23:55 +00008884 TheSelect->getValueType(0),
Chris Lattnerf1658062010-09-21 15:58:55 +00008885 // FIXME: Discards pointer info.
8886 LLD->getChain(), Addr, MachinePointerInfo(),
8887 LLD->getMemoryVT(), LLD->isVolatile(),
8888 LLD->isNonTemporal(), LLD->getAlignment());
Chris Lattner40c62d52005-10-18 06:04:22 +00008889 }
Chris Lattnerf1658062010-09-21 15:58:55 +00008890
8891 // Users of the select now use the result of the load.
8892 CombineTo(TheSelect, Load);
8893
8894 // Users of the old loads now use the new load's chain. We know the
8895 // old-load value is dead now.
8896 CombineTo(LHS.getNode(), Load.getValue(0), Load.getValue(1));
8897 CombineTo(RHS.getNode(), Load.getValue(0), Load.getValue(1));
8898 return true;
Chris Lattner40c62d52005-10-18 06:04:22 +00008899 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008900
Chris Lattner40c62d52005-10-18 06:04:22 +00008901 return false;
8902}
8903
Chris Lattner600fec32009-03-11 05:08:08 +00008904/// SimplifySelectCC - Simplify an expression of the form (N0 cond N1) ? N2 : N3
8905/// where 'cond' is the comparison specified by CC.
Scott Michelfdc40a02009-02-17 22:15:04 +00008906SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1,
Dan Gohman475871a2008-07-27 21:46:04 +00008907 SDValue N2, SDValue N3,
8908 ISD::CondCode CC, bool NotExtCompare) {
Chris Lattner600fec32009-03-11 05:08:08 +00008909 // (x ? y : y) -> y.
8910 if (N2 == N3) return N2;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008911
Owen Andersone50ed302009-08-10 22:56:29 +00008912 EVT VT = N2.getValueType();
Gabor Greifba36cb52008-08-28 21:40:38 +00008913 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
8914 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
8915 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00008916
8917 // Determine if the condition we're dealing with is constant
Duncan Sands5480c042009-01-01 15:52:00 +00008918 SDValue SCC = SimplifySetCC(TLI.getSetCCResultType(N0.getValueType()),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00008919 N0, N1, CC, DL, false);
Gabor Greifba36cb52008-08-28 21:40:38 +00008920 if (SCC.getNode()) AddToWorkList(SCC.getNode());
8921 ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00008922
8923 // fold select_cc true, x, y -> x
Dan Gohman002e5d02008-03-13 22:13:53 +00008924 if (SCCC && !SCCC->isNullValue())
Nate Begemanf845b452005-10-08 00:29:44 +00008925 return N2;
8926 // fold select_cc false, x, y -> y
Dan Gohman002e5d02008-03-13 22:13:53 +00008927 if (SCCC && SCCC->isNullValue())
Nate Begemanf845b452005-10-08 00:29:44 +00008928 return N3;
Scott Michelfdc40a02009-02-17 22:15:04 +00008929
Nate Begemanf845b452005-10-08 00:29:44 +00008930 // Check to see if we can simplify the select into an fabs node
8931 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) {
8932 // Allow either -0.0 or 0.0
Dale Johannesen87503a62007-08-25 22:10:57 +00008933 if (CFP->getValueAPF().isZero()) {
Nate Begemanf845b452005-10-08 00:29:44 +00008934 // select (setg[te] X, +/-0.0), X, fneg(X) -> fabs
8935 if ((CC == ISD::SETGE || CC == ISD::SETGT) &&
8936 N0 == N2 && N3.getOpcode() == ISD::FNEG &&
8937 N2 == N3.getOperand(0))
Bill Wendling836ca7d2009-01-30 23:59:18 +00008938 return DAG.getNode(ISD::FABS, DL, VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00008939
Nate Begemanf845b452005-10-08 00:29:44 +00008940 // select (setl[te] X, +/-0.0), fneg(X), X -> fabs
8941 if ((CC == ISD::SETLT || CC == ISD::SETLE) &&
8942 N0 == N3 && N2.getOpcode() == ISD::FNEG &&
8943 N2.getOperand(0) == N3)
Bill Wendling836ca7d2009-01-30 23:59:18 +00008944 return DAG.getNode(ISD::FABS, DL, VT, N3);
Nate Begemanf845b452005-10-08 00:29:44 +00008945 }
8946 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008947
Chris Lattner600fec32009-03-11 05:08:08 +00008948 // Turn "(a cond b) ? 1.0f : 2.0f" into "load (tmp + ((a cond b) ? 0 : 4)"
8949 // where "tmp" is a constant pool entry containing an array with 1.0 and 2.0
8950 // in it. This is a win when the constant is not otherwise available because
8951 // it replaces two constant pool loads with one. We only do this if the FP
8952 // type is known to be legal, because if it isn't, then we are before legalize
8953 // types an we want the other legalization to happen first (e.g. to avoid
Mon P Wang0b7a7862009-03-14 00:25:19 +00008954 // messing with soft float) and if the ConstantFP is not legal, because if
8955 // it is legal, we may not need to store the FP constant in a constant pool.
Chris Lattner600fec32009-03-11 05:08:08 +00008956 if (ConstantFPSDNode *TV = dyn_cast<ConstantFPSDNode>(N2))
8957 if (ConstantFPSDNode *FV = dyn_cast<ConstantFPSDNode>(N3)) {
8958 if (TLI.isTypeLegal(N2.getValueType()) &&
Mon P Wang0b7a7862009-03-14 00:25:19 +00008959 (TLI.getOperationAction(ISD::ConstantFP, N2.getValueType()) !=
8960 TargetLowering::Legal) &&
Chris Lattner600fec32009-03-11 05:08:08 +00008961 // If both constants have multiple uses, then we won't need to do an
8962 // extra load, they are likely around in registers for other users.
8963 (TV->hasOneUse() || FV->hasOneUse())) {
8964 Constant *Elts[] = {
8965 const_cast<ConstantFP*>(FV->getConstantFPValue()),
8966 const_cast<ConstantFP*>(TV->getConstantFPValue())
8967 };
Chris Lattnerdb125cf2011-07-18 04:54:35 +00008968 Type *FPTy = Elts[0]->getType();
Chris Lattner600fec32009-03-11 05:08:08 +00008969 const TargetData &TD = *TLI.getTargetData();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008970
Chris Lattner600fec32009-03-11 05:08:08 +00008971 // Create a ConstantArray of the two constants.
Jay Foad26701082011-06-22 09:24:39 +00008972 Constant *CA = ConstantArray::get(ArrayType::get(FPTy, 2), Elts);
Chris Lattner600fec32009-03-11 05:08:08 +00008973 SDValue CPIdx = DAG.getConstantPool(CA, TLI.getPointerTy(),
8974 TD.getPrefTypeAlignment(FPTy));
Evan Cheng1606e8e2009-03-13 07:51:59 +00008975 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
Chris Lattner600fec32009-03-11 05:08:08 +00008976
8977 // Get the offsets to the 0 and 1 element of the array so that we can
8978 // select between them.
8979 SDValue Zero = DAG.getIntPtrConstant(0);
Duncan Sands777d2302009-05-09 07:06:46 +00008980 unsigned EltSize = (unsigned)TD.getTypeAllocSize(Elts[0]->getType());
Chris Lattner600fec32009-03-11 05:08:08 +00008981 SDValue One = DAG.getIntPtrConstant(EltSize);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008982
Chris Lattner600fec32009-03-11 05:08:08 +00008983 SDValue Cond = DAG.getSetCC(DL,
8984 TLI.getSetCCResultType(N0.getValueType()),
8985 N0, N1, CC);
Dan Gohman7b316c92011-09-22 23:01:29 +00008986 AddToWorkList(Cond.getNode());
Chris Lattner600fec32009-03-11 05:08:08 +00008987 SDValue CstOffset = DAG.getNode(ISD::SELECT, DL, Zero.getValueType(),
8988 Cond, One, Zero);
Dan Gohman7b316c92011-09-22 23:01:29 +00008989 AddToWorkList(CstOffset.getNode());
Chris Lattner600fec32009-03-11 05:08:08 +00008990 CPIdx = DAG.getNode(ISD::ADD, DL, TLI.getPointerTy(), CPIdx,
8991 CstOffset);
Dan Gohman7b316c92011-09-22 23:01:29 +00008992 AddToWorkList(CPIdx.getNode());
Chris Lattner600fec32009-03-11 05:08:08 +00008993 return DAG.getLoad(TV->getValueType(0), DL, DAG.getEntryNode(), CPIdx,
Chris Lattner85ca1062010-09-21 07:32:19 +00008994 MachinePointerInfo::getConstantPool(), false,
Pete Cooperd752e0f2011-11-08 18:42:53 +00008995 false, false, Alignment);
Chris Lattner600fec32009-03-11 05:08:08 +00008996
8997 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008998 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008999
Nate Begemanf845b452005-10-08 00:29:44 +00009000 // Check to see if we can perform the "gzip trick", transforming
Bill Wendling836ca7d2009-01-30 23:59:18 +00009001 // (select_cc setlt X, 0, A, 0) -> (and (sra X, (sub size(X), 1), A)
Chris Lattnere3152e52006-09-20 06:41:35 +00009002 if (N1C && N3C && N3C->isNullValue() && CC == ISD::SETLT &&
Dan Gohman002e5d02008-03-13 22:13:53 +00009003 (N1C->isNullValue() || // (a < 0) ? b : 0
9004 (N1C->getAPIntValue() == 1 && N0 == N2))) { // (a < 1) ? a : 0
Owen Andersone50ed302009-08-10 22:56:29 +00009005 EVT XType = N0.getValueType();
9006 EVT AType = N2.getValueType();
Duncan Sands8e4eb092008-06-08 20:54:56 +00009007 if (XType.bitsGE(AType)) {
Sylvestre Ledru94c22712012-09-27 10:14:43 +00009008 // and (sra X, size(X)-1, A) -> "and (srl X, C2), A" iff A is a
Nate Begeman07ed4172005-10-10 21:26:48 +00009009 // single-bit constant.
Dan Gohman002e5d02008-03-13 22:13:53 +00009010 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue()-1)) == 0)) {
9011 unsigned ShCtV = N2C->getAPIntValue().logBase2();
Duncan Sands83ec4b62008-06-06 12:08:01 +00009012 ShCtV = XType.getSizeInBits()-ShCtV-1;
Owen Anderson95771af2011-02-25 21:41:48 +00009013 SDValue ShCt = DAG.getConstant(ShCtV,
9014 getShiftAmountTy(N0.getValueType()));
Bill Wendling9729c5a2009-01-31 03:12:48 +00009015 SDValue Shift = DAG.getNode(ISD::SRL, N0.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00009016 XType, N0, ShCt);
Gabor Greifba36cb52008-08-28 21:40:38 +00009017 AddToWorkList(Shift.getNode());
Bill Wendling836ca7d2009-01-30 23:59:18 +00009018
Duncan Sands8e4eb092008-06-08 20:54:56 +00009019 if (XType.bitsGT(AType)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00009020 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
Gabor Greifba36cb52008-08-28 21:40:38 +00009021 AddToWorkList(Shift.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00009022 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00009023
9024 return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
Nate Begemanf845b452005-10-08 00:29:44 +00009025 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00009026
Bill Wendling9729c5a2009-01-31 03:12:48 +00009027 SDValue Shift = DAG.getNode(ISD::SRA, N0.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00009028 XType, N0,
9029 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00009030 getShiftAmountTy(N0.getValueType())));
Gabor Greifba36cb52008-08-28 21:40:38 +00009031 AddToWorkList(Shift.getNode());
Bill Wendling836ca7d2009-01-30 23:59:18 +00009032
Duncan Sands8e4eb092008-06-08 20:54:56 +00009033 if (XType.bitsGT(AType)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00009034 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
Gabor Greifba36cb52008-08-28 21:40:38 +00009035 AddToWorkList(Shift.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00009036 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00009037
9038 return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
Nate Begemanf845b452005-10-08 00:29:44 +00009039 }
9040 }
Scott Michelfdc40a02009-02-17 22:15:04 +00009041
Owen Andersoned1088a2010-09-22 22:58:22 +00009042 // fold (select_cc seteq (and x, y), 0, 0, A) -> (and (shr (shl x)) A)
9043 // where y is has a single bit set.
9044 // A plaintext description would be, we can turn the SELECT_CC into an AND
9045 // when the condition can be materialized as an all-ones register. Any
9046 // single bit-test can be materialized as an all-ones register with
9047 // shift-left and shift-right-arith.
9048 if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND &&
9049 N0->getValueType(0) == VT &&
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009050 N1C && N1C->isNullValue() &&
Owen Andersoned1088a2010-09-22 22:58:22 +00009051 N2C && N2C->isNullValue()) {
9052 SDValue AndLHS = N0->getOperand(0);
9053 ConstantSDNode *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->getOperand(1));
9054 if (ConstAndRHS && ConstAndRHS->getAPIntValue().countPopulation() == 1) {
9055 // Shift the tested bit over the sign bit.
9056 APInt AndMask = ConstAndRHS->getAPIntValue();
9057 SDValue ShlAmt =
Owen Anderson95771af2011-02-25 21:41:48 +00009058 DAG.getConstant(AndMask.countLeadingZeros(),
9059 getShiftAmountTy(AndLHS.getValueType()));
Owen Andersoned1088a2010-09-22 22:58:22 +00009060 SDValue Shl = DAG.getNode(ISD::SHL, N0.getDebugLoc(), VT, AndLHS, ShlAmt);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009061
Owen Andersoned1088a2010-09-22 22:58:22 +00009062 // Now arithmetic right shift it all the way over, so the result is either
9063 // all-ones, or zero.
9064 SDValue ShrAmt =
Owen Anderson95771af2011-02-25 21:41:48 +00009065 DAG.getConstant(AndMask.getBitWidth()-1,
9066 getShiftAmountTy(Shl.getValueType()));
Owen Andersoned1088a2010-09-22 22:58:22 +00009067 SDValue Shr = DAG.getNode(ISD::SRA, N0.getDebugLoc(), VT, Shl, ShrAmt);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009068
Owen Andersoned1088a2010-09-22 22:58:22 +00009069 return DAG.getNode(ISD::AND, DL, VT, Shr, N3);
9070 }
9071 }
9072
Nate Begeman07ed4172005-10-10 21:26:48 +00009073 // fold select C, 16, 0 -> shl C, 4
Dan Gohman002e5d02008-03-13 22:13:53 +00009074 if (N2C && N3C && N3C->isNullValue() && N2C->getAPIntValue().isPowerOf2() &&
Duncan Sands28b77e92011-09-06 19:07:46 +00009075 TLI.getBooleanContents(N0.getValueType().isVector()) ==
9076 TargetLowering::ZeroOrOneBooleanContent) {
Scott Michelfdc40a02009-02-17 22:15:04 +00009077
Chris Lattner1eba01e2007-04-11 06:50:51 +00009078 // If the caller doesn't want us to simplify this into a zext of a compare,
9079 // don't do it.
Dan Gohman002e5d02008-03-13 22:13:53 +00009080 if (NotExtCompare && N2C->getAPIntValue() == 1)
Dan Gohman475871a2008-07-27 21:46:04 +00009081 return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00009082
Nate Begeman07ed4172005-10-10 21:26:48 +00009083 // Get a SetCC of the condition
9084 // FIXME: Should probably make sure that setcc is legal if we ever have a
9085 // target where it isn't.
Dan Gohman475871a2008-07-27 21:46:04 +00009086 SDValue Temp, SCC;
Nate Begeman07ed4172005-10-10 21:26:48 +00009087 // cast from setcc result type to select result type
Duncan Sands25cf2272008-11-24 14:53:14 +00009088 if (LegalTypes) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00009089 SCC = DAG.getSetCC(DL, TLI.getSetCCResultType(N0.getValueType()),
Duncan Sands5480c042009-01-01 15:52:00 +00009090 N0, N1, CC);
Duncan Sands8e4eb092008-06-08 20:54:56 +00009091 if (N2.getValueType().bitsLT(SCC.getValueType()))
Bill Wendling9729c5a2009-01-31 03:12:48 +00009092 Temp = DAG.getZeroExtendInReg(SCC, N2.getDebugLoc(), N2.getValueType());
Chris Lattner555d8d62006-12-07 22:36:47 +00009093 else
Bill Wendling9729c5a2009-01-31 03:12:48 +00009094 Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00009095 N2.getValueType(), SCC);
Nate Begemanb0d04a72006-02-18 02:40:58 +00009096 } else {
Owen Anderson825b72b2009-08-11 20:47:22 +00009097 SCC = DAG.getSetCC(N0.getDebugLoc(), MVT::i1, N0, N1, CC);
Bill Wendling9729c5a2009-01-31 03:12:48 +00009098 Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00009099 N2.getValueType(), SCC);
Nate Begemanb0d04a72006-02-18 02:40:58 +00009100 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00009101
Gabor Greifba36cb52008-08-28 21:40:38 +00009102 AddToWorkList(SCC.getNode());
9103 AddToWorkList(Temp.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00009104
Dan Gohman002e5d02008-03-13 22:13:53 +00009105 if (N2C->getAPIntValue() == 1)
Chris Lattnerc56a81d2007-04-11 06:43:25 +00009106 return Temp;
Bill Wendling836ca7d2009-01-30 23:59:18 +00009107
Nate Begeman07ed4172005-10-10 21:26:48 +00009108 // shl setcc result by log2 n2c
Bill Wendling836ca7d2009-01-30 23:59:18 +00009109 return DAG.getNode(ISD::SHL, DL, N2.getValueType(), Temp,
Dan Gohman002e5d02008-03-13 22:13:53 +00009110 DAG.getConstant(N2C->getAPIntValue().logBase2(),
Owen Anderson95771af2011-02-25 21:41:48 +00009111 getShiftAmountTy(Temp.getValueType())));
Nate Begeman07ed4172005-10-10 21:26:48 +00009112 }
Scott Michelfdc40a02009-02-17 22:15:04 +00009113
Nate Begemanf845b452005-10-08 00:29:44 +00009114 // Check to see if this is the equivalent of setcc
9115 // FIXME: Turn all of these into setcc if setcc if setcc is legal
9116 // otherwise, go ahead with the folds.
Dan Gohman002e5d02008-03-13 22:13:53 +00009117 if (0 && N3C && N3C->isNullValue() && N2C && (N2C->getAPIntValue() == 1ULL)) {
Owen Andersone50ed302009-08-10 22:56:29 +00009118 EVT XType = N0.getValueType();
Duncan Sands25cf2272008-11-24 14:53:14 +00009119 if (!LegalOperations ||
Duncan Sands5480c042009-01-01 15:52:00 +00009120 TLI.isOperationLegal(ISD::SETCC, TLI.getSetCCResultType(XType))) {
Bill Wendling836ca7d2009-01-30 23:59:18 +00009121 SDValue Res = DAG.getSetCC(DL, TLI.getSetCCResultType(XType), N0, N1, CC);
Nate Begemanf845b452005-10-08 00:29:44 +00009122 if (Res.getValueType() != VT)
Bill Wendling836ca7d2009-01-30 23:59:18 +00009123 Res = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Res);
Nate Begemanf845b452005-10-08 00:29:44 +00009124 return Res;
9125 }
Scott Michelfdc40a02009-02-17 22:15:04 +00009126
Bill Wendling836ca7d2009-01-30 23:59:18 +00009127 // fold (seteq X, 0) -> (srl (ctlz X, log2(size(X))))
Scott Michelfdc40a02009-02-17 22:15:04 +00009128 if (N1C && N1C->isNullValue() && CC == ISD::SETEQ &&
Duncan Sands25cf2272008-11-24 14:53:14 +00009129 (!LegalOperations ||
Duncan Sands184a8762008-06-14 17:48:34 +00009130 TLI.isOperationLegal(ISD::CTLZ, XType))) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00009131 SDValue Ctlz = DAG.getNode(ISD::CTLZ, N0.getDebugLoc(), XType, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00009132 return DAG.getNode(ISD::SRL, DL, XType, Ctlz,
Duncan Sands83ec4b62008-06-06 12:08:01 +00009133 DAG.getConstant(Log2_32(XType.getSizeInBits()),
Owen Anderson95771af2011-02-25 21:41:48 +00009134 getShiftAmountTy(Ctlz.getValueType())));
Nate Begemanf845b452005-10-08 00:29:44 +00009135 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00009136 // fold (setgt X, 0) -> (srl (and (-X, ~X), size(X)-1))
Scott Michelfdc40a02009-02-17 22:15:04 +00009137 if (N1C && N1C->isNullValue() && CC == ISD::SETGT) {
Bill Wendling836ca7d2009-01-30 23:59:18 +00009138 SDValue NegN0 = DAG.getNode(ISD::SUB, N0.getDebugLoc(),
9139 XType, DAG.getConstant(0, XType), N0);
Bill Wendling7581bfa2009-01-30 23:03:19 +00009140 SDValue NotN0 = DAG.getNOT(N0.getDebugLoc(), N0, XType);
Bill Wendling836ca7d2009-01-30 23:59:18 +00009141 return DAG.getNode(ISD::SRL, DL, XType,
Bill Wendlingfc4b6772009-02-01 11:19:36 +00009142 DAG.getNode(ISD::AND, DL, XType, NegN0, NotN0),
Duncan Sands83ec4b62008-06-06 12:08:01 +00009143 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00009144 getShiftAmountTy(XType)));
Nate Begemanf845b452005-10-08 00:29:44 +00009145 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00009146 // fold (setgt X, -1) -> (xor (srl (X, size(X)-1), 1))
Nate Begemanf845b452005-10-08 00:29:44 +00009147 if (N1C && N1C->isAllOnesValue() && CC == ISD::SETGT) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00009148 SDValue Sign = DAG.getNode(ISD::SRL, N0.getDebugLoc(), XType, N0,
Bill Wendling836ca7d2009-01-30 23:59:18 +00009149 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00009150 getShiftAmountTy(N0.getValueType())));
Bill Wendling836ca7d2009-01-30 23:59:18 +00009151 return DAG.getNode(ISD::XOR, DL, XType, Sign, DAG.getConstant(1, XType));
Nate Begemanf845b452005-10-08 00:29:44 +00009152 }
9153 }
Scott Michelfdc40a02009-02-17 22:15:04 +00009154
Benjamin Kramercde51102010-07-08 12:09:56 +00009155 // Check to see if this is an integer abs.
9156 // select_cc setg[te] X, 0, X, -X ->
9157 // select_cc setgt X, -1, X, -X ->
9158 // select_cc setl[te] X, 0, -X, X ->
9159 // select_cc setlt X, 1, -X, X ->
Nate Begemanf845b452005-10-08 00:29:44 +00009160 // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
Benjamin Kramercde51102010-07-08 12:09:56 +00009161 if (N1C) {
9162 ConstantSDNode *SubC = NULL;
9163 if (((N1C->isNullValue() && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
9164 (N1C->isAllOnesValue() && CC == ISD::SETGT)) &&
9165 N0 == N2 && N3.getOpcode() == ISD::SUB && N0 == N3.getOperand(1))
9166 SubC = dyn_cast<ConstantSDNode>(N3.getOperand(0));
9167 else if (((N1C->isNullValue() && (CC == ISD::SETLT || CC == ISD::SETLE)) ||
9168 (N1C->isOne() && CC == ISD::SETLT)) &&
9169 N0 == N3 && N2.getOpcode() == ISD::SUB && N0 == N2.getOperand(1))
9170 SubC = dyn_cast<ConstantSDNode>(N2.getOperand(0));
9171
Owen Andersone50ed302009-08-10 22:56:29 +00009172 EVT XType = N0.getValueType();
Benjamin Kramercde51102010-07-08 12:09:56 +00009173 if (SubC && SubC->isNullValue() && XType.isInteger()) {
9174 SDValue Shift = DAG.getNode(ISD::SRA, N0.getDebugLoc(), XType,
9175 N0,
9176 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00009177 getShiftAmountTy(N0.getValueType())));
Benjamin Kramercde51102010-07-08 12:09:56 +00009178 SDValue Add = DAG.getNode(ISD::ADD, N0.getDebugLoc(),
9179 XType, N0, Shift);
9180 AddToWorkList(Shift.getNode());
9181 AddToWorkList(Add.getNode());
9182 return DAG.getNode(ISD::XOR, DL, XType, Add, Shift);
Nate Begemanf845b452005-10-08 00:29:44 +00009183 }
9184 }
Scott Michelfdc40a02009-02-17 22:15:04 +00009185
Dan Gohman475871a2008-07-27 21:46:04 +00009186 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00009187}
9188
Evan Chengfa1eb272007-02-08 22:13:59 +00009189/// SimplifySetCC - This is a stub for TargetLowering::SimplifySetCC.
Owen Andersone50ed302009-08-10 22:56:29 +00009190SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0,
Dan Gohman475871a2008-07-27 21:46:04 +00009191 SDValue N1, ISD::CondCode Cond,
Dale Johannesenff97d4f2009-02-03 00:47:48 +00009192 DebugLoc DL, bool foldBooleans) {
Scott Michelfdc40a02009-02-17 22:15:04 +00009193 TargetLowering::DAGCombinerInfo
Jakob Stoklund Olesen78d12642009-07-24 18:22:59 +00009194 DagCombineInfo(DAG, !LegalTypes, !LegalOperations, false, this);
Dale Johannesenff97d4f2009-02-03 00:47:48 +00009195 return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL);
Nate Begeman452d7beb2005-09-16 00:54:12 +00009196}
9197
Nate Begeman69575232005-10-20 02:15:44 +00009198/// BuildSDIVSequence - Given an ISD::SDIV node expressing a divide by constant,
9199/// return a DAG expression to select that will generate the same value by
9200/// multiplying by a magic number. See:
9201/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html>
Dan Gohman475871a2008-07-27 21:46:04 +00009202SDValue DAGCombiner::BuildSDIV(SDNode *N) {
Andrew Lenharth232c9102006-06-12 16:07:18 +00009203 std::vector<SDNode*> Built;
Richard Osborne19a4daf2011-11-07 17:09:05 +00009204 SDValue S = TLI.BuildSDIV(N, DAG, LegalOperations, &Built);
Andrew Lenharthdae9cbe2006-05-16 17:42:15 +00009205
Andrew Lenharth232c9102006-06-12 16:07:18 +00009206 for (std::vector<SDNode*>::iterator ii = Built.begin(), ee = Built.end();
Andrew Lenharthdae9cbe2006-05-16 17:42:15 +00009207 ii != ee; ++ii)
9208 AddToWorkList(*ii);
9209 return S;
Nate Begeman69575232005-10-20 02:15:44 +00009210}
9211
9212/// BuildUDIVSequence - Given an ISD::UDIV node expressing a divide by constant,
9213/// return a DAG expression to select that will generate the same value by
9214/// multiplying by a magic number. See:
9215/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html>
Dan Gohman475871a2008-07-27 21:46:04 +00009216SDValue DAGCombiner::BuildUDIV(SDNode *N) {
Andrew Lenharth232c9102006-06-12 16:07:18 +00009217 std::vector<SDNode*> Built;
Richard Osborne19a4daf2011-11-07 17:09:05 +00009218 SDValue S = TLI.BuildUDIV(N, DAG, LegalOperations, &Built);
Nate Begeman69575232005-10-20 02:15:44 +00009219
Andrew Lenharth232c9102006-06-12 16:07:18 +00009220 for (std::vector<SDNode*>::iterator ii = Built.begin(), ee = Built.end();
Andrew Lenharthdae9cbe2006-05-16 17:42:15 +00009221 ii != ee; ++ii)
9222 AddToWorkList(*ii);
9223 return S;
Nate Begeman69575232005-10-20 02:15:44 +00009224}
9225
Nate Begemancc66cdd2009-09-25 06:05:26 +00009226/// FindBaseOffset - Return true if base is a frame index, which is known not
Eric Christopher503a64d2010-12-09 04:48:06 +00009227// to alias with anything but itself. Provides base object and offset as
9228// results.
Nate Begemancc66cdd2009-09-25 06:05:26 +00009229static bool FindBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset,
Roman Divacky2943e372012-09-05 22:15:49 +00009230 const GlobalValue *&GV, const void *&CV) {
Jim Laskey71382342006-10-07 23:37:56 +00009231 // Assume it is a primitive operation.
Nate Begemancc66cdd2009-09-25 06:05:26 +00009232 Base = Ptr; Offset = 0; GV = 0; CV = 0;
Scott Michelfdc40a02009-02-17 22:15:04 +00009233
Jim Laskey71382342006-10-07 23:37:56 +00009234 // If it's an adding a simple constant then integrate the offset.
9235 if (Base.getOpcode() == ISD::ADD) {
9236 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Base.getOperand(1))) {
9237 Base = Base.getOperand(0);
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00009238 Offset += C->getZExtValue();
Jim Laskey71382342006-10-07 23:37:56 +00009239 }
9240 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009241
Nate Begemancc66cdd2009-09-25 06:05:26 +00009242 // Return the underlying GlobalValue, and update the Offset. Return false
9243 // for GlobalAddressSDNode since the same GlobalAddress may be represented
9244 // by multiple nodes with different offsets.
9245 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Base)) {
9246 GV = G->getGlobal();
9247 Offset += G->getOffset();
9248 return false;
9249 }
Scott Michelfdc40a02009-02-17 22:15:04 +00009250
Nate Begemancc66cdd2009-09-25 06:05:26 +00009251 // Return the underlying Constant value, and update the Offset. Return false
9252 // for ConstantSDNodes since the same constant pool entry may be represented
9253 // by multiple nodes with different offsets.
9254 if (ConstantPoolSDNode *C = dyn_cast<ConstantPoolSDNode>(Base)) {
Roman Divacky2943e372012-09-05 22:15:49 +00009255 CV = C->isMachineConstantPoolEntry() ? (const void *)C->getMachineCPVal()
9256 : (const void *)C->getConstVal();
Nate Begemancc66cdd2009-09-25 06:05:26 +00009257 Offset += C->getOffset();
9258 return false;
9259 }
Jim Laskey71382342006-10-07 23:37:56 +00009260 // If it's any of the following then it can't alias with anything but itself.
Nate Begemancc66cdd2009-09-25 06:05:26 +00009261 return isa<FrameIndexSDNode>(Base);
Jim Laskey71382342006-10-07 23:37:56 +00009262}
9263
9264/// isAlias - Return true if there is any possibility that the two addresses
9265/// overlap.
Dan Gohman475871a2008-07-27 21:46:04 +00009266bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1,
Jim Laskey096c22e2006-10-18 12:29:57 +00009267 const Value *SrcValue1, int SrcValueOffset1,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009268 unsigned SrcValueAlign1,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00009269 const MDNode *TBAAInfo1,
Dan Gohman475871a2008-07-27 21:46:04 +00009270 SDValue Ptr2, int64_t Size2,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009271 const Value *SrcValue2, int SrcValueOffset2,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00009272 unsigned SrcValueAlign2,
9273 const MDNode *TBAAInfo2) const {
Jim Laskey71382342006-10-07 23:37:56 +00009274 // If they are the same then they must be aliases.
9275 if (Ptr1 == Ptr2) return true;
Scott Michelfdc40a02009-02-17 22:15:04 +00009276
Jim Laskey71382342006-10-07 23:37:56 +00009277 // Gather base node and offset information.
Dan Gohman475871a2008-07-27 21:46:04 +00009278 SDValue Base1, Base2;
Jim Laskey71382342006-10-07 23:37:56 +00009279 int64_t Offset1, Offset2;
Dan Gohman46510a72010-04-15 01:51:59 +00009280 const GlobalValue *GV1, *GV2;
Roman Divacky2943e372012-09-05 22:15:49 +00009281 const void *CV1, *CV2;
Nate Begemancc66cdd2009-09-25 06:05:26 +00009282 bool isFrameIndex1 = FindBaseOffset(Ptr1, Base1, Offset1, GV1, CV1);
9283 bool isFrameIndex2 = FindBaseOffset(Ptr2, Base2, Offset2, GV2, CV2);
Scott Michelfdc40a02009-02-17 22:15:04 +00009284
Nate Begemancc66cdd2009-09-25 06:05:26 +00009285 // If they have a same base address then check to see if they overlap.
9286 if (Base1 == Base2 || (GV1 && (GV1 == GV2)) || (CV1 && (CV1 == CV2)))
Bill Wendling836ca7d2009-01-30 23:59:18 +00009287 return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1);
Scott Michelfdc40a02009-02-17 22:15:04 +00009288
Owen Anderson4a9f1502010-09-20 20:39:59 +00009289 // It is possible for different frame indices to alias each other, mostly
9290 // when tail call optimization reuses return address slots for arguments.
9291 // To catch this case, look up the actual index of frame indices to compute
9292 // the real alias relationship.
9293 if (isFrameIndex1 && isFrameIndex2) {
9294 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
9295 Offset1 += MFI->getObjectOffset(cast<FrameIndexSDNode>(Base1)->getIndex());
9296 Offset2 += MFI->getObjectOffset(cast<FrameIndexSDNode>(Base2)->getIndex());
9297 return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1);
9298 }
9299
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009300 // Otherwise, if we know what the bases are, and they aren't identical, then
Owen Anderson4a9f1502010-09-20 20:39:59 +00009301 // we know they cannot alias.
Nate Begemancc66cdd2009-09-25 06:05:26 +00009302 if ((isFrameIndex1 || CV1 || GV1) && (isFrameIndex2 || CV2 || GV2))
9303 return false;
Jim Laskey096c22e2006-10-18 12:29:57 +00009304
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009305 // If we know required SrcValue1 and SrcValue2 have relatively large alignment
9306 // compared to the size and offset of the access, we may be able to prove they
9307 // do not alias. This check is conservative for now to catch cases created by
9308 // splitting vector types.
9309 if ((SrcValueAlign1 == SrcValueAlign2) &&
9310 (SrcValueOffset1 != SrcValueOffset2) &&
9311 (Size1 == Size2) && (SrcValueAlign1 > Size1)) {
9312 int64_t OffAlign1 = SrcValueOffset1 % SrcValueAlign1;
9313 int64_t OffAlign2 = SrcValueOffset2 % SrcValueAlign1;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009314
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009315 // There is no overlap between these relatively aligned accesses of similar
9316 // size, return no alias.
9317 if ((OffAlign1 + Size1) <= OffAlign2 || (OffAlign2 + Size2) <= OffAlign1)
9318 return false;
9319 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009320
Jim Laskey07a27092006-10-18 19:08:31 +00009321 if (CombinerGlobalAA) {
9322 // Use alias analysis information.
Dan Gohmane9c8fa02007-08-27 16:32:11 +00009323 int64_t MinOffset = std::min(SrcValueOffset1, SrcValueOffset2);
9324 int64_t Overlap1 = Size1 + SrcValueOffset1 - MinOffset;
9325 int64_t Overlap2 = Size2 + SrcValueOffset2 - MinOffset;
Scott Michelfdc40a02009-02-17 22:15:04 +00009326 AliasAnalysis::AliasResult AAResult =
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00009327 AA.alias(AliasAnalysis::Location(SrcValue1, Overlap1, TBAAInfo1),
9328 AliasAnalysis::Location(SrcValue2, Overlap2, TBAAInfo2));
Jim Laskey07a27092006-10-18 19:08:31 +00009329 if (AAResult == AliasAnalysis::NoAlias)
9330 return false;
9331 }
Jim Laskey096c22e2006-10-18 12:29:57 +00009332
9333 // Otherwise we have to assume they alias.
9334 return true;
Jim Laskey71382342006-10-07 23:37:56 +00009335}
9336
9337/// FindAliasInfo - Extracts the relevant alias information from the memory
9338/// node. Returns true if the operand was a load.
Jim Laskey7ca56af2006-10-11 13:47:09 +00009339bool DAGCombiner::FindAliasInfo(SDNode *N,
Benjamin Kramerae4746b2012-01-15 11:50:43 +00009340 SDValue &Ptr, int64_t &Size,
9341 const Value *&SrcValue,
9342 int &SrcValueOffset,
9343 unsigned &SrcValueAlign,
9344 const MDNode *&TBAAInfo) const {
9345 LSBaseSDNode *LS = cast<LSBaseSDNode>(N);
9346
9347 Ptr = LS->getBasePtr();
9348 Size = LS->getMemoryVT().getSizeInBits() >> 3;
9349 SrcValue = LS->getSrcValue();
9350 SrcValueOffset = LS->getSrcValueOffset();
9351 SrcValueAlign = LS->getOriginalAlignment();
9352 TBAAInfo = LS->getTBAAInfo();
9353 return isa<LoadSDNode>(LS);
Jim Laskey71382342006-10-07 23:37:56 +00009354}
9355
Jim Laskey6ff23e52006-10-04 16:53:27 +00009356/// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
9357/// looking for aliasing nodes and adding them to the Aliases vector.
Dan Gohman475871a2008-07-27 21:46:04 +00009358void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
9359 SmallVector<SDValue, 8> &Aliases) {
9360 SmallVector<SDValue, 8> Chains; // List of chains to visit.
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009361 SmallPtrSet<SDNode *, 16> Visited; // Visited node set.
Scott Michelfdc40a02009-02-17 22:15:04 +00009362
Jim Laskey279f0532006-09-25 16:29:54 +00009363 // Get alias information for node.
Dan Gohman475871a2008-07-27 21:46:04 +00009364 SDValue Ptr;
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009365 int64_t Size;
9366 const Value *SrcValue;
9367 int SrcValueOffset;
9368 unsigned SrcValueAlign;
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00009369 const MDNode *SrcTBAAInfo;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009370 bool IsLoad = FindAliasInfo(N, Ptr, Size, SrcValue, SrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00009371 SrcValueAlign, SrcTBAAInfo);
Jim Laskey279f0532006-09-25 16:29:54 +00009372
Jim Laskey6ff23e52006-10-04 16:53:27 +00009373 // Starting off.
Jim Laskeybc588b82006-10-05 15:07:25 +00009374 Chains.push_back(OriginalChain);
Nate Begeman677c89d2009-10-12 05:53:58 +00009375 unsigned Depth = 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009376
Jim Laskeybc588b82006-10-05 15:07:25 +00009377 // Look at each chain and determine if it is an alias. If so, add it to the
9378 // aliases list. If not, then continue up the chain looking for the next
Scott Michelfdc40a02009-02-17 22:15:04 +00009379 // candidate.
Jim Laskeybc588b82006-10-05 15:07:25 +00009380 while (!Chains.empty()) {
Dan Gohman475871a2008-07-27 21:46:04 +00009381 SDValue Chain = Chains.back();
Jim Laskeybc588b82006-10-05 15:07:25 +00009382 Chains.pop_back();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009383
9384 // For TokenFactor nodes, look at each operand and only continue up the
9385 // chain until we find two aliases. If we've seen two aliases, assume we'll
Nate Begeman677c89d2009-10-12 05:53:58 +00009386 // find more and revert to original chain since the xform is unlikely to be
9387 // profitable.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009388 //
9389 // FIXME: The depth check could be made to return the last non-aliasing
Nate Begeman677c89d2009-10-12 05:53:58 +00009390 // chain we found before we hit a tokenfactor rather than the original
9391 // chain.
9392 if (Depth > 6 || Aliases.size() == 2) {
9393 Aliases.clear();
9394 Aliases.push_back(OriginalChain);
9395 break;
9396 }
Scott Michelfdc40a02009-02-17 22:15:04 +00009397
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009398 // Don't bother if we've been before.
9399 if (!Visited.insert(Chain.getNode()))
9400 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00009401
Jim Laskeybc588b82006-10-05 15:07:25 +00009402 switch (Chain.getOpcode()) {
9403 case ISD::EntryToken:
9404 // Entry token is ideal chain operand, but handled in FindBetterChain.
9405 break;
Scott Michelfdc40a02009-02-17 22:15:04 +00009406
Jim Laskeybc588b82006-10-05 15:07:25 +00009407 case ISD::LOAD:
9408 case ISD::STORE: {
9409 // Get alias information for Chain.
Dan Gohman475871a2008-07-27 21:46:04 +00009410 SDValue OpPtr;
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009411 int64_t OpSize;
9412 const Value *OpSrcValue;
9413 int OpSrcValueOffset;
9414 unsigned OpSrcValueAlign;
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00009415 const MDNode *OpSrcTBAAInfo;
Gabor Greifba36cb52008-08-28 21:40:38 +00009416 bool IsOpLoad = FindAliasInfo(Chain.getNode(), OpPtr, OpSize,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009417 OpSrcValue, OpSrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00009418 OpSrcValueAlign,
9419 OpSrcTBAAInfo);
Scott Michelfdc40a02009-02-17 22:15:04 +00009420
Jim Laskeybc588b82006-10-05 15:07:25 +00009421 // If chain is alias then stop here.
9422 if (!(IsLoad && IsOpLoad) &&
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009423 isAlias(Ptr, Size, SrcValue, SrcValueOffset, SrcValueAlign,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00009424 SrcTBAAInfo,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009425 OpPtr, OpSize, OpSrcValue, OpSrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00009426 OpSrcValueAlign, OpSrcTBAAInfo)) {
Jim Laskeybc588b82006-10-05 15:07:25 +00009427 Aliases.push_back(Chain);
9428 } else {
9429 // Look further up the chain.
Scott Michelfdc40a02009-02-17 22:15:04 +00009430 Chains.push_back(Chain.getOperand(0));
Nate Begeman677c89d2009-10-12 05:53:58 +00009431 ++Depth;
Jim Laskey279f0532006-09-25 16:29:54 +00009432 }
Jim Laskeybc588b82006-10-05 15:07:25 +00009433 break;
9434 }
Scott Michelfdc40a02009-02-17 22:15:04 +00009435
Jim Laskeybc588b82006-10-05 15:07:25 +00009436 case ISD::TokenFactor:
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009437 // We have to check each of the operands of the token factor for "small"
9438 // token factors, so we queue them up. Adding the operands to the queue
9439 // (stack) in reverse order maintains the original order and increases the
9440 // likelihood that getNode will find a matching token factor (CSE.)
9441 if (Chain.getNumOperands() > 16) {
9442 Aliases.push_back(Chain);
9443 break;
9444 }
Jim Laskeybc588b82006-10-05 15:07:25 +00009445 for (unsigned n = Chain.getNumOperands(); n;)
9446 Chains.push_back(Chain.getOperand(--n));
Nate Begeman677c89d2009-10-12 05:53:58 +00009447 ++Depth;
Jim Laskeybc588b82006-10-05 15:07:25 +00009448 break;
Scott Michelfdc40a02009-02-17 22:15:04 +00009449
Jim Laskeybc588b82006-10-05 15:07:25 +00009450 default:
9451 // For all other instructions we will just have to take what we can get.
9452 Aliases.push_back(Chain);
9453 break;
Jim Laskey279f0532006-09-25 16:29:54 +00009454 }
9455 }
Jim Laskey6ff23e52006-10-04 16:53:27 +00009456}
9457
9458/// FindBetterChain - Walk up chain skipping non-aliasing memory nodes, looking
9459/// for a better chain (aliasing node.)
Dan Gohman475871a2008-07-27 21:46:04 +00009460SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) {
9461 SmallVector<SDValue, 8> Aliases; // Ops for replacing token factor.
Scott Michelfdc40a02009-02-17 22:15:04 +00009462
Jim Laskey6ff23e52006-10-04 16:53:27 +00009463 // Accumulate all the aliases to this node.
9464 GatherAllAliases(N, OldChain, Aliases);
Scott Michelfdc40a02009-02-17 22:15:04 +00009465
Dan Gohman71dc7c92011-05-17 22:20:36 +00009466 // If no operands then chain to entry token.
9467 if (Aliases.size() == 0)
Jim Laskey6ff23e52006-10-04 16:53:27 +00009468 return DAG.getEntryNode();
Dan Gohman71dc7c92011-05-17 22:20:36 +00009469
9470 // If a single operand then chain to it. We don't need to revisit it.
9471 if (Aliases.size() == 1)
Jim Laskey6ff23e52006-10-04 16:53:27 +00009472 return Aliases[0];
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009473
Jim Laskey6ff23e52006-10-04 16:53:27 +00009474 // Construct a custom tailored token factor.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00009475 return DAG.getNode(ISD::TokenFactor, N->getDebugLoc(), MVT::Other,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00009476 &Aliases[0], Aliases.size());
Jim Laskey279f0532006-09-25 16:29:54 +00009477}
9478
Nate Begeman1d4d4142005-09-01 00:19:25 +00009479// SelectionDAG::Combine - This is the entry point for the file.
9480//
Bill Wendlingbe8cc2a2009-04-29 00:15:41 +00009481void SelectionDAG::Combine(CombineLevel Level, AliasAnalysis &AA,
Bill Wendling98a366d2009-04-29 23:29:43 +00009482 CodeGenOpt::Level OptLevel) {
Nate Begeman1d4d4142005-09-01 00:19:25 +00009483 /// run - This is the main entry point to this class.
9484 ///
Bill Wendlingbe8cc2a2009-04-29 00:15:41 +00009485 DAGCombiner(*this, AA, OptLevel).Run(Level);
Nate Begeman1d4d4142005-09-01 00:19:25 +00009486}