blob: 4b56e41b72cbc78fe65982c46c9cab4c4297eabd [file] [log] [blame]
Nate Begeman4ebd8052005-09-01 23:24:04 +00001//===-- DAGCombiner.cpp - Implement a DAG node combiner -------------------===//
Nate Begeman1d4d4142005-09-01 00:19:25 +00002//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner4ee451d2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Nate Begeman1d4d4142005-09-01 00:19:25 +00007//
8//===----------------------------------------------------------------------===//
9//
10// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
11// both before and after the DAG is legalized.
Scott Michelfdc40a02009-02-17 22:15:04 +000012//
Dan Gohman41287002009-04-25 17:09:45 +000013// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
14// primarily intended to handle simplification opportunities that are implicit
15// in the LLVM IR and exposed by the various codegen lowering phases.
16//
Nate Begeman1d4d4142005-09-01 00:19:25 +000017//===----------------------------------------------------------------------===//
18
19#define DEBUG_TYPE "dagcombine"
Nate Begeman1d4d4142005-09-01 00:19:25 +000020#include "llvm/CodeGen/SelectionDAG.h"
Chris Lattner600fec32009-03-11 05:08:08 +000021#include "llvm/DerivedTypes.h"
Owen Andersona90b3dc2009-07-15 21:51:10 +000022#include "llvm/LLVMContext.h"
Chris Lattner00161a62008-01-25 07:20:16 +000023#include "llvm/CodeGen/MachineFunction.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000025#include "llvm/Analysis/AliasAnalysis.h"
Evan Cheng59d5b682007-05-07 21:27:48 +000026#include "llvm/Target/TargetData.h"
Nate Begeman1d4d4142005-09-01 00:19:25 +000027#include "llvm/Target/TargetLowering.h"
Evan Cheng59d5b682007-05-07 21:27:48 +000028#include "llvm/Target/TargetMachine.h"
Chris Lattnerddae4bd2007-01-08 23:04:05 +000029#include "llvm/Target/TargetOptions.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000030#include "llvm/ADT/SmallPtrSet.h"
31#include "llvm/ADT/Statistic.h"
Jim Laskeyd1aed7a2006-09-21 16:28:59 +000032#include "llvm/Support/CommandLine.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000033#include "llvm/Support/Debug.h"
Torok Edwin7d696d82009-07-11 13:10:19 +000034#include "llvm/Support/ErrorHandling.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000035#include "llvm/Support/MathExtras.h"
Chris Lattnerbbbfa992009-08-23 06:35:02 +000036#include "llvm/Support/raw_ostream.h"
Chris Lattnera500fc62005-09-09 23:53:39 +000037#include <algorithm>
Nate Begeman1d4d4142005-09-01 00:19:25 +000038using namespace llvm;
39
Chris Lattnercd3245a2006-12-19 22:41:21 +000040STATISTIC(NodesCombined , "Number of dag nodes combined");
41STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
42STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
Evan Cheng8b944d32009-05-28 00:35:15 +000043STATISTIC(OpsNarrowed , "Number of load/op/store narrowed");
Evan Cheng31959b12011-02-02 01:06:55 +000044STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int");
Chris Lattnercd3245a2006-12-19 22:41:21 +000045
Nate Begeman1d4d4142005-09-01 00:19:25 +000046namespace {
Jim Laskey71382342006-10-07 23:37:56 +000047 static cl::opt<bool>
Owen Anderson0dcc8142010-09-19 21:01:26 +000048 CombinerAA("combiner-alias-analysis", cl::Hidden,
Jim Laskey26f7fa72006-10-17 19:33:52 +000049 cl::desc("Turn on alias analysis during testing"));
Jim Laskey3ad175b2006-10-12 15:22:24 +000050
Jim Laskey07a27092006-10-18 19:08:31 +000051 static cl::opt<bool>
52 CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
53 cl::desc("Include global information in alias analysis"));
54
Jim Laskeybc588b82006-10-05 15:07:25 +000055//------------------------------ DAGCombiner ---------------------------------//
56
Nick Lewycky6726b6d2009-10-25 06:33:48 +000057 class DAGCombiner {
Nate Begeman1d4d4142005-09-01 00:19:25 +000058 SelectionDAG &DAG;
Dan Gohman79ce2762009-01-15 19:20:50 +000059 const TargetLowering &TLI;
Duncan Sands25cf2272008-11-24 14:53:14 +000060 CombineLevel Level;
Bill Wendling98a366d2009-04-29 23:29:43 +000061 CodeGenOpt::Level OptLevel;
Duncan Sands25cf2272008-11-24 14:53:14 +000062 bool LegalOperations;
63 bool LegalTypes;
Nate Begeman1d4d4142005-09-01 00:19:25 +000064
65 // Worklist of all of the nodes that need to be simplified.
James Molloy6660c052012-02-16 09:17:04 +000066 //
67 // This has the semantics that when adding to the worklist,
68 // the item added must be next to be processed. It should
69 // also only appear once. The naive approach to this takes
70 // linear time.
71 //
72 // To reduce the insert/remove time to logarithmic, we use
73 // a set and a vector to maintain our worklist.
74 //
75 // The set contains the items on the worklist, but does not
76 // maintain the order they should be visited.
77 //
78 // The vector maintains the order nodes should be visited, but may
79 // contain duplicate or removed nodes. When choosing a node to
80 // visit, we pop off the order stack until we find an item that is
81 // also in the contents set. All operations are O(log N).
82 SmallPtrSet<SDNode*, 64> WorkListContents;
Benjamin Kramerd5f76902012-03-10 00:23:58 +000083 SmallVector<SDNode*, 64> WorkListOrder;
Nate Begeman1d4d4142005-09-01 00:19:25 +000084
Jim Laskeyc7c3f112006-10-16 20:52:31 +000085 // AA - Used for DAG load/store alias analysis.
86 AliasAnalysis &AA;
87
Nate Begeman1d4d4142005-09-01 00:19:25 +000088 /// AddUsersToWorkList - When an instruction is simplified, add all users of
89 /// the instruction to the work lists because they might get more simplified
90 /// now.
91 ///
92 void AddUsersToWorkList(SDNode *N) {
93 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
Nate Begeman4ebd8052005-09-01 23:24:04 +000094 UI != UE; ++UI)
Dan Gohman89684502008-07-27 20:43:25 +000095 AddToWorkList(*UI);
Nate Begeman1d4d4142005-09-01 00:19:25 +000096 }
97
Dan Gohman389079b2007-10-08 17:57:15 +000098 /// visit - call the node-specific routine that knows how to fold each
99 /// particular type of node.
Dan Gohman475871a2008-07-27 21:46:04 +0000100 SDValue visit(SDNode *N);
Dan Gohman389079b2007-10-08 17:57:15 +0000101
Chris Lattner24664722006-03-01 04:53:38 +0000102 public:
James Molloy6afa3f72012-02-16 09:48:07 +0000103 /// AddToWorkList - Add to the work list making sure its instance is at the
James Molloy6660c052012-02-16 09:17:04 +0000104 /// back (next to be processed.)
Chris Lattner5750df92006-03-01 04:03:14 +0000105 void AddToWorkList(SDNode *N) {
James Molloy6660c052012-02-16 09:17:04 +0000106 WorkListContents.insert(N);
107 WorkListOrder.push_back(N);
Chris Lattner5750df92006-03-01 04:03:14 +0000108 }
Jim Laskey6ff23e52006-10-04 16:53:27 +0000109
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000110 /// removeFromWorkList - remove all instances of N from the worklist.
111 ///
112 void removeFromWorkList(SDNode *N) {
James Molloy6660c052012-02-16 09:17:04 +0000113 WorkListContents.erase(N);
Chris Lattner01a22022005-10-10 22:04:48 +0000114 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000115
Dan Gohman475871a2008-07-27 21:46:04 +0000116 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
Evan Cheng0b0cd912009-03-28 05:57:29 +0000117 bool AddTo = true);
Scott Michelfdc40a02009-02-17 22:15:04 +0000118
Dan Gohman475871a2008-07-27 21:46:04 +0000119 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
Jim Laskey274062c2006-10-13 23:32:28 +0000120 return CombineTo(N, &Res, 1, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000121 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000122
Dan Gohman475871a2008-07-27 21:46:04 +0000123 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
Evan Cheng0b0cd912009-03-28 05:57:29 +0000124 bool AddTo = true) {
Dan Gohman475871a2008-07-27 21:46:04 +0000125 SDValue To[] = { Res0, Res1 };
Jim Laskey274062c2006-10-13 23:32:28 +0000126 return CombineTo(N, To, 2, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000127 }
Dan Gohmane5af2d32009-01-29 01:59:02 +0000128
129 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
Scott Michelfdc40a02009-02-17 22:15:04 +0000130
131 private:
132
Chris Lattner012f2412006-02-17 21:58:01 +0000133 /// SimplifyDemandedBits - Check the specified integer node value to see if
Chris Lattnerb2742f42006-03-01 19:55:35 +0000134 /// it can be simplified or if things it uses can be simplified by bit
Chris Lattner012f2412006-02-17 21:58:01 +0000135 /// propagation. If so, return true.
Dan Gohman475871a2008-07-27 21:46:04 +0000136 bool SimplifyDemandedBits(SDValue Op) {
Dan Gohman87862e72009-12-11 21:31:27 +0000137 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
138 APInt Demanded = APInt::getAllOnesValue(BitWidth);
Dan Gohman7b8d4a92008-02-27 00:25:32 +0000139 return SimplifyDemandedBits(Op, Demanded);
140 }
141
Dan Gohman475871a2008-07-27 21:46:04 +0000142 bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
Chris Lattner87514ca2005-10-10 22:31:19 +0000143
Chris Lattner448f2192006-11-11 00:39:41 +0000144 bool CombineToPreIndexedLoadStore(SDNode *N);
145 bool CombineToPostIndexedLoadStore(SDNode *N);
Scott Michelfdc40a02009-02-17 22:15:04 +0000146
Evan Cheng95c57ea2010-04-24 04:43:44 +0000147 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
148 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
149 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
150 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
Evan Cheng64b7bf72010-04-16 06:14:10 +0000151 SDValue PromoteIntBinOp(SDValue Op);
Evan Cheng07c4e102010-04-22 20:19:46 +0000152 SDValue PromoteIntShiftOp(SDValue Op);
Evan Cheng4c26e932010-04-19 19:29:22 +0000153 SDValue PromoteExtend(SDValue Op);
154 bool PromoteLoad(SDValue Op);
Scott Michelfdc40a02009-02-17 22:15:04 +0000155
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +0000156 void ExtendSetCCUses(SmallVector<SDNode*, 4> SetCCs,
157 SDValue Trunc, SDValue ExtLoad, DebugLoc DL,
158 ISD::NodeType ExtType);
159
Dan Gohman389079b2007-10-08 17:57:15 +0000160 /// combine - call the node-specific routine that knows how to fold each
161 /// particular type of node. If that doesn't do anything, try the
162 /// target-specific DAG combines.
Dan Gohman475871a2008-07-27 21:46:04 +0000163 SDValue combine(SDNode *N);
Nate Begeman1d4d4142005-09-01 00:19:25 +0000164
165 // Visitation implementation - Implement dag node combining for different
166 // node types. The semantics are as follows:
167 // Return Value:
Evan Cheng17a568b2008-08-29 22:21:44 +0000168 // SDValue.getNode() == 0 - No change was made
169 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
170 // otherwise - N should be replaced by the returned Operand.
Nate Begeman1d4d4142005-09-01 00:19:25 +0000171 //
Dan Gohman475871a2008-07-27 21:46:04 +0000172 SDValue visitTokenFactor(SDNode *N);
173 SDValue visitMERGE_VALUES(SDNode *N);
174 SDValue visitADD(SDNode *N);
175 SDValue visitSUB(SDNode *N);
176 SDValue visitADDC(SDNode *N);
Craig Toppercc274522012-01-07 09:06:39 +0000177 SDValue visitSUBC(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000178 SDValue visitADDE(SDNode *N);
Craig Toppercc274522012-01-07 09:06:39 +0000179 SDValue visitSUBE(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000180 SDValue visitMUL(SDNode *N);
181 SDValue visitSDIV(SDNode *N);
182 SDValue visitUDIV(SDNode *N);
183 SDValue visitSREM(SDNode *N);
184 SDValue visitUREM(SDNode *N);
185 SDValue visitMULHU(SDNode *N);
186 SDValue visitMULHS(SDNode *N);
187 SDValue visitSMUL_LOHI(SDNode *N);
188 SDValue visitUMUL_LOHI(SDNode *N);
Benjamin Kramerf55d26e2011-05-21 18:31:55 +0000189 SDValue visitSMULO(SDNode *N);
190 SDValue visitUMULO(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000191 SDValue visitSDIVREM(SDNode *N);
192 SDValue visitUDIVREM(SDNode *N);
193 SDValue visitAND(SDNode *N);
194 SDValue visitOR(SDNode *N);
195 SDValue visitXOR(SDNode *N);
196 SDValue SimplifyVBinOp(SDNode *N);
197 SDValue visitSHL(SDNode *N);
198 SDValue visitSRA(SDNode *N);
199 SDValue visitSRL(SDNode *N);
200 SDValue visitCTLZ(SDNode *N);
Chandler Carruth63974b22011-12-13 01:56:10 +0000201 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000202 SDValue visitCTTZ(SDNode *N);
Chandler Carruth63974b22011-12-13 01:56:10 +0000203 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000204 SDValue visitCTPOP(SDNode *N);
205 SDValue visitSELECT(SDNode *N);
206 SDValue visitSELECT_CC(SDNode *N);
207 SDValue visitSETCC(SDNode *N);
208 SDValue visitSIGN_EXTEND(SDNode *N);
209 SDValue visitZERO_EXTEND(SDNode *N);
210 SDValue visitANY_EXTEND(SDNode *N);
211 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
212 SDValue visitTRUNCATE(SDNode *N);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000213 SDValue visitBITCAST(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000214 SDValue visitBUILD_PAIR(SDNode *N);
215 SDValue visitFADD(SDNode *N);
216 SDValue visitFSUB(SDNode *N);
217 SDValue visitFMUL(SDNode *N);
Owen Anderson062c0a52012-05-02 22:17:40 +0000218 SDValue visitFMA(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000219 SDValue visitFDIV(SDNode *N);
220 SDValue visitFREM(SDNode *N);
221 SDValue visitFCOPYSIGN(SDNode *N);
222 SDValue visitSINT_TO_FP(SDNode *N);
223 SDValue visitUINT_TO_FP(SDNode *N);
224 SDValue visitFP_TO_SINT(SDNode *N);
225 SDValue visitFP_TO_UINT(SDNode *N);
226 SDValue visitFP_ROUND(SDNode *N);
227 SDValue visitFP_ROUND_INREG(SDNode *N);
228 SDValue visitFP_EXTEND(SDNode *N);
229 SDValue visitFNEG(SDNode *N);
230 SDValue visitFABS(SDNode *N);
231 SDValue visitBRCOND(SDNode *N);
232 SDValue visitBR_CC(SDNode *N);
233 SDValue visitLOAD(SDNode *N);
234 SDValue visitSTORE(SDNode *N);
235 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
236 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
237 SDValue visitBUILD_VECTOR(SDNode *N);
238 SDValue visitCONCAT_VECTORS(SDNode *N);
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +0000239 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000240 SDValue visitVECTOR_SHUFFLE(SDNode *N);
Jim Grosbach9a526492010-06-23 16:07:42 +0000241 SDValue visitMEMBARRIER(SDNode *N);
Chris Lattner01a22022005-10-10 22:04:48 +0000242
Dan Gohman475871a2008-07-27 21:46:04 +0000243 SDValue XformToShuffleWithZero(SDNode *N);
Bill Wendling35247c32009-01-30 00:45:56 +0000244 SDValue ReassociateOps(unsigned Opc, DebugLoc DL, SDValue LHS, SDValue RHS);
Scott Michelfdc40a02009-02-17 22:15:04 +0000245
Dan Gohman475871a2008-07-27 21:46:04 +0000246 SDValue visitShiftByConstant(SDNode *N, unsigned Amt);
Chris Lattnere70da202007-12-06 07:33:36 +0000247
Dan Gohman475871a2008-07-27 21:46:04 +0000248 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
249 SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
Bill Wendling836ca7d2009-01-30 23:59:18 +0000250 SDValue SimplifySelect(DebugLoc DL, SDValue N0, SDValue N1, SDValue N2);
Scott Michelfdc40a02009-02-17 22:15:04 +0000251 SDValue SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1, SDValue N2,
252 SDValue N3, ISD::CondCode CC,
Bill Wendling836ca7d2009-01-30 23:59:18 +0000253 bool NotExtCompare = false);
Owen Andersone50ed302009-08-10 22:56:29 +0000254 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
Dale Johannesenff97d4f2009-02-03 00:47:48 +0000255 DebugLoc DL, bool foldBooleans = true);
Scott Michelfdc40a02009-02-17 22:15:04 +0000256 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
Chris Lattner5eee4272008-01-26 01:09:19 +0000257 unsigned HiOp);
Owen Andersone50ed302009-08-10 22:56:29 +0000258 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000259 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
Dan Gohman475871a2008-07-27 21:46:04 +0000260 SDValue BuildSDIV(SDNode *N);
261 SDValue BuildUDIV(SDNode *N);
Evan Cheng9568e5c2011-06-21 06:01:08 +0000262 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
263 bool DemandHighBits = true);
264 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
Bill Wendling317bd702009-01-30 21:14:50 +0000265 SDNode *MatchRotate(SDValue LHS, SDValue RHS, DebugLoc DL);
Dan Gohman475871a2008-07-27 21:46:04 +0000266 SDValue ReduceLoadWidth(SDNode *N);
Evan Cheng8b944d32009-05-28 00:35:15 +0000267 SDValue ReduceLoadOpStoreWidth(SDNode *N);
Evan Cheng31959b12011-02-02 01:06:55 +0000268 SDValue TransformFPLoadStorePair(SDNode *N);
Scott Michelfdc40a02009-02-17 22:15:04 +0000269
Dan Gohman475871a2008-07-27 21:46:04 +0000270 SDValue GetDemandedBits(SDValue V, const APInt &Mask);
Scott Michelfdc40a02009-02-17 22:15:04 +0000271
Jim Laskey6ff23e52006-10-04 16:53:27 +0000272 /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
273 /// looking for aliasing nodes and adding them to the Aliases vector.
Dan Gohman475871a2008-07-27 21:46:04 +0000274 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
275 SmallVector<SDValue, 8> &Aliases);
Jim Laskey6ff23e52006-10-04 16:53:27 +0000276
Jim Laskey096c22e2006-10-18 12:29:57 +0000277 /// isAlias - Return true if there is any possibility that the two addresses
278 /// overlap.
Dan Gohman475871a2008-07-27 21:46:04 +0000279 bool isAlias(SDValue Ptr1, int64_t Size1,
Jim Laskey096c22e2006-10-18 12:29:57 +0000280 const Value *SrcValue1, int SrcValueOffset1,
Nate Begemanb6aef5c2009-09-15 00:18:30 +0000281 unsigned SrcValueAlign1,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +0000282 const MDNode *TBAAInfo1,
Dan Gohman475871a2008-07-27 21:46:04 +0000283 SDValue Ptr2, int64_t Size2,
Nate Begemanb6aef5c2009-09-15 00:18:30 +0000284 const Value *SrcValue2, int SrcValueOffset2,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +0000285 unsigned SrcValueAlign2,
286 const MDNode *TBAAInfo2) const;
Scott Michelfdc40a02009-02-17 22:15:04 +0000287
Jim Laskey7ca56af2006-10-11 13:47:09 +0000288 /// FindAliasInfo - Extracts the relevant alias information from the memory
289 /// node. Returns true if the operand was a load.
290 bool FindAliasInfo(SDNode *N,
Dan Gohman475871a2008-07-27 21:46:04 +0000291 SDValue &Ptr, int64_t &Size,
Nate Begemanb6aef5c2009-09-15 00:18:30 +0000292 const Value *&SrcValue, int &SrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +0000293 unsigned &SrcValueAlignment,
294 const MDNode *&TBAAInfo) const;
Scott Michelfdc40a02009-02-17 22:15:04 +0000295
Jim Laskey279f0532006-09-25 16:29:54 +0000296 /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes,
Jim Laskey6ff23e52006-10-04 16:53:27 +0000297 /// looking for a better chain (aliasing node.)
Dan Gohman475871a2008-07-27 21:46:04 +0000298 SDValue FindBetterChain(SDNode *N, SDValue Chain);
Duncan Sands92abc622009-01-31 15:50:11 +0000299
Chris Lattner2392ae72010-04-15 04:48:01 +0000300 public:
Bill Wendling98a366d2009-04-29 23:29:43 +0000301 DAGCombiner(SelectionDAG &D, AliasAnalysis &A, CodeGenOpt::Level OL)
Eli Friedman50185242011-11-12 00:35:34 +0000302 : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
Chris Lattner2392ae72010-04-15 04:48:01 +0000303 OptLevel(OL), LegalOperations(false), LegalTypes(false), AA(A) {}
Scott Michelfdc40a02009-02-17 22:15:04 +0000304
Nate Begeman1d4d4142005-09-01 00:19:25 +0000305 /// Run - runs the dag combiner on all nodes in the work list
Duncan Sands25cf2272008-11-24 14:53:14 +0000306 void Run(CombineLevel AtLevel);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000307
Chris Lattner2392ae72010-04-15 04:48:01 +0000308 SelectionDAG &getDAG() const { return DAG; }
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000309
Chris Lattner2392ae72010-04-15 04:48:01 +0000310 /// getShiftAmountTy - Returns a type large enough to hold any valid
311 /// shift amount - before type legalization these can be huge.
Owen Anderson95771af2011-02-25 21:41:48 +0000312 EVT getShiftAmountTy(EVT LHSTy) {
313 return LegalTypes ? TLI.getShiftAmountTy(LHSTy) : TLI.getPointerTy();
Chris Lattner2392ae72010-04-15 04:48:01 +0000314 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000315
Chris Lattner2392ae72010-04-15 04:48:01 +0000316 /// isTypeLegal - This method returns true if we are running before type
317 /// legalization or if the specified VT is legal.
318 bool isTypeLegal(const EVT &VT) {
319 if (!LegalTypes) return true;
320 return TLI.isTypeLegal(VT);
321 }
Nate Begeman1d4d4142005-09-01 00:19:25 +0000322 };
323}
324
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000325
326namespace {
327/// WorkListRemover - This class is a DAGUpdateListener that removes any deleted
328/// nodes from the worklist.
Nick Lewycky6726b6d2009-10-25 06:33:48 +0000329class WorkListRemover : public SelectionDAG::DAGUpdateListener {
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000330 DAGCombiner &DC;
331public:
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000332 explicit WorkListRemover(DAGCombiner &dc)
333 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
Scott Michelfdc40a02009-02-17 22:15:04 +0000334
Duncan Sandsedfcf592008-06-11 11:42:12 +0000335 virtual void NodeDeleted(SDNode *N, SDNode *E) {
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000336 DC.removeFromWorkList(N);
337 }
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000338};
339}
340
Chris Lattner24664722006-03-01 04:53:38 +0000341//===----------------------------------------------------------------------===//
342// TargetLowering::DAGCombinerInfo implementation
343//===----------------------------------------------------------------------===//
344
345void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
346 ((DAGCombiner*)DC)->AddToWorkList(N);
347}
348
Cameron Zwariched3caf92011-04-02 02:40:26 +0000349void TargetLowering::DAGCombinerInfo::RemoveFromWorklist(SDNode *N) {
350 ((DAGCombiner*)DC)->removeFromWorkList(N);
351}
352
Dan Gohman475871a2008-07-27 21:46:04 +0000353SDValue TargetLowering::DAGCombinerInfo::
Evan Cheng0b0cd912009-03-28 05:57:29 +0000354CombineTo(SDNode *N, const std::vector<SDValue> &To, bool AddTo) {
355 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000356}
357
Dan Gohman475871a2008-07-27 21:46:04 +0000358SDValue TargetLowering::DAGCombinerInfo::
Evan Cheng0b0cd912009-03-28 05:57:29 +0000359CombineTo(SDNode *N, SDValue Res, bool AddTo) {
360 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000361}
362
363
Dan Gohman475871a2008-07-27 21:46:04 +0000364SDValue TargetLowering::DAGCombinerInfo::
Evan Cheng0b0cd912009-03-28 05:57:29 +0000365CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
366 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000367}
368
Dan Gohmane5af2d32009-01-29 01:59:02 +0000369void TargetLowering::DAGCombinerInfo::
370CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
371 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
372}
Chris Lattner24664722006-03-01 04:53:38 +0000373
Chris Lattner24664722006-03-01 04:53:38 +0000374//===----------------------------------------------------------------------===//
Chris Lattner29446522007-05-14 22:04:50 +0000375// Helper Functions
376//===----------------------------------------------------------------------===//
377
378/// isNegatibleForFree - Return 1 if we can compute the negated form of the
379/// specified expression for the same cost as the expression itself, or 2 if we
380/// can compute the negated form more cheaply than the expression itself.
Duncan Sands25cf2272008-11-24 14:53:14 +0000381static char isNegatibleForFree(SDValue Op, bool LegalOperations,
Owen Andersonafd3d562012-03-06 00:29:31 +0000382 const TargetLowering &TLI,
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000383 const TargetOptions *Options,
Chris Lattner0254e702008-02-26 07:04:54 +0000384 unsigned Depth = 0) {
Dale Johannesendb44bf82007-10-16 23:38:29 +0000385 // No compile time optimizations on this type.
Owen Anderson825b72b2009-08-11 20:47:22 +0000386 if (Op.getValueType() == MVT::ppcf128)
Dale Johannesendb44bf82007-10-16 23:38:29 +0000387 return 0;
388
Chris Lattner29446522007-05-14 22:04:50 +0000389 // fneg is removable even if it has multiple uses.
390 if (Op.getOpcode() == ISD::FNEG) return 2;
Scott Michelfdc40a02009-02-17 22:15:04 +0000391
Chris Lattner29446522007-05-14 22:04:50 +0000392 // Don't allow anything with multiple uses.
393 if (!Op.hasOneUse()) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000394
Chris Lattner3adf9512007-05-25 02:19:06 +0000395 // Don't recurse exponentially.
396 if (Depth > 6) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000397
Chris Lattner29446522007-05-14 22:04:50 +0000398 switch (Op.getOpcode()) {
399 default: return false;
400 case ISD::ConstantFP:
Chris Lattner0254e702008-02-26 07:04:54 +0000401 // Don't invert constant FP values after legalize. The negated constant
402 // isn't necessarily legal.
Duncan Sands25cf2272008-11-24 14:53:14 +0000403 return LegalOperations ? 0 : 1;
Chris Lattner29446522007-05-14 22:04:50 +0000404 case ISD::FADD:
405 // FIXME: determine better conditions for this xform.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000406 if (!Options->UnsafeFPMath) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000407
Owen Andersonafd3d562012-03-06 00:29:31 +0000408 // After operation legalization, it might not be legal to create new FSUBs.
409 if (LegalOperations &&
410 !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType()))
411 return 0;
412
Bill Wendlingd34470c2009-01-30 23:10:18 +0000413 // fold (fsub (fadd A, B)) -> (fsub (fneg A), B)
Owen Andersonafd3d562012-03-06 00:29:31 +0000414 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
415 Options, Depth + 1))
Chris Lattner29446522007-05-14 22:04:50 +0000416 return V;
Bill Wendlingd34470c2009-01-30 23:10:18 +0000417 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
Owen Andersonafd3d562012-03-06 00:29:31 +0000418 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000419 Depth + 1);
Chris Lattner29446522007-05-14 22:04:50 +0000420 case ISD::FSUB:
Scott Michelfdc40a02009-02-17 22:15:04 +0000421 // We can't turn -(A-B) into B-A when we honor signed zeros.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000422 if (!Options->UnsafeFPMath) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000423
Bill Wendlingd34470c2009-01-30 23:10:18 +0000424 // fold (fneg (fsub A, B)) -> (fsub B, A)
Chris Lattner29446522007-05-14 22:04:50 +0000425 return 1;
Scott Michelfdc40a02009-02-17 22:15:04 +0000426
Chris Lattner29446522007-05-14 22:04:50 +0000427 case ISD::FMUL:
428 case ISD::FDIV:
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000429 if (Options->HonorSignDependentRoundingFPMath()) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000430
Bill Wendlingd34470c2009-01-30 23:10:18 +0000431 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
Owen Andersonafd3d562012-03-06 00:29:31 +0000432 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
433 Options, Depth + 1))
Chris Lattner29446522007-05-14 22:04:50 +0000434 return V;
Scott Michelfdc40a02009-02-17 22:15:04 +0000435
Owen Andersonafd3d562012-03-06 00:29:31 +0000436 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000437 Depth + 1);
Scott Michelfdc40a02009-02-17 22:15:04 +0000438
Chris Lattner29446522007-05-14 22:04:50 +0000439 case ISD::FP_EXTEND:
440 case ISD::FP_ROUND:
441 case ISD::FSIN:
Owen Andersonafd3d562012-03-06 00:29:31 +0000442 return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000443 Depth + 1);
Chris Lattner29446522007-05-14 22:04:50 +0000444 }
445}
446
447/// GetNegatedExpression - If isNegatibleForFree returns true, this function
448/// returns the newly negated expression.
Dan Gohman475871a2008-07-27 21:46:04 +0000449static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000450 bool LegalOperations, unsigned Depth = 0) {
Chris Lattner29446522007-05-14 22:04:50 +0000451 // fneg is removable even if it has multiple uses.
452 if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
Scott Michelfdc40a02009-02-17 22:15:04 +0000453
Chris Lattner29446522007-05-14 22:04:50 +0000454 // Don't allow anything with multiple uses.
455 assert(Op.hasOneUse() && "Unknown reuse!");
Scott Michelfdc40a02009-02-17 22:15:04 +0000456
Chris Lattner3adf9512007-05-25 02:19:06 +0000457 assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
Chris Lattner29446522007-05-14 22:04:50 +0000458 switch (Op.getOpcode()) {
Torok Edwinc23197a2009-07-14 16:55:14 +0000459 default: llvm_unreachable("Unknown code");
Dale Johannesenc4dd3c32007-08-31 23:34:27 +0000460 case ISD::ConstantFP: {
461 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
462 V.changeSign();
463 return DAG.getConstantFP(V, Op.getValueType());
464 }
Chris Lattner29446522007-05-14 22:04:50 +0000465 case ISD::FADD:
466 // FIXME: determine better conditions for this xform.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000467 assert(DAG.getTarget().Options.UnsafeFPMath);
Scott Michelfdc40a02009-02-17 22:15:04 +0000468
Bill Wendlingd34470c2009-01-30 23:10:18 +0000469 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000470 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
Owen Andersonafd3d562012-03-06 00:29:31 +0000471 DAG.getTargetLoweringInfo(),
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000472 &DAG.getTarget().Options, Depth+1))
Bill Wendling35247c32009-01-30 00:45:56 +0000473 return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000474 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000475 LegalOperations, Depth+1),
Chris Lattner29446522007-05-14 22:04:50 +0000476 Op.getOperand(1));
Bill Wendlingd34470c2009-01-30 23:10:18 +0000477 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
Bill Wendling35247c32009-01-30 00:45:56 +0000478 return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000479 GetNegatedExpression(Op.getOperand(1), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000480 LegalOperations, Depth+1),
Chris Lattner29446522007-05-14 22:04:50 +0000481 Op.getOperand(0));
482 case ISD::FSUB:
Scott Michelfdc40a02009-02-17 22:15:04 +0000483 // We can't turn -(A-B) into B-A when we honor signed zeros.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000484 assert(DAG.getTarget().Options.UnsafeFPMath);
Dan Gohman23ff1822007-07-02 15:48:56 +0000485
Bill Wendlingd34470c2009-01-30 23:10:18 +0000486 // fold (fneg (fsub 0, B)) -> B
Dan Gohman23ff1822007-07-02 15:48:56 +0000487 if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
Dale Johannesenc4dd3c32007-08-31 23:34:27 +0000488 if (N0CFP->getValueAPF().isZero())
Dan Gohman23ff1822007-07-02 15:48:56 +0000489 return Op.getOperand(1);
Scott Michelfdc40a02009-02-17 22:15:04 +0000490
Bill Wendlingd34470c2009-01-30 23:10:18 +0000491 // fold (fneg (fsub A, B)) -> (fsub B, A)
Bill Wendling35247c32009-01-30 00:45:56 +0000492 return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(),
493 Op.getOperand(1), Op.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +0000494
Chris Lattner29446522007-05-14 22:04:50 +0000495 case ISD::FMUL:
496 case ISD::FDIV:
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000497 assert(!DAG.getTarget().Options.HonorSignDependentRoundingFPMath());
Scott Michelfdc40a02009-02-17 22:15:04 +0000498
Bill Wendlingd34470c2009-01-30 23:10:18 +0000499 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000500 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
Owen Andersonafd3d562012-03-06 00:29:31 +0000501 DAG.getTargetLoweringInfo(),
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000502 &DAG.getTarget().Options, Depth+1))
Bill Wendling35247c32009-01-30 00:45:56 +0000503 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000504 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000505 LegalOperations, Depth+1),
Chris Lattner29446522007-05-14 22:04:50 +0000506 Op.getOperand(1));
Scott Michelfdc40a02009-02-17 22:15:04 +0000507
Bill Wendlingd34470c2009-01-30 23:10:18 +0000508 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
Bill Wendling35247c32009-01-30 00:45:56 +0000509 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(),
Chris Lattner29446522007-05-14 22:04:50 +0000510 Op.getOperand(0),
Chris Lattner0254e702008-02-26 07:04:54 +0000511 GetNegatedExpression(Op.getOperand(1), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000512 LegalOperations, Depth+1));
Scott Michelfdc40a02009-02-17 22:15:04 +0000513
Chris Lattner29446522007-05-14 22:04:50 +0000514 case ISD::FP_EXTEND:
Chris Lattner29446522007-05-14 22:04:50 +0000515 case ISD::FSIN:
Bill Wendling35247c32009-01-30 00:45:56 +0000516 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000517 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000518 LegalOperations, Depth+1));
Chris Lattner0bd48932008-01-17 07:00:52 +0000519 case ISD::FP_ROUND:
Bill Wendling35247c32009-01-30 00:45:56 +0000520 return DAG.getNode(ISD::FP_ROUND, Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000521 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000522 LegalOperations, Depth+1),
Chris Lattner0bd48932008-01-17 07:00:52 +0000523 Op.getOperand(1));
Chris Lattner29446522007-05-14 22:04:50 +0000524 }
525}
Chris Lattner24664722006-03-01 04:53:38 +0000526
527
Nate Begeman4ebd8052005-09-01 23:24:04 +0000528// isSetCCEquivalent - Return true if this node is a setcc, or is a select_cc
529// that selects between the values 1 and 0, making it equivalent to a setcc.
Scott Michelfdc40a02009-02-17 22:15:04 +0000530// Also, set the incoming LHS, RHS, and CC references to the appropriate
Nate Begeman646d7e22005-09-02 21:18:40 +0000531// nodes based on the type of node we are checking. This simplifies life a
532// bit for the callers.
Dan Gohman475871a2008-07-27 21:46:04 +0000533static bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
534 SDValue &CC) {
Nate Begeman646d7e22005-09-02 21:18:40 +0000535 if (N.getOpcode() == ISD::SETCC) {
536 LHS = N.getOperand(0);
537 RHS = N.getOperand(1);
538 CC = N.getOperand(2);
Nate Begeman4ebd8052005-09-01 23:24:04 +0000539 return true;
Nate Begeman646d7e22005-09-02 21:18:40 +0000540 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000541 if (N.getOpcode() == ISD::SELECT_CC &&
Nate Begeman1d4d4142005-09-01 00:19:25 +0000542 N.getOperand(2).getOpcode() == ISD::Constant &&
543 N.getOperand(3).getOpcode() == ISD::Constant &&
Dan Gohman002e5d02008-03-13 22:13:53 +0000544 cast<ConstantSDNode>(N.getOperand(2))->getAPIntValue() == 1 &&
Nate Begeman646d7e22005-09-02 21:18:40 +0000545 cast<ConstantSDNode>(N.getOperand(3))->isNullValue()) {
546 LHS = N.getOperand(0);
547 RHS = N.getOperand(1);
548 CC = N.getOperand(4);
Nate Begeman1d4d4142005-09-01 00:19:25 +0000549 return true;
Nate Begeman646d7e22005-09-02 21:18:40 +0000550 }
Nate Begeman1d4d4142005-09-01 00:19:25 +0000551 return false;
552}
553
Nate Begeman99801192005-09-07 23:25:52 +0000554// isOneUseSetCC - Return true if this is a SetCC-equivalent operation with only
555// one use. If this is true, it allows the users to invert the operation for
556// free when it is profitable to do so.
Dan Gohman475871a2008-07-27 21:46:04 +0000557static bool isOneUseSetCC(SDValue N) {
558 SDValue N0, N1, N2;
Gabor Greifba36cb52008-08-28 21:40:38 +0000559 if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
Nate Begeman4ebd8052005-09-01 23:24:04 +0000560 return true;
561 return false;
562}
563
Bill Wendling35247c32009-01-30 00:45:56 +0000564SDValue DAGCombiner::ReassociateOps(unsigned Opc, DebugLoc DL,
565 SDValue N0, SDValue N1) {
Owen Andersone50ed302009-08-10 22:56:29 +0000566 EVT VT = N0.getValueType();
Nate Begemancd4d58c2006-02-03 06:46:56 +0000567 if (N0.getOpcode() == Opc && isa<ConstantSDNode>(N0.getOperand(1))) {
568 if (isa<ConstantSDNode>(N1)) {
Bill Wendling35247c32009-01-30 00:45:56 +0000569 // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
Bill Wendling6af76182009-01-30 20:50:00 +0000570 SDValue OpNode =
571 DAG.FoldConstantArithmetic(Opc, VT,
572 cast<ConstantSDNode>(N0.getOperand(1)),
573 cast<ConstantSDNode>(N1));
Bill Wendlingd69c3142009-01-30 02:23:43 +0000574 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
Dan Gohman71dc7c92011-05-17 22:20:36 +0000575 }
576 if (N0.hasOneUse()) {
Bill Wendling35247c32009-01-30 00:45:56 +0000577 // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one use
578 SDValue OpNode = DAG.getNode(Opc, N0.getDebugLoc(), VT,
579 N0.getOperand(0), N1);
Gabor Greifba36cb52008-08-28 21:40:38 +0000580 AddToWorkList(OpNode.getNode());
Bill Wendling35247c32009-01-30 00:45:56 +0000581 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
Nate Begemancd4d58c2006-02-03 06:46:56 +0000582 }
583 }
Bill Wendling35247c32009-01-30 00:45:56 +0000584
Nate Begemancd4d58c2006-02-03 06:46:56 +0000585 if (N1.getOpcode() == Opc && isa<ConstantSDNode>(N1.getOperand(1))) {
586 if (isa<ConstantSDNode>(N0)) {
Bill Wendling35247c32009-01-30 00:45:56 +0000587 // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
Bill Wendling6af76182009-01-30 20:50:00 +0000588 SDValue OpNode =
589 DAG.FoldConstantArithmetic(Opc, VT,
590 cast<ConstantSDNode>(N1.getOperand(1)),
591 cast<ConstantSDNode>(N0));
Bill Wendlingd69c3142009-01-30 02:23:43 +0000592 return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
Dan Gohman71dc7c92011-05-17 22:20:36 +0000593 }
594 if (N1.hasOneUse()) {
Bill Wendling35247c32009-01-30 00:45:56 +0000595 // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one use
Bill Wendlingd69c3142009-01-30 02:23:43 +0000596 SDValue OpNode = DAG.getNode(Opc, N0.getDebugLoc(), VT,
Bill Wendling35247c32009-01-30 00:45:56 +0000597 N1.getOperand(0), N0);
Gabor Greifba36cb52008-08-28 21:40:38 +0000598 AddToWorkList(OpNode.getNode());
Bill Wendling35247c32009-01-30 00:45:56 +0000599 return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
Nate Begemancd4d58c2006-02-03 06:46:56 +0000600 }
601 }
Bill Wendling35247c32009-01-30 00:45:56 +0000602
Dan Gohman475871a2008-07-27 21:46:04 +0000603 return SDValue();
Nate Begemancd4d58c2006-02-03 06:46:56 +0000604}
605
Dan Gohman475871a2008-07-27 21:46:04 +0000606SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
607 bool AddTo) {
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000608 assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
609 ++NodesCombined;
David Greenef1090292010-01-05 01:25:00 +0000610 DEBUG(dbgs() << "\nReplacing.1 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000611 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000612 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000613 To[0].getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000614 dbgs() << " and " << NumTo-1 << " other values\n";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000615 for (unsigned i = 0, e = NumTo; i != e; ++i)
Jakob Stoklund Olesen9f0d4e62009-12-03 05:15:35 +0000616 assert((!To[i].getNode() ||
617 N->getValueType(i) == To[i].getValueType()) &&
Dan Gohman764fd0c2009-01-21 15:17:51 +0000618 "Cannot combine value to value of different type!"));
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000619 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000620 DAG.ReplaceAllUsesWith(N, To);
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000621 if (AddTo) {
622 // Push the new nodes and any users onto the worklist
623 for (unsigned i = 0, e = NumTo; i != e; ++i) {
Chris Lattnerd1980a52009-03-12 06:52:53 +0000624 if (To[i].getNode()) {
625 AddToWorkList(To[i].getNode());
626 AddUsersToWorkList(To[i].getNode());
627 }
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000628 }
629 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000630
Dan Gohmandbe664a2009-01-19 21:44:21 +0000631 // Finally, if the node is now dead, remove it from the graph. The node
632 // may not be dead if the replacement process recursively simplified to
633 // something else needing this node.
634 if (N->use_empty()) {
635 // Nodes can be reintroduced into the worklist. Make sure we do not
636 // process a node that has been replaced.
637 removeFromWorkList(N);
Scott Michelfdc40a02009-02-17 22:15:04 +0000638
Dan Gohmandbe664a2009-01-19 21:44:21 +0000639 // Finally, since the node is now dead, remove it from the graph.
640 DAG.DeleteNode(N);
641 }
Dan Gohman475871a2008-07-27 21:46:04 +0000642 return SDValue(N, 0);
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000643}
644
Evan Chenge5b51ac2010-04-17 06:13:15 +0000645void DAGCombiner::
646CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
Scott Michelfdc40a02009-02-17 22:15:04 +0000647 // Replace all uses. If any nodes become isomorphic to other nodes and
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000648 // are deleted, make sure to remove them from our worklist.
649 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000650 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
Dan Gohmane5af2d32009-01-29 01:59:02 +0000651
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000652 // Push the new node and any (possibly new) users onto the worklist.
Gabor Greifba36cb52008-08-28 21:40:38 +0000653 AddToWorkList(TLO.New.getNode());
654 AddUsersToWorkList(TLO.New.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000655
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000656 // Finally, if the node is now dead, remove it from the graph. The node
657 // may not be dead if the replacement process recursively simplified to
658 // something else needing this node.
Gabor Greifba36cb52008-08-28 21:40:38 +0000659 if (TLO.Old.getNode()->use_empty()) {
660 removeFromWorkList(TLO.Old.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000661
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000662 // If the operands of this node are only used by the node, they will now
663 // be dead. Make sure to visit them first to delete dead nodes early.
Gabor Greifba36cb52008-08-28 21:40:38 +0000664 for (unsigned i = 0, e = TLO.Old.getNode()->getNumOperands(); i != e; ++i)
665 if (TLO.Old.getNode()->getOperand(i).getNode()->hasOneUse())
666 AddToWorkList(TLO.Old.getNode()->getOperand(i).getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000667
Gabor Greifba36cb52008-08-28 21:40:38 +0000668 DAG.DeleteNode(TLO.Old.getNode());
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000669 }
Dan Gohmane5af2d32009-01-29 01:59:02 +0000670}
671
672/// SimplifyDemandedBits - Check the specified integer node value to see if
673/// it can be simplified or if things it uses can be simplified by bit
674/// propagation. If so, return true.
675bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
Evan Chenge5b51ac2010-04-17 06:13:15 +0000676 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
Dan Gohmane5af2d32009-01-29 01:59:02 +0000677 APInt KnownZero, KnownOne;
678 if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
679 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +0000680
Dan Gohmane5af2d32009-01-29 01:59:02 +0000681 // Revisit the node.
682 AddToWorkList(Op.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000683
Dan Gohmane5af2d32009-01-29 01:59:02 +0000684 // Replace the old value with the new one.
685 ++NodesCombined;
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000686 DEBUG(dbgs() << "\nReplacing.2 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000687 TLO.Old.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000688 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000689 TLO.New.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000690 dbgs() << '\n');
Scott Michelfdc40a02009-02-17 22:15:04 +0000691
Dan Gohmane5af2d32009-01-29 01:59:02 +0000692 CommitTargetLoweringOpt(TLO);
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000693 return true;
694}
695
Evan Cheng95c57ea2010-04-24 04:43:44 +0000696void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
697 DebugLoc dl = Load->getDebugLoc();
698 EVT VT = Load->getValueType(0);
699 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0));
Evan Cheng4c26e932010-04-19 19:29:22 +0000700
Evan Cheng95c57ea2010-04-24 04:43:44 +0000701 DEBUG(dbgs() << "\nReplacing.9 ";
702 Load->dump(&DAG);
703 dbgs() << "\nWith: ";
704 Trunc.getNode()->dump(&DAG);
705 dbgs() << '\n');
706 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000707 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
708 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
Evan Cheng95c57ea2010-04-24 04:43:44 +0000709 removeFromWorkList(Load);
710 DAG.DeleteNode(Load);
Evan Chengac7eae52010-04-27 19:48:13 +0000711 AddToWorkList(Trunc.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000712}
713
714SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
715 Replace = false;
Evan Cheng4c26e932010-04-19 19:29:22 +0000716 DebugLoc dl = Op.getDebugLoc();
Evan Chenge5b51ac2010-04-17 06:13:15 +0000717 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
Evan Chengac7eae52010-04-27 19:48:13 +0000718 EVT MemVT = LD->getMemoryVT();
719 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
Owen Anderson95771af2011-02-25 21:41:48 +0000720 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
Eric Christopher503a64d2010-12-09 04:48:06 +0000721 : ISD::EXTLOAD)
Evan Chengac7eae52010-04-27 19:48:13 +0000722 : LD->getExtensionType();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000723 Replace = true;
Stuart Hastingsa9011292011-02-16 16:23:55 +0000724 return DAG.getExtLoad(ExtType, dl, PVT,
Evan Chenge5b51ac2010-04-17 06:13:15 +0000725 LD->getChain(), LD->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +0000726 LD->getPointerInfo(),
Evan Chengac7eae52010-04-27 19:48:13 +0000727 MemVT, LD->isVolatile(),
Evan Chenge5b51ac2010-04-17 06:13:15 +0000728 LD->isNonTemporal(), LD->getAlignment());
729 }
730
Evan Cheng4c26e932010-04-19 19:29:22 +0000731 unsigned Opc = Op.getOpcode();
Evan Chengcaf77402010-04-23 19:10:30 +0000732 switch (Opc) {
733 default: break;
734 case ISD::AssertSext:
Evan Cheng4c26e932010-04-19 19:29:22 +0000735 return DAG.getNode(ISD::AssertSext, dl, PVT,
Evan Cheng95c57ea2010-04-24 04:43:44 +0000736 SExtPromoteOperand(Op.getOperand(0), PVT),
Evan Cheng4c26e932010-04-19 19:29:22 +0000737 Op.getOperand(1));
Evan Chengcaf77402010-04-23 19:10:30 +0000738 case ISD::AssertZext:
Evan Cheng4c26e932010-04-19 19:29:22 +0000739 return DAG.getNode(ISD::AssertZext, dl, PVT,
Evan Cheng95c57ea2010-04-24 04:43:44 +0000740 ZExtPromoteOperand(Op.getOperand(0), PVT),
Evan Cheng4c26e932010-04-19 19:29:22 +0000741 Op.getOperand(1));
Evan Chengcaf77402010-04-23 19:10:30 +0000742 case ISD::Constant: {
743 unsigned ExtOpc =
Evan Cheng4c26e932010-04-19 19:29:22 +0000744 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
Evan Chengcaf77402010-04-23 19:10:30 +0000745 return DAG.getNode(ExtOpc, dl, PVT, Op);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000746 }
Evan Chengcaf77402010-04-23 19:10:30 +0000747 }
748
749 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
Evan Chenge5b51ac2010-04-17 06:13:15 +0000750 return SDValue();
Evan Chengcaf77402010-04-23 19:10:30 +0000751 return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op);
Evan Cheng64b7bf72010-04-16 06:14:10 +0000752}
753
Evan Cheng95c57ea2010-04-24 04:43:44 +0000754SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
Evan Chenge5b51ac2010-04-17 06:13:15 +0000755 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
756 return SDValue();
757 EVT OldVT = Op.getValueType();
758 DebugLoc dl = Op.getDebugLoc();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000759 bool Replace = false;
760 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
761 if (NewOp.getNode() == 0)
Evan Chenge5b51ac2010-04-17 06:13:15 +0000762 return SDValue();
Evan Chengac7eae52010-04-27 19:48:13 +0000763 AddToWorkList(NewOp.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000764
765 if (Replace)
766 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
767 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp,
Evan Chenge5b51ac2010-04-17 06:13:15 +0000768 DAG.getValueType(OldVT));
769}
770
Evan Cheng95c57ea2010-04-24 04:43:44 +0000771SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
Evan Chenge5b51ac2010-04-17 06:13:15 +0000772 EVT OldVT = Op.getValueType();
773 DebugLoc dl = Op.getDebugLoc();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000774 bool Replace = false;
775 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
776 if (NewOp.getNode() == 0)
Evan Chenge5b51ac2010-04-17 06:13:15 +0000777 return SDValue();
Evan Chengac7eae52010-04-27 19:48:13 +0000778 AddToWorkList(NewOp.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000779
780 if (Replace)
781 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
782 return DAG.getZeroExtendInReg(NewOp, dl, OldVT);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000783}
784
Evan Cheng64b7bf72010-04-16 06:14:10 +0000785/// PromoteIntBinOp - Promote the specified integer binary operation if the
786/// target indicates it is beneficial. e.g. On x86, it's usually better to
787/// promote i16 operations to i32 since i16 instructions are longer.
788SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
789 if (!LegalOperations)
790 return SDValue();
791
792 EVT VT = Op.getValueType();
793 if (VT.isVector() || !VT.isInteger())
794 return SDValue();
795
Evan Chenge5b51ac2010-04-17 06:13:15 +0000796 // If operation type is 'undesirable', e.g. i16 on x86, consider
797 // promoting it.
798 unsigned Opc = Op.getOpcode();
799 if (TLI.isTypeDesirableForOp(Opc, VT))
800 return SDValue();
801
Evan Cheng64b7bf72010-04-16 06:14:10 +0000802 EVT PVT = VT;
Evan Chenge5b51ac2010-04-17 06:13:15 +0000803 // Consult target whether it is a good idea to promote this operation and
804 // what's the right type to promote it to.
805 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
Evan Cheng64b7bf72010-04-16 06:14:10 +0000806 assert(PVT != VT && "Don't know what type to promote to!");
807
Evan Cheng95c57ea2010-04-24 04:43:44 +0000808 bool Replace0 = false;
809 SDValue N0 = Op.getOperand(0);
810 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
811 if (NN0.getNode() == 0)
Evan Cheng07c4e102010-04-22 20:19:46 +0000812 return SDValue();
813
Evan Cheng95c57ea2010-04-24 04:43:44 +0000814 bool Replace1 = false;
815 SDValue N1 = Op.getOperand(1);
Evan Chengaad753b2010-05-10 19:03:57 +0000816 SDValue NN1;
817 if (N0 == N1)
818 NN1 = NN0;
819 else {
820 NN1 = PromoteOperand(N1, PVT, Replace1);
821 if (NN1.getNode() == 0)
822 return SDValue();
823 }
Evan Cheng07c4e102010-04-22 20:19:46 +0000824
Evan Cheng95c57ea2010-04-24 04:43:44 +0000825 AddToWorkList(NN0.getNode());
Evan Chengaad753b2010-05-10 19:03:57 +0000826 if (NN1.getNode())
827 AddToWorkList(NN1.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000828
829 if (Replace0)
830 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
831 if (Replace1)
832 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
Evan Cheng07c4e102010-04-22 20:19:46 +0000833
Evan Chengac7eae52010-04-27 19:48:13 +0000834 DEBUG(dbgs() << "\nPromoting ";
835 Op.getNode()->dump(&DAG));
Evan Cheng07c4e102010-04-22 20:19:46 +0000836 DebugLoc dl = Op.getDebugLoc();
837 return DAG.getNode(ISD::TRUNCATE, dl, VT,
Evan Cheng95c57ea2010-04-24 04:43:44 +0000838 DAG.getNode(Opc, dl, PVT, NN0, NN1));
Evan Cheng07c4e102010-04-22 20:19:46 +0000839 }
840 return SDValue();
841}
842
843/// PromoteIntShiftOp - Promote the specified integer shift operation if the
844/// target indicates it is beneficial. e.g. On x86, it's usually better to
845/// promote i16 operations to i32 since i16 instructions are longer.
846SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
847 if (!LegalOperations)
848 return SDValue();
849
850 EVT VT = Op.getValueType();
851 if (VT.isVector() || !VT.isInteger())
852 return SDValue();
853
854 // If operation type is 'undesirable', e.g. i16 on x86, consider
855 // promoting it.
856 unsigned Opc = Op.getOpcode();
857 if (TLI.isTypeDesirableForOp(Opc, VT))
858 return SDValue();
859
860 EVT PVT = VT;
861 // Consult target whether it is a good idea to promote this operation and
862 // what's the right type to promote it to.
863 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
864 assert(PVT != VT && "Don't know what type to promote to!");
865
Evan Cheng95c57ea2010-04-24 04:43:44 +0000866 bool Replace = false;
Evan Chenge5b51ac2010-04-17 06:13:15 +0000867 SDValue N0 = Op.getOperand(0);
868 if (Opc == ISD::SRA)
Evan Cheng95c57ea2010-04-24 04:43:44 +0000869 N0 = SExtPromoteOperand(Op.getOperand(0), PVT);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000870 else if (Opc == ISD::SRL)
Evan Cheng95c57ea2010-04-24 04:43:44 +0000871 N0 = ZExtPromoteOperand(Op.getOperand(0), PVT);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000872 else
Evan Cheng95c57ea2010-04-24 04:43:44 +0000873 N0 = PromoteOperand(N0, PVT, Replace);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000874 if (N0.getNode() == 0)
875 return SDValue();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000876
Evan Chenge5b51ac2010-04-17 06:13:15 +0000877 AddToWorkList(N0.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000878 if (Replace)
879 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
Evan Cheng64b7bf72010-04-16 06:14:10 +0000880
Evan Chengac7eae52010-04-27 19:48:13 +0000881 DEBUG(dbgs() << "\nPromoting ";
882 Op.getNode()->dump(&DAG));
Evan Cheng64b7bf72010-04-16 06:14:10 +0000883 DebugLoc dl = Op.getDebugLoc();
884 return DAG.getNode(ISD::TRUNCATE, dl, VT,
Evan Cheng07c4e102010-04-22 20:19:46 +0000885 DAG.getNode(Opc, dl, PVT, N0, Op.getOperand(1)));
Evan Cheng64b7bf72010-04-16 06:14:10 +0000886 }
887 return SDValue();
888}
889
Evan Cheng4c26e932010-04-19 19:29:22 +0000890SDValue DAGCombiner::PromoteExtend(SDValue Op) {
891 if (!LegalOperations)
892 return SDValue();
893
894 EVT VT = Op.getValueType();
895 if (VT.isVector() || !VT.isInteger())
896 return SDValue();
897
898 // If operation type is 'undesirable', e.g. i16 on x86, consider
899 // promoting it.
900 unsigned Opc = Op.getOpcode();
901 if (TLI.isTypeDesirableForOp(Opc, VT))
902 return SDValue();
903
904 EVT PVT = VT;
905 // Consult target whether it is a good idea to promote this operation and
906 // what's the right type to promote it to.
907 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
908 assert(PVT != VT && "Don't know what type to promote to!");
909 // fold (aext (aext x)) -> (aext x)
910 // fold (aext (zext x)) -> (zext x)
911 // fold (aext (sext x)) -> (sext x)
Evan Chengac7eae52010-04-27 19:48:13 +0000912 DEBUG(dbgs() << "\nPromoting ";
913 Op.getNode()->dump(&DAG));
Evan Cheng4c26e932010-04-19 19:29:22 +0000914 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), VT, Op.getOperand(0));
915 }
916 return SDValue();
917}
918
919bool DAGCombiner::PromoteLoad(SDValue Op) {
920 if (!LegalOperations)
921 return false;
922
923 EVT VT = Op.getValueType();
924 if (VT.isVector() || !VT.isInteger())
925 return false;
926
927 // If operation type is 'undesirable', e.g. i16 on x86, consider
928 // promoting it.
929 unsigned Opc = Op.getOpcode();
930 if (TLI.isTypeDesirableForOp(Opc, VT))
931 return false;
932
933 EVT PVT = VT;
934 // Consult target whether it is a good idea to promote this operation and
935 // what's the right type to promote it to.
936 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
937 assert(PVT != VT && "Don't know what type to promote to!");
938
939 DebugLoc dl = Op.getDebugLoc();
940 SDNode *N = Op.getNode();
941 LoadSDNode *LD = cast<LoadSDNode>(N);
Evan Chengac7eae52010-04-27 19:48:13 +0000942 EVT MemVT = LD->getMemoryVT();
943 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
Owen Anderson95771af2011-02-25 21:41:48 +0000944 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
Eric Christopher503a64d2010-12-09 04:48:06 +0000945 : ISD::EXTLOAD)
Evan Chengac7eae52010-04-27 19:48:13 +0000946 : LD->getExtensionType();
Stuart Hastingsa9011292011-02-16 16:23:55 +0000947 SDValue NewLD = DAG.getExtLoad(ExtType, dl, PVT,
Evan Cheng4c26e932010-04-19 19:29:22 +0000948 LD->getChain(), LD->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +0000949 LD->getPointerInfo(),
Evan Chengac7eae52010-04-27 19:48:13 +0000950 MemVT, LD->isVolatile(),
Evan Cheng4c26e932010-04-19 19:29:22 +0000951 LD->isNonTemporal(), LD->getAlignment());
952 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD);
953
Evan Cheng95c57ea2010-04-24 04:43:44 +0000954 DEBUG(dbgs() << "\nPromoting ";
Evan Cheng4c26e932010-04-19 19:29:22 +0000955 N->dump(&DAG);
Evan Cheng95c57ea2010-04-24 04:43:44 +0000956 dbgs() << "\nTo: ";
Evan Cheng4c26e932010-04-19 19:29:22 +0000957 Result.getNode()->dump(&DAG);
958 dbgs() << '\n');
959 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +0000960 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
961 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
Evan Cheng4c26e932010-04-19 19:29:22 +0000962 removeFromWorkList(N);
963 DAG.DeleteNode(N);
Evan Chengac7eae52010-04-27 19:48:13 +0000964 AddToWorkList(Result.getNode());
Evan Cheng4c26e932010-04-19 19:29:22 +0000965 return true;
966 }
967 return false;
968}
969
Evan Chenge5b51ac2010-04-17 06:13:15 +0000970
Chris Lattner29446522007-05-14 22:04:50 +0000971//===----------------------------------------------------------------------===//
972// Main DAG Combiner implementation
973//===----------------------------------------------------------------------===//
974
Duncan Sands25cf2272008-11-24 14:53:14 +0000975void DAGCombiner::Run(CombineLevel AtLevel) {
976 // set the instance variables, so that the various visit routines may use it.
977 Level = AtLevel;
Eli Friedman50185242011-11-12 00:35:34 +0000978 LegalOperations = Level >= AfterLegalizeVectorOps;
979 LegalTypes = Level >= AfterLegalizeTypes;
Nate Begeman4ebd8052005-09-01 23:24:04 +0000980
Evan Cheng17a568b2008-08-29 22:21:44 +0000981 // Add all the dag nodes to the worklist.
Evan Cheng17a568b2008-08-29 22:21:44 +0000982 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
983 E = DAG.allnodes_end(); I != E; ++I)
James Molloy6660c052012-02-16 09:17:04 +0000984 AddToWorkList(I);
Duncan Sands25cf2272008-11-24 14:53:14 +0000985
Evan Cheng17a568b2008-08-29 22:21:44 +0000986 // Create a dummy node (which is not added to allnodes), that adds a reference
987 // to the root node, preventing it from being deleted, and tracking any
988 // changes of the root.
989 HandleSDNode Dummy(DAG.getRoot());
Scott Michelfdc40a02009-02-17 22:15:04 +0000990
Jim Laskey26f7fa72006-10-17 19:33:52 +0000991 // The root of the dag may dangle to deleted nodes until the dag combiner is
992 // done. Set it to null to avoid confusion.
Dan Gohman475871a2008-07-27 21:46:04 +0000993 DAG.setRoot(SDValue());
Scott Michelfdc40a02009-02-17 22:15:04 +0000994
James Molloy6660c052012-02-16 09:17:04 +0000995 // while the worklist isn't empty, find a node and
Evan Cheng17a568b2008-08-29 22:21:44 +0000996 // try and combine it.
James Molloy6660c052012-02-16 09:17:04 +0000997 while (!WorkListContents.empty()) {
998 SDNode *N;
999 // The WorkListOrder holds the SDNodes in order, but it may contain duplicates.
1000 // In order to avoid a linear scan, we use a set (O(log N)) to hold what the
1001 // worklist *should* contain, and check the node we want to visit is should
1002 // actually be visited.
1003 do {
Benjamin Kramerd5f76902012-03-10 00:23:58 +00001004 N = WorkListOrder.pop_back_val();
James Molloy6660c052012-02-16 09:17:04 +00001005 } while (!WorkListContents.erase(N));
Scott Michelfdc40a02009-02-17 22:15:04 +00001006
Evan Cheng17a568b2008-08-29 22:21:44 +00001007 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1008 // N is deleted from the DAG, since they too may now be dead or may have a
1009 // reduced number of uses, allowing other xforms.
1010 if (N->use_empty() && N != &Dummy) {
1011 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1012 AddToWorkList(N->getOperand(i).getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00001013
Evan Cheng17a568b2008-08-29 22:21:44 +00001014 DAG.DeleteNode(N);
1015 continue;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001016 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001017
Evan Cheng17a568b2008-08-29 22:21:44 +00001018 SDValue RV = combine(N);
Scott Michelfdc40a02009-02-17 22:15:04 +00001019
Evan Cheng17a568b2008-08-29 22:21:44 +00001020 if (RV.getNode() == 0)
1021 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00001022
Evan Cheng17a568b2008-08-29 22:21:44 +00001023 ++NodesCombined;
Scott Michelfdc40a02009-02-17 22:15:04 +00001024
Evan Cheng17a568b2008-08-29 22:21:44 +00001025 // If we get back the same node we passed in, rather than a new node or
1026 // zero, we know that the node must have defined multiple values and
Scott Michelfdc40a02009-02-17 22:15:04 +00001027 // CombineTo was used. Since CombineTo takes care of the worklist
Evan Cheng17a568b2008-08-29 22:21:44 +00001028 // mechanics for us, we have no work to do in this case.
1029 if (RV.getNode() == N)
1030 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00001031
Evan Cheng17a568b2008-08-29 22:21:44 +00001032 assert(N->getOpcode() != ISD::DELETED_NODE &&
1033 RV.getNode()->getOpcode() != ISD::DELETED_NODE &&
1034 "Node was deleted but visit returned new node!");
Chris Lattner729c6d12006-05-27 00:43:02 +00001035
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001036 DEBUG(dbgs() << "\nReplacing.3 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00001037 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00001038 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00001039 RV.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00001040 dbgs() << '\n');
Eric Christopher7332e6e2011-07-14 01:12:15 +00001041
Devang Patel9728ea22011-05-23 22:04:42 +00001042 // Transfer debug value.
1043 DAG.TransferDbgValues(SDValue(N, 0), RV);
Evan Cheng17a568b2008-08-29 22:21:44 +00001044 WorkListRemover DeadNodes(*this);
1045 if (N->getNumValues() == RV.getNode()->getNumValues())
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00001046 DAG.ReplaceAllUsesWith(N, RV.getNode());
Evan Cheng17a568b2008-08-29 22:21:44 +00001047 else {
1048 assert(N->getValueType(0) == RV.getValueType() &&
1049 N->getNumValues() == 1 && "Type mismatch");
1050 SDValue OpV = RV;
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00001051 DAG.ReplaceAllUsesWith(N, &OpV);
Evan Cheng17a568b2008-08-29 22:21:44 +00001052 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001053
Evan Cheng17a568b2008-08-29 22:21:44 +00001054 // Push the new node and any users onto the worklist
1055 AddToWorkList(RV.getNode());
1056 AddUsersToWorkList(RV.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00001057
Evan Cheng17a568b2008-08-29 22:21:44 +00001058 // Add any uses of the old node to the worklist in case this node is the
1059 // last one that uses them. They may become dead after this node is
1060 // deleted.
1061 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1062 AddToWorkList(N->getOperand(i).getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00001063
Dan Gohmandbe664a2009-01-19 21:44:21 +00001064 // Finally, if the node is now dead, remove it from the graph. The node
1065 // may not be dead if the replacement process recursively simplified to
1066 // something else needing this node.
1067 if (N->use_empty()) {
1068 // Nodes can be reintroduced into the worklist. Make sure we do not
1069 // process a node that has been replaced.
1070 removeFromWorkList(N);
Scott Michelfdc40a02009-02-17 22:15:04 +00001071
Dan Gohmandbe664a2009-01-19 21:44:21 +00001072 // Finally, since the node is now dead, remove it from the graph.
1073 DAG.DeleteNode(N);
1074 }
Evan Cheng17a568b2008-08-29 22:21:44 +00001075 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001076
Chris Lattner95038592005-10-05 06:35:28 +00001077 // If the root changed (e.g. it was a dead load, update the root).
1078 DAG.setRoot(Dummy.getValue());
Hal Finkel31490ba2012-04-16 03:33:22 +00001079 DAG.RemoveDeadNodes();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001080}
1081
Dan Gohman475871a2008-07-27 21:46:04 +00001082SDValue DAGCombiner::visit(SDNode *N) {
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001083 switch (N->getOpcode()) {
Nate Begeman1d4d4142005-09-01 00:19:25 +00001084 default: break;
Nate Begeman4942a962005-09-01 00:33:32 +00001085 case ISD::TokenFactor: return visitTokenFactor(N);
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001086 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001087 case ISD::ADD: return visitADD(N);
1088 case ISD::SUB: return visitSUB(N);
Chris Lattner91153682007-03-04 20:03:15 +00001089 case ISD::ADDC: return visitADDC(N);
Craig Toppercc274522012-01-07 09:06:39 +00001090 case ISD::SUBC: return visitSUBC(N);
Chris Lattner91153682007-03-04 20:03:15 +00001091 case ISD::ADDE: return visitADDE(N);
Craig Toppercc274522012-01-07 09:06:39 +00001092 case ISD::SUBE: return visitSUBE(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001093 case ISD::MUL: return visitMUL(N);
1094 case ISD::SDIV: return visitSDIV(N);
1095 case ISD::UDIV: return visitUDIV(N);
1096 case ISD::SREM: return visitSREM(N);
1097 case ISD::UREM: return visitUREM(N);
1098 case ISD::MULHU: return visitMULHU(N);
1099 case ISD::MULHS: return visitMULHS(N);
Dan Gohman389079b2007-10-08 17:57:15 +00001100 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1101 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
Benjamin Kramerf55d26e2011-05-21 18:31:55 +00001102 case ISD::SMULO: return visitSMULO(N);
1103 case ISD::UMULO: return visitUMULO(N);
Dan Gohman389079b2007-10-08 17:57:15 +00001104 case ISD::SDIVREM: return visitSDIVREM(N);
1105 case ISD::UDIVREM: return visitUDIVREM(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001106 case ISD::AND: return visitAND(N);
1107 case ISD::OR: return visitOR(N);
1108 case ISD::XOR: return visitXOR(N);
1109 case ISD::SHL: return visitSHL(N);
1110 case ISD::SRA: return visitSRA(N);
1111 case ISD::SRL: return visitSRL(N);
1112 case ISD::CTLZ: return visitCTLZ(N);
Chandler Carruth63974b22011-12-13 01:56:10 +00001113 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001114 case ISD::CTTZ: return visitCTTZ(N);
Chandler Carruth63974b22011-12-13 01:56:10 +00001115 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001116 case ISD::CTPOP: return visitCTPOP(N);
Nate Begeman452d7beb2005-09-16 00:54:12 +00001117 case ISD::SELECT: return visitSELECT(N);
1118 case ISD::SELECT_CC: return visitSELECT_CC(N);
1119 case ISD::SETCC: return visitSETCC(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001120 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1121 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
Chris Lattner5ffc0662006-05-05 05:58:59 +00001122 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001123 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1124 case ISD::TRUNCATE: return visitTRUNCATE(N);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001125 case ISD::BITCAST: return visitBITCAST(N);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00001126 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
Chris Lattner01b3d732005-09-28 22:28:18 +00001127 case ISD::FADD: return visitFADD(N);
1128 case ISD::FSUB: return visitFSUB(N);
1129 case ISD::FMUL: return visitFMUL(N);
Owen Anderson062c0a52012-05-02 22:17:40 +00001130 case ISD::FMA: return visitFMA(N);
Chris Lattner01b3d732005-09-28 22:28:18 +00001131 case ISD::FDIV: return visitFDIV(N);
1132 case ISD::FREM: return visitFREM(N);
Chris Lattner12d83032006-03-05 05:30:57 +00001133 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001134 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1135 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1136 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1137 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1138 case ISD::FP_ROUND: return visitFP_ROUND(N);
1139 case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
1140 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1141 case ISD::FNEG: return visitFNEG(N);
1142 case ISD::FABS: return visitFABS(N);
Nate Begeman44728a72005-09-19 22:34:01 +00001143 case ISD::BRCOND: return visitBRCOND(N);
Nate Begeman44728a72005-09-19 22:34:01 +00001144 case ISD::BR_CC: return visitBR_CC(N);
Chris Lattner01a22022005-10-10 22:04:48 +00001145 case ISD::LOAD: return visitLOAD(N);
Chris Lattner87514ca2005-10-10 22:31:19 +00001146 case ISD::STORE: return visitSTORE(N);
Chris Lattnerca242442006-03-19 01:27:56 +00001147 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
Evan Cheng513da432007-10-06 08:19:55 +00001148 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
Dan Gohman7f321562007-06-25 16:23:39 +00001149 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1150 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00001151 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
Chris Lattner66445d32006-03-28 22:11:53 +00001152 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
Jim Grosbach9a526492010-06-23 16:07:42 +00001153 case ISD::MEMBARRIER: return visitMEMBARRIER(N);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001154 }
Dan Gohman475871a2008-07-27 21:46:04 +00001155 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001156}
1157
Dan Gohman475871a2008-07-27 21:46:04 +00001158SDValue DAGCombiner::combine(SDNode *N) {
Dan Gohman475871a2008-07-27 21:46:04 +00001159 SDValue RV = visit(N);
Dan Gohman389079b2007-10-08 17:57:15 +00001160
1161 // If nothing happened, try a target-specific DAG combine.
Gabor Greifba36cb52008-08-28 21:40:38 +00001162 if (RV.getNode() == 0) {
Dan Gohman389079b2007-10-08 17:57:15 +00001163 assert(N->getOpcode() != ISD::DELETED_NODE &&
1164 "Node was deleted but visit returned NULL!");
1165
1166 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1167 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1168
1169 // Expose the DAG combiner to the target combiner impls.
Scott Michelfdc40a02009-02-17 22:15:04 +00001170 TargetLowering::DAGCombinerInfo
Jakob Stoklund Olesen78d12642009-07-24 18:22:59 +00001171 DagCombineInfo(DAG, !LegalTypes, !LegalOperations, false, this);
Dan Gohman389079b2007-10-08 17:57:15 +00001172
1173 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1174 }
1175 }
1176
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001177 // If nothing happened still, try promoting the operation.
1178 if (RV.getNode() == 0) {
1179 switch (N->getOpcode()) {
1180 default: break;
1181 case ISD::ADD:
1182 case ISD::SUB:
1183 case ISD::MUL:
1184 case ISD::AND:
1185 case ISD::OR:
1186 case ISD::XOR:
1187 RV = PromoteIntBinOp(SDValue(N, 0));
1188 break;
1189 case ISD::SHL:
1190 case ISD::SRA:
1191 case ISD::SRL:
1192 RV = PromoteIntShiftOp(SDValue(N, 0));
1193 break;
1194 case ISD::SIGN_EXTEND:
1195 case ISD::ZERO_EXTEND:
1196 case ISD::ANY_EXTEND:
1197 RV = PromoteExtend(SDValue(N, 0));
1198 break;
1199 case ISD::LOAD:
1200 if (PromoteLoad(SDValue(N, 0)))
1201 RV = SDValue(N, 0);
1202 break;
1203 }
1204 }
1205
Scott Michelfdc40a02009-02-17 22:15:04 +00001206 // If N is a commutative binary node, try commuting it to enable more
Evan Cheng08b11732008-03-22 01:55:50 +00001207 // sdisel CSE.
Scott Michelfdc40a02009-02-17 22:15:04 +00001208 if (RV.getNode() == 0 &&
Evan Cheng08b11732008-03-22 01:55:50 +00001209 SelectionDAG::isCommutativeBinOp(N->getOpcode()) &&
1210 N->getNumValues() == 1) {
Dan Gohman475871a2008-07-27 21:46:04 +00001211 SDValue N0 = N->getOperand(0);
1212 SDValue N1 = N->getOperand(1);
Bill Wendling5c71acf2009-01-30 01:13:16 +00001213
Evan Cheng08b11732008-03-22 01:55:50 +00001214 // Constant operands are canonicalized to RHS.
1215 if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
Dan Gohman475871a2008-07-27 21:46:04 +00001216 SDValue Ops[] = { N1, N0 };
Evan Cheng08b11732008-03-22 01:55:50 +00001217 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(),
1218 Ops, 2);
Evan Chengea100462008-03-24 23:55:16 +00001219 if (CSENode)
Dan Gohman475871a2008-07-27 21:46:04 +00001220 return SDValue(CSENode, 0);
Evan Cheng08b11732008-03-22 01:55:50 +00001221 }
1222 }
1223
Dan Gohman389079b2007-10-08 17:57:15 +00001224 return RV;
Scott Michelfdc40a02009-02-17 22:15:04 +00001225}
Dan Gohman389079b2007-10-08 17:57:15 +00001226
Chris Lattner6270f682006-10-08 22:57:01 +00001227/// getInputChainForNode - Given a node, return its input chain if it has one,
1228/// otherwise return a null sd operand.
Dan Gohman475871a2008-07-27 21:46:04 +00001229static SDValue getInputChainForNode(SDNode *N) {
Chris Lattner6270f682006-10-08 22:57:01 +00001230 if (unsigned NumOps = N->getNumOperands()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00001231 if (N->getOperand(0).getValueType() == MVT::Other)
Chris Lattner6270f682006-10-08 22:57:01 +00001232 return N->getOperand(0);
Owen Anderson825b72b2009-08-11 20:47:22 +00001233 else if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
Chris Lattner6270f682006-10-08 22:57:01 +00001234 return N->getOperand(NumOps-1);
1235 for (unsigned i = 1; i < NumOps-1; ++i)
Owen Anderson825b72b2009-08-11 20:47:22 +00001236 if (N->getOperand(i).getValueType() == MVT::Other)
Chris Lattner6270f682006-10-08 22:57:01 +00001237 return N->getOperand(i);
1238 }
Bill Wendling5c71acf2009-01-30 01:13:16 +00001239 return SDValue();
Chris Lattner6270f682006-10-08 22:57:01 +00001240}
1241
Dan Gohman475871a2008-07-27 21:46:04 +00001242SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
Chris Lattner6270f682006-10-08 22:57:01 +00001243 // If N has two operands, where one has an input chain equal to the other,
1244 // the 'other' chain is redundant.
1245 if (N->getNumOperands() == 2) {
Gabor Greifba36cb52008-08-28 21:40:38 +00001246 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
Chris Lattner6270f682006-10-08 22:57:01 +00001247 return N->getOperand(0);
Gabor Greifba36cb52008-08-28 21:40:38 +00001248 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
Chris Lattner6270f682006-10-08 22:57:01 +00001249 return N->getOperand(1);
1250 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001251
Chris Lattnerc76d4412007-05-16 06:37:59 +00001252 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
Dan Gohman475871a2008-07-27 21:46:04 +00001253 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
Scott Michelfdc40a02009-02-17 22:15:04 +00001254 SmallPtrSet<SDNode*, 16> SeenOps;
Chris Lattnerc76d4412007-05-16 06:37:59 +00001255 bool Changed = false; // If we should replace this token factor.
Scott Michelfdc40a02009-02-17 22:15:04 +00001256
Jim Laskey6ff23e52006-10-04 16:53:27 +00001257 // Start out with this token factor.
Jim Laskey279f0532006-09-25 16:29:54 +00001258 TFs.push_back(N);
Scott Michelfdc40a02009-02-17 22:15:04 +00001259
Jim Laskey71382342006-10-07 23:37:56 +00001260 // Iterate through token factors. The TFs grows when new token factors are
Jim Laskeybc588b82006-10-05 15:07:25 +00001261 // encountered.
1262 for (unsigned i = 0; i < TFs.size(); ++i) {
1263 SDNode *TF = TFs[i];
Scott Michelfdc40a02009-02-17 22:15:04 +00001264
Jim Laskey6ff23e52006-10-04 16:53:27 +00001265 // Check each of the operands.
1266 for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00001267 SDValue Op = TF->getOperand(i);
Scott Michelfdc40a02009-02-17 22:15:04 +00001268
Jim Laskey6ff23e52006-10-04 16:53:27 +00001269 switch (Op.getOpcode()) {
1270 case ISD::EntryToken:
Jim Laskeybc588b82006-10-05 15:07:25 +00001271 // Entry tokens don't need to be added to the list. They are
1272 // rededundant.
1273 Changed = true;
Jim Laskey6ff23e52006-10-04 16:53:27 +00001274 break;
Scott Michelfdc40a02009-02-17 22:15:04 +00001275
Jim Laskey6ff23e52006-10-04 16:53:27 +00001276 case ISD::TokenFactor:
Nate Begemanb6aef5c2009-09-15 00:18:30 +00001277 if (Op.hasOneUse() &&
Gabor Greifba36cb52008-08-28 21:40:38 +00001278 std::find(TFs.begin(), TFs.end(), Op.getNode()) == TFs.end()) {
Jim Laskey6ff23e52006-10-04 16:53:27 +00001279 // Queue up for processing.
Gabor Greifba36cb52008-08-28 21:40:38 +00001280 TFs.push_back(Op.getNode());
Jim Laskey6ff23e52006-10-04 16:53:27 +00001281 // Clean up in case the token factor is removed.
Gabor Greifba36cb52008-08-28 21:40:38 +00001282 AddToWorkList(Op.getNode());
Jim Laskey6ff23e52006-10-04 16:53:27 +00001283 Changed = true;
1284 break;
Jim Laskey279f0532006-09-25 16:29:54 +00001285 }
Jim Laskey6ff23e52006-10-04 16:53:27 +00001286 // Fall thru
Scott Michelfdc40a02009-02-17 22:15:04 +00001287
Jim Laskey6ff23e52006-10-04 16:53:27 +00001288 default:
Chris Lattnerc76d4412007-05-16 06:37:59 +00001289 // Only add if it isn't already in the list.
Gabor Greifba36cb52008-08-28 21:40:38 +00001290 if (SeenOps.insert(Op.getNode()))
Jim Laskeybc588b82006-10-05 15:07:25 +00001291 Ops.push_back(Op);
Chris Lattnerc76d4412007-05-16 06:37:59 +00001292 else
1293 Changed = true;
Jim Laskey6ff23e52006-10-04 16:53:27 +00001294 break;
Jim Laskey279f0532006-09-25 16:29:54 +00001295 }
1296 }
Jim Laskey6ff23e52006-10-04 16:53:27 +00001297 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001298
Dan Gohman475871a2008-07-27 21:46:04 +00001299 SDValue Result;
Jim Laskey6ff23e52006-10-04 16:53:27 +00001300
1301 // If we've change things around then replace token factor.
1302 if (Changed) {
Dan Gohman30359592008-01-29 13:02:09 +00001303 if (Ops.empty()) {
Jim Laskey6ff23e52006-10-04 16:53:27 +00001304 // The entry token is the only possible outcome.
1305 Result = DAG.getEntryNode();
1306 } else {
1307 // New and improved token factor.
Bill Wendling5c71acf2009-01-30 01:13:16 +00001308 Result = DAG.getNode(ISD::TokenFactor, N->getDebugLoc(),
Owen Anderson825b72b2009-08-11 20:47:22 +00001309 MVT::Other, &Ops[0], Ops.size());
Nate Begemanded49632005-10-13 03:11:28 +00001310 }
Bill Wendling5c71acf2009-01-30 01:13:16 +00001311
Jim Laskey274062c2006-10-13 23:32:28 +00001312 // Don't add users to work list.
1313 return CombineTo(N, Result, false);
Nate Begemanded49632005-10-13 03:11:28 +00001314 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001315
Jim Laskey6ff23e52006-10-04 16:53:27 +00001316 return Result;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001317}
1318
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001319/// MERGE_VALUES can always be eliminated.
Dan Gohman475871a2008-07-27 21:46:04 +00001320SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001321 WorkListRemover DeadNodes(*this);
Dan Gohman00edf392009-08-10 23:43:19 +00001322 // Replacing results may cause a different MERGE_VALUES to suddenly
1323 // be CSE'd with N, and carry its uses with it. Iterate until no
1324 // uses remain, to ensure that the node can be safely deleted.
1325 do {
1326 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00001327 DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i));
Dan Gohman00edf392009-08-10 23:43:19 +00001328 } while (!N->use_empty());
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001329 removeFromWorkList(N);
1330 DAG.DeleteNode(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001331 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001332}
1333
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001334static
Bill Wendlingd69c3142009-01-30 02:23:43 +00001335SDValue combineShlAddConstant(DebugLoc DL, SDValue N0, SDValue N1,
1336 SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00001337 EVT VT = N0.getValueType();
Dan Gohman475871a2008-07-27 21:46:04 +00001338 SDValue N00 = N0.getOperand(0);
1339 SDValue N01 = N0.getOperand(1);
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001340 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N01);
Bill Wendlingd69c3142009-01-30 02:23:43 +00001341
Gabor Greifba36cb52008-08-28 21:40:38 +00001342 if (N01C && N00.getOpcode() == ISD::ADD && N00.getNode()->hasOneUse() &&
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001343 isa<ConstantSDNode>(N00.getOperand(1))) {
Bill Wendlingd69c3142009-01-30 02:23:43 +00001344 // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<<c2), )
1345 N0 = DAG.getNode(ISD::ADD, N0.getDebugLoc(), VT,
1346 DAG.getNode(ISD::SHL, N00.getDebugLoc(), VT,
1347 N00.getOperand(0), N01),
1348 DAG.getNode(ISD::SHL, N01.getDebugLoc(), VT,
1349 N00.getOperand(1), N01));
1350 return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001351 }
Bill Wendlingd69c3142009-01-30 02:23:43 +00001352
Dan Gohman475871a2008-07-27 21:46:04 +00001353 return SDValue();
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001354}
1355
Dan Gohman475871a2008-07-27 21:46:04 +00001356SDValue DAGCombiner::visitADD(SDNode *N) {
1357 SDValue N0 = N->getOperand(0);
1358 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00001359 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1360 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001361 EVT VT = N0.getValueType();
Dan Gohman7f321562007-06-25 16:23:39 +00001362
1363 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001364 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001365 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001366 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001367 }
Bill Wendling2476e5d2008-12-10 22:36:00 +00001368
Dan Gohman613e0d82007-07-03 14:03:57 +00001369 // fold (add x, undef) -> undef
Dan Gohman70fb1ae2007-07-10 15:19:29 +00001370 if (N0.getOpcode() == ISD::UNDEF)
1371 return N0;
1372 if (N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00001373 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001374 // fold (add c1, c2) -> c1+c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001375 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001376 return DAG.FoldConstantArithmetic(ISD::ADD, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00001377 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00001378 if (N0C && !N1C)
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001379 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001380 // fold (add x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00001381 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00001382 return N0;
Dan Gohman6520e202008-10-18 02:06:02 +00001383 // fold (add Sym, c) -> Sym+c
1384 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
Duncan Sands25cf2272008-11-24 14:53:14 +00001385 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA) && N1C &&
Dan Gohman6520e202008-10-18 02:06:02 +00001386 GA->getOpcode() == ISD::GlobalAddress)
Devang Patel0d881da2010-07-06 22:08:15 +00001387 return DAG.getGlobalAddress(GA->getGlobal(), N1C->getDebugLoc(), VT,
Dan Gohman6520e202008-10-18 02:06:02 +00001388 GA->getOffset() +
1389 (uint64_t)N1C->getSExtValue());
Chris Lattner4aafb4f2006-01-12 20:22:43 +00001390 // fold ((c1-A)+c2) -> (c1+c2)-A
1391 if (N1C && N0.getOpcode() == ISD::SUB)
1392 if (ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getOperand(0)))
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001393 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
Dan Gohman002e5d02008-03-13 22:13:53 +00001394 DAG.getConstant(N1C->getAPIntValue()+
1395 N0C->getAPIntValue(), VT),
Chris Lattner4aafb4f2006-01-12 20:22:43 +00001396 N0.getOperand(1));
Nate Begemancd4d58c2006-02-03 06:46:56 +00001397 // reassociate add
Bill Wendling35247c32009-01-30 00:45:56 +00001398 SDValue RADD = ReassociateOps(ISD::ADD, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001399 if (RADD.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00001400 return RADD;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001401 // fold ((0-A) + B) -> B-A
1402 if (N0.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N0.getOperand(0)) &&
1403 cast<ConstantSDNode>(N0.getOperand(0))->isNullValue())
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001404 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1, N0.getOperand(1));
Nate Begeman1d4d4142005-09-01 00:19:25 +00001405 // fold (A + (0-B)) -> A-B
1406 if (N1.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N1.getOperand(0)) &&
1407 cast<ConstantSDNode>(N1.getOperand(0))->isNullValue())
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001408 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, N1.getOperand(1));
Chris Lattner01b3d732005-09-28 22:28:18 +00001409 // fold (A+(B-A)) -> B
1410 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
Nate Begeman83e75ec2005-09-06 04:43:02 +00001411 return N1.getOperand(0);
Dale Johannesen56eca912008-11-27 00:43:21 +00001412 // fold ((B-A)+A) -> B
1413 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
1414 return N0.getOperand(0);
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001415 // fold (A+(B-(A+C))) to (B-C)
1416 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001417 N0 == N1.getOperand(1).getOperand(0))
1418 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1.getOperand(0),
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001419 N1.getOperand(1).getOperand(1));
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001420 // fold (A+(B-(C+A))) to (B-C)
1421 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001422 N0 == N1.getOperand(1).getOperand(1))
1423 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1.getOperand(0),
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001424 N1.getOperand(1).getOperand(0));
Dale Johannesen7c7bc722008-12-23 23:47:22 +00001425 // fold (A+((B-A)+or-C)) to (B+or-C)
Dale Johannesen34d79852008-12-02 18:40:40 +00001426 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
1427 N1.getOperand(0).getOpcode() == ISD::SUB &&
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001428 N0 == N1.getOperand(0).getOperand(1))
1429 return DAG.getNode(N1.getOpcode(), N->getDebugLoc(), VT,
1430 N1.getOperand(0).getOperand(0), N1.getOperand(1));
Dale Johannesen34d79852008-12-02 18:40:40 +00001431
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001432 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
1433 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
1434 SDValue N00 = N0.getOperand(0);
1435 SDValue N01 = N0.getOperand(1);
1436 SDValue N10 = N1.getOperand(0);
1437 SDValue N11 = N1.getOperand(1);
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001438
1439 if (isa<ConstantSDNode>(N00) || isa<ConstantSDNode>(N10))
1440 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1441 DAG.getNode(ISD::ADD, N0.getDebugLoc(), VT, N00, N10),
1442 DAG.getNode(ISD::ADD, N1.getDebugLoc(), VT, N01, N11));
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001443 }
Chris Lattner947c2892006-03-13 06:51:27 +00001444
Dan Gohman475871a2008-07-27 21:46:04 +00001445 if (!VT.isVector() && SimplifyDemandedBits(SDValue(N, 0)))
1446 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001447
Chris Lattner947c2892006-03-13 06:51:27 +00001448 // fold (a+b) -> (a|b) iff a and b share no bits.
Duncan Sands83ec4b62008-06-06 12:08:01 +00001449 if (VT.isInteger() && !VT.isVector()) {
Dan Gohman948d8ea2008-02-20 16:33:30 +00001450 APInt LHSZero, LHSOne;
1451 APInt RHSZero, RHSOne;
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001452 DAG.ComputeMaskedBits(N0, LHSZero, LHSOne);
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001453
Dan Gohman948d8ea2008-02-20 16:33:30 +00001454 if (LHSZero.getBoolValue()) {
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001455 DAG.ComputeMaskedBits(N1, RHSZero, RHSOne);
Scott Michelfdc40a02009-02-17 22:15:04 +00001456
Chris Lattner947c2892006-03-13 06:51:27 +00001457 // If all possibly-set bits on the LHS are clear on the RHS, return an OR.
1458 // If all possibly-set bits on the RHS are clear on the LHS, return an OR.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001459 if ((RHSZero & ~LHSZero) == ~LHSZero || (LHSZero & ~RHSZero) == ~RHSZero)
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001460 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N1);
Chris Lattner947c2892006-03-13 06:51:27 +00001461 }
1462 }
Evan Cheng3ef554d2006-11-06 08:14:30 +00001463
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001464 // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<<c2), )
Gabor Greifba36cb52008-08-28 21:40:38 +00001465 if (N0.getOpcode() == ISD::SHL && N0.getNode()->hasOneUse()) {
Bill Wendlingd69c3142009-01-30 02:23:43 +00001466 SDValue Result = combineShlAddConstant(N->getDebugLoc(), N0, N1, DAG);
Gabor Greifba36cb52008-08-28 21:40:38 +00001467 if (Result.getNode()) return Result;
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001468 }
Gabor Greifba36cb52008-08-28 21:40:38 +00001469 if (N1.getOpcode() == ISD::SHL && N1.getNode()->hasOneUse()) {
Bill Wendlingd69c3142009-01-30 02:23:43 +00001470 SDValue Result = combineShlAddConstant(N->getDebugLoc(), N1, N0, DAG);
Gabor Greifba36cb52008-08-28 21:40:38 +00001471 if (Result.getNode()) return Result;
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001472 }
1473
Dan Gohmancd9e1552010-01-19 23:30:49 +00001474 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
1475 if (N1.getOpcode() == ISD::SHL &&
1476 N1.getOperand(0).getOpcode() == ISD::SUB)
1477 if (ConstantSDNode *C =
1478 dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(0)))
1479 if (C->getAPIntValue() == 0)
1480 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0,
1481 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1482 N1.getOperand(0).getOperand(1),
1483 N1.getOperand(1)));
1484 if (N0.getOpcode() == ISD::SHL &&
1485 N0.getOperand(0).getOpcode() == ISD::SUB)
1486 if (ConstantSDNode *C =
1487 dyn_cast<ConstantSDNode>(N0.getOperand(0).getOperand(0)))
1488 if (C->getAPIntValue() == 0)
1489 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1,
1490 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1491 N0.getOperand(0).getOperand(1),
1492 N0.getOperand(1)));
1493
Owen Andersonbc146b02010-09-21 20:42:50 +00001494 if (N1.getOpcode() == ISD::AND) {
1495 SDValue AndOp0 = N1.getOperand(0);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001496 ConstantSDNode *AndOp1 = dyn_cast<ConstantSDNode>(N1->getOperand(1));
Owen Andersonbc146b02010-09-21 20:42:50 +00001497 unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
1498 unsigned DestBits = VT.getScalarType().getSizeInBits();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001499
Owen Andersonbc146b02010-09-21 20:42:50 +00001500 // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
1501 // and similar xforms where the inner op is either ~0 or 0.
1502 if (NumSignBits == DestBits && AndOp1 && AndOp1->isOne()) {
1503 DebugLoc DL = N->getDebugLoc();
1504 return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), AndOp0);
1505 }
1506 }
1507
Benjamin Kramerf50125e2010-12-22 23:17:45 +00001508 // add (sext i1), X -> sub X, (zext i1)
1509 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
1510 N0.getOperand(0).getValueType() == MVT::i1 &&
1511 !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
1512 DebugLoc DL = N->getDebugLoc();
1513 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
1514 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
1515 }
1516
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001517 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001518}
1519
Dan Gohman475871a2008-07-27 21:46:04 +00001520SDValue DAGCombiner::visitADDC(SDNode *N) {
1521 SDValue N0 = N->getOperand(0);
1522 SDValue N1 = N->getOperand(1);
Chris Lattner91153682007-03-04 20:03:15 +00001523 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1524 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001525 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00001526
Chris Lattner91153682007-03-04 20:03:15 +00001527 // If the flag result is dead, turn this into an ADD.
Craig Topper704e1a02012-01-07 18:31:09 +00001528 if (!N->hasAnyUseOfValue(1))
Craig Toppercc274522012-01-07 09:06:39 +00001529 return CombineTo(N, DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N0, N1),
Dale Johannesen874ae252009-06-02 03:12:52 +00001530 DAG.getNode(ISD::CARRY_FALSE,
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00001531 N->getDebugLoc(), MVT::Glue));
Scott Michelfdc40a02009-02-17 22:15:04 +00001532
Chris Lattner91153682007-03-04 20:03:15 +00001533 // canonicalize constant to RHS.
Dan Gohman0a4627d2008-06-23 15:29:14 +00001534 if (N0C && !N1C)
Bill Wendling14036c02009-01-30 02:38:00 +00001535 return DAG.getNode(ISD::ADDC, N->getDebugLoc(), N->getVTList(), N1, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001536
Chris Lattnerb6541762007-03-04 20:40:38 +00001537 // fold (addc x, 0) -> x + no carry out
1538 if (N1C && N1C->isNullValue())
Dale Johannesen874ae252009-06-02 03:12:52 +00001539 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00001540 N->getDebugLoc(), MVT::Glue));
Scott Michelfdc40a02009-02-17 22:15:04 +00001541
Dale Johannesen874ae252009-06-02 03:12:52 +00001542 // fold (addc a, b) -> (or a, b), CARRY_FALSE iff a and b share no bits.
Dan Gohman948d8ea2008-02-20 16:33:30 +00001543 APInt LHSZero, LHSOne;
1544 APInt RHSZero, RHSOne;
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001545 DAG.ComputeMaskedBits(N0, LHSZero, LHSOne);
Bill Wendling14036c02009-01-30 02:38:00 +00001546
Dan Gohman948d8ea2008-02-20 16:33:30 +00001547 if (LHSZero.getBoolValue()) {
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001548 DAG.ComputeMaskedBits(N1, RHSZero, RHSOne);
Scott Michelfdc40a02009-02-17 22:15:04 +00001549
Chris Lattnerb6541762007-03-04 20:40:38 +00001550 // If all possibly-set bits on the LHS are clear on the RHS, return an OR.
1551 // If all possibly-set bits on the RHS are clear on the LHS, return an OR.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00001552 if ((RHSZero & ~LHSZero) == ~LHSZero || (LHSZero & ~RHSZero) == ~RHSZero)
Bill Wendling14036c02009-01-30 02:38:00 +00001553 return CombineTo(N, DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N1),
Dale Johannesen874ae252009-06-02 03:12:52 +00001554 DAG.getNode(ISD::CARRY_FALSE,
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00001555 N->getDebugLoc(), MVT::Glue));
Chris Lattnerb6541762007-03-04 20:40:38 +00001556 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001557
Dan Gohman475871a2008-07-27 21:46:04 +00001558 return SDValue();
Chris Lattner91153682007-03-04 20:03:15 +00001559}
1560
Dan Gohman475871a2008-07-27 21:46:04 +00001561SDValue DAGCombiner::visitADDE(SDNode *N) {
1562 SDValue N0 = N->getOperand(0);
1563 SDValue N1 = N->getOperand(1);
1564 SDValue CarryIn = N->getOperand(2);
Chris Lattner91153682007-03-04 20:03:15 +00001565 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1566 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001567
Chris Lattner91153682007-03-04 20:03:15 +00001568 // canonicalize constant to RHS
Dan Gohman0a4627d2008-06-23 15:29:14 +00001569 if (N0C && !N1C)
Bill Wendling14036c02009-01-30 02:38:00 +00001570 return DAG.getNode(ISD::ADDE, N->getDebugLoc(), N->getVTList(),
1571 N1, N0, CarryIn);
Scott Michelfdc40a02009-02-17 22:15:04 +00001572
Chris Lattnerb6541762007-03-04 20:40:38 +00001573 // fold (adde x, y, false) -> (addc x, y)
Dale Johannesen874ae252009-06-02 03:12:52 +00001574 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
Craig Toppercc274522012-01-07 09:06:39 +00001575 return DAG.getNode(ISD::ADDC, N->getDebugLoc(), N->getVTList(), N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001576
Dan Gohman475871a2008-07-27 21:46:04 +00001577 return SDValue();
Chris Lattner91153682007-03-04 20:03:15 +00001578}
1579
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001580// Since it may not be valid to emit a fold to zero for vector initializers
1581// check if we can before folding.
1582static SDValue tryFoldToZero(DebugLoc DL, const TargetLowering &TLI, EVT VT,
Owen Anderson95771af2011-02-25 21:41:48 +00001583 SelectionDAG &DAG, bool LegalOperations) {
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001584 if (!VT.isVector()) {
1585 return DAG.getConstant(0, VT);
Dan Gohman71dc7c92011-05-17 22:20:36 +00001586 }
1587 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) {
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001588 // Produce a vector of zeros.
1589 SDValue El = DAG.getConstant(0, VT.getVectorElementType());
1590 std::vector<SDValue> Ops(VT.getVectorNumElements(), El);
1591 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT,
1592 &Ops[0], Ops.size());
1593 }
1594 return SDValue();
1595}
1596
Dan Gohman475871a2008-07-27 21:46:04 +00001597SDValue DAGCombiner::visitSUB(SDNode *N) {
1598 SDValue N0 = N->getOperand(0);
1599 SDValue N1 = N->getOperand(1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001600 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode());
1601 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
Eric Christopher7332e6e2011-07-14 01:12:15 +00001602 ConstantSDNode *N1C1 = N1.getOpcode() != ISD::ADD ? 0 :
1603 dyn_cast<ConstantSDNode>(N1.getOperand(1).getNode());
Owen Andersone50ed302009-08-10 22:56:29 +00001604 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00001605
Dan Gohman7f321562007-06-25 16:23:39 +00001606 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001607 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001608 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001609 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001610 }
Bill Wendling2476e5d2008-12-10 22:36:00 +00001611
Chris Lattner854077d2005-10-17 01:07:11 +00001612 // fold (sub x, x) -> 0
Eric Christopher169e1552011-02-16 01:10:03 +00001613 // FIXME: Refactor this and xor and other similar operations together.
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001614 if (N0 == N1)
1615 return tryFoldToZero(N->getDebugLoc(), TLI, VT, DAG, LegalOperations);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001616 // fold (sub c1, c2) -> c1-c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001617 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001618 return DAG.FoldConstantArithmetic(ISD::SUB, VT, N0C, N1C);
Chris Lattner05b57432005-10-11 06:07:15 +00001619 // fold (sub x, c) -> (add x, -c)
1620 if (N1C)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001621 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00001622 DAG.getConstant(-N1C->getAPIntValue(), VT));
Evan Cheng1ad0e8b2010-01-18 21:38:44 +00001623 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
1624 if (N0C && N0C->isAllOnesValue())
1625 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N1, N0);
Benjamin Kramer2c94b422011-01-29 12:34:05 +00001626 // fold A-(A-B) -> B
1627 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
1628 return N1.getOperand(1);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001629 // fold (A+B)-A -> B
Chris Lattner01b3d732005-09-28 22:28:18 +00001630 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
Nate Begeman83e75ec2005-09-06 04:43:02 +00001631 return N0.getOperand(1);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001632 // fold (A+B)-B -> A
Chris Lattner01b3d732005-09-28 22:28:18 +00001633 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
Scott Michelfdc40a02009-02-17 22:15:04 +00001634 return N0.getOperand(0);
Eric Christopher7332e6e2011-07-14 01:12:15 +00001635 // fold C2-(A+C1) -> (C2-C1)-A
1636 if (N1.getOpcode() == ISD::ADD && N0C && N1C1) {
1637 SDValue NewC = DAG.getConstant((N0C->getAPIntValue() - N1C1->getAPIntValue()), VT);
1638 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, NewC,
1639 N1.getOperand(0));
1640 }
Dale Johannesen7c7bc722008-12-23 23:47:22 +00001641 // fold ((A+(B+or-C))-B) -> A+or-C
Dale Johannesenfd3b7b72008-12-16 22:13:49 +00001642 if (N0.getOpcode() == ISD::ADD &&
Dale Johannesenf9cbc1f2008-12-23 23:01:27 +00001643 (N0.getOperand(1).getOpcode() == ISD::SUB ||
1644 N0.getOperand(1).getOpcode() == ISD::ADD) &&
Dale Johannesenfd3b7b72008-12-16 22:13:49 +00001645 N0.getOperand(1).getOperand(0) == N1)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001646 return DAG.getNode(N0.getOperand(1).getOpcode(), N->getDebugLoc(), VT,
1647 N0.getOperand(0), N0.getOperand(1).getOperand(1));
Dale Johannesenf9cbc1f2008-12-23 23:01:27 +00001648 // fold ((A+(C+B))-B) -> A+C
1649 if (N0.getOpcode() == ISD::ADD &&
1650 N0.getOperand(1).getOpcode() == ISD::ADD &&
1651 N0.getOperand(1).getOperand(1) == N1)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001652 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT,
1653 N0.getOperand(0), N0.getOperand(1).getOperand(0));
Dale Johannesen58e39b02008-12-23 01:59:54 +00001654 // fold ((A-(B-C))-C) -> A-B
1655 if (N0.getOpcode() == ISD::SUB &&
1656 N0.getOperand(1).getOpcode() == ISD::SUB &&
1657 N0.getOperand(1).getOperand(1) == N1)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001658 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1659 N0.getOperand(0), N0.getOperand(1).getOperand(0));
Bill Wendlingb0702e02009-01-30 02:42:10 +00001660
Dan Gohman613e0d82007-07-03 14:03:57 +00001661 // If either operand of a sub is undef, the result is undef
Dan Gohman70fb1ae2007-07-10 15:19:29 +00001662 if (N0.getOpcode() == ISD::UNDEF)
1663 return N0;
1664 if (N1.getOpcode() == ISD::UNDEF)
1665 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001666
Dan Gohman6520e202008-10-18 02:06:02 +00001667 // If the relocation model supports it, consider symbol offsets.
1668 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
Duncan Sands25cf2272008-11-24 14:53:14 +00001669 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
Dan Gohman6520e202008-10-18 02:06:02 +00001670 // fold (sub Sym, c) -> Sym-c
1671 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
Devang Patel0d881da2010-07-06 22:08:15 +00001672 return DAG.getGlobalAddress(GA->getGlobal(), N1C->getDebugLoc(), VT,
Dan Gohman6520e202008-10-18 02:06:02 +00001673 GA->getOffset() -
1674 (uint64_t)N1C->getSExtValue());
1675 // fold (sub Sym+c1, Sym+c2) -> c1-c2
1676 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
1677 if (GA->getGlobal() == GB->getGlobal())
1678 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
1679 VT);
1680 }
1681
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001682 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001683}
1684
Craig Toppercc274522012-01-07 09:06:39 +00001685SDValue DAGCombiner::visitSUBC(SDNode *N) {
1686 SDValue N0 = N->getOperand(0);
1687 SDValue N1 = N->getOperand(1);
1688 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1689 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1690 EVT VT = N0.getValueType();
1691
1692 // If the flag result is dead, turn this into an SUB.
Craig Topper704e1a02012-01-07 18:31:09 +00001693 if (!N->hasAnyUseOfValue(1))
Craig Toppercc274522012-01-07 09:06:39 +00001694 return CombineTo(N, DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, N1),
1695 DAG.getNode(ISD::CARRY_FALSE, N->getDebugLoc(),
1696 MVT::Glue));
1697
1698 // fold (subc x, x) -> 0 + no borrow
1699 if (N0 == N1)
1700 return CombineTo(N, DAG.getConstant(0, VT),
1701 DAG.getNode(ISD::CARRY_FALSE, N->getDebugLoc(),
1702 MVT::Glue));
1703
1704 // fold (subc x, 0) -> x + no borrow
1705 if (N1C && N1C->isNullValue())
1706 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, N->getDebugLoc(),
1707 MVT::Glue));
1708
1709 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
1710 if (N0C && N0C->isAllOnesValue())
1711 return CombineTo(N, DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N1, N0),
1712 DAG.getNode(ISD::CARRY_FALSE, N->getDebugLoc(),
1713 MVT::Glue));
1714
1715 return SDValue();
1716}
1717
1718SDValue DAGCombiner::visitSUBE(SDNode *N) {
1719 SDValue N0 = N->getOperand(0);
1720 SDValue N1 = N->getOperand(1);
1721 SDValue CarryIn = N->getOperand(2);
1722
1723 // fold (sube x, y, false) -> (subc x, y)
1724 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
1725 return DAG.getNode(ISD::SUBC, N->getDebugLoc(), N->getVTList(), N0, N1);
1726
1727 return SDValue();
1728}
1729
Dan Gohman475871a2008-07-27 21:46:04 +00001730SDValue DAGCombiner::visitMUL(SDNode *N) {
1731 SDValue N0 = N->getOperand(0);
1732 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00001733 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1734 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001735 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00001736
Dan Gohman7f321562007-06-25 16:23:39 +00001737 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001738 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001739 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001740 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001741 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001742
Dan Gohman613e0d82007-07-03 14:03:57 +00001743 // fold (mul x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00001744 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00001745 return DAG.getConstant(0, VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001746 // fold (mul c1, c2) -> c1*c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001747 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001748 return DAG.FoldConstantArithmetic(ISD::MUL, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00001749 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00001750 if (N0C && !N1C)
Bill Wendling9c8148a2009-01-30 02:45:56 +00001751 return DAG.getNode(ISD::MUL, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001752 // fold (mul x, 0) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00001753 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00001754 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001755 // fold (mul x, -1) -> 0-x
Nate Begeman646d7e22005-09-02 21:18:40 +00001756 if (N1C && N1C->isAllOnesValue())
Bill Wendling9c8148a2009-01-30 02:45:56 +00001757 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1758 DAG.getConstant(0, VT), N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001759 // fold (mul x, (1 << c)) -> x << c
Dan Gohman002e5d02008-03-13 22:13:53 +00001760 if (N1C && N1C->getAPIntValue().isPowerOf2())
Bill Wendling9c8148a2009-01-30 02:45:56 +00001761 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00001762 DAG.getConstant(N1C->getAPIntValue().logBase2(),
Owen Anderson95771af2011-02-25 21:41:48 +00001763 getShiftAmountTy(N0.getValueType())));
Chris Lattner3e6099b2005-10-30 06:41:49 +00001764 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
Chris Lattner66b8bc32009-03-09 20:22:18 +00001765 if (N1C && (-N1C->getAPIntValue()).isPowerOf2()) {
1766 unsigned Log2Val = (-N1C->getAPIntValue()).logBase2();
Scott Michelfdc40a02009-02-17 22:15:04 +00001767 // FIXME: If the input is something that is easily negated (e.g. a
Chris Lattner3e6099b2005-10-30 06:41:49 +00001768 // single-use add), we should put the negate there.
Bill Wendling9c8148a2009-01-30 02:45:56 +00001769 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1770 DAG.getConstant(0, VT),
Bill Wendling73e16b22009-01-30 02:49:26 +00001771 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0,
Owen Anderson95771af2011-02-25 21:41:48 +00001772 DAG.getConstant(Log2Val,
1773 getShiftAmountTy(N0.getValueType()))));
Chris Lattner66b8bc32009-03-09 20:22:18 +00001774 }
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001775 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
Bill Wendling73e16b22009-01-30 02:49:26 +00001776 if (N1C && N0.getOpcode() == ISD::SHL &&
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001777 isa<ConstantSDNode>(N0.getOperand(1))) {
Bill Wendling9c8148a2009-01-30 02:45:56 +00001778 SDValue C3 = DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1779 N1, N0.getOperand(1));
Gabor Greifba36cb52008-08-28 21:40:38 +00001780 AddToWorkList(C3.getNode());
Bill Wendling9c8148a2009-01-30 02:45:56 +00001781 return DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
1782 N0.getOperand(0), C3);
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001783 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001784
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001785 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
1786 // use.
1787 {
Dan Gohman475871a2008-07-27 21:46:04 +00001788 SDValue Sh(0,0), Y(0,0);
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001789 // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
1790 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
Gabor Greifba36cb52008-08-28 21:40:38 +00001791 N0.getNode()->hasOneUse()) {
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001792 Sh = N0; Y = N1;
Scott Michelfdc40a02009-02-17 22:15:04 +00001793 } else if (N1.getOpcode() == ISD::SHL &&
Gabor Greif12632d22008-08-30 19:29:20 +00001794 isa<ConstantSDNode>(N1.getOperand(1)) &&
1795 N1.getNode()->hasOneUse()) {
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001796 Sh = N1; Y = N0;
1797 }
Bill Wendling73e16b22009-01-30 02:49:26 +00001798
Gabor Greifba36cb52008-08-28 21:40:38 +00001799 if (Sh.getNode()) {
Bill Wendling9c8148a2009-01-30 02:45:56 +00001800 SDValue Mul = DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
1801 Sh.getOperand(0), Y);
1802 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1803 Mul, Sh.getOperand(1));
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001804 }
1805 }
Bill Wendling73e16b22009-01-30 02:49:26 +00001806
Chris Lattnera1deca32006-03-04 23:33:26 +00001807 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
Scott Michelfdc40a02009-02-17 22:15:04 +00001808 if (N1C && N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse() &&
Bill Wendling9c8148a2009-01-30 02:45:56 +00001809 isa<ConstantSDNode>(N0.getOperand(1)))
1810 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT,
1811 DAG.getNode(ISD::MUL, N0.getDebugLoc(), VT,
1812 N0.getOperand(0), N1),
1813 DAG.getNode(ISD::MUL, N1.getDebugLoc(), VT,
1814 N0.getOperand(1), N1));
Scott Michelfdc40a02009-02-17 22:15:04 +00001815
Nate Begemancd4d58c2006-02-03 06:46:56 +00001816 // reassociate mul
Bill Wendling35247c32009-01-30 00:45:56 +00001817 SDValue RMUL = ReassociateOps(ISD::MUL, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001818 if (RMUL.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00001819 return RMUL;
Dan Gohman7f321562007-06-25 16:23:39 +00001820
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001821 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001822}
1823
Dan Gohman475871a2008-07-27 21:46:04 +00001824SDValue DAGCombiner::visitSDIV(SDNode *N) {
1825 SDValue N0 = N->getOperand(0);
1826 SDValue N1 = N->getOperand(1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001827 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode());
1828 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
Owen Andersone50ed302009-08-10 22:56:29 +00001829 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001830
Dan Gohman7f321562007-06-25 16:23:39 +00001831 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001832 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001833 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001834 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001835 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001836
Nate Begeman1d4d4142005-09-01 00:19:25 +00001837 // fold (sdiv c1, c2) -> c1/c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001838 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001839 return DAG.FoldConstantArithmetic(ISD::SDIV, VT, N0C, N1C);
Nate Begeman405e3ec2005-10-21 00:02:42 +00001840 // fold (sdiv X, 1) -> X
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001841 if (N1C && N1C->getAPIntValue() == 1LL)
Nate Begeman405e3ec2005-10-21 00:02:42 +00001842 return N0;
1843 // fold (sdiv X, -1) -> 0-X
1844 if (N1C && N1C->isAllOnesValue())
Bill Wendling944d34b2009-01-30 02:52:17 +00001845 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1846 DAG.getConstant(0, VT), N0);
Chris Lattner094c8fc2005-10-07 06:10:46 +00001847 // If we know the sign bits of both operands are zero, strength reduce to a
1848 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
Duncan Sands83ec4b62008-06-06 12:08:01 +00001849 if (!VT.isVector()) {
Dan Gohman2e68b6f2008-02-25 21:11:39 +00001850 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
Bill Wendling944d34b2009-01-30 02:52:17 +00001851 return DAG.getNode(ISD::UDIV, N->getDebugLoc(), N1.getValueType(),
1852 N0, N1);
Chris Lattnerf32aac32008-01-27 23:32:17 +00001853 }
Nate Begemancd6a6ed2006-02-17 07:26:20 +00001854 // fold (sdiv X, pow2) -> simple ops after legalize
Eli Friedman1c663fe2011-12-07 03:55:52 +00001855 if (N1C && !N1C->isNullValue() &&
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001856 (N1C->getAPIntValue().isPowerOf2() ||
1857 (-N1C->getAPIntValue()).isPowerOf2())) {
Nate Begeman405e3ec2005-10-21 00:02:42 +00001858 // If dividing by powers of two is cheap, then don't perform the following
1859 // fold.
1860 if (TLI.isPow2DivCheap())
Dan Gohman475871a2008-07-27 21:46:04 +00001861 return SDValue();
Bill Wendling944d34b2009-01-30 02:52:17 +00001862
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001863 unsigned lg2 = N1C->getAPIntValue().countTrailingZeros();
Bill Wendling944d34b2009-01-30 02:52:17 +00001864
Chris Lattner8f4880b2006-02-16 08:02:36 +00001865 // Splat the sign bit into the register
Bill Wendling944d34b2009-01-30 02:52:17 +00001866 SDValue SGN = DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, N0,
1867 DAG.getConstant(VT.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00001868 getShiftAmountTy(N0.getValueType())));
Gabor Greifba36cb52008-08-28 21:40:38 +00001869 AddToWorkList(SGN.getNode());
Bill Wendling944d34b2009-01-30 02:52:17 +00001870
Chris Lattner8f4880b2006-02-16 08:02:36 +00001871 // Add (N0 < 0) ? abs2 - 1 : 0;
Bill Wendling944d34b2009-01-30 02:52:17 +00001872 SDValue SRL = DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, SGN,
1873 DAG.getConstant(VT.getSizeInBits() - lg2,
Owen Anderson95771af2011-02-25 21:41:48 +00001874 getShiftAmountTy(SGN.getValueType())));
Bill Wendling944d34b2009-01-30 02:52:17 +00001875 SDValue ADD = DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N0, SRL);
Gabor Greifba36cb52008-08-28 21:40:38 +00001876 AddToWorkList(SRL.getNode());
1877 AddToWorkList(ADD.getNode()); // Divide by pow2
Bill Wendling944d34b2009-01-30 02:52:17 +00001878 SDValue SRA = DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, ADD,
Owen Anderson95771af2011-02-25 21:41:48 +00001879 DAG.getConstant(lg2, getShiftAmountTy(ADD.getValueType())));
Bill Wendling944d34b2009-01-30 02:52:17 +00001880
Nate Begeman405e3ec2005-10-21 00:02:42 +00001881 // If we're dividing by a positive value, we're done. Otherwise, we must
1882 // negate the result.
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001883 if (N1C->getAPIntValue().isNonNegative())
Nate Begeman405e3ec2005-10-21 00:02:42 +00001884 return SRA;
Bill Wendling944d34b2009-01-30 02:52:17 +00001885
Gabor Greifba36cb52008-08-28 21:40:38 +00001886 AddToWorkList(SRA.getNode());
Bill Wendling944d34b2009-01-30 02:52:17 +00001887 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1888 DAG.getConstant(0, VT), SRA);
Nate Begeman405e3ec2005-10-21 00:02:42 +00001889 }
Bill Wendling944d34b2009-01-30 02:52:17 +00001890
Nate Begeman69575232005-10-20 02:15:44 +00001891 // if integer divide is expensive and we satisfy the requirements, emit an
1892 // alternate sequence.
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001893 if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001894 SDValue Op = BuildSDIV(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001895 if (Op.getNode()) return Op;
Nate Begeman69575232005-10-20 02:15:44 +00001896 }
Dan Gohman7f321562007-06-25 16:23:39 +00001897
Dan Gohman613e0d82007-07-03 14:03:57 +00001898 // undef / X -> 0
1899 if (N0.getOpcode() == ISD::UNDEF)
1900 return DAG.getConstant(0, VT);
1901 // X / undef -> undef
1902 if (N1.getOpcode() == ISD::UNDEF)
1903 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001904
Dan Gohman475871a2008-07-27 21:46:04 +00001905 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001906}
1907
Dan Gohman475871a2008-07-27 21:46:04 +00001908SDValue DAGCombiner::visitUDIV(SDNode *N) {
1909 SDValue N0 = N->getOperand(0);
1910 SDValue N1 = N->getOperand(1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001911 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode());
1912 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
Owen Andersone50ed302009-08-10 22:56:29 +00001913 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001914
Dan Gohman7f321562007-06-25 16:23:39 +00001915 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001916 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001917 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001918 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001919 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001920
Nate Begeman1d4d4142005-09-01 00:19:25 +00001921 // fold (udiv c1, c2) -> c1/c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001922 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001923 return DAG.FoldConstantArithmetic(ISD::UDIV, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001924 // fold (udiv x, (1 << c)) -> x >>u c
Dan Gohman002e5d02008-03-13 22:13:53 +00001925 if (N1C && N1C->getAPIntValue().isPowerOf2())
Scott Michelfdc40a02009-02-17 22:15:04 +00001926 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00001927 DAG.getConstant(N1C->getAPIntValue().logBase2(),
Owen Anderson95771af2011-02-25 21:41:48 +00001928 getShiftAmountTy(N0.getValueType())));
Nate Begemanfb5e4bd2006-02-05 07:20:23 +00001929 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
1930 if (N1.getOpcode() == ISD::SHL) {
1931 if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00001932 if (SHC->getAPIntValue().isPowerOf2()) {
Owen Andersone50ed302009-08-10 22:56:29 +00001933 EVT ADDVT = N1.getOperand(1).getValueType();
Bill Wendling07d85142009-01-30 02:55:25 +00001934 SDValue Add = DAG.getNode(ISD::ADD, N->getDebugLoc(), ADDVT,
1935 N1.getOperand(1),
1936 DAG.getConstant(SHC->getAPIntValue()
1937 .logBase2(),
1938 ADDVT));
Gabor Greifba36cb52008-08-28 21:40:38 +00001939 AddToWorkList(Add.getNode());
Bill Wendling07d85142009-01-30 02:55:25 +00001940 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0, Add);
Nate Begemanfb5e4bd2006-02-05 07:20:23 +00001941 }
1942 }
1943 }
Nate Begeman69575232005-10-20 02:15:44 +00001944 // fold (udiv x, c) -> alternate
Dan Gohman002e5d02008-03-13 22:13:53 +00001945 if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001946 SDValue Op = BuildUDIV(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001947 if (Op.getNode()) return Op;
Chris Lattnere9936d12005-10-22 18:50:15 +00001948 }
Dan Gohman7f321562007-06-25 16:23:39 +00001949
Dan Gohman613e0d82007-07-03 14:03:57 +00001950 // undef / X -> 0
1951 if (N0.getOpcode() == ISD::UNDEF)
1952 return DAG.getConstant(0, VT);
1953 // X / undef -> undef
1954 if (N1.getOpcode() == ISD::UNDEF)
1955 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001956
Dan Gohman475871a2008-07-27 21:46:04 +00001957 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001958}
1959
Dan Gohman475871a2008-07-27 21:46:04 +00001960SDValue DAGCombiner::visitSREM(SDNode *N) {
1961 SDValue N0 = N->getOperand(0);
1962 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00001963 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1964 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001965 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001966
Nate Begeman1d4d4142005-09-01 00:19:25 +00001967 // fold (srem c1, c2) -> c1%c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001968 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001969 return DAG.FoldConstantArithmetic(ISD::SREM, VT, N0C, N1C);
Nate Begeman07ed4172005-10-10 21:26:48 +00001970 // If we know the sign bits of both operands are zero, strength reduce to a
1971 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
Duncan Sands83ec4b62008-06-06 12:08:01 +00001972 if (!VT.isVector()) {
Dan Gohman2e68b6f2008-02-25 21:11:39 +00001973 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001974 return DAG.getNode(ISD::UREM, N->getDebugLoc(), VT, N0, N1);
Chris Lattneree339f42008-01-27 23:21:58 +00001975 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001976
Dan Gohman77003042007-11-26 23:46:11 +00001977 // If X/C can be simplified by the division-by-constant logic, lower
1978 // X%C to the equivalent of X-X/C*C.
Chris Lattner26d29902006-10-12 20:58:32 +00001979 if (N1C && !N1C->isNullValue()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001980 SDValue Div = DAG.getNode(ISD::SDIV, N->getDebugLoc(), VT, N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001981 AddToWorkList(Div.getNode());
1982 SDValue OptimizedDiv = combine(Div.getNode());
1983 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001984 SDValue Mul = DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
1985 OptimizedDiv, N1);
1986 SDValue Sub = DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, Mul);
Gabor Greifba36cb52008-08-28 21:40:38 +00001987 AddToWorkList(Mul.getNode());
Dan Gohman77003042007-11-26 23:46:11 +00001988 return Sub;
1989 }
Chris Lattner26d29902006-10-12 20:58:32 +00001990 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001991
Dan Gohman613e0d82007-07-03 14:03:57 +00001992 // undef % X -> 0
1993 if (N0.getOpcode() == ISD::UNDEF)
1994 return DAG.getConstant(0, VT);
1995 // X % undef -> undef
1996 if (N1.getOpcode() == ISD::UNDEF)
1997 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001998
Dan Gohman475871a2008-07-27 21:46:04 +00001999 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002000}
2001
Dan Gohman475871a2008-07-27 21:46:04 +00002002SDValue DAGCombiner::visitUREM(SDNode *N) {
2003 SDValue N0 = N->getOperand(0);
2004 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00002005 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2006 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002007 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00002008
Nate Begeman1d4d4142005-09-01 00:19:25 +00002009 // fold (urem c1, c2) -> c1%c2
Nate Begeman646d7e22005-09-02 21:18:40 +00002010 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00002011 return DAG.FoldConstantArithmetic(ISD::UREM, VT, N0C, N1C);
Nate Begeman07ed4172005-10-10 21:26:48 +00002012 // fold (urem x, pow2) -> (and x, pow2-1)
Dan Gohman002e5d02008-03-13 22:13:53 +00002013 if (N1C && !N1C->isNullValue() && N1C->getAPIntValue().isPowerOf2())
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00002014 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00002015 DAG.getConstant(N1C->getAPIntValue()-1,VT));
Nate Begemanc031e332006-02-05 07:36:48 +00002016 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
2017 if (N1.getOpcode() == ISD::SHL) {
2018 if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00002019 if (SHC->getAPIntValue().isPowerOf2()) {
Dan Gohman475871a2008-07-27 21:46:04 +00002020 SDValue Add =
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00002021 DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N1,
Duncan Sands83ec4b62008-06-06 12:08:01 +00002022 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()),
Dan Gohman002e5d02008-03-13 22:13:53 +00002023 VT));
Gabor Greifba36cb52008-08-28 21:40:38 +00002024 AddToWorkList(Add.getNode());
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00002025 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0, Add);
Nate Begemanc031e332006-02-05 07:36:48 +00002026 }
2027 }
2028 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002029
Dan Gohman77003042007-11-26 23:46:11 +00002030 // If X/C can be simplified by the division-by-constant logic, lower
2031 // X%C to the equivalent of X-X/C*C.
Chris Lattner26d29902006-10-12 20:58:32 +00002032 if (N1C && !N1C->isNullValue()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00002033 SDValue Div = DAG.getNode(ISD::UDIV, N->getDebugLoc(), VT, N0, N1);
Dan Gohman942ca7f2008-09-08 16:59:01 +00002034 AddToWorkList(Div.getNode());
Gabor Greifba36cb52008-08-28 21:40:38 +00002035 SDValue OptimizedDiv = combine(Div.getNode());
2036 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00002037 SDValue Mul = DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
2038 OptimizedDiv, N1);
2039 SDValue Sub = DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, Mul);
Gabor Greifba36cb52008-08-28 21:40:38 +00002040 AddToWorkList(Mul.getNode());
Dan Gohman77003042007-11-26 23:46:11 +00002041 return Sub;
2042 }
Chris Lattner26d29902006-10-12 20:58:32 +00002043 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002044
Dan Gohman613e0d82007-07-03 14:03:57 +00002045 // undef % X -> 0
2046 if (N0.getOpcode() == ISD::UNDEF)
2047 return DAG.getConstant(0, VT);
2048 // X % undef -> undef
2049 if (N1.getOpcode() == ISD::UNDEF)
2050 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00002051
Dan Gohman475871a2008-07-27 21:46:04 +00002052 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002053}
2054
Dan Gohman475871a2008-07-27 21:46:04 +00002055SDValue DAGCombiner::visitMULHS(SDNode *N) {
2056 SDValue N0 = N->getOperand(0);
2057 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00002058 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002059 EVT VT = N->getValueType(0);
Chris Lattnerde1c3602010-12-13 08:39:01 +00002060 DebugLoc DL = N->getDebugLoc();
Scott Michelfdc40a02009-02-17 22:15:04 +00002061
Nate Begeman1d4d4142005-09-01 00:19:25 +00002062 // fold (mulhs x, 0) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00002063 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002064 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00002065 // fold (mulhs x, 1) -> (sra x, size(x)-1)
Dan Gohman002e5d02008-03-13 22:13:53 +00002066 if (N1C && N1C->getAPIntValue() == 1)
Bill Wendling326411d2009-01-30 03:00:18 +00002067 return DAG.getNode(ISD::SRA, N->getDebugLoc(), N0.getValueType(), N0,
2068 DAG.getConstant(N0.getValueType().getSizeInBits() - 1,
Owen Anderson95771af2011-02-25 21:41:48 +00002069 getShiftAmountTy(N0.getValueType())));
Dan Gohman613e0d82007-07-03 14:03:57 +00002070 // fold (mulhs x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00002071 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00002072 return DAG.getConstant(0, VT);
Dan Gohman7f321562007-06-25 16:23:39 +00002073
Chris Lattnerde1c3602010-12-13 08:39:01 +00002074 // If the type twice as wide is legal, transform the mulhs to a wider multiply
2075 // plus a shift.
2076 if (VT.isSimple() && !VT.isVector()) {
2077 MVT Simple = VT.getSimpleVT();
2078 unsigned SimpleSize = Simple.getSizeInBits();
2079 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2080 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2081 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
2082 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
2083 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
Chris Lattner1a0fbe22010-12-15 05:51:39 +00002084 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
Owen Anderson95771af2011-02-25 21:41:48 +00002085 DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType())));
Chris Lattnerde1c3602010-12-13 08:39:01 +00002086 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
2087 }
2088 }
Owen Anderson95771af2011-02-25 21:41:48 +00002089
Dan Gohman475871a2008-07-27 21:46:04 +00002090 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002091}
2092
Dan Gohman475871a2008-07-27 21:46:04 +00002093SDValue DAGCombiner::visitMULHU(SDNode *N) {
2094 SDValue N0 = N->getOperand(0);
2095 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00002096 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002097 EVT VT = N->getValueType(0);
Chris Lattnerde1c3602010-12-13 08:39:01 +00002098 DebugLoc DL = N->getDebugLoc();
Scott Michelfdc40a02009-02-17 22:15:04 +00002099
Nate Begeman1d4d4142005-09-01 00:19:25 +00002100 // fold (mulhu x, 0) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00002101 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002102 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00002103 // fold (mulhu x, 1) -> 0
Dan Gohman002e5d02008-03-13 22:13:53 +00002104 if (N1C && N1C->getAPIntValue() == 1)
Nate Begeman83e75ec2005-09-06 04:43:02 +00002105 return DAG.getConstant(0, N0.getValueType());
Dan Gohman613e0d82007-07-03 14:03:57 +00002106 // fold (mulhu x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00002107 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00002108 return DAG.getConstant(0, VT);
Dan Gohman7f321562007-06-25 16:23:39 +00002109
Chris Lattnerde1c3602010-12-13 08:39:01 +00002110 // If the type twice as wide is legal, transform the mulhu to a wider multiply
2111 // plus a shift.
2112 if (VT.isSimple() && !VT.isVector()) {
2113 MVT Simple = VT.getSimpleVT();
2114 unsigned SimpleSize = Simple.getSizeInBits();
2115 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2116 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2117 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
2118 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
2119 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
2120 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
Owen Anderson95771af2011-02-25 21:41:48 +00002121 DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType())));
Chris Lattnerde1c3602010-12-13 08:39:01 +00002122 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
2123 }
2124 }
Owen Anderson95771af2011-02-25 21:41:48 +00002125
Dan Gohman475871a2008-07-27 21:46:04 +00002126 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002127}
2128
Dan Gohman389079b2007-10-08 17:57:15 +00002129/// SimplifyNodeWithTwoResults - Perform optimizations common to nodes that
2130/// compute two values. LoOp and HiOp give the opcodes for the two computations
2131/// that are being performed. Return true if a simplification was made.
2132///
Scott Michelfdc40a02009-02-17 22:15:04 +00002133SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
Dan Gohman475871a2008-07-27 21:46:04 +00002134 unsigned HiOp) {
Dan Gohman389079b2007-10-08 17:57:15 +00002135 // If the high half is not needed, just compute the low half.
Evan Cheng44711942007-11-08 09:25:29 +00002136 bool HiExists = N->hasAnyUseOfValue(1);
2137 if (!HiExists &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002138 (!LegalOperations ||
Dan Gohman389079b2007-10-08 17:57:15 +00002139 TLI.isOperationLegal(LoOp, N->getValueType(0)))) {
Bill Wendling826d1142009-01-30 03:08:40 +00002140 SDValue Res = DAG.getNode(LoOp, N->getDebugLoc(), N->getValueType(0),
2141 N->op_begin(), N->getNumOperands());
Chris Lattner5eee4272008-01-26 01:09:19 +00002142 return CombineTo(N, Res, Res);
Dan Gohman389079b2007-10-08 17:57:15 +00002143 }
2144
2145 // If the low half is not needed, just compute the high half.
Evan Cheng44711942007-11-08 09:25:29 +00002146 bool LoExists = N->hasAnyUseOfValue(0);
2147 if (!LoExists &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002148 (!LegalOperations ||
Dan Gohman389079b2007-10-08 17:57:15 +00002149 TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
Bill Wendling826d1142009-01-30 03:08:40 +00002150 SDValue Res = DAG.getNode(HiOp, N->getDebugLoc(), N->getValueType(1),
2151 N->op_begin(), N->getNumOperands());
Chris Lattner5eee4272008-01-26 01:09:19 +00002152 return CombineTo(N, Res, Res);
Dan Gohman389079b2007-10-08 17:57:15 +00002153 }
2154
Evan Cheng44711942007-11-08 09:25:29 +00002155 // If both halves are used, return as it is.
2156 if (LoExists && HiExists)
Dan Gohman475871a2008-07-27 21:46:04 +00002157 return SDValue();
Evan Cheng44711942007-11-08 09:25:29 +00002158
2159 // If the two computed results can be simplified separately, separate them.
Evan Cheng44711942007-11-08 09:25:29 +00002160 if (LoExists) {
Bill Wendling826d1142009-01-30 03:08:40 +00002161 SDValue Lo = DAG.getNode(LoOp, N->getDebugLoc(), N->getValueType(0),
2162 N->op_begin(), N->getNumOperands());
Gabor Greifba36cb52008-08-28 21:40:38 +00002163 AddToWorkList(Lo.getNode());
2164 SDValue LoOpt = combine(Lo.getNode());
2165 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002166 (!LegalOperations ||
Duncan Sandsd4b9c172008-06-13 19:07:40 +00002167 TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
Chris Lattner5eee4272008-01-26 01:09:19 +00002168 return CombineTo(N, LoOpt, LoOpt);
Dan Gohman389079b2007-10-08 17:57:15 +00002169 }
2170
Evan Cheng44711942007-11-08 09:25:29 +00002171 if (HiExists) {
Bill Wendling826d1142009-01-30 03:08:40 +00002172 SDValue Hi = DAG.getNode(HiOp, N->getDebugLoc(), N->getValueType(1),
Duncan Sands25cf2272008-11-24 14:53:14 +00002173 N->op_begin(), N->getNumOperands());
Gabor Greifba36cb52008-08-28 21:40:38 +00002174 AddToWorkList(Hi.getNode());
2175 SDValue HiOpt = combine(Hi.getNode());
2176 if (HiOpt.getNode() && HiOpt != Hi &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002177 (!LegalOperations ||
Duncan Sandsd4b9c172008-06-13 19:07:40 +00002178 TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
Chris Lattner5eee4272008-01-26 01:09:19 +00002179 return CombineTo(N, HiOpt, HiOpt);
Evan Cheng44711942007-11-08 09:25:29 +00002180 }
Bill Wendling826d1142009-01-30 03:08:40 +00002181
Dan Gohman475871a2008-07-27 21:46:04 +00002182 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002183}
2184
Dan Gohman475871a2008-07-27 21:46:04 +00002185SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
2186 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS);
Gabor Greifba36cb52008-08-28 21:40:38 +00002187 if (Res.getNode()) return Res;
Dan Gohman389079b2007-10-08 17:57:15 +00002188
Chris Lattner33e77d32010-12-15 06:04:19 +00002189 EVT VT = N->getValueType(0);
2190 DebugLoc DL = N->getDebugLoc();
2191
2192 // If the type twice as wide is legal, transform the mulhu to a wider multiply
2193 // plus a shift.
2194 if (VT.isSimple() && !VT.isVector()) {
2195 MVT Simple = VT.getSimpleVT();
2196 unsigned SimpleSize = Simple.getSizeInBits();
2197 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2198 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2199 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
2200 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
2201 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
2202 // Compute the high part as N1.
2203 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
Owen Anderson95771af2011-02-25 21:41:48 +00002204 DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType())));
Chris Lattner33e77d32010-12-15 06:04:19 +00002205 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
2206 // Compute the low part as N0.
2207 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
2208 return CombineTo(N, Lo, Hi);
2209 }
2210 }
Owen Anderson95771af2011-02-25 21:41:48 +00002211
Dan Gohman475871a2008-07-27 21:46:04 +00002212 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002213}
2214
Dan Gohman475871a2008-07-27 21:46:04 +00002215SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
2216 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU);
Gabor Greifba36cb52008-08-28 21:40:38 +00002217 if (Res.getNode()) return Res;
Dan Gohman389079b2007-10-08 17:57:15 +00002218
Chris Lattner33e77d32010-12-15 06:04:19 +00002219 EVT VT = N->getValueType(0);
2220 DebugLoc DL = N->getDebugLoc();
Owen Anderson95771af2011-02-25 21:41:48 +00002221
Chris Lattner33e77d32010-12-15 06:04:19 +00002222 // If the type twice as wide is legal, transform the mulhu to a wider multiply
2223 // plus a shift.
2224 if (VT.isSimple() && !VT.isVector()) {
2225 MVT Simple = VT.getSimpleVT();
2226 unsigned SimpleSize = Simple.getSizeInBits();
2227 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2228 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2229 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
2230 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
2231 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
2232 // Compute the high part as N1.
2233 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
Owen Anderson95771af2011-02-25 21:41:48 +00002234 DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType())));
Chris Lattner33e77d32010-12-15 06:04:19 +00002235 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
2236 // Compute the low part as N0.
2237 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
2238 return CombineTo(N, Lo, Hi);
2239 }
2240 }
Owen Anderson95771af2011-02-25 21:41:48 +00002241
Dan Gohman475871a2008-07-27 21:46:04 +00002242 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002243}
2244
Benjamin Kramerf55d26e2011-05-21 18:31:55 +00002245SDValue DAGCombiner::visitSMULO(SDNode *N) {
2246 // (smulo x, 2) -> (saddo x, x)
2247 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2248 if (C2->getAPIntValue() == 2)
2249 return DAG.getNode(ISD::SADDO, N->getDebugLoc(), N->getVTList(),
2250 N->getOperand(0), N->getOperand(0));
2251
2252 return SDValue();
2253}
2254
2255SDValue DAGCombiner::visitUMULO(SDNode *N) {
2256 // (umulo x, 2) -> (uaddo x, x)
2257 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2258 if (C2->getAPIntValue() == 2)
2259 return DAG.getNode(ISD::UADDO, N->getDebugLoc(), N->getVTList(),
2260 N->getOperand(0), N->getOperand(0));
2261
2262 return SDValue();
2263}
2264
Dan Gohman475871a2008-07-27 21:46:04 +00002265SDValue DAGCombiner::visitSDIVREM(SDNode *N) {
2266 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::SDIV, ISD::SREM);
Gabor Greifba36cb52008-08-28 21:40:38 +00002267 if (Res.getNode()) return Res;
Scott Michelfdc40a02009-02-17 22:15:04 +00002268
Dan Gohman475871a2008-07-27 21:46:04 +00002269 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002270}
2271
Dan Gohman475871a2008-07-27 21:46:04 +00002272SDValue DAGCombiner::visitUDIVREM(SDNode *N) {
2273 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::UDIV, ISD::UREM);
Gabor Greifba36cb52008-08-28 21:40:38 +00002274 if (Res.getNode()) return Res;
Scott Michelfdc40a02009-02-17 22:15:04 +00002275
Dan Gohman475871a2008-07-27 21:46:04 +00002276 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002277}
2278
Chris Lattner35e5c142006-05-05 05:51:50 +00002279/// SimplifyBinOpWithSameOpcodeHands - If this is a binary operator with
2280/// two operands of the same opcode, try to simplify it.
Dan Gohman475871a2008-07-27 21:46:04 +00002281SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
2282 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
Owen Andersone50ed302009-08-10 22:56:29 +00002283 EVT VT = N0.getValueType();
Chris Lattner35e5c142006-05-05 05:51:50 +00002284 assert(N0.getOpcode() == N1.getOpcode() && "Bad input!");
Scott Michelfdc40a02009-02-17 22:15:04 +00002285
Dan Gohmanff00a552010-01-14 03:08:49 +00002286 // Bail early if none of these transforms apply.
2287 if (N0.getNode()->getNumOperands() == 0) return SDValue();
2288
Chris Lattner540121f2006-05-05 06:31:05 +00002289 // For each of OP in AND/OR/XOR:
2290 // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
2291 // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
2292 // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
Dan Gohman4e39e9d2010-06-24 14:30:44 +00002293 // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
Nate Begeman93e0ed32009-12-03 07:11:29 +00002294 //
2295 // do not sink logical op inside of a vector extend, since it may combine
2296 // into a vsetcc.
Evan Chengd40d03e2010-01-06 19:38:29 +00002297 EVT Op0VT = N0.getOperand(0).getValueType();
2298 if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
Dan Gohman97121ba2009-04-08 00:15:30 +00002299 N0.getOpcode() == ISD::SIGN_EXTEND ||
Evan Chenge5b51ac2010-04-17 06:13:15 +00002300 // Avoid infinite looping with PromoteIntBinOp.
2301 (N0.getOpcode() == ISD::ANY_EXTEND &&
2302 (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
Dan Gohman4e39e9d2010-06-24 14:30:44 +00002303 (N0.getOpcode() == ISD::TRUNCATE &&
2304 (!TLI.isZExtFree(VT, Op0VT) ||
2305 !TLI.isTruncateFree(Op0VT, VT)) &&
2306 TLI.isTypeLegal(Op0VT))) &&
Nate Begeman93e0ed32009-12-03 07:11:29 +00002307 !VT.isVector() &&
Evan Chengd40d03e2010-01-06 19:38:29 +00002308 Op0VT == N1.getOperand(0).getValueType() &&
2309 (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
Bill Wendlingb74c8672009-01-30 19:25:47 +00002310 SDValue ORNode = DAG.getNode(N->getOpcode(), N0.getDebugLoc(),
2311 N0.getOperand(0).getValueType(),
2312 N0.getOperand(0), N1.getOperand(0));
Gabor Greifba36cb52008-08-28 21:40:38 +00002313 AddToWorkList(ORNode.getNode());
Bill Wendlingb74c8672009-01-30 19:25:47 +00002314 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT, ORNode);
Chris Lattner35e5c142006-05-05 05:51:50 +00002315 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002316
Chris Lattnera3dc3f62006-05-05 06:10:43 +00002317 // For each of OP in SHL/SRL/SRA/AND...
2318 // fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
2319 // fold (or (OP x, z), (OP y, z)) -> (OP (or x, y), z)
2320 // fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
Chris Lattner35e5c142006-05-05 05:51:50 +00002321 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
Chris Lattnera3dc3f62006-05-05 06:10:43 +00002322 N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
Chris Lattner35e5c142006-05-05 05:51:50 +00002323 N0.getOperand(1) == N1.getOperand(1)) {
Bill Wendlingb74c8672009-01-30 19:25:47 +00002324 SDValue ORNode = DAG.getNode(N->getOpcode(), N0.getDebugLoc(),
2325 N0.getOperand(0).getValueType(),
2326 N0.getOperand(0), N1.getOperand(0));
Gabor Greifba36cb52008-08-28 21:40:38 +00002327 AddToWorkList(ORNode.getNode());
Bill Wendlingb74c8672009-01-30 19:25:47 +00002328 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
2329 ORNode, N0.getOperand(1));
Chris Lattner35e5c142006-05-05 05:51:50 +00002330 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002331
Nadav Rotem4ac90812012-04-01 19:31:22 +00002332 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
2333 // Only perform this optimization after type legalization and before
2334 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
2335 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
2336 // we don't want to undo this promotion.
2337 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
2338 // on scalars.
2339 if ((N0.getOpcode() == ISD::BITCAST || N0.getOpcode() == ISD::SCALAR_TO_VECTOR)
2340 && Level == AfterLegalizeVectorOps) {
2341 SDValue In0 = N0.getOperand(0);
2342 SDValue In1 = N1.getOperand(0);
2343 EVT In0Ty = In0.getValueType();
2344 EVT In1Ty = In1.getValueType();
2345 // If both incoming values are integers, and the original types are the same.
2346 if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
2347 SDValue Op = DAG.getNode(N->getOpcode(), N->getDebugLoc(), In0Ty, In0, In1);
2348 SDValue BC = DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT, Op);
2349 AddToWorkList(Op.getNode());
2350 return BC;
2351 }
2352 }
2353
2354 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
2355 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
2356 // If both shuffles use the same mask, and both shuffle within a single
2357 // vector, then it is worthwhile to move the swizzle after the operation.
2358 // The type-legalizer generates this pattern when loading illegal
2359 // vector types from memory. In many cases this allows additional shuffle
2360 // optimizations.
Craig Topperf9204232012-04-09 07:19:09 +00002361 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG &&
2362 N0.getOperand(1).getOpcode() == ISD::UNDEF &&
2363 N1.getOperand(1).getOpcode() == ISD::UNDEF) {
Nadav Rotem4ac90812012-04-01 19:31:22 +00002364 ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0);
2365 ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1);
Craig Topperf9204232012-04-09 07:19:09 +00002366
2367 assert(N0.getOperand(0).getValueType() == N1.getOperand(1).getValueType() &&
2368 "Inputs to shuffles are not the same type");
Nadav Rotem4ac90812012-04-01 19:31:22 +00002369
2370 unsigned NumElts = VT.getVectorNumElements();
Nadav Rotem4ac90812012-04-01 19:31:22 +00002371
2372 // Check that both shuffles use the same mask. The masks are known to be of
2373 // the same length because the result vector type is the same.
2374 bool SameMask = true;
2375 for (unsigned i = 0; i != NumElts; ++i) {
2376 int Idx0 = SVN0->getMaskElt(i);
2377 int Idx1 = SVN1->getMaskElt(i);
2378 if (Idx0 != Idx1) {
2379 SameMask = false;
2380 break;
2381 }
2382 }
2383
Craig Topperf9204232012-04-09 07:19:09 +00002384 if (SameMask) {
2385 SDValue Op = DAG.getNode(N->getOpcode(), N->getDebugLoc(), VT,
2386 N0.getOperand(0), N1.getOperand(0));
Nadav Rotem4ac90812012-04-01 19:31:22 +00002387 AddToWorkList(Op.getNode());
Craig Topperf9204232012-04-09 07:19:09 +00002388 return DAG.getVectorShuffle(VT, N->getDebugLoc(), Op,
2389 DAG.getUNDEF(VT), &SVN0->getMask()[0]);
Nadav Rotem4ac90812012-04-01 19:31:22 +00002390 }
2391 }
Craig Topperf9204232012-04-09 07:19:09 +00002392
Dan Gohman475871a2008-07-27 21:46:04 +00002393 return SDValue();
Chris Lattner35e5c142006-05-05 05:51:50 +00002394}
2395
Dan Gohman475871a2008-07-27 21:46:04 +00002396SDValue DAGCombiner::visitAND(SDNode *N) {
2397 SDValue N0 = N->getOperand(0);
2398 SDValue N1 = N->getOperand(1);
2399 SDValue LL, LR, RL, RR, CC0, CC1;
Nate Begeman646d7e22005-09-02 21:18:40 +00002400 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2401 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002402 EVT VT = N1.getValueType();
Dan Gohman6900a392010-03-04 00:23:16 +00002403 unsigned BitWidth = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00002404
Dan Gohman7f321562007-06-25 16:23:39 +00002405 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00002406 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00002407 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002408 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00002409 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002410
Dan Gohman613e0d82007-07-03 14:03:57 +00002411 // fold (and x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00002412 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00002413 return DAG.getConstant(0, VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00002414 // fold (and c1, c2) -> c1&c2
Nate Begeman646d7e22005-09-02 21:18:40 +00002415 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00002416 return DAG.FoldConstantArithmetic(ISD::AND, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00002417 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00002418 if (N0C && !N1C)
Bill Wendlingfc4b6772009-02-01 11:19:36 +00002419 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00002420 // fold (and x, -1) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00002421 if (N1C && N1C->isAllOnesValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002422 return N0;
2423 // if (and x, c) is known to be zero, return 0
Dan Gohman475871a2008-07-27 21:46:04 +00002424 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002425 APInt::getAllOnesValue(BitWidth)))
Nate Begeman83e75ec2005-09-06 04:43:02 +00002426 return DAG.getConstant(0, VT);
Nate Begemancd4d58c2006-02-03 06:46:56 +00002427 // reassociate and
Bill Wendling35247c32009-01-30 00:45:56 +00002428 SDValue RAND = ReassociateOps(ISD::AND, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00002429 if (RAND.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00002430 return RAND;
Bill Wendling7d9f2b92010-03-03 00:35:56 +00002431 // fold (and (or x, C), D) -> D if (C & D) == D
Nate Begeman5dc7e862005-11-02 18:42:59 +00002432 if (N1C && N0.getOpcode() == ISD::OR)
Nate Begeman1d4d4142005-09-01 00:19:25 +00002433 if (ConstantSDNode *ORI = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
Dan Gohman002e5d02008-03-13 22:13:53 +00002434 if ((ORI->getAPIntValue() & N1C->getAPIntValue()) == N1C->getAPIntValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002435 return N1;
Chris Lattner3603cd62006-02-02 07:17:31 +00002436 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
2437 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
Dan Gohman475871a2008-07-27 21:46:04 +00002438 SDValue N0Op0 = N0.getOperand(0);
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002439 APInt Mask = ~N1C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00002440 Mask = Mask.trunc(N0Op0.getValueSizeInBits());
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002441 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
Bill Wendling2627a882009-01-30 20:43:18 +00002442 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(),
2443 N0.getValueType(), N0Op0);
Scott Michelfdc40a02009-02-17 22:15:04 +00002444
Chris Lattner1ec05d12006-03-01 21:47:21 +00002445 // Replace uses of the AND with uses of the Zero extend node.
2446 CombineTo(N, Zext);
Scott Michelfdc40a02009-02-17 22:15:04 +00002447
Chris Lattner3603cd62006-02-02 07:17:31 +00002448 // We actually want to replace all uses of the any_extend with the
2449 // zero_extend, to avoid duplicating things. This will later cause this
2450 // AND to be folded.
Gabor Greifba36cb52008-08-28 21:40:38 +00002451 CombineTo(N0.getNode(), Zext);
Dan Gohman475871a2008-07-27 21:46:04 +00002452 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattner3603cd62006-02-02 07:17:31 +00002453 }
2454 }
James Molloy6259dcd2012-02-20 12:02:38 +00002455 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
2456 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
2457 // already be zero by virtue of the width of the base type of the load.
2458 //
2459 // the 'X' node here can either be nothing or an extract_vector_elt to catch
2460 // more cases.
2461 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
2462 N0.getOperand(0).getOpcode() == ISD::LOAD) ||
2463 N0.getOpcode() == ISD::LOAD) {
2464 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
2465 N0 : N0.getOperand(0) );
2466
2467 // Get the constant (if applicable) the zero'th operand is being ANDed with.
2468 // This can be a pure constant or a vector splat, in which case we treat the
2469 // vector as a scalar and use the splat value.
2470 APInt Constant = APInt::getNullValue(1);
2471 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
2472 Constant = C->getAPIntValue();
2473 } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
2474 APInt SplatValue, SplatUndef;
2475 unsigned SplatBitSize;
2476 bool HasAnyUndefs;
2477 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
2478 SplatBitSize, HasAnyUndefs);
2479 if (IsSplat) {
2480 // Undef bits can contribute to a possible optimisation if set, so
2481 // set them.
2482 SplatValue |= SplatUndef;
2483
2484 // The splat value may be something like "0x00FFFFFF", which means 0 for
2485 // the first vector value and FF for the rest, repeating. We need a mask
2486 // that will apply equally to all members of the vector, so AND all the
2487 // lanes of the constant together.
2488 EVT VT = Vector->getValueType(0);
2489 unsigned BitWidth = VT.getVectorElementType().getSizeInBits();
2490 Constant = APInt::getAllOnesValue(BitWidth);
2491 for (unsigned i = 0, n = VT.getVectorNumElements(); i < n; ++i)
2492 Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
2493 }
2494 }
2495
2496 // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
2497 // actually legal and isn't going to get expanded, else this is a false
2498 // optimisation.
2499 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
2500 Load->getMemoryVT());
2501
2502 // Resize the constant to the same size as the original memory access before
2503 // extension. If it is still the AllOnesValue then this AND is completely
2504 // unneeded.
2505 Constant =
2506 Constant.zextOrTrunc(Load->getMemoryVT().getScalarType().getSizeInBits());
2507
2508 bool B;
2509 switch (Load->getExtensionType()) {
2510 default: B = false; break;
2511 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
2512 case ISD::ZEXTLOAD:
2513 case ISD::NON_EXTLOAD: B = true; break;
2514 }
2515
2516 if (B && Constant.isAllOnesValue()) {
2517 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
2518 // preserve semantics once we get rid of the AND.
2519 SDValue NewLoad(Load, 0);
2520 if (Load->getExtensionType() == ISD::EXTLOAD) {
2521 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
2522 Load->getValueType(0), Load->getDebugLoc(),
2523 Load->getChain(), Load->getBasePtr(),
2524 Load->getOffset(), Load->getMemoryVT(),
2525 Load->getMemOperand());
2526 // Replace uses of the EXTLOAD with the new ZEXTLOAD.
Hal Finkeld65e4632012-06-20 15:42:48 +00002527 if (Load->getNumValues() == 3) {
2528 // PRE/POST_INC loads have 3 values.
2529 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
2530 NewLoad.getValue(2) };
2531 CombineTo(Load, To, 3, true);
2532 } else {
2533 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
2534 }
James Molloy6259dcd2012-02-20 12:02:38 +00002535 }
2536
2537 // Fold the AND away, taking care not to fold to the old load node if we
2538 // replaced it.
2539 CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
2540
2541 return SDValue(N, 0); // Return N so it doesn't get rechecked!
2542 }
2543 }
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002544 // fold (and (setcc x), (setcc y)) -> (setcc (and x, y))
2545 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
2546 ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
2547 ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
Scott Michelfdc40a02009-02-17 22:15:04 +00002548
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002549 if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00002550 LL.getValueType().isInteger()) {
Bill Wendling2627a882009-01-30 20:43:18 +00002551 // fold (and (seteq X, 0), (seteq Y, 0)) -> (seteq (or X, Y), 0)
Dan Gohman002e5d02008-03-13 22:13:53 +00002552 if (cast<ConstantSDNode>(LR)->isNullValue() && Op1 == ISD::SETEQ) {
Bill Wendling2627a882009-01-30 20:43:18 +00002553 SDValue ORNode = DAG.getNode(ISD::OR, N0.getDebugLoc(),
2554 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00002555 AddToWorkList(ORNode.getNode());
Bill Wendling2627a882009-01-30 20:43:18 +00002556 return DAG.getSetCC(N->getDebugLoc(), VT, ORNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002557 }
Bill Wendling2627a882009-01-30 20:43:18 +00002558 // fold (and (seteq X, -1), (seteq Y, -1)) -> (seteq (and X, Y), -1)
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002559 if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETEQ) {
Bill Wendling2627a882009-01-30 20:43:18 +00002560 SDValue ANDNode = DAG.getNode(ISD::AND, N0.getDebugLoc(),
2561 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00002562 AddToWorkList(ANDNode.getNode());
Bill Wendling2627a882009-01-30 20:43:18 +00002563 return DAG.getSetCC(N->getDebugLoc(), VT, ANDNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002564 }
Bill Wendling2627a882009-01-30 20:43:18 +00002565 // fold (and (setgt X, -1), (setgt Y, -1)) -> (setgt (or X, Y), -1)
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002566 if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETGT) {
Bill Wendling2627a882009-01-30 20:43:18 +00002567 SDValue ORNode = DAG.getNode(ISD::OR, N0.getDebugLoc(),
2568 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00002569 AddToWorkList(ORNode.getNode());
Bill Wendling2627a882009-01-30 20:43:18 +00002570 return DAG.getSetCC(N->getDebugLoc(), VT, ORNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002571 }
2572 }
2573 // canonicalize equivalent to ll == rl
2574 if (LL == RR && LR == RL) {
2575 Op1 = ISD::getSetCCSwappedOperands(Op1);
2576 std::swap(RL, RR);
2577 }
2578 if (LL == RL && LR == RR) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00002579 bool isInteger = LL.getValueType().isInteger();
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002580 ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger);
Chris Lattner6e1c6232008-10-28 07:11:07 +00002581 if (Result != ISD::SETCC_INVALID &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002582 (!LegalOperations || TLI.isCondCodeLegal(Result, LL.getValueType())))
Bill Wendling2627a882009-01-30 20:43:18 +00002583 return DAG.getSetCC(N->getDebugLoc(), N0.getValueType(),
2584 LL, LR, Result);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002585 }
2586 }
Chris Lattner35e5c142006-05-05 05:51:50 +00002587
Bill Wendling2627a882009-01-30 20:43:18 +00002588 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
Chris Lattner35e5c142006-05-05 05:51:50 +00002589 if (N0.getOpcode() == N1.getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00002590 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002591 if (Tmp.getNode()) return Tmp;
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002592 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002593
Nate Begemande996292006-02-03 22:24:05 +00002594 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
2595 // fold (and (sra)) -> (and (srl)) when possible.
Duncan Sands83ec4b62008-06-06 12:08:01 +00002596 if (!VT.isVector() &&
Dan Gohman475871a2008-07-27 21:46:04 +00002597 SimplifyDemandedBits(SDValue(N, 0)))
2598 return SDValue(N, 0);
Evan Chengd40d03e2010-01-06 19:38:29 +00002599
Nate Begemanded49632005-10-13 03:11:28 +00002600 // fold (zext_inreg (extload x)) -> (zextload x)
Gabor Greifba36cb52008-08-28 21:40:38 +00002601 if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
Evan Cheng466685d2006-10-09 20:57:25 +00002602 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00002603 EVT MemVT = LN0->getMemoryVT();
Nate Begemanbfd65a02005-10-13 18:34:58 +00002604 // If we zero all the possible extended bits, then we can turn this into
2605 // a zextload if we are running before legalize or the operation is legal.
Dan Gohman6900a392010-03-04 00:23:16 +00002606 unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits();
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002607 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
Dan Gohman6900a392010-03-04 00:23:16 +00002608 BitWidth - MemVT.getScalarType().getSizeInBits())) &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002609 ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00002610 TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT))) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00002611 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N0.getDebugLoc(), VT,
Bill Wendling2627a882009-01-30 20:43:18 +00002612 LN0->getChain(), LN0->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002613 LN0->getPointerInfo(), MemVT,
David Greene1e559442010-02-15 17:00:31 +00002614 LN0->isVolatile(), LN0->isNonTemporal(),
2615 LN0->getAlignment());
Chris Lattner5750df92006-03-01 04:03:14 +00002616 AddToWorkList(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002617 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00002618 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00002619 }
2620 }
2621 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
Gabor Greifba36cb52008-08-28 21:40:38 +00002622 if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
Evan Cheng83060c52007-03-07 08:07:03 +00002623 N0.hasOneUse()) {
Evan Cheng466685d2006-10-09 20:57:25 +00002624 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00002625 EVT MemVT = LN0->getMemoryVT();
Nate Begemanbfd65a02005-10-13 18:34:58 +00002626 // If we zero all the possible extended bits, then we can turn this into
2627 // a zextload if we are running before legalize or the operation is legal.
Dan Gohman6900a392010-03-04 00:23:16 +00002628 unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits();
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002629 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
Dan Gohman6900a392010-03-04 00:23:16 +00002630 BitWidth - MemVT.getScalarType().getSizeInBits())) &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002631 ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00002632 TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT))) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00002633 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N0.getDebugLoc(), VT,
Bill Wendling2627a882009-01-30 20:43:18 +00002634 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002635 LN0->getBasePtr(), LN0->getPointerInfo(),
2636 MemVT,
David Greene1e559442010-02-15 17:00:31 +00002637 LN0->isVolatile(), LN0->isNonTemporal(),
2638 LN0->getAlignment());
Chris Lattner5750df92006-03-01 04:03:14 +00002639 AddToWorkList(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002640 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00002641 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00002642 }
2643 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002644
Chris Lattner35a9f5a2006-02-28 06:49:37 +00002645 // fold (and (load x), 255) -> (zextload x, i8)
2646 // fold (and (extload x, i16), 255) -> (zextload x, i8)
Evan Chengd40d03e2010-01-06 19:38:29 +00002647 // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
2648 if (N1C && (N0.getOpcode() == ISD::LOAD ||
2649 (N0.getOpcode() == ISD::ANY_EXTEND &&
2650 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
2651 bool HasAnyExt = N0.getOpcode() == ISD::ANY_EXTEND;
2652 LoadSDNode *LN0 = HasAnyExt
2653 ? cast<LoadSDNode>(N0.getOperand(0))
2654 : cast<LoadSDNode>(N0);
Evan Cheng466685d2006-10-09 20:57:25 +00002655 if (LN0->getExtensionType() != ISD::SEXTLOAD &&
Chris Lattnerbd1fccf2010-01-07 21:59:23 +00002656 LN0->isUnindexed() && N0.hasOneUse() && LN0->hasOneUse()) {
Duncan Sands8eab8a22008-06-09 11:32:28 +00002657 uint32_t ActiveBits = N1C->getAPIntValue().getActiveBits();
Evan Chengd40d03e2010-01-06 19:38:29 +00002658 if (ActiveBits > 0 && APIntOps::isMask(ActiveBits, N1C->getAPIntValue())){
2659 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
2660 EVT LoadedVT = LN0->getMemoryVT();
Duncan Sands8eab8a22008-06-09 11:32:28 +00002661
Evan Chengd40d03e2010-01-06 19:38:29 +00002662 if (ExtVT == LoadedVT &&
2663 (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT))) {
Chris Lattneref7634c2010-01-07 21:53:27 +00002664 EVT LoadResultTy = HasAnyExt ? LN0->getValueType(0) : VT;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002665
2666 SDValue NewLoad =
Stuart Hastingsa9011292011-02-16 16:23:55 +00002667 DAG.getExtLoad(ISD::ZEXTLOAD, LN0->getDebugLoc(), LoadResultTy,
Chris Lattneref7634c2010-01-07 21:53:27 +00002668 LN0->getChain(), LN0->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002669 LN0->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00002670 ExtVT, LN0->isVolatile(), LN0->isNonTemporal(),
2671 LN0->getAlignment());
Chris Lattneref7634c2010-01-07 21:53:27 +00002672 AddToWorkList(N);
2673 CombineTo(LN0, NewLoad, NewLoad.getValue(1));
2674 return SDValue(N, 0); // Return N so it doesn't get rechecked!
2675 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002676
Chris Lattneref7634c2010-01-07 21:53:27 +00002677 // Do not change the width of a volatile load.
2678 // Do not generate loads of non-round integer types since these can
2679 // be expensive (and would be wrong if the type is not byte sized).
2680 if (!LN0->isVolatile() && LoadedVT.bitsGT(ExtVT) && ExtVT.isRound() &&
2681 (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT))) {
2682 EVT PtrType = LN0->getOperand(1).getValueType();
Bill Wendling2627a882009-01-30 20:43:18 +00002683
Chris Lattneref7634c2010-01-07 21:53:27 +00002684 unsigned Alignment = LN0->getAlignment();
2685 SDValue NewPtr = LN0->getBasePtr();
2686
2687 // For big endian targets, we need to add an offset to the pointer
2688 // to load the correct bytes. For little endian systems, we merely
2689 // need to read fewer bytes from the same pointer.
2690 if (TLI.isBigEndian()) {
Evan Chengd40d03e2010-01-06 19:38:29 +00002691 unsigned LVTStoreBytes = LoadedVT.getStoreSize();
2692 unsigned EVTStoreBytes = ExtVT.getStoreSize();
2693 unsigned PtrOff = LVTStoreBytes - EVTStoreBytes;
Chris Lattneref7634c2010-01-07 21:53:27 +00002694 NewPtr = DAG.getNode(ISD::ADD, LN0->getDebugLoc(), PtrType,
2695 NewPtr, DAG.getConstant(PtrOff, PtrType));
2696 Alignment = MinAlign(Alignment, PtrOff);
Evan Chengd40d03e2010-01-06 19:38:29 +00002697 }
Chris Lattneref7634c2010-01-07 21:53:27 +00002698
2699 AddToWorkList(NewPtr.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002700
Chris Lattneref7634c2010-01-07 21:53:27 +00002701 EVT LoadResultTy = HasAnyExt ? LN0->getValueType(0) : VT;
2702 SDValue Load =
Stuart Hastingsa9011292011-02-16 16:23:55 +00002703 DAG.getExtLoad(ISD::ZEXTLOAD, LN0->getDebugLoc(), LoadResultTy,
Chris Lattneref7634c2010-01-07 21:53:27 +00002704 LN0->getChain(), NewPtr,
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002705 LN0->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00002706 ExtVT, LN0->isVolatile(), LN0->isNonTemporal(),
2707 Alignment);
Chris Lattneref7634c2010-01-07 21:53:27 +00002708 AddToWorkList(N);
2709 CombineTo(LN0, Load, Load.getValue(1));
2710 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Duncan Sandsdc846502007-10-28 12:59:45 +00002711 }
Evan Cheng466685d2006-10-09 20:57:25 +00002712 }
Chris Lattner15045b62006-02-28 06:35:35 +00002713 }
2714 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002715
Evan Chengb3a3d5e2010-04-28 07:10:39 +00002716 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002717}
2718
Evan Cheng9568e5c2011-06-21 06:01:08 +00002719/// MatchBSwapHWord - Match (a >> 8) | (a << 8) as (bswap a) >> 16
2720///
2721SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
2722 bool DemandHighBits) {
2723 if (!LegalOperations)
2724 return SDValue();
2725
2726 EVT VT = N->getValueType(0);
2727 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
2728 return SDValue();
2729 if (!TLI.isOperationLegal(ISD::BSWAP, VT))
2730 return SDValue();
2731
2732 // Recognize (and (shl a, 8), 0xff), (and (srl a, 8), 0xff00)
2733 bool LookPassAnd0 = false;
2734 bool LookPassAnd1 = false;
2735 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
2736 std::swap(N0, N1);
2737 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
2738 std::swap(N0, N1);
2739 if (N0.getOpcode() == ISD::AND) {
2740 if (!N0.getNode()->hasOneUse())
2741 return SDValue();
2742 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2743 if (!N01C || N01C->getZExtValue() != 0xFF00)
2744 return SDValue();
2745 N0 = N0.getOperand(0);
2746 LookPassAnd0 = true;
2747 }
2748
2749 if (N1.getOpcode() == ISD::AND) {
2750 if (!N1.getNode()->hasOneUse())
2751 return SDValue();
2752 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
2753 if (!N11C || N11C->getZExtValue() != 0xFF)
2754 return SDValue();
2755 N1 = N1.getOperand(0);
2756 LookPassAnd1 = true;
2757 }
2758
2759 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
2760 std::swap(N0, N1);
2761 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
2762 return SDValue();
2763 if (!N0.getNode()->hasOneUse() ||
2764 !N1.getNode()->hasOneUse())
2765 return SDValue();
2766
2767 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2768 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
2769 if (!N01C || !N11C)
2770 return SDValue();
2771 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
2772 return SDValue();
2773
2774 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
2775 SDValue N00 = N0->getOperand(0);
2776 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
2777 if (!N00.getNode()->hasOneUse())
2778 return SDValue();
2779 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
2780 if (!N001C || N001C->getZExtValue() != 0xFF)
2781 return SDValue();
2782 N00 = N00.getOperand(0);
2783 LookPassAnd0 = true;
2784 }
2785
2786 SDValue N10 = N1->getOperand(0);
2787 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
2788 if (!N10.getNode()->hasOneUse())
2789 return SDValue();
2790 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
2791 if (!N101C || N101C->getZExtValue() != 0xFF00)
2792 return SDValue();
2793 N10 = N10.getOperand(0);
2794 LookPassAnd1 = true;
2795 }
2796
2797 if (N00 != N10)
2798 return SDValue();
2799
2800 // Make sure everything beyond the low halfword is zero since the SRL 16
2801 // will clear the top bits.
2802 unsigned OpSizeInBits = VT.getSizeInBits();
2803 if (DemandHighBits && OpSizeInBits > 16 &&
2804 (!LookPassAnd0 || !LookPassAnd1) &&
2805 !DAG.MaskedValueIsZero(N10, APInt::getHighBitsSet(OpSizeInBits, 16)))
2806 return SDValue();
Eric Christopher7332e6e2011-07-14 01:12:15 +00002807
Evan Cheng9568e5c2011-06-21 06:01:08 +00002808 SDValue Res = DAG.getNode(ISD::BSWAP, N->getDebugLoc(), VT, N00);
2809 if (OpSizeInBits > 16)
2810 Res = DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, Res,
2811 DAG.getConstant(OpSizeInBits-16, getShiftAmountTy(VT)));
2812 return Res;
2813}
2814
2815/// isBSwapHWordElement - Return true if the specified node is an element
2816/// that makes up a 32-bit packed halfword byteswap. i.e.
2817/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8)
2818static bool isBSwapHWordElement(SDValue N, SmallVector<SDNode*,4> &Parts) {
2819 if (!N.getNode()->hasOneUse())
2820 return false;
2821
2822 unsigned Opc = N.getOpcode();
2823 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
2824 return false;
2825
2826 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2827 if (!N1C)
2828 return false;
2829
2830 unsigned Num;
2831 switch (N1C->getZExtValue()) {
2832 default:
2833 return false;
2834 case 0xFF: Num = 0; break;
2835 case 0xFF00: Num = 1; break;
2836 case 0xFF0000: Num = 2; break;
2837 case 0xFF000000: Num = 3; break;
2838 }
2839
2840 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
2841 SDValue N0 = N.getOperand(0);
2842 if (Opc == ISD::AND) {
2843 if (Num == 0 || Num == 2) {
2844 // (x >> 8) & 0xff
2845 // (x >> 8) & 0xff0000
2846 if (N0.getOpcode() != ISD::SRL)
2847 return false;
2848 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2849 if (!C || C->getZExtValue() != 8)
2850 return false;
2851 } else {
2852 // (x << 8) & 0xff00
2853 // (x << 8) & 0xff000000
2854 if (N0.getOpcode() != ISD::SHL)
2855 return false;
2856 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2857 if (!C || C->getZExtValue() != 8)
2858 return false;
2859 }
2860 } else if (Opc == ISD::SHL) {
2861 // (x & 0xff) << 8
2862 // (x & 0xff0000) << 8
2863 if (Num != 0 && Num != 2)
2864 return false;
2865 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2866 if (!C || C->getZExtValue() != 8)
2867 return false;
2868 } else { // Opc == ISD::SRL
2869 // (x & 0xff00) >> 8
2870 // (x & 0xff000000) >> 8
2871 if (Num != 1 && Num != 3)
2872 return false;
2873 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2874 if (!C || C->getZExtValue() != 8)
2875 return false;
2876 }
2877
2878 if (Parts[Num])
2879 return false;
2880
2881 Parts[Num] = N0.getOperand(0).getNode();
2882 return true;
2883}
2884
2885/// MatchBSwapHWord - Match a 32-bit packed halfword bswap. That is
2886/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8)
2887/// => (rotl (bswap x), 16)
2888SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
2889 if (!LegalOperations)
2890 return SDValue();
2891
2892 EVT VT = N->getValueType(0);
2893 if (VT != MVT::i32)
2894 return SDValue();
2895 if (!TLI.isOperationLegal(ISD::BSWAP, VT))
2896 return SDValue();
2897
2898 SmallVector<SDNode*,4> Parts(4, (SDNode*)0);
2899 // Look for either
2900 // (or (or (and), (and)), (or (and), (and)))
2901 // (or (or (or (and), (and)), (and)), (and))
2902 if (N0.getOpcode() != ISD::OR)
2903 return SDValue();
2904 SDValue N00 = N0.getOperand(0);
2905 SDValue N01 = N0.getOperand(1);
2906
2907 if (N1.getOpcode() == ISD::OR) {
2908 // (or (or (and), (and)), (or (and), (and)))
2909 SDValue N000 = N00.getOperand(0);
2910 if (!isBSwapHWordElement(N000, Parts))
2911 return SDValue();
2912
2913 SDValue N001 = N00.getOperand(1);
2914 if (!isBSwapHWordElement(N001, Parts))
2915 return SDValue();
2916 SDValue N010 = N01.getOperand(0);
2917 if (!isBSwapHWordElement(N010, Parts))
2918 return SDValue();
2919 SDValue N011 = N01.getOperand(1);
2920 if (!isBSwapHWordElement(N011, Parts))
2921 return SDValue();
2922 } else {
2923 // (or (or (or (and), (and)), (and)), (and))
2924 if (!isBSwapHWordElement(N1, Parts))
2925 return SDValue();
2926 if (!isBSwapHWordElement(N01, Parts))
2927 return SDValue();
2928 if (N00.getOpcode() != ISD::OR)
2929 return SDValue();
2930 SDValue N000 = N00.getOperand(0);
2931 if (!isBSwapHWordElement(N000, Parts))
2932 return SDValue();
2933 SDValue N001 = N00.getOperand(1);
2934 if (!isBSwapHWordElement(N001, Parts))
2935 return SDValue();
2936 }
2937
2938 // Make sure the parts are all coming from the same node.
2939 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
2940 return SDValue();
2941
2942 SDValue BSwap = DAG.getNode(ISD::BSWAP, N->getDebugLoc(), VT,
2943 SDValue(Parts[0],0));
2944
2945 // Result of the bswap should be rotated by 16. If it's not legal, than
2946 // do (x << 16) | (x >> 16).
2947 SDValue ShAmt = DAG.getConstant(16, getShiftAmountTy(VT));
2948 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
2949 return DAG.getNode(ISD::ROTL, N->getDebugLoc(), VT, BSwap, ShAmt);
2950 else if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
2951 return DAG.getNode(ISD::ROTR, N->getDebugLoc(), VT, BSwap, ShAmt);
2952 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT,
2953 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, BSwap, ShAmt),
2954 DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, BSwap, ShAmt));
2955}
2956
Dan Gohman475871a2008-07-27 21:46:04 +00002957SDValue DAGCombiner::visitOR(SDNode *N) {
2958 SDValue N0 = N->getOperand(0);
2959 SDValue N1 = N->getOperand(1);
2960 SDValue LL, LR, RL, RR, CC0, CC1;
Nate Begeman646d7e22005-09-02 21:18:40 +00002961 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2962 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002963 EVT VT = N1.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00002964
Dan Gohman7f321562007-06-25 16:23:39 +00002965 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00002966 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00002967 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002968 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00002969 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002970
Dan Gohman613e0d82007-07-03 14:03:57 +00002971 // fold (or x, undef) -> -1
Bob Wilson86749492010-06-28 23:40:25 +00002972 if (!LegalOperations &&
2973 (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)) {
Nate Begeman93e0ed32009-12-03 07:11:29 +00002974 EVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT;
2975 return DAG.getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
2976 }
Nate Begeman1d4d4142005-09-01 00:19:25 +00002977 // fold (or c1, c2) -> c1|c2
Nate Begeman646d7e22005-09-02 21:18:40 +00002978 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00002979 return DAG.FoldConstantArithmetic(ISD::OR, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00002980 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00002981 if (N0C && !N1C)
Bill Wendling09025642009-01-30 20:59:34 +00002982 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00002983 // fold (or x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00002984 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002985 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00002986 // fold (or x, -1) -> -1
Nate Begeman646d7e22005-09-02 21:18:40 +00002987 if (N1C && N1C->isAllOnesValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002988 return N1;
2989 // fold (or x, c) -> c iff (x & ~c) == 0
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002990 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
Nate Begeman83e75ec2005-09-06 04:43:02 +00002991 return N1;
Evan Cheng9568e5c2011-06-21 06:01:08 +00002992
2993 // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
2994 SDValue BSwap = MatchBSwapHWord(N, N0, N1);
2995 if (BSwap.getNode() != 0)
2996 return BSwap;
2997 BSwap = MatchBSwapHWordLow(N, N0, N1);
2998 if (BSwap.getNode() != 0)
2999 return BSwap;
3000
Nate Begemancd4d58c2006-02-03 06:46:56 +00003001 // reassociate or
Bill Wendling35247c32009-01-30 00:45:56 +00003002 SDValue ROR = ReassociateOps(ISD::OR, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00003003 if (ROR.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00003004 return ROR;
3005 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
Bill Wendling7d9f2b92010-03-03 00:35:56 +00003006 // iff (c1 & c2) == 0.
Gabor Greifba36cb52008-08-28 21:40:38 +00003007 if (N1C && N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
Chris Lattner731d3482005-10-27 05:06:38 +00003008 isa<ConstantSDNode>(N0.getOperand(1))) {
Chris Lattner731d3482005-10-27 05:06:38 +00003009 ConstantSDNode *C1 = cast<ConstantSDNode>(N0.getOperand(1));
Bill Wendling32f9eb22010-03-03 01:58:01 +00003010 if ((C1->getAPIntValue() & N1C->getAPIntValue()) != 0)
Bill Wendling7d9f2b92010-03-03 00:35:56 +00003011 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
3012 DAG.getNode(ISD::OR, N0.getDebugLoc(), VT,
3013 N0.getOperand(0), N1),
3014 DAG.FoldConstantArithmetic(ISD::OR, VT, N1C, C1));
Nate Begeman223df222005-09-08 20:18:10 +00003015 }
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003016 // fold (or (setcc x), (setcc y)) -> (setcc (or x, y))
3017 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
3018 ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
3019 ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
Scott Michelfdc40a02009-02-17 22:15:04 +00003020
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003021 if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00003022 LL.getValueType().isInteger()) {
Bill Wendling09025642009-01-30 20:59:34 +00003023 // fold (or (setne X, 0), (setne Y, 0)) -> (setne (or X, Y), 0)
3024 // fold (or (setlt X, 0), (setlt Y, 0)) -> (setne (or X, Y), 0)
Scott Michelfdc40a02009-02-17 22:15:04 +00003025 if (cast<ConstantSDNode>(LR)->isNullValue() &&
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003026 (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) {
Bill Wendling09025642009-01-30 20:59:34 +00003027 SDValue ORNode = DAG.getNode(ISD::OR, LR.getDebugLoc(),
3028 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00003029 AddToWorkList(ORNode.getNode());
Bill Wendling09025642009-01-30 20:59:34 +00003030 return DAG.getSetCC(N->getDebugLoc(), VT, ORNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003031 }
Bill Wendling09025642009-01-30 20:59:34 +00003032 // fold (or (setne X, -1), (setne Y, -1)) -> (setne (and X, Y), -1)
3033 // fold (or (setgt X, -1), (setgt Y -1)) -> (setgt (and X, Y), -1)
Scott Michelfdc40a02009-02-17 22:15:04 +00003034 if (cast<ConstantSDNode>(LR)->isAllOnesValue() &&
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003035 (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) {
Bill Wendling09025642009-01-30 20:59:34 +00003036 SDValue ANDNode = DAG.getNode(ISD::AND, LR.getDebugLoc(),
3037 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00003038 AddToWorkList(ANDNode.getNode());
Bill Wendling09025642009-01-30 20:59:34 +00003039 return DAG.getSetCC(N->getDebugLoc(), VT, ANDNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003040 }
3041 }
3042 // canonicalize equivalent to ll == rl
3043 if (LL == RR && LR == RL) {
3044 Op1 = ISD::getSetCCSwappedOperands(Op1);
3045 std::swap(RL, RR);
3046 }
3047 if (LL == RL && LR == RR) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00003048 bool isInteger = LL.getValueType().isInteger();
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003049 ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger);
Chris Lattner6e1c6232008-10-28 07:11:07 +00003050 if (Result != ISD::SETCC_INVALID &&
Duncan Sands25cf2272008-11-24 14:53:14 +00003051 (!LegalOperations || TLI.isCondCodeLegal(Result, LL.getValueType())))
Bill Wendling09025642009-01-30 20:59:34 +00003052 return DAG.getSetCC(N->getDebugLoc(), N0.getValueType(),
3053 LL, LR, Result);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003054 }
3055 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003056
Bill Wendling09025642009-01-30 20:59:34 +00003057 // Simplify: (or (op x...), (op y...)) -> (op (or x, y))
Chris Lattner35e5c142006-05-05 05:51:50 +00003058 if (N0.getOpcode() == N1.getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003059 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00003060 if (Tmp.getNode()) return Tmp;
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003061 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003062
Bill Wendling09025642009-01-30 20:59:34 +00003063 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible.
Chris Lattner1ec72732006-09-14 21:11:37 +00003064 if (N0.getOpcode() == ISD::AND &&
3065 N1.getOpcode() == ISD::AND &&
3066 N0.getOperand(1).getOpcode() == ISD::Constant &&
3067 N1.getOperand(1).getOpcode() == ISD::Constant &&
3068 // Don't increase # computations.
Gabor Greifba36cb52008-08-28 21:40:38 +00003069 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
Chris Lattner1ec72732006-09-14 21:11:37 +00003070 // We can only do this xform if we know that bits from X that are set in C2
3071 // but not in C1 are already zero. Likewise for Y.
Dan Gohman2e68b6f2008-02-25 21:11:39 +00003072 const APInt &LHSMask =
3073 cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
3074 const APInt &RHSMask =
3075 cast<ConstantSDNode>(N1.getOperand(1))->getAPIntValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00003076
Dan Gohmanea859be2007-06-22 14:59:07 +00003077 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
3078 DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
Bill Wendling09025642009-01-30 20:59:34 +00003079 SDValue X = DAG.getNode(ISD::OR, N0.getDebugLoc(), VT,
3080 N0.getOperand(0), N1.getOperand(0));
3081 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, X,
3082 DAG.getConstant(LHSMask | RHSMask, VT));
Chris Lattner1ec72732006-09-14 21:11:37 +00003083 }
3084 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003085
Chris Lattner516b9622006-09-14 20:50:57 +00003086 // See if this is some rotate idiom.
Bill Wendling317bd702009-01-30 21:14:50 +00003087 if (SDNode *Rot = MatchRotate(N0, N1, N->getDebugLoc()))
Dan Gohman475871a2008-07-27 21:46:04 +00003088 return SDValue(Rot, 0);
Chris Lattner35e5c142006-05-05 05:51:50 +00003089
Dan Gohman4e39e9d2010-06-24 14:30:44 +00003090 // Simplify the operands using demanded-bits information.
3091 if (!VT.isVector() &&
3092 SimplifyDemandedBits(SDValue(N, 0)))
3093 return SDValue(N, 0);
3094
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003095 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003096}
3097
Chris Lattner516b9622006-09-14 20:50:57 +00003098/// MatchRotateHalf - Match "(X shl/srl V1) & V2" where V2 may not be present.
Dan Gohman475871a2008-07-27 21:46:04 +00003099static bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) {
Chris Lattner516b9622006-09-14 20:50:57 +00003100 if (Op.getOpcode() == ISD::AND) {
Reid Spencer3ed469c2006-11-02 20:25:50 +00003101 if (isa<ConstantSDNode>(Op.getOperand(1))) {
Chris Lattner516b9622006-09-14 20:50:57 +00003102 Mask = Op.getOperand(1);
3103 Op = Op.getOperand(0);
3104 } else {
3105 return false;
3106 }
3107 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003108
Chris Lattner516b9622006-09-14 20:50:57 +00003109 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
3110 Shift = Op;
3111 return true;
3112 }
Bill Wendling09025642009-01-30 20:59:34 +00003113
Scott Michelfdc40a02009-02-17 22:15:04 +00003114 return false;
Chris Lattner516b9622006-09-14 20:50:57 +00003115}
3116
Chris Lattner516b9622006-09-14 20:50:57 +00003117// MatchRotate - Handle an 'or' of two operands. If this is one of the many
3118// idioms for rotate, and if the target supports rotation instructions, generate
3119// a rot[lr].
Bill Wendling317bd702009-01-30 21:14:50 +00003120SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, DebugLoc DL) {
Duncan Sandsd4b9c172008-06-13 19:07:40 +00003121 // Must be a legal type. Expanded 'n promoted things won't work with rotates.
Owen Andersone50ed302009-08-10 22:56:29 +00003122 EVT VT = LHS.getValueType();
Chris Lattner516b9622006-09-14 20:50:57 +00003123 if (!TLI.isTypeLegal(VT)) return 0;
3124
3125 // The target must have at least one rotate flavor.
Dan Gohmanf560ffa2009-01-28 17:46:25 +00003126 bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT);
3127 bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT);
Chris Lattner516b9622006-09-14 20:50:57 +00003128 if (!HasROTL && !HasROTR) return 0;
Duncan Sandsd4b9c172008-06-13 19:07:40 +00003129
Chris Lattner516b9622006-09-14 20:50:57 +00003130 // Match "(X shl/srl V1) & V2" where V2 may not be present.
Dan Gohman475871a2008-07-27 21:46:04 +00003131 SDValue LHSShift; // The shift.
3132 SDValue LHSMask; // AND value if any.
Chris Lattner516b9622006-09-14 20:50:57 +00003133 if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
3134 return 0; // Not part of a rotate.
3135
Dan Gohman475871a2008-07-27 21:46:04 +00003136 SDValue RHSShift; // The shift.
3137 SDValue RHSMask; // AND value if any.
Chris Lattner516b9622006-09-14 20:50:57 +00003138 if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
3139 return 0; // Not part of a rotate.
Scott Michelfdc40a02009-02-17 22:15:04 +00003140
Chris Lattner516b9622006-09-14 20:50:57 +00003141 if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
3142 return 0; // Not shifting the same value.
3143
3144 if (LHSShift.getOpcode() == RHSShift.getOpcode())
3145 return 0; // Shifts must disagree.
Scott Michelfdc40a02009-02-17 22:15:04 +00003146
Chris Lattner516b9622006-09-14 20:50:57 +00003147 // Canonicalize shl to left side in a shl/srl pair.
3148 if (RHSShift.getOpcode() == ISD::SHL) {
3149 std::swap(LHS, RHS);
3150 std::swap(LHSShift, RHSShift);
3151 std::swap(LHSMask , RHSMask );
3152 }
3153
Duncan Sands83ec4b62008-06-06 12:08:01 +00003154 unsigned OpSizeInBits = VT.getSizeInBits();
Dan Gohman475871a2008-07-27 21:46:04 +00003155 SDValue LHSShiftArg = LHSShift.getOperand(0);
3156 SDValue LHSShiftAmt = LHSShift.getOperand(1);
3157 SDValue RHSShiftAmt = RHSShift.getOperand(1);
Chris Lattner516b9622006-09-14 20:50:57 +00003158
3159 // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
3160 // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
Scott Michelc9dc1142007-04-02 21:36:32 +00003161 if (LHSShiftAmt.getOpcode() == ISD::Constant &&
3162 RHSShiftAmt.getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003163 uint64_t LShVal = cast<ConstantSDNode>(LHSShiftAmt)->getZExtValue();
3164 uint64_t RShVal = cast<ConstantSDNode>(RHSShiftAmt)->getZExtValue();
Chris Lattner516b9622006-09-14 20:50:57 +00003165 if ((LShVal + RShVal) != OpSizeInBits)
3166 return 0;
3167
Dan Gohman475871a2008-07-27 21:46:04 +00003168 SDValue Rot;
Chris Lattner516b9622006-09-14 20:50:57 +00003169 if (HasROTL)
Bill Wendling317bd702009-01-30 21:14:50 +00003170 Rot = DAG.getNode(ISD::ROTL, DL, VT, LHSShiftArg, LHSShiftAmt);
Chris Lattner516b9622006-09-14 20:50:57 +00003171 else
Bill Wendling317bd702009-01-30 21:14:50 +00003172 Rot = DAG.getNode(ISD::ROTR, DL, VT, LHSShiftArg, RHSShiftAmt);
Scott Michelfdc40a02009-02-17 22:15:04 +00003173
Chris Lattner516b9622006-09-14 20:50:57 +00003174 // If there is an AND of either shifted operand, apply it to the result.
Gabor Greifba36cb52008-08-28 21:40:38 +00003175 if (LHSMask.getNode() || RHSMask.getNode()) {
Dan Gohman220a8232008-03-03 23:51:38 +00003176 APInt Mask = APInt::getAllOnesValue(OpSizeInBits);
Scott Michelfdc40a02009-02-17 22:15:04 +00003177
Gabor Greifba36cb52008-08-28 21:40:38 +00003178 if (LHSMask.getNode()) {
Dan Gohman220a8232008-03-03 23:51:38 +00003179 APInt RHSBits = APInt::getLowBitsSet(OpSizeInBits, LShVal);
3180 Mask &= cast<ConstantSDNode>(LHSMask)->getAPIntValue() | RHSBits;
Chris Lattner516b9622006-09-14 20:50:57 +00003181 }
Gabor Greifba36cb52008-08-28 21:40:38 +00003182 if (RHSMask.getNode()) {
Dan Gohman220a8232008-03-03 23:51:38 +00003183 APInt LHSBits = APInt::getHighBitsSet(OpSizeInBits, RShVal);
3184 Mask &= cast<ConstantSDNode>(RHSMask)->getAPIntValue() | LHSBits;
Chris Lattner516b9622006-09-14 20:50:57 +00003185 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003186
Bill Wendling317bd702009-01-30 21:14:50 +00003187 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, DAG.getConstant(Mask, VT));
Chris Lattner516b9622006-09-14 20:50:57 +00003188 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003189
Gabor Greifba36cb52008-08-28 21:40:38 +00003190 return Rot.getNode();
Chris Lattner516b9622006-09-14 20:50:57 +00003191 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003192
Chris Lattner516b9622006-09-14 20:50:57 +00003193 // If there is a mask here, and we have a variable shift, we can't be sure
3194 // that we're masking out the right stuff.
Gabor Greifba36cb52008-08-28 21:40:38 +00003195 if (LHSMask.getNode() || RHSMask.getNode())
Chris Lattner516b9622006-09-14 20:50:57 +00003196 return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +00003197
Chris Lattner516b9622006-09-14 20:50:57 +00003198 // fold (or (shl x, y), (srl x, (sub 32, y))) -> (rotl x, y)
3199 // fold (or (shl x, y), (srl x, (sub 32, y))) -> (rotr x, (sub 32, y))
Scott Michelc9dc1142007-04-02 21:36:32 +00003200 if (RHSShiftAmt.getOpcode() == ISD::SUB &&
3201 LHSShiftAmt == RHSShiftAmt.getOperand(1)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00003202 if (ConstantSDNode *SUBC =
Scott Michelc9dc1142007-04-02 21:36:32 +00003203 dyn_cast<ConstantSDNode>(RHSShiftAmt.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003204 if (SUBC->getAPIntValue() == OpSizeInBits) {
Chris Lattner516b9622006-09-14 20:50:57 +00003205 if (HasROTL)
Bill Wendling317bd702009-01-30 21:14:50 +00003206 return DAG.getNode(ISD::ROTL, DL, VT,
3207 LHSShiftArg, LHSShiftAmt).getNode();
Chris Lattner516b9622006-09-14 20:50:57 +00003208 else
Bill Wendling317bd702009-01-30 21:14:50 +00003209 return DAG.getNode(ISD::ROTR, DL, VT,
3210 LHSShiftArg, RHSShiftAmt).getNode();
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00003211 }
Chris Lattner516b9622006-09-14 20:50:57 +00003212 }
3213 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003214
Chris Lattner516b9622006-09-14 20:50:57 +00003215 // fold (or (shl x, (sub 32, y)), (srl x, r)) -> (rotr x, y)
3216 // fold (or (shl x, (sub 32, y)), (srl x, r)) -> (rotl x, (sub 32, y))
Scott Michelc9dc1142007-04-02 21:36:32 +00003217 if (LHSShiftAmt.getOpcode() == ISD::SUB &&
3218 RHSShiftAmt == LHSShiftAmt.getOperand(1)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00003219 if (ConstantSDNode *SUBC =
Scott Michelc9dc1142007-04-02 21:36:32 +00003220 dyn_cast<ConstantSDNode>(LHSShiftAmt.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003221 if (SUBC->getAPIntValue() == OpSizeInBits) {
Bill Wendling2692d592008-08-31 01:13:31 +00003222 if (HasROTR)
Bill Wendling317bd702009-01-30 21:14:50 +00003223 return DAG.getNode(ISD::ROTR, DL, VT,
3224 LHSShiftArg, RHSShiftAmt).getNode();
Bill Wendling2692d592008-08-31 01:13:31 +00003225 else
Bill Wendling317bd702009-01-30 21:14:50 +00003226 return DAG.getNode(ISD::ROTL, DL, VT,
3227 LHSShiftArg, LHSShiftAmt).getNode();
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00003228 }
Scott Michelc9dc1142007-04-02 21:36:32 +00003229 }
3230 }
3231
Dan Gohman74feef22008-10-17 01:23:35 +00003232 // Look for sign/zext/any-extended or truncate cases:
Scott Michelc9dc1142007-04-02 21:36:32 +00003233 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND
3234 || LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND
Dan Gohman74feef22008-10-17 01:23:35 +00003235 || LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND
3236 || LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
Scott Michelc9dc1142007-04-02 21:36:32 +00003237 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND
3238 || RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND
Dan Gohman74feef22008-10-17 01:23:35 +00003239 || RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND
3240 || RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
Dan Gohman475871a2008-07-27 21:46:04 +00003241 SDValue LExtOp0 = LHSShiftAmt.getOperand(0);
3242 SDValue RExtOp0 = RHSShiftAmt.getOperand(0);
Scott Michelc9dc1142007-04-02 21:36:32 +00003243 if (RExtOp0.getOpcode() == ISD::SUB &&
3244 RExtOp0.getOperand(1) == LExtOp0) {
3245 // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003246 // (rotl x, y)
Scott Michelc9dc1142007-04-02 21:36:32 +00003247 // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003248 // (rotr x, (sub 32, y))
Dan Gohman74feef22008-10-17 01:23:35 +00003249 if (ConstantSDNode *SUBC =
3250 dyn_cast<ConstantSDNode>(RExtOp0.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003251 if (SUBC->getAPIntValue() == OpSizeInBits) {
Bill Wendling317bd702009-01-30 21:14:50 +00003252 return DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
3253 LHSShiftArg,
Gabor Greif12632d22008-08-30 19:29:20 +00003254 HasROTL ? LHSShiftAmt : RHSShiftAmt).getNode();
Scott Michelc9dc1142007-04-02 21:36:32 +00003255 }
3256 }
3257 } else if (LExtOp0.getOpcode() == ISD::SUB &&
3258 RExtOp0 == LExtOp0.getOperand(1)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00003259 // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext y))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003260 // (rotr x, y)
Bill Wendling353dea22008-08-31 01:04:56 +00003261 // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext y))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003262 // (rotl x, (sub 32, y))
Dan Gohman74feef22008-10-17 01:23:35 +00003263 if (ConstantSDNode *SUBC =
3264 dyn_cast<ConstantSDNode>(LExtOp0.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003265 if (SUBC->getAPIntValue() == OpSizeInBits) {
Bill Wendling317bd702009-01-30 21:14:50 +00003266 return DAG.getNode(HasROTR ? ISD::ROTR : ISD::ROTL, DL, VT,
3267 LHSShiftArg,
Bill Wendling353dea22008-08-31 01:04:56 +00003268 HasROTR ? RHSShiftAmt : LHSShiftAmt).getNode();
Scott Michelc9dc1142007-04-02 21:36:32 +00003269 }
3270 }
Chris Lattner516b9622006-09-14 20:50:57 +00003271 }
3272 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003273
Chris Lattner516b9622006-09-14 20:50:57 +00003274 return 0;
3275}
3276
Dan Gohman475871a2008-07-27 21:46:04 +00003277SDValue DAGCombiner::visitXOR(SDNode *N) {
3278 SDValue N0 = N->getOperand(0);
3279 SDValue N1 = N->getOperand(1);
3280 SDValue LHS, RHS, CC;
Nate Begeman646d7e22005-09-02 21:18:40 +00003281 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3282 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003283 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00003284
Dan Gohman7f321562007-06-25 16:23:39 +00003285 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00003286 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003287 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00003288 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00003289 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003290
Evan Cheng26471c42008-03-25 20:08:07 +00003291 // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
3292 if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF)
3293 return DAG.getConstant(0, VT);
Dan Gohman613e0d82007-07-03 14:03:57 +00003294 // fold (xor x, undef) -> undef
Dan Gohman70fb1ae2007-07-10 15:19:29 +00003295 if (N0.getOpcode() == ISD::UNDEF)
3296 return N0;
3297 if (N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00003298 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003299 // fold (xor c1, c2) -> c1^c2
Nate Begeman646d7e22005-09-02 21:18:40 +00003300 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003301 return DAG.FoldConstantArithmetic(ISD::XOR, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00003302 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00003303 if (N0C && !N1C)
Bill Wendling317bd702009-01-30 21:14:50 +00003304 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003305 // fold (xor x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003306 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003307 return N0;
Nate Begemancd4d58c2006-02-03 06:46:56 +00003308 // reassociate xor
Bill Wendling35247c32009-01-30 00:45:56 +00003309 SDValue RXOR = ReassociateOps(ISD::XOR, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00003310 if (RXOR.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00003311 return RXOR;
Bill Wendlingae89bb12008-11-11 08:25:46 +00003312
Nate Begeman1d4d4142005-09-01 00:19:25 +00003313 // fold !(x cc y) -> (x !cc y)
Dan Gohman002e5d02008-03-13 22:13:53 +00003314 if (N1C && N1C->getAPIntValue() == 1 && isSetCCEquivalent(N0, LHS, RHS, CC)) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00003315 bool isInt = LHS.getValueType().isInteger();
Nate Begeman646d7e22005-09-02 21:18:40 +00003316 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
3317 isInt);
Bill Wendlingae89bb12008-11-11 08:25:46 +00003318
Duncan Sands25cf2272008-11-24 14:53:14 +00003319 if (!LegalOperations || TLI.isCondCodeLegal(NotCC, LHS.getValueType())) {
Bill Wendlingae89bb12008-11-11 08:25:46 +00003320 switch (N0.getOpcode()) {
3321 default:
Torok Edwinc23197a2009-07-14 16:55:14 +00003322 llvm_unreachable("Unhandled SetCC Equivalent!");
Bill Wendlingae89bb12008-11-11 08:25:46 +00003323 case ISD::SETCC:
Bill Wendling317bd702009-01-30 21:14:50 +00003324 return DAG.getSetCC(N->getDebugLoc(), VT, LHS, RHS, NotCC);
Bill Wendlingae89bb12008-11-11 08:25:46 +00003325 case ISD::SELECT_CC:
Bill Wendling317bd702009-01-30 21:14:50 +00003326 return DAG.getSelectCC(N->getDebugLoc(), LHS, RHS, N0.getOperand(2),
Bill Wendlingae89bb12008-11-11 08:25:46 +00003327 N0.getOperand(3), NotCC);
3328 }
3329 }
Nate Begeman1d4d4142005-09-01 00:19:25 +00003330 }
Bill Wendlingae89bb12008-11-11 08:25:46 +00003331
Chris Lattner61c5ff42007-09-10 21:39:07 +00003332 // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
Dan Gohman002e5d02008-03-13 22:13:53 +00003333 if (N1C && N1C->getAPIntValue() == 1 && N0.getOpcode() == ISD::ZERO_EXTEND &&
Gabor Greif12632d22008-08-30 19:29:20 +00003334 N0.getNode()->hasOneUse() &&
3335 isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
Dan Gohman475871a2008-07-27 21:46:04 +00003336 SDValue V = N0.getOperand(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003337 V = DAG.getNode(ISD::XOR, N0.getDebugLoc(), V.getValueType(), V,
Duncan Sands272dce02007-10-10 09:54:50 +00003338 DAG.getConstant(1, V.getValueType()));
Gabor Greifba36cb52008-08-28 21:40:38 +00003339 AddToWorkList(V.getNode());
Bill Wendling317bd702009-01-30 21:14:50 +00003340 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, V);
Chris Lattner61c5ff42007-09-10 21:39:07 +00003341 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003342
Bill Wendling317bd702009-01-30 21:14:50 +00003343 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
Owen Anderson825b72b2009-08-11 20:47:22 +00003344 if (N1C && N1C->getAPIntValue() == 1 && VT == MVT::i1 &&
Nate Begeman99801192005-09-07 23:25:52 +00003345 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
Dan Gohman475871a2008-07-27 21:46:04 +00003346 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
Nate Begeman99801192005-09-07 23:25:52 +00003347 if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
3348 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
Bill Wendling317bd702009-01-30 21:14:50 +00003349 LHS = DAG.getNode(ISD::XOR, LHS.getDebugLoc(), VT, LHS, N1); // LHS = ~LHS
3350 RHS = DAG.getNode(ISD::XOR, RHS.getDebugLoc(), VT, RHS, N1); // RHS = ~RHS
Gabor Greifba36cb52008-08-28 21:40:38 +00003351 AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode());
Bill Wendling317bd702009-01-30 21:14:50 +00003352 return DAG.getNode(NewOpcode, N->getDebugLoc(), VT, LHS, RHS);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003353 }
3354 }
Bill Wendling317bd702009-01-30 21:14:50 +00003355 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
Scott Michelfdc40a02009-02-17 22:15:04 +00003356 if (N1C && N1C->isAllOnesValue() &&
Nate Begeman99801192005-09-07 23:25:52 +00003357 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
Dan Gohman475871a2008-07-27 21:46:04 +00003358 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
Nate Begeman99801192005-09-07 23:25:52 +00003359 if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
3360 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
Bill Wendling317bd702009-01-30 21:14:50 +00003361 LHS = DAG.getNode(ISD::XOR, LHS.getDebugLoc(), VT, LHS, N1); // LHS = ~LHS
3362 RHS = DAG.getNode(ISD::XOR, RHS.getDebugLoc(), VT, RHS, N1); // RHS = ~RHS
Gabor Greifba36cb52008-08-28 21:40:38 +00003363 AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode());
Bill Wendling317bd702009-01-30 21:14:50 +00003364 return DAG.getNode(NewOpcode, N->getDebugLoc(), VT, LHS, RHS);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003365 }
3366 }
Bill Wendling317bd702009-01-30 21:14:50 +00003367 // fold (xor (xor x, c1), c2) -> (xor x, (xor c1, c2))
Nate Begeman223df222005-09-08 20:18:10 +00003368 if (N1C && N0.getOpcode() == ISD::XOR) {
3369 ConstantSDNode *N00C = dyn_cast<ConstantSDNode>(N0.getOperand(0));
3370 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3371 if (N00C)
Bill Wendling317bd702009-01-30 21:14:50 +00003372 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N0.getOperand(1),
3373 DAG.getConstant(N1C->getAPIntValue() ^
Dan Gohman002e5d02008-03-13 22:13:53 +00003374 N00C->getAPIntValue(), VT));
Nate Begeman223df222005-09-08 20:18:10 +00003375 if (N01C)
Bill Wendling317bd702009-01-30 21:14:50 +00003376 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N0.getOperand(0),
3377 DAG.getConstant(N1C->getAPIntValue() ^
Dan Gohman002e5d02008-03-13 22:13:53 +00003378 N01C->getAPIntValue(), VT));
Nate Begeman223df222005-09-08 20:18:10 +00003379 }
3380 // fold (xor x, x) -> 0
Eric Christopher7bccf6a2011-02-16 04:50:12 +00003381 if (N0 == N1)
3382 return tryFoldToZero(N->getDebugLoc(), TLI, VT, DAG, LegalOperations);
Scott Michelfdc40a02009-02-17 22:15:04 +00003383
Chris Lattner35e5c142006-05-05 05:51:50 +00003384 // Simplify: xor (op x...), (op y...) -> (op (xor x, y))
3385 if (N0.getOpcode() == N1.getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003386 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00003387 if (Tmp.getNode()) return Tmp;
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003388 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003389
Chris Lattner3e104b12006-04-08 04:15:24 +00003390 // Simplify the expression using non-local knowledge.
Duncan Sands83ec4b62008-06-06 12:08:01 +00003391 if (!VT.isVector() &&
Dan Gohman475871a2008-07-27 21:46:04 +00003392 SimplifyDemandedBits(SDValue(N, 0)))
3393 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003394
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003395 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003396}
3397
Chris Lattnere70da202007-12-06 07:33:36 +00003398/// visitShiftByConstant - Handle transforms common to the three shifts, when
3399/// the shift amount is a constant.
Dan Gohman475871a2008-07-27 21:46:04 +00003400SDValue DAGCombiner::visitShiftByConstant(SDNode *N, unsigned Amt) {
Gabor Greifba36cb52008-08-28 21:40:38 +00003401 SDNode *LHS = N->getOperand(0).getNode();
Dan Gohman475871a2008-07-27 21:46:04 +00003402 if (!LHS->hasOneUse()) return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00003403
Chris Lattnere70da202007-12-06 07:33:36 +00003404 // We want to pull some binops through shifts, so that we have (and (shift))
3405 // instead of (shift (and)), likewise for add, or, xor, etc. This sort of
3406 // thing happens with address calculations, so it's important to canonicalize
3407 // it.
3408 bool HighBitSet = false; // Can we transform this if the high bit is set?
Scott Michelfdc40a02009-02-17 22:15:04 +00003409
Chris Lattnere70da202007-12-06 07:33:36 +00003410 switch (LHS->getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003411 default: return SDValue();
Chris Lattnere70da202007-12-06 07:33:36 +00003412 case ISD::OR:
3413 case ISD::XOR:
3414 HighBitSet = false; // We can only transform sra if the high bit is clear.
3415 break;
3416 case ISD::AND:
3417 HighBitSet = true; // We can only transform sra if the high bit is set.
3418 break;
3419 case ISD::ADD:
Scott Michelfdc40a02009-02-17 22:15:04 +00003420 if (N->getOpcode() != ISD::SHL)
Dan Gohman475871a2008-07-27 21:46:04 +00003421 return SDValue(); // only shl(add) not sr[al](add).
Chris Lattnere70da202007-12-06 07:33:36 +00003422 HighBitSet = false; // We can only transform sra if the high bit is clear.
3423 break;
3424 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003425
Chris Lattnere70da202007-12-06 07:33:36 +00003426 // We require the RHS of the binop to be a constant as well.
3427 ConstantSDNode *BinOpCst = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
Dan Gohman475871a2008-07-27 21:46:04 +00003428 if (!BinOpCst) return SDValue();
Bill Wendling88103372009-01-30 21:37:17 +00003429
3430 // FIXME: disable this unless the input to the binop is a shift by a constant.
3431 // If it is not a shift, it pessimizes some common cases like:
Chris Lattnerd3fd6d22007-12-06 07:47:55 +00003432 //
Bill Wendling88103372009-01-30 21:37:17 +00003433 // void foo(int *X, int i) { X[i & 1235] = 1; }
3434 // int bar(int *X, int i) { return X[i & 255]; }
Gabor Greifba36cb52008-08-28 21:40:38 +00003435 SDNode *BinOpLHSVal = LHS->getOperand(0).getNode();
Scott Michelfdc40a02009-02-17 22:15:04 +00003436 if ((BinOpLHSVal->getOpcode() != ISD::SHL &&
Chris Lattnerd3fd6d22007-12-06 07:47:55 +00003437 BinOpLHSVal->getOpcode() != ISD::SRA &&
3438 BinOpLHSVal->getOpcode() != ISD::SRL) ||
3439 !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1)))
Dan Gohman475871a2008-07-27 21:46:04 +00003440 return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00003441
Owen Andersone50ed302009-08-10 22:56:29 +00003442 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003443
Bill Wendling88103372009-01-30 21:37:17 +00003444 // If this is a signed shift right, and the high bit is modified by the
3445 // logical operation, do not perform the transformation. The highBitSet
3446 // boolean indicates the value of the high bit of the constant which would
3447 // cause it to be modified for this operation.
Chris Lattnere70da202007-12-06 07:33:36 +00003448 if (N->getOpcode() == ISD::SRA) {
Dan Gohman220a8232008-03-03 23:51:38 +00003449 bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative();
3450 if (BinOpRHSSignSet != HighBitSet)
Dan Gohman475871a2008-07-27 21:46:04 +00003451 return SDValue();
Chris Lattnere70da202007-12-06 07:33:36 +00003452 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003453
Chris Lattnere70da202007-12-06 07:33:36 +00003454 // Fold the constants, shifting the binop RHS by the shift amount.
Bill Wendling88103372009-01-30 21:37:17 +00003455 SDValue NewRHS = DAG.getNode(N->getOpcode(), LHS->getOperand(1).getDebugLoc(),
3456 N->getValueType(0),
3457 LHS->getOperand(1), N->getOperand(1));
Chris Lattnere70da202007-12-06 07:33:36 +00003458
3459 // Create the new shift.
Eric Christopher503a64d2010-12-09 04:48:06 +00003460 SDValue NewShift = DAG.getNode(N->getOpcode(),
3461 LHS->getOperand(0).getDebugLoc(),
Bill Wendling88103372009-01-30 21:37:17 +00003462 VT, LHS->getOperand(0), N->getOperand(1));
Chris Lattnere70da202007-12-06 07:33:36 +00003463
3464 // Create the new binop.
Bill Wendling88103372009-01-30 21:37:17 +00003465 return DAG.getNode(LHS->getOpcode(), N->getDebugLoc(), VT, NewShift, NewRHS);
Chris Lattnere70da202007-12-06 07:33:36 +00003466}
3467
Dan Gohman475871a2008-07-27 21:46:04 +00003468SDValue DAGCombiner::visitSHL(SDNode *N) {
3469 SDValue N0 = N->getOperand(0);
3470 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00003471 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3472 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003473 EVT VT = N0.getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00003474 unsigned OpSizeInBits = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00003475
Nate Begeman1d4d4142005-09-01 00:19:25 +00003476 // fold (shl c1, c2) -> c1<<c2
Nate Begeman646d7e22005-09-02 21:18:40 +00003477 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003478 return DAG.FoldConstantArithmetic(ISD::SHL, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003479 // fold (shl 0, x) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00003480 if (N0C && N0C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003481 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003482 // fold (shl x, c >= size(x)) -> undef
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003483 if (N1C && N1C->getZExtValue() >= OpSizeInBits)
Dale Johannesene8d72302009-02-06 23:05:02 +00003484 return DAG.getUNDEF(VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003485 // fold (shl x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003486 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003487 return N0;
Chad Rosier92bcd962011-06-14 22:29:10 +00003488 // fold (shl undef, x) -> 0
3489 if (N0.getOpcode() == ISD::UNDEF)
3490 return DAG.getConstant(0, VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003491 // if (shl x, c) is known to be zero, return 0
Dan Gohman475871a2008-07-27 21:46:04 +00003492 if (DAG.MaskedValueIsZero(SDValue(N, 0),
Dan Gohman87862e72009-12-11 21:31:27 +00003493 APInt::getAllOnesValue(OpSizeInBits)))
Nate Begeman83e75ec2005-09-06 04:43:02 +00003494 return DAG.getConstant(0, VT);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003495 // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
Evan Chengeb9f8922008-08-30 02:03:58 +00003496 if (N1.getOpcode() == ISD::TRUNCATE &&
Evan Cheng242ebd12008-09-22 18:19:24 +00003497 N1.getOperand(0).getOpcode() == ISD::AND &&
3498 N1.hasOneUse() && N1.getOperand(0).hasOneUse()) {
Evan Chengeb9f8922008-08-30 02:03:58 +00003499 SDValue N101 = N1.getOperand(0).getOperand(1);
Evan Cheng242ebd12008-09-22 18:19:24 +00003500 if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) {
Owen Andersone50ed302009-08-10 22:56:29 +00003501 EVT TruncVT = N1.getValueType();
Evan Cheng242ebd12008-09-22 18:19:24 +00003502 SDValue N100 = N1.getOperand(0).getOperand(0);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003503 APInt TruncC = N101C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00003504 TruncC = TruncC.trunc(TruncVT.getSizeInBits());
Bill Wendling88103372009-01-30 21:37:17 +00003505 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0,
Bill Wendlingfc4b6772009-02-01 11:19:36 +00003506 DAG.getNode(ISD::AND, N->getDebugLoc(), TruncVT,
3507 DAG.getNode(ISD::TRUNCATE,
3508 N->getDebugLoc(),
3509 TruncVT, N100),
Dan Gohmance9bc122009-01-27 20:39:34 +00003510 DAG.getConstant(TruncC, TruncVT)));
Evan Chengeb9f8922008-08-30 02:03:58 +00003511 }
3512 }
3513
Dan Gohman475871a2008-07-27 21:46:04 +00003514 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
3515 return SDValue(N, 0);
Bill Wendling88103372009-01-30 21:37:17 +00003516
3517 // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
Scott Michelfdc40a02009-02-17 22:15:04 +00003518 if (N1C && N0.getOpcode() == ISD::SHL &&
Nate Begeman1d4d4142005-09-01 00:19:25 +00003519 N0.getOperand(1).getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003520 uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
3521 uint64_t c2 = N1C->getZExtValue();
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003522 if (c1 + c2 >= OpSizeInBits)
Nate Begeman83e75ec2005-09-06 04:43:02 +00003523 return DAG.getConstant(0, VT);
Bill Wendling88103372009-01-30 21:37:17 +00003524 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0.getOperand(0),
Nate Begeman83e75ec2005-09-06 04:43:02 +00003525 DAG.getConstant(c1 + c2, N1.getValueType()));
Nate Begeman1d4d4142005-09-01 00:19:25 +00003526 }
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003527
3528 // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2)))
3529 // For this to be valid, the second form must not preserve any of the bits
3530 // that are shifted out by the inner shift in the first form. This means
3531 // the outer shift size must be >= the number of bits added by the ext.
3532 // As a corollary, we don't care what kind of ext it is.
3533 if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
3534 N0.getOpcode() == ISD::ANY_EXTEND ||
3535 N0.getOpcode() == ISD::SIGN_EXTEND) &&
3536 N0.getOperand(0).getOpcode() == ISD::SHL &&
3537 isa<ConstantSDNode>(N0.getOperand(0)->getOperand(1))) {
Owen Anderson95771af2011-02-25 21:41:48 +00003538 uint64_t c1 =
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003539 cast<ConstantSDNode>(N0.getOperand(0)->getOperand(1))->getZExtValue();
3540 uint64_t c2 = N1C->getZExtValue();
3541 EVT InnerShiftVT = N0.getOperand(0).getValueType();
3542 uint64_t InnerShiftSize = InnerShiftVT.getScalarType().getSizeInBits();
3543 if (c2 >= OpSizeInBits - InnerShiftSize) {
3544 if (c1 + c2 >= OpSizeInBits)
3545 return DAG.getConstant(0, VT);
3546 return DAG.getNode(ISD::SHL, N0->getDebugLoc(), VT,
3547 DAG.getNode(N0.getOpcode(), N0->getDebugLoc(), VT,
3548 N0.getOperand(0)->getOperand(0)),
3549 DAG.getConstant(c1 + c2, N1.getValueType()));
3550 }
3551 }
3552
Eli Friedman2a6d9eb2011-06-09 22:14:44 +00003553 // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
3554 // (and (srl x, (sub c1, c2), MASK)
Chandler Carruth62dfc512012-01-05 11:05:55 +00003555 // Only fold this if the inner shift has no other uses -- if it does, folding
3556 // this will increase the total number of instructions.
3557 if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse() &&
Nate Begeman1d4d4142005-09-01 00:19:25 +00003558 N0.getOperand(1).getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003559 uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
Evan Chengd101a722009-07-21 05:40:15 +00003560 if (c1 < VT.getSizeInBits()) {
3561 uint64_t c2 = N1C->getZExtValue();
Eli Friedman2a6d9eb2011-06-09 22:14:44 +00003562 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
3563 VT.getSizeInBits() - c1);
3564 SDValue Shift;
3565 if (c2 > c1) {
3566 Mask = Mask.shl(c2-c1);
3567 Shift = DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0.getOperand(0),
3568 DAG.getConstant(c2-c1, N1.getValueType()));
3569 } else {
3570 Mask = Mask.lshr(c1-c2);
3571 Shift = DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0.getOperand(0),
3572 DAG.getConstant(c1-c2, N1.getValueType()));
3573 }
3574 return DAG.getNode(ISD::AND, N0.getDebugLoc(), VT, Shift,
3575 DAG.getConstant(Mask, VT));
Evan Chengd101a722009-07-21 05:40:15 +00003576 }
Nate Begeman1d4d4142005-09-01 00:19:25 +00003577 }
Bill Wendling88103372009-01-30 21:37:17 +00003578 // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
Dan Gohman5cbd37e2009-08-06 09:18:59 +00003579 if (N1C && N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1)) {
3580 SDValue HiBitsMask =
3581 DAG.getConstant(APInt::getHighBitsSet(VT.getSizeInBits(),
3582 VT.getSizeInBits() -
3583 N1C->getZExtValue()),
3584 VT);
Bill Wendling88103372009-01-30 21:37:17 +00003585 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0.getOperand(0),
Dan Gohman5cbd37e2009-08-06 09:18:59 +00003586 HiBitsMask);
3587 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003588
Evan Chenge5b51ac2010-04-17 06:13:15 +00003589 if (N1C) {
3590 SDValue NewSHL = visitShiftByConstant(N, N1C->getZExtValue());
3591 if (NewSHL.getNode())
3592 return NewSHL;
3593 }
3594
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003595 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003596}
3597
Dan Gohman475871a2008-07-27 21:46:04 +00003598SDValue DAGCombiner::visitSRA(SDNode *N) {
3599 SDValue N0 = N->getOperand(0);
3600 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00003601 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3602 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003603 EVT VT = N0.getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00003604 unsigned OpSizeInBits = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00003605
Bill Wendling88103372009-01-30 21:37:17 +00003606 // fold (sra c1, c2) -> (sra c1, c2)
Nate Begeman646d7e22005-09-02 21:18:40 +00003607 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003608 return DAG.FoldConstantArithmetic(ISD::SRA, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003609 // fold (sra 0, x) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00003610 if (N0C && N0C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003611 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003612 // fold (sra -1, x) -> -1
Nate Begeman646d7e22005-09-02 21:18:40 +00003613 if (N0C && N0C->isAllOnesValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003614 return N0;
Bill Wendling88103372009-01-30 21:37:17 +00003615 // fold (sra x, (setge c, size(x))) -> undef
Dan Gohman87862e72009-12-11 21:31:27 +00003616 if (N1C && N1C->getZExtValue() >= OpSizeInBits)
Dale Johannesene8d72302009-02-06 23:05:02 +00003617 return DAG.getUNDEF(VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003618 // fold (sra x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003619 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003620 return N0;
Nate Begemanfb7217b2006-02-17 19:54:08 +00003621 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
3622 // sext_inreg.
3623 if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
Dan Gohman87862e72009-12-11 21:31:27 +00003624 unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
Dan Gohmand1996362010-01-09 02:13:55 +00003625 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
3626 if (VT.isVector())
3627 ExtVT = EVT::getVectorVT(*DAG.getContext(),
3628 ExtVT, VT.getVectorNumElements());
3629 if ((!LegalOperations ||
3630 TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT)))
Bill Wendling88103372009-01-30 21:37:17 +00003631 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT,
Dan Gohmand1996362010-01-09 02:13:55 +00003632 N0.getOperand(0), DAG.getValueType(ExtVT));
Nate Begemanfb7217b2006-02-17 19:54:08 +00003633 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00003634
Bill Wendling88103372009-01-30 21:37:17 +00003635 // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
Chris Lattner71d9ebc2006-02-28 06:23:04 +00003636 if (N1C && N0.getOpcode() == ISD::SRA) {
3637 if (ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003638 unsigned Sum = N1C->getZExtValue() + C1->getZExtValue();
Dan Gohman87862e72009-12-11 21:31:27 +00003639 if (Sum >= OpSizeInBits) Sum = OpSizeInBits-1;
Bill Wendling88103372009-01-30 21:37:17 +00003640 return DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, N0.getOperand(0),
Chris Lattner71d9ebc2006-02-28 06:23:04 +00003641 DAG.getConstant(Sum, N1C->getValueType(0)));
3642 }
3643 }
Christopher Lamb15cbde32008-03-19 08:30:06 +00003644
Bill Wendling88103372009-01-30 21:37:17 +00003645 // fold (sra (shl X, m), (sub result_size, n))
3646 // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for
Scott Michelfdc40a02009-02-17 22:15:04 +00003647 // result_size - n != m.
3648 // If truncate is free for the target sext(shl) is likely to result in better
Christopher Lambb9b04282008-03-20 04:31:39 +00003649 // code.
Christopher Lamb15cbde32008-03-19 08:30:06 +00003650 if (N0.getOpcode() == ISD::SHL) {
3651 // Get the two constanst of the shifts, CN0 = m, CN = n.
3652 const ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3653 if (N01C && N1C) {
Christopher Lambb9b04282008-03-20 04:31:39 +00003654 // Determine what the truncate's result bitsize and type would be.
Owen Andersone50ed302009-08-10 22:56:29 +00003655 EVT TruncVT =
Eric Christopher503a64d2010-12-09 04:48:06 +00003656 EVT::getIntegerVT(*DAG.getContext(),
3657 OpSizeInBits - N1C->getZExtValue());
Christopher Lambb9b04282008-03-20 04:31:39 +00003658 // Determine the residual right-shift amount.
Torok Edwin6bb49582009-05-23 17:29:48 +00003659 signed ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue();
Duncan Sandsd4b9c172008-06-13 19:07:40 +00003660
Scott Michelfdc40a02009-02-17 22:15:04 +00003661 // If the shift is not a no-op (in which case this should be just a sign
3662 // extend already), the truncated to type is legal, sign_extend is legal
Dan Gohmanf451cb82010-02-10 16:03:48 +00003663 // on that type, and the truncate to that type is both legal and free,
Christopher Lambb9b04282008-03-20 04:31:39 +00003664 // perform the transform.
Torok Edwin6bb49582009-05-23 17:29:48 +00003665 if ((ShiftAmt > 0) &&
Dan Gohmanf560ffa2009-01-28 17:46:25 +00003666 TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) &&
3667 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) &&
Evan Cheng260e07e2008-03-20 02:18:41 +00003668 TLI.isTruncateFree(VT, TruncVT)) {
Christopher Lambb9b04282008-03-20 04:31:39 +00003669
Owen Anderson95771af2011-02-25 21:41:48 +00003670 SDValue Amt = DAG.getConstant(ShiftAmt,
3671 getShiftAmountTy(N0.getOperand(0).getValueType()));
Bill Wendling88103372009-01-30 21:37:17 +00003672 SDValue Shift = DAG.getNode(ISD::SRL, N0.getDebugLoc(), VT,
3673 N0.getOperand(0), Amt);
3674 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(), TruncVT,
3675 Shift);
3676 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(),
3677 N->getValueType(0), Trunc);
Christopher Lamb15cbde32008-03-19 08:30:06 +00003678 }
3679 }
3680 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003681
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003682 // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))).
Evan Chengeb9f8922008-08-30 02:03:58 +00003683 if (N1.getOpcode() == ISD::TRUNCATE &&
Evan Cheng242ebd12008-09-22 18:19:24 +00003684 N1.getOperand(0).getOpcode() == ISD::AND &&
3685 N1.hasOneUse() && N1.getOperand(0).hasOneUse()) {
Evan Chengeb9f8922008-08-30 02:03:58 +00003686 SDValue N101 = N1.getOperand(0).getOperand(1);
Evan Cheng242ebd12008-09-22 18:19:24 +00003687 if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) {
Owen Andersone50ed302009-08-10 22:56:29 +00003688 EVT TruncVT = N1.getValueType();
Evan Cheng242ebd12008-09-22 18:19:24 +00003689 SDValue N100 = N1.getOperand(0).getOperand(0);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003690 APInt TruncC = N101C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00003691 TruncC = TruncC.trunc(TruncVT.getScalarType().getSizeInBits());
Bill Wendling88103372009-01-30 21:37:17 +00003692 return DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, N0,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003693 DAG.getNode(ISD::AND, N->getDebugLoc(),
Bill Wendling88103372009-01-30 21:37:17 +00003694 TruncVT,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003695 DAG.getNode(ISD::TRUNCATE,
3696 N->getDebugLoc(),
3697 TruncVT, N100),
Dan Gohmance9bc122009-01-27 20:39:34 +00003698 DAG.getConstant(TruncC, TruncVT)));
Evan Chengeb9f8922008-08-30 02:03:58 +00003699 }
3700 }
3701
Benjamin Kramer9b108a32011-01-30 16:38:43 +00003702 // fold (sra (trunc (sr x, c1)), c2) -> (trunc (sra x, c1+c2))
3703 // if c1 is equal to the number of bits the trunc removes
3704 if (N0.getOpcode() == ISD::TRUNCATE &&
3705 (N0.getOperand(0).getOpcode() == ISD::SRL ||
3706 N0.getOperand(0).getOpcode() == ISD::SRA) &&
3707 N0.getOperand(0).hasOneUse() &&
3708 N0.getOperand(0).getOperand(1).hasOneUse() &&
3709 N1C && isa<ConstantSDNode>(N0.getOperand(0).getOperand(1))) {
3710 EVT LargeVT = N0.getOperand(0).getValueType();
3711 ConstantSDNode *LargeShiftAmt =
3712 cast<ConstantSDNode>(N0.getOperand(0).getOperand(1));
3713
3714 if (LargeVT.getScalarType().getSizeInBits() - OpSizeInBits ==
3715 LargeShiftAmt->getZExtValue()) {
3716 SDValue Amt =
3717 DAG.getConstant(LargeShiftAmt->getZExtValue() + N1C->getZExtValue(),
Owen Anderson95771af2011-02-25 21:41:48 +00003718 getShiftAmountTy(N0.getOperand(0).getOperand(0).getValueType()));
Benjamin Kramer9b108a32011-01-30 16:38:43 +00003719 SDValue SRA = DAG.getNode(ISD::SRA, N->getDebugLoc(), LargeVT,
3720 N0.getOperand(0).getOperand(0), Amt);
3721 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, SRA);
3722 }
3723 }
3724
Scott Michelfdc40a02009-02-17 22:15:04 +00003725 // Simplify, based on bits shifted out of the LHS.
Dan Gohman475871a2008-07-27 21:46:04 +00003726 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
3727 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003728
3729
Nate Begeman1d4d4142005-09-01 00:19:25 +00003730 // If the sign bit is known to be zero, switch this to a SRL.
Dan Gohman2e68b6f2008-02-25 21:11:39 +00003731 if (DAG.SignBitIsZero(N0))
Bill Wendling88103372009-01-30 21:37:17 +00003732 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0, N1);
Chris Lattnere70da202007-12-06 07:33:36 +00003733
Evan Chenge5b51ac2010-04-17 06:13:15 +00003734 if (N1C) {
3735 SDValue NewSRA = visitShiftByConstant(N, N1C->getZExtValue());
3736 if (NewSRA.getNode())
3737 return NewSRA;
3738 }
3739
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003740 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003741}
3742
Dan Gohman475871a2008-07-27 21:46:04 +00003743SDValue DAGCombiner::visitSRL(SDNode *N) {
3744 SDValue N0 = N->getOperand(0);
3745 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00003746 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3747 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003748 EVT VT = N0.getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00003749 unsigned OpSizeInBits = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00003750
Nate Begeman1d4d4142005-09-01 00:19:25 +00003751 // fold (srl c1, c2) -> c1 >>u c2
Nate Begeman646d7e22005-09-02 21:18:40 +00003752 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003753 return DAG.FoldConstantArithmetic(ISD::SRL, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003754 // fold (srl 0, x) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00003755 if (N0C && N0C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003756 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003757 // fold (srl x, c >= size(x)) -> undef
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003758 if (N1C && N1C->getZExtValue() >= OpSizeInBits)
Dale Johannesene8d72302009-02-06 23:05:02 +00003759 return DAG.getUNDEF(VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003760 // fold (srl x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003761 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003762 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003763 // if (srl x, c) is known to be zero, return 0
Dan Gohman475871a2008-07-27 21:46:04 +00003764 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
Dan Gohman2e68b6f2008-02-25 21:11:39 +00003765 APInt::getAllOnesValue(OpSizeInBits)))
Nate Begeman83e75ec2005-09-06 04:43:02 +00003766 return DAG.getConstant(0, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00003767
Bill Wendling88103372009-01-30 21:37:17 +00003768 // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
Scott Michelfdc40a02009-02-17 22:15:04 +00003769 if (N1C && N0.getOpcode() == ISD::SRL &&
Nate Begeman1d4d4142005-09-01 00:19:25 +00003770 N0.getOperand(1).getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003771 uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
3772 uint64_t c2 = N1C->getZExtValue();
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003773 if (c1 + c2 >= OpSizeInBits)
Nate Begeman83e75ec2005-09-06 04:43:02 +00003774 return DAG.getConstant(0, VT);
Bill Wendling88103372009-01-30 21:37:17 +00003775 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0.getOperand(0),
Nate Begeman83e75ec2005-09-06 04:43:02 +00003776 DAG.getConstant(c1 + c2, N1.getValueType()));
Nate Begeman1d4d4142005-09-01 00:19:25 +00003777 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003778
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003779 // fold (srl (trunc (srl x, c1)), c2) -> 0 or (trunc (srl x, (add c1, c2)))
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003780 if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
3781 N0.getOperand(0).getOpcode() == ISD::SRL &&
Dale Johannesen025cc6e2010-12-20 20:10:50 +00003782 isa<ConstantSDNode>(N0.getOperand(0)->getOperand(1))) {
Owen Anderson95771af2011-02-25 21:41:48 +00003783 uint64_t c1 =
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003784 cast<ConstantSDNode>(N0.getOperand(0)->getOperand(1))->getZExtValue();
3785 uint64_t c2 = N1C->getZExtValue();
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003786 EVT InnerShiftVT = N0.getOperand(0).getValueType();
3787 EVT ShiftCountVT = N0.getOperand(0)->getOperand(1).getValueType();
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003788 uint64_t InnerShiftSize = InnerShiftVT.getScalarType().getSizeInBits();
Dale Johannesen025cc6e2010-12-20 20:10:50 +00003789 // This is only valid if the OpSizeInBits + c1 = size of inner shift.
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003790 if (c1 + OpSizeInBits == InnerShiftSize) {
3791 if (c1 + c2 >= InnerShiftSize)
3792 return DAG.getConstant(0, VT);
3793 return DAG.getNode(ISD::TRUNCATE, N0->getDebugLoc(), VT,
Owen Anderson95771af2011-02-25 21:41:48 +00003794 DAG.getNode(ISD::SRL, N0->getDebugLoc(), InnerShiftVT,
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003795 N0.getOperand(0)->getOperand(0),
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003796 DAG.getConstant(c1 + c2, ShiftCountVT)));
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003797 }
3798 }
3799
Chris Lattnerefcddc32010-04-15 05:28:43 +00003800 // fold (srl (shl x, c), c) -> (and x, cst2)
3801 if (N1C && N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1 &&
3802 N0.getValueSizeInBits() <= 64) {
3803 uint64_t ShAmt = N1C->getZExtValue()+64-N0.getValueSizeInBits();
3804 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0.getOperand(0),
3805 DAG.getConstant(~0ULL >> ShAmt, VT));
3806 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003807
Scott Michelfdc40a02009-02-17 22:15:04 +00003808
Chris Lattner06afe072006-05-05 22:53:17 +00003809 // fold (srl (anyextend x), c) -> (anyextend (srl x, c))
3810 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
3811 // Shifting in all undef bits?
Owen Andersone50ed302009-08-10 22:56:29 +00003812 EVT SmallVT = N0.getOperand(0).getValueType();
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003813 if (N1C->getZExtValue() >= SmallVT.getSizeInBits())
Dale Johannesene8d72302009-02-06 23:05:02 +00003814 return DAG.getUNDEF(VT);
Chris Lattner06afe072006-05-05 22:53:17 +00003815
Evan Chenge5b51ac2010-04-17 06:13:15 +00003816 if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
Owen Andersona34d9362011-04-14 17:30:49 +00003817 uint64_t ShiftAmt = N1C->getZExtValue();
Evan Chenge5b51ac2010-04-17 06:13:15 +00003818 SDValue SmallShift = DAG.getNode(ISD::SRL, N0.getDebugLoc(), SmallVT,
Owen Andersona34d9362011-04-14 17:30:49 +00003819 N0.getOperand(0),
3820 DAG.getConstant(ShiftAmt, getShiftAmountTy(SmallVT)));
Evan Chenge5b51ac2010-04-17 06:13:15 +00003821 AddToWorkList(SmallShift.getNode());
3822 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, SmallShift);
3823 }
Chris Lattner06afe072006-05-05 22:53:17 +00003824 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003825
Chris Lattner3657ffe2006-10-12 20:23:19 +00003826 // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign
3827 // bit, which is unmodified by sra.
Bill Wendling88103372009-01-30 21:37:17 +00003828 if (N1C && N1C->getZExtValue() + 1 == VT.getSizeInBits()) {
Chris Lattner3657ffe2006-10-12 20:23:19 +00003829 if (N0.getOpcode() == ISD::SRA)
Bill Wendling88103372009-01-30 21:37:17 +00003830 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0.getOperand(0), N1);
Chris Lattner3657ffe2006-10-12 20:23:19 +00003831 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003832
Chris Lattner350bec02006-04-02 06:11:11 +00003833 // fold (srl (ctlz x), "5") -> x iff x has one bit set (the low bit).
Scott Michelfdc40a02009-02-17 22:15:04 +00003834 if (N1C && N0.getOpcode() == ISD::CTLZ &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00003835 N1C->getAPIntValue() == Log2_32(VT.getSizeInBits())) {
Dan Gohman948d8ea2008-02-20 16:33:30 +00003836 APInt KnownZero, KnownOne;
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00003837 DAG.ComputeMaskedBits(N0.getOperand(0), KnownZero, KnownOne);
Scott Michelfdc40a02009-02-17 22:15:04 +00003838
Chris Lattner350bec02006-04-02 06:11:11 +00003839 // If any of the input bits are KnownOne, then the input couldn't be all
3840 // zeros, thus the result of the srl will always be zero.
Dan Gohman948d8ea2008-02-20 16:33:30 +00003841 if (KnownOne.getBoolValue()) return DAG.getConstant(0, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00003842
Chris Lattner350bec02006-04-02 06:11:11 +00003843 // If all of the bits input the to ctlz node are known to be zero, then
3844 // the result of the ctlz is "32" and the result of the shift is one.
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00003845 APInt UnknownBits = ~KnownZero;
Chris Lattner350bec02006-04-02 06:11:11 +00003846 if (UnknownBits == 0) return DAG.getConstant(1, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00003847
Chris Lattner350bec02006-04-02 06:11:11 +00003848 // Otherwise, check to see if there is exactly one bit input to the ctlz.
Bill Wendling88103372009-01-30 21:37:17 +00003849 if ((UnknownBits & (UnknownBits - 1)) == 0) {
Chris Lattner350bec02006-04-02 06:11:11 +00003850 // Okay, we know that only that the single bit specified by UnknownBits
Bill Wendling88103372009-01-30 21:37:17 +00003851 // could be set on input to the CTLZ node. If this bit is set, the SRL
3852 // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
3853 // to an SRL/XOR pair, which is likely to simplify more.
Dan Gohman948d8ea2008-02-20 16:33:30 +00003854 unsigned ShAmt = UnknownBits.countTrailingZeros();
Dan Gohman475871a2008-07-27 21:46:04 +00003855 SDValue Op = N0.getOperand(0);
Bill Wendling88103372009-01-30 21:37:17 +00003856
Chris Lattner350bec02006-04-02 06:11:11 +00003857 if (ShAmt) {
Bill Wendling88103372009-01-30 21:37:17 +00003858 Op = DAG.getNode(ISD::SRL, N0.getDebugLoc(), VT, Op,
Owen Anderson95771af2011-02-25 21:41:48 +00003859 DAG.getConstant(ShAmt, getShiftAmountTy(Op.getValueType())));
Gabor Greifba36cb52008-08-28 21:40:38 +00003860 AddToWorkList(Op.getNode());
Chris Lattner350bec02006-04-02 06:11:11 +00003861 }
Bill Wendling88103372009-01-30 21:37:17 +00003862
3863 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT,
3864 Op, DAG.getConstant(1, VT));
Chris Lattner350bec02006-04-02 06:11:11 +00003865 }
3866 }
Evan Chengeb9f8922008-08-30 02:03:58 +00003867
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003868 // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))).
Evan Chengeb9f8922008-08-30 02:03:58 +00003869 if (N1.getOpcode() == ISD::TRUNCATE &&
Evan Cheng242ebd12008-09-22 18:19:24 +00003870 N1.getOperand(0).getOpcode() == ISD::AND &&
3871 N1.hasOneUse() && N1.getOperand(0).hasOneUse()) {
Evan Chengeb9f8922008-08-30 02:03:58 +00003872 SDValue N101 = N1.getOperand(0).getOperand(1);
Evan Cheng242ebd12008-09-22 18:19:24 +00003873 if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) {
Owen Andersone50ed302009-08-10 22:56:29 +00003874 EVT TruncVT = N1.getValueType();
Evan Cheng242ebd12008-09-22 18:19:24 +00003875 SDValue N100 = N1.getOperand(0).getOperand(0);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003876 APInt TruncC = N101C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00003877 TruncC = TruncC.trunc(TruncVT.getSizeInBits());
Bill Wendling88103372009-01-30 21:37:17 +00003878 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003879 DAG.getNode(ISD::AND, N->getDebugLoc(),
Bill Wendling88103372009-01-30 21:37:17 +00003880 TruncVT,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003881 DAG.getNode(ISD::TRUNCATE,
3882 N->getDebugLoc(),
3883 TruncVT, N100),
Dan Gohmance9bc122009-01-27 20:39:34 +00003884 DAG.getConstant(TruncC, TruncVT)));
Evan Chengeb9f8922008-08-30 02:03:58 +00003885 }
3886 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003887
Chris Lattner61a4c072007-04-18 03:06:49 +00003888 // fold operands of srl based on knowledge that the low bits are not
3889 // demanded.
Dan Gohman475871a2008-07-27 21:46:04 +00003890 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
3891 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003892
Evan Cheng9ab2b982009-12-18 21:31:31 +00003893 if (N1C) {
3894 SDValue NewSRL = visitShiftByConstant(N, N1C->getZExtValue());
3895 if (NewSRL.getNode())
3896 return NewSRL;
3897 }
3898
Dan Gohman4e39e9d2010-06-24 14:30:44 +00003899 // Attempt to convert a srl of a load into a narrower zero-extending load.
3900 SDValue NarrowLoad = ReduceLoadWidth(N);
3901 if (NarrowLoad.getNode())
3902 return NarrowLoad;
3903
Evan Cheng9ab2b982009-12-18 21:31:31 +00003904 // Here is a common situation. We want to optimize:
3905 //
3906 // %a = ...
3907 // %b = and i32 %a, 2
3908 // %c = srl i32 %b, 1
3909 // brcond i32 %c ...
3910 //
3911 // into
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003912 //
Evan Cheng9ab2b982009-12-18 21:31:31 +00003913 // %a = ...
3914 // %b = and %a, 2
3915 // %c = setcc eq %b, 0
3916 // brcond %c ...
3917 //
3918 // However when after the source operand of SRL is optimized into AND, the SRL
3919 // itself may not be optimized further. Look for it and add the BRCOND into
3920 // the worklist.
Evan Chengd40d03e2010-01-06 19:38:29 +00003921 if (N->hasOneUse()) {
3922 SDNode *Use = *N->use_begin();
3923 if (Use->getOpcode() == ISD::BRCOND)
3924 AddToWorkList(Use);
3925 else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) {
3926 // Also look pass the truncate.
3927 Use = *Use->use_begin();
3928 if (Use->getOpcode() == ISD::BRCOND)
3929 AddToWorkList(Use);
3930 }
3931 }
Evan Cheng9ab2b982009-12-18 21:31:31 +00003932
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003933 return SDValue();
Evan Cheng4c26e932010-04-19 19:29:22 +00003934}
3935
Dan Gohman475871a2008-07-27 21:46:04 +00003936SDValue DAGCombiner::visitCTLZ(SDNode *N) {
3937 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00003938 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003939
3940 // fold (ctlz c1) -> c2
Chris Lattner310b5782006-05-06 23:06:26 +00003941 if (isa<ConstantSDNode>(N0))
Bill Wendling34584e62009-01-30 22:02:18 +00003942 return DAG.getNode(ISD::CTLZ, N->getDebugLoc(), VT, N0);
Dan Gohman475871a2008-07-27 21:46:04 +00003943 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003944}
3945
Chandler Carruth63974b22011-12-13 01:56:10 +00003946SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
3947 SDValue N0 = N->getOperand(0);
3948 EVT VT = N->getValueType(0);
3949
3950 // fold (ctlz_zero_undef c1) -> c2
3951 if (isa<ConstantSDNode>(N0))
3952 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, N->getDebugLoc(), VT, N0);
3953 return SDValue();
3954}
3955
Dan Gohman475871a2008-07-27 21:46:04 +00003956SDValue DAGCombiner::visitCTTZ(SDNode *N) {
3957 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00003958 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003959
Nate Begeman1d4d4142005-09-01 00:19:25 +00003960 // fold (cttz c1) -> c2
Chris Lattner310b5782006-05-06 23:06:26 +00003961 if (isa<ConstantSDNode>(N0))
Bill Wendling34584e62009-01-30 22:02:18 +00003962 return DAG.getNode(ISD::CTTZ, N->getDebugLoc(), VT, N0);
Dan Gohman475871a2008-07-27 21:46:04 +00003963 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003964}
3965
Chandler Carruth63974b22011-12-13 01:56:10 +00003966SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
3967 SDValue N0 = N->getOperand(0);
3968 EVT VT = N->getValueType(0);
3969
3970 // fold (cttz_zero_undef c1) -> c2
3971 if (isa<ConstantSDNode>(N0))
3972 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, N->getDebugLoc(), VT, N0);
3973 return SDValue();
3974}
3975
Dan Gohman475871a2008-07-27 21:46:04 +00003976SDValue DAGCombiner::visitCTPOP(SDNode *N) {
3977 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00003978 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003979
Nate Begeman1d4d4142005-09-01 00:19:25 +00003980 // fold (ctpop c1) -> c2
Chris Lattner310b5782006-05-06 23:06:26 +00003981 if (isa<ConstantSDNode>(N0))
Bill Wendling34584e62009-01-30 22:02:18 +00003982 return DAG.getNode(ISD::CTPOP, N->getDebugLoc(), VT, N0);
Dan Gohman475871a2008-07-27 21:46:04 +00003983 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003984}
3985
Dan Gohman475871a2008-07-27 21:46:04 +00003986SDValue DAGCombiner::visitSELECT(SDNode *N) {
3987 SDValue N0 = N->getOperand(0);
3988 SDValue N1 = N->getOperand(1);
3989 SDValue N2 = N->getOperand(2);
Nate Begeman452d7beb2005-09-16 00:54:12 +00003990 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3991 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3992 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
Owen Andersone50ed302009-08-10 22:56:29 +00003993 EVT VT = N->getValueType(0);
3994 EVT VT0 = N0.getValueType();
Nate Begeman44728a72005-09-19 22:34:01 +00003995
Bill Wendling34584e62009-01-30 22:02:18 +00003996 // fold (select C, X, X) -> X
Nate Begeman452d7beb2005-09-16 00:54:12 +00003997 if (N1 == N2)
3998 return N1;
Bill Wendling34584e62009-01-30 22:02:18 +00003999 // fold (select true, X, Y) -> X
Nate Begeman452d7beb2005-09-16 00:54:12 +00004000 if (N0C && !N0C->isNullValue())
4001 return N1;
Bill Wendling34584e62009-01-30 22:02:18 +00004002 // fold (select false, X, Y) -> Y
Nate Begeman452d7beb2005-09-16 00:54:12 +00004003 if (N0C && N0C->isNullValue())
4004 return N2;
Bill Wendling34584e62009-01-30 22:02:18 +00004005 // fold (select C, 1, X) -> (or C, X)
Owen Anderson825b72b2009-08-11 20:47:22 +00004006 if (VT == MVT::i1 && N1C && N1C->getAPIntValue() == 1)
Bill Wendling34584e62009-01-30 22:02:18 +00004007 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N2);
4008 // fold (select C, 0, 1) -> (xor C, 1)
Bob Wilson67ba2232009-01-22 22:05:48 +00004009 if (VT.isInteger() &&
Owen Anderson825b72b2009-08-11 20:47:22 +00004010 (VT0 == MVT::i1 ||
Bob Wilson67ba2232009-01-22 22:05:48 +00004011 (VT0.isInteger() &&
Duncan Sands28b77e92011-09-06 19:07:46 +00004012 TLI.getBooleanContents(false) == TargetLowering::ZeroOrOneBooleanContent)) &&
Dan Gohman002e5d02008-03-13 22:13:53 +00004013 N1C && N2C && N1C->isNullValue() && N2C->getAPIntValue() == 1) {
Bill Wendling34584e62009-01-30 22:02:18 +00004014 SDValue XORNode;
Evan Cheng571c4782007-08-18 05:57:05 +00004015 if (VT == VT0)
Bill Wendling34584e62009-01-30 22:02:18 +00004016 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT0,
4017 N0, DAG.getConstant(1, VT0));
4018 XORNode = DAG.getNode(ISD::XOR, N0.getDebugLoc(), VT0,
4019 N0, DAG.getConstant(1, VT0));
Gabor Greifba36cb52008-08-28 21:40:38 +00004020 AddToWorkList(XORNode.getNode());
Duncan Sands8e4eb092008-06-08 20:54:56 +00004021 if (VT.bitsGT(VT0))
Bill Wendling34584e62009-01-30 22:02:18 +00004022 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, XORNode);
4023 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, XORNode);
Evan Cheng571c4782007-08-18 05:57:05 +00004024 }
Bill Wendling34584e62009-01-30 22:02:18 +00004025 // fold (select C, 0, X) -> (and (not C), X)
Owen Anderson825b72b2009-08-11 20:47:22 +00004026 if (VT == VT0 && VT == MVT::i1 && N1C && N1C->isNullValue()) {
Bill Wendling7581bfa2009-01-30 23:03:19 +00004027 SDValue NOTNode = DAG.getNOT(N0.getDebugLoc(), N0, VT);
Bob Wilson4c245462009-01-22 17:39:32 +00004028 AddToWorkList(NOTNode.getNode());
Bill Wendling7581bfa2009-01-30 23:03:19 +00004029 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, NOTNode, N2);
Nate Begeman452d7beb2005-09-16 00:54:12 +00004030 }
Bill Wendling34584e62009-01-30 22:02:18 +00004031 // fold (select C, X, 1) -> (or (not C), X)
Owen Anderson825b72b2009-08-11 20:47:22 +00004032 if (VT == VT0 && VT == MVT::i1 && N2C && N2C->getAPIntValue() == 1) {
Bill Wendling34584e62009-01-30 22:02:18 +00004033 SDValue NOTNode = DAG.getNOT(N0.getDebugLoc(), N0, VT);
Bob Wilson4c245462009-01-22 17:39:32 +00004034 AddToWorkList(NOTNode.getNode());
Bill Wendling7581bfa2009-01-30 23:03:19 +00004035 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, NOTNode, N1);
Nate Begeman452d7beb2005-09-16 00:54:12 +00004036 }
Bill Wendling34584e62009-01-30 22:02:18 +00004037 // fold (select C, X, 0) -> (and C, X)
Owen Anderson825b72b2009-08-11 20:47:22 +00004038 if (VT == MVT::i1 && N2C && N2C->isNullValue())
Bill Wendling34584e62009-01-30 22:02:18 +00004039 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0, N1);
4040 // fold (select X, X, Y) -> (or X, Y)
4041 // fold (select X, 1, Y) -> (or X, Y)
Owen Anderson825b72b2009-08-11 20:47:22 +00004042 if (VT == MVT::i1 && (N0 == N1 || (N1C && N1C->getAPIntValue() == 1)))
Bill Wendling34584e62009-01-30 22:02:18 +00004043 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N2);
4044 // fold (select X, Y, X) -> (and X, Y)
4045 // fold (select X, Y, 0) -> (and X, Y)
Owen Anderson825b72b2009-08-11 20:47:22 +00004046 if (VT == MVT::i1 && (N0 == N2 || (N2C && N2C->getAPIntValue() == 0)))
Bill Wendling34584e62009-01-30 22:02:18 +00004047 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00004048
Chris Lattner40c62d52005-10-18 06:04:22 +00004049 // If we can fold this based on the true/false value, do so.
4050 if (SimplifySelectOps(N, N1, N2))
Dan Gohman475871a2008-07-27 21:46:04 +00004051 return SDValue(N, 0); // Don't revisit N.
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004052
Nate Begeman44728a72005-09-19 22:34:01 +00004053 // fold selects based on a setcc into other things, such as min/max/abs
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00004054 if (N0.getOpcode() == ISD::SETCC) {
Nate Begeman750ac1b2006-02-01 07:19:44 +00004055 // FIXME:
Owen Anderson825b72b2009-08-11 20:47:22 +00004056 // Check against MVT::Other for SELECT_CC, which is a workaround for targets
Nate Begeman750ac1b2006-02-01 07:19:44 +00004057 // having to say they don't support SELECT_CC on every type the DAG knows
4058 // about, since there is no way to mark an opcode illegal at all value types
Owen Anderson825b72b2009-08-11 20:47:22 +00004059 if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, MVT::Other) &&
Dan Gohman4ea48042009-08-02 16:19:38 +00004060 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))
Bill Wendling34584e62009-01-30 22:02:18 +00004061 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), VT,
4062 N0.getOperand(0), N0.getOperand(1),
Nate Begeman750ac1b2006-02-01 07:19:44 +00004063 N1, N2, N0.getOperand(2));
Chris Lattner600fec32009-03-11 05:08:08 +00004064 return SimplifySelect(N->getDebugLoc(), N0, N1, N2);
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00004065 }
Bill Wendling34584e62009-01-30 22:02:18 +00004066
Dan Gohman475871a2008-07-27 21:46:04 +00004067 return SDValue();
Nate Begeman452d7beb2005-09-16 00:54:12 +00004068}
4069
Dan Gohman475871a2008-07-27 21:46:04 +00004070SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
4071 SDValue N0 = N->getOperand(0);
4072 SDValue N1 = N->getOperand(1);
4073 SDValue N2 = N->getOperand(2);
4074 SDValue N3 = N->getOperand(3);
4075 SDValue N4 = N->getOperand(4);
Nate Begeman44728a72005-09-19 22:34:01 +00004076 ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get();
Scott Michelfdc40a02009-02-17 22:15:04 +00004077
Nate Begeman44728a72005-09-19 22:34:01 +00004078 // fold select_cc lhs, rhs, x, x, cc -> x
4079 if (N2 == N3)
4080 return N2;
Scott Michelfdc40a02009-02-17 22:15:04 +00004081
Chris Lattner5f42a242006-09-20 06:19:26 +00004082 // Determine if the condition we're dealing with is constant
Duncan Sands5480c042009-01-01 15:52:00 +00004083 SDValue SCC = SimplifySetCC(TLI.getSetCCResultType(N0.getValueType()),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00004084 N0, N1, CC, N->getDebugLoc(), false);
Gabor Greifba36cb52008-08-28 21:40:38 +00004085 if (SCC.getNode()) AddToWorkList(SCC.getNode());
Chris Lattner5f42a242006-09-20 06:19:26 +00004086
Gabor Greifba36cb52008-08-28 21:40:38 +00004087 if (ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.getNode())) {
Dan Gohman002e5d02008-03-13 22:13:53 +00004088 if (!SCCC->isNullValue())
Chris Lattner5f42a242006-09-20 06:19:26 +00004089 return N2; // cond always true -> true val
4090 else
4091 return N3; // cond always false -> false val
4092 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004093
Chris Lattner5f42a242006-09-20 06:19:26 +00004094 // Fold to a simpler select_cc
Gabor Greifba36cb52008-08-28 21:40:38 +00004095 if (SCC.getNode() && SCC.getOpcode() == ISD::SETCC)
Scott Michelfdc40a02009-02-17 22:15:04 +00004096 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), N2.getValueType(),
4097 SCC.getOperand(0), SCC.getOperand(1), N2, N3,
Chris Lattner5f42a242006-09-20 06:19:26 +00004098 SCC.getOperand(2));
Scott Michelfdc40a02009-02-17 22:15:04 +00004099
Chris Lattner40c62d52005-10-18 06:04:22 +00004100 // If we can fold this based on the true/false value, do so.
4101 if (SimplifySelectOps(N, N2, N3))
Dan Gohman475871a2008-07-27 21:46:04 +00004102 return SDValue(N, 0); // Don't revisit N.
Scott Michelfdc40a02009-02-17 22:15:04 +00004103
Nate Begeman44728a72005-09-19 22:34:01 +00004104 // fold select_cc into other things, such as min/max/abs
Bill Wendling836ca7d2009-01-30 23:59:18 +00004105 return SimplifySelectCC(N->getDebugLoc(), N0, N1, N2, N3, CC);
Nate Begeman452d7beb2005-09-16 00:54:12 +00004106}
4107
Dan Gohman475871a2008-07-27 21:46:04 +00004108SDValue DAGCombiner::visitSETCC(SDNode *N) {
Nate Begeman452d7beb2005-09-16 00:54:12 +00004109 return SimplifySetCC(N->getValueType(0), N->getOperand(0), N->getOperand(1),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00004110 cast<CondCodeSDNode>(N->getOperand(2))->get(),
4111 N->getDebugLoc());
Nate Begeman452d7beb2005-09-16 00:54:12 +00004112}
4113
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004114// ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
Dan Gohman57fc82d2009-04-09 03:51:29 +00004115// "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004116// transformation. Returns true if extension are possible and the above
Scott Michelfdc40a02009-02-17 22:15:04 +00004117// mentioned transformation is profitable.
Dan Gohman475871a2008-07-27 21:46:04 +00004118static bool ExtendUsesToFormExtLoad(SDNode *N, SDValue N0,
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004119 unsigned ExtOpc,
4120 SmallVector<SDNode*, 4> &ExtendNodes,
Dan Gohman79ce2762009-01-15 19:20:50 +00004121 const TargetLowering &TLI) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004122 bool HasCopyToRegUses = false;
4123 bool isTruncFree = TLI.isTruncateFree(N->getValueType(0), N0.getValueType());
Gabor Greif12632d22008-08-30 19:29:20 +00004124 for (SDNode::use_iterator UI = N0.getNode()->use_begin(),
4125 UE = N0.getNode()->use_end();
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004126 UI != UE; ++UI) {
Dan Gohman89684502008-07-27 20:43:25 +00004127 SDNode *User = *UI;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004128 if (User == N)
4129 continue;
Dan Gohman57fc82d2009-04-09 03:51:29 +00004130 if (UI.getUse().getResNo() != N0.getResNo())
4131 continue;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004132 // FIXME: Only extend SETCC N, N and SETCC N, c for now.
Dan Gohman57fc82d2009-04-09 03:51:29 +00004133 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004134 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get();
4135 if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC))
4136 // Sign bits will be lost after a zext.
4137 return false;
4138 bool Add = false;
4139 for (unsigned i = 0; i != 2; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00004140 SDValue UseOp = User->getOperand(i);
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004141 if (UseOp == N0)
4142 continue;
4143 if (!isa<ConstantSDNode>(UseOp))
4144 return false;
4145 Add = true;
4146 }
4147 if (Add)
4148 ExtendNodes.push_back(User);
Dan Gohman57fc82d2009-04-09 03:51:29 +00004149 continue;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004150 }
Dan Gohman57fc82d2009-04-09 03:51:29 +00004151 // If truncates aren't free and there are users we can't
4152 // extend, it isn't worthwhile.
4153 if (!isTruncFree)
4154 return false;
4155 // Remember if this value is live-out.
4156 if (User->getOpcode() == ISD::CopyToReg)
4157 HasCopyToRegUses = true;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004158 }
4159
4160 if (HasCopyToRegUses) {
4161 bool BothLiveOut = false;
4162 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
4163 UI != UE; ++UI) {
Dan Gohman57fc82d2009-04-09 03:51:29 +00004164 SDUse &Use = UI.getUse();
4165 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) {
4166 BothLiveOut = true;
4167 break;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004168 }
4169 }
4170 if (BothLiveOut)
4171 // Both unextended and extended values are live out. There had better be
Bob Wilsonbebfbc52010-11-28 06:51:19 +00004172 // a good reason for the transformation.
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004173 return ExtendNodes.size();
4174 }
4175 return true;
4176}
4177
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004178void DAGCombiner::ExtendSetCCUses(SmallVector<SDNode*, 4> SetCCs,
4179 SDValue Trunc, SDValue ExtLoad, DebugLoc DL,
4180 ISD::NodeType ExtType) {
4181 // Extend SetCC uses if necessary.
4182 for (unsigned i = 0, e = SetCCs.size(); i != e; ++i) {
4183 SDNode *SetCC = SetCCs[i];
4184 SmallVector<SDValue, 4> Ops;
4185
4186 for (unsigned j = 0; j != 2; ++j) {
4187 SDValue SOp = SetCC->getOperand(j);
4188 if (SOp == Trunc)
4189 Ops.push_back(ExtLoad);
4190 else
4191 Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp));
4192 }
4193
4194 Ops.push_back(SetCC->getOperand(2));
4195 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0),
4196 &Ops[0], Ops.size()));
4197 }
4198}
4199
Dan Gohman475871a2008-07-27 21:46:04 +00004200SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
4201 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004202 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00004203
Nate Begeman1d4d4142005-09-01 00:19:25 +00004204 // fold (sext c1) -> c1
Reid Spencer3ed469c2006-11-02 20:25:50 +00004205 if (isa<ConstantSDNode>(N0))
Bill Wendling6ce610f2009-01-30 22:23:15 +00004206 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00004207
Nate Begeman1d4d4142005-09-01 00:19:25 +00004208 // fold (sext (sext x)) -> (sext x)
Chris Lattner310b5782006-05-06 23:06:26 +00004209 // fold (sext (aext x)) -> (sext x)
4210 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004211 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT,
4212 N0.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00004213
Chris Lattner22558872007-02-26 03:13:59 +00004214 if (N0.getOpcode() == ISD::TRUNCATE) {
Dan Gohman1fdfa6a2008-05-20 20:56:33 +00004215 // fold (sext (truncate (load x))) -> (sext (smaller load x))
4216 // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
Gabor Greifba36cb52008-08-28 21:40:38 +00004217 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
4218 if (NarrowLoad.getNode()) {
Dale Johannesen61734eb2010-05-25 17:50:03 +00004219 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4220 if (NarrowLoad.getNode() != N0.getNode()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004221 CombineTo(N0.getNode(), NarrowLoad);
Dale Johannesen61734eb2010-05-25 17:50:03 +00004222 // CombineTo deleted the truncate, if needed, but not what's under it.
4223 AddToWorkList(oye);
4224 }
Dan Gohmanc7b34442009-04-27 02:00:55 +00004225 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng0b063de2007-03-23 02:16:52 +00004226 }
Evan Chengc88138f2007-03-22 01:54:19 +00004227
Dan Gohman1fdfa6a2008-05-20 20:56:33 +00004228 // See if the value being truncated is already sign extended. If so, just
4229 // eliminate the trunc/sext pair.
Dan Gohman475871a2008-07-27 21:46:04 +00004230 SDValue Op = N0.getOperand(0);
Dan Gohmand1996362010-01-09 02:13:55 +00004231 unsigned OpBits = Op.getValueType().getScalarType().getSizeInBits();
4232 unsigned MidBits = N0.getValueType().getScalarType().getSizeInBits();
4233 unsigned DestBits = VT.getScalarType().getSizeInBits();
Dan Gohmanea859be2007-06-22 14:59:07 +00004234 unsigned NumSignBits = DAG.ComputeNumSignBits(Op);
Scott Michelfdc40a02009-02-17 22:15:04 +00004235
Chris Lattner22558872007-02-26 03:13:59 +00004236 if (OpBits == DestBits) {
4237 // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign
4238 // bits, it is already ready.
4239 if (NumSignBits > DestBits-MidBits)
4240 return Op;
4241 } else if (OpBits < DestBits) {
4242 // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign
4243 // bits, just sext from i32.
4244 if (NumSignBits > OpBits-MidBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004245 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, Op);
Chris Lattner22558872007-02-26 03:13:59 +00004246 } else {
4247 // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign
4248 // bits, just truncate to i32.
4249 if (NumSignBits > OpBits-MidBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004250 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, Op);
Chris Lattner6007b842006-09-21 06:00:20 +00004251 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004252
Chris Lattner22558872007-02-26 03:13:59 +00004253 // fold (sext (truncate x)) -> (sextinreg x).
Duncan Sands25cf2272008-11-24 14:53:14 +00004254 if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG,
4255 N0.getValueType())) {
Dan Gohmand1996362010-01-09 02:13:55 +00004256 if (OpBits < DestBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004257 Op = DAG.getNode(ISD::ANY_EXTEND, N0.getDebugLoc(), VT, Op);
Dan Gohmand1996362010-01-09 02:13:55 +00004258 else if (OpBits > DestBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004259 Op = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(), VT, Op);
4260 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT, Op,
Dan Gohmand1996362010-01-09 02:13:55 +00004261 DAG.getValueType(N0.getValueType()));
Chris Lattner22558872007-02-26 03:13:59 +00004262 }
Chris Lattner6007b842006-09-21 06:00:20 +00004263 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004264
Evan Cheng110dec22005-12-14 02:19:23 +00004265 // fold (sext (load x)) -> (sext (truncate (sextload x)))
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004266 // None of the supported targets knows how to perform load and sign extend
Nadav Rotemfcd96192011-02-27 07:40:43 +00004267 // on vectors in one instruction. We only perform this transformation on
4268 // scalars.
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004269 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004270 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00004271 TLI.isLoadExtLegal(ISD::SEXTLOAD, N0.getValueType()))) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004272 bool DoXform = true;
4273 SmallVector<SDNode*, 4> SetCCs;
4274 if (!N0.hasOneUse())
4275 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::SIGN_EXTEND, SetCCs, TLI);
4276 if (DoXform) {
4277 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00004278 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Dan Gohman57fc82d2009-04-09 03:51:29 +00004279 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004280 LN0->getBasePtr(), LN0->getPointerInfo(),
Duncan Sands25cf2272008-11-24 14:53:14 +00004281 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00004282 LN0->isVolatile(), LN0->isNonTemporal(),
4283 LN0->getAlignment());
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004284 CombineTo(N, ExtLoad);
Bill Wendling6ce610f2009-01-30 22:23:15 +00004285 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4286 N0.getValueType(), ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00004287 CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1));
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004288 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4289 ISD::SIGN_EXTEND);
Dan Gohman475871a2008-07-27 21:46:04 +00004290 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004291 }
Nate Begeman3df4d522005-10-12 20:40:40 +00004292 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004293
4294 // fold (sext (sextload x)) -> (sext (truncate (sextload x)))
4295 // fold (sext ( extload x)) -> (sext (truncate (sextload x)))
Gabor Greifba36cb52008-08-28 21:40:38 +00004296 if ((ISD::isSEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) &&
4297 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
Evan Cheng466685d2006-10-09 20:57:25 +00004298 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00004299 EVT MemVT = LN0->getMemoryVT();
Duncan Sands25cf2272008-11-24 14:53:14 +00004300 if ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00004301 TLI.isLoadExtLegal(ISD::SEXTLOAD, MemVT)) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00004302 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling6ce610f2009-01-30 22:23:15 +00004303 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004304 LN0->getBasePtr(), LN0->getPointerInfo(),
4305 MemVT,
David Greene1e559442010-02-15 17:00:31 +00004306 LN0->isVolatile(), LN0->isNonTemporal(),
4307 LN0->getAlignment());
Jim Laskeyf6c4ccf2006-12-15 21:38:30 +00004308 CombineTo(N, ExtLoad);
Gabor Greif12632d22008-08-30 19:29:20 +00004309 CombineTo(N0.getNode(),
Bill Wendling6ce610f2009-01-30 22:23:15 +00004310 DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4311 N0.getValueType(), ExtLoad),
Jim Laskeyf6c4ccf2006-12-15 21:38:30 +00004312 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00004313 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Jim Laskeyf6c4ccf2006-12-15 21:38:30 +00004314 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004315 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004316
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004317 // fold (sext (and/or/xor (load x), cst)) ->
4318 // (and/or/xor (sextload x), (sext cst))
4319 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
4320 N0.getOpcode() == ISD::XOR) &&
4321 isa<LoadSDNode>(N0.getOperand(0)) &&
4322 N0.getOperand(1).getOpcode() == ISD::Constant &&
4323 TLI.isLoadExtLegal(ISD::SEXTLOAD, N0.getValueType()) &&
4324 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
4325 LoadSDNode *LN0 = cast<LoadSDNode>(N0.getOperand(0));
4326 if (LN0->getExtensionType() != ISD::ZEXTLOAD) {
4327 bool DoXform = true;
4328 SmallVector<SDNode*, 4> SetCCs;
4329 if (!N0.hasOneUse())
4330 DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::SIGN_EXTEND,
4331 SetCCs, TLI);
4332 if (DoXform) {
4333 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, LN0->getDebugLoc(), VT,
4334 LN0->getChain(), LN0->getBasePtr(),
4335 LN0->getPointerInfo(),
4336 LN0->getMemoryVT(),
4337 LN0->isVolatile(),
4338 LN0->isNonTemporal(),
4339 LN0->getAlignment());
4340 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
4341 Mask = Mask.sext(VT.getSizeInBits());
4342 SDValue And = DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
4343 ExtLoad, DAG.getConstant(Mask, VT));
4344 SDValue Trunc = DAG.getNode(ISD::TRUNCATE,
4345 N0.getOperand(0).getDebugLoc(),
4346 N0.getOperand(0).getValueType(), ExtLoad);
4347 CombineTo(N, And);
4348 CombineTo(N0.getOperand(0).getNode(), Trunc, ExtLoad.getValue(1));
4349 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4350 ISD::SIGN_EXTEND);
4351 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4352 }
4353 }
4354 }
4355
Chris Lattner20a35c32007-04-11 05:32:27 +00004356 if (N0.getOpcode() == ISD::SETCC) {
Chris Lattner2b7a2712009-07-08 00:31:33 +00004357 // sext(setcc) -> sext_in_reg(vsetcc) for vectors.
Dan Gohman3ce89f42010-04-30 17:19:19 +00004358 // Only do this before legalize for now.
4359 if (VT.isVector() && !LegalOperations) {
4360 EVT N0VT = N0.getOperand(0).getValueType();
Nadav Rotem2e506192012-04-11 08:26:11 +00004361 // On some architectures (such as SSE/NEON/etc) the SETCC result type is
4362 // of the same size as the compared operands. Only optimize sext(setcc())
4363 // if this is the case.
4364 EVT SVT = TLI.getSetCCResultType(N0VT);
4365
4366 // We know that the # elements of the results is the same as the
4367 // # elements of the compare (and the # elements of the compare result
4368 // for that matter). Check to see that they are the same size. If so,
4369 // we know that the element size of the sext'd result matches the
4370 // element size of the compare operands.
4371 if (VT.getSizeInBits() == SVT.getSizeInBits())
Duncan Sands28b77e92011-09-06 19:07:46 +00004372 return DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
Duncan Sands34727662010-07-12 08:16:59 +00004373 N0.getOperand(1),
4374 cast<CondCodeSDNode>(N0.getOperand(2))->get());
Dan Gohman3ce89f42010-04-30 17:19:19 +00004375 // If the desired elements are smaller or larger than the source
4376 // elements we can use a matching integer vector type and then
4377 // truncate/sign extend
4378 else {
Duncan Sands34727662010-07-12 08:16:59 +00004379 EVT MatchingElementType =
4380 EVT::getIntegerVT(*DAG.getContext(),
4381 N0VT.getScalarType().getSizeInBits());
4382 EVT MatchingVectorType =
4383 EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
4384 N0VT.getVectorNumElements());
Nadav Rotem2e506192012-04-11 08:26:11 +00004385
4386 if (SVT == MatchingVectorType) {
4387 SDValue VsetCC = DAG.getSetCC(N->getDebugLoc(), MatchingVectorType,
4388 N0.getOperand(0), N0.getOperand(1),
4389 cast<CondCodeSDNode>(N0.getOperand(2))->get());
4390 return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT);
4391 }
Dan Gohman3ce89f42010-04-30 17:19:19 +00004392 }
Chris Lattner2b7a2712009-07-08 00:31:33 +00004393 }
Dan Gohman3ce89f42010-04-30 17:19:19 +00004394
Chris Lattner2b7a2712009-07-08 00:31:33 +00004395 // sext(setcc x, y, cc) -> (select_cc x, y, -1, 0, cc)
Dan Gohmana7bcef12010-04-24 01:17:30 +00004396 unsigned ElementWidth = VT.getScalarType().getSizeInBits();
Dan Gohman5cbd37e2009-08-06 09:18:59 +00004397 SDValue NegOne =
Dan Gohmana7bcef12010-04-24 01:17:30 +00004398 DAG.getConstant(APInt::getAllOnesValue(ElementWidth), VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00004399 SDValue SCC =
Bill Wendling836ca7d2009-01-30 23:59:18 +00004400 SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1),
Dan Gohman5cbd37e2009-08-06 09:18:59 +00004401 NegOne, DAG.getConstant(0, VT),
Chris Lattner1eba01e2007-04-11 06:50:51 +00004402 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
Gabor Greifba36cb52008-08-28 21:40:38 +00004403 if (SCC.getNode()) return SCC;
Evan Cheng8c7ecaf2010-01-26 02:00:44 +00004404 if (!LegalOperations ||
4405 TLI.isOperationLegal(ISD::SETCC, TLI.getSetCCResultType(VT)))
4406 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), VT,
4407 DAG.getSetCC(N->getDebugLoc(),
4408 TLI.getSetCCResultType(VT),
4409 N0.getOperand(0), N0.getOperand(1),
4410 cast<CondCodeSDNode>(N0.getOperand(2))->get()),
4411 NegOne, DAG.getConstant(0, VT));
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004412 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004413
Dan Gohman8f0ad582008-04-28 16:58:24 +00004414 // fold (sext x) -> (zext x) if the sign bit is known zero.
Duncan Sands25cf2272008-11-24 14:53:14 +00004415 if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) &&
Dan Gohman187db7b2008-04-28 18:47:17 +00004416 DAG.SignBitIsZero(N0))
Bill Wendling6ce610f2009-01-30 22:23:15 +00004417 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00004418
Evan Chengb3a3d5e2010-04-28 07:10:39 +00004419 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00004420}
4421
Rafael Espindoladecbc432012-04-09 16:06:03 +00004422// isTruncateOf - If N is a truncate of some other value, return true, record
4423// the value being truncated in Op and which of Op's bits are zero in KnownZero.
4424// This function computes KnownZero to avoid a duplicated call to
4425// ComputeMaskedBits in the caller.
4426static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
4427 APInt &KnownZero) {
4428 APInt KnownOne;
4429 if (N->getOpcode() == ISD::TRUNCATE) {
4430 Op = N->getOperand(0);
4431 DAG.ComputeMaskedBits(Op, KnownZero, KnownOne);
4432 return true;
4433 }
4434
4435 if (N->getOpcode() != ISD::SETCC || N->getValueType(0) != MVT::i1 ||
4436 cast<CondCodeSDNode>(N->getOperand(2))->get() != ISD::SETNE)
4437 return false;
4438
4439 SDValue Op0 = N->getOperand(0);
4440 SDValue Op1 = N->getOperand(1);
4441 assert(Op0.getValueType() == Op1.getValueType());
4442
4443 ConstantSDNode *COp0 = dyn_cast<ConstantSDNode>(Op0);
4444 ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1);
Rafael Espindolafdb230a2012-04-10 00:16:22 +00004445 if (COp0 && COp0->isNullValue())
Rafael Espindoladecbc432012-04-09 16:06:03 +00004446 Op = Op1;
Rafael Espindolafdb230a2012-04-10 00:16:22 +00004447 else if (COp1 && COp1->isNullValue())
Rafael Espindoladecbc432012-04-09 16:06:03 +00004448 Op = Op0;
4449 else
4450 return false;
4451
4452 DAG.ComputeMaskedBits(Op, KnownZero, KnownOne);
4453
4454 if (!(KnownZero | APInt(Op.getValueSizeInBits(), 1)).isAllOnesValue())
4455 return false;
4456
4457 return true;
4458}
4459
Dan Gohman475871a2008-07-27 21:46:04 +00004460SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
4461 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004462 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00004463
Nate Begeman1d4d4142005-09-01 00:19:25 +00004464 // fold (zext c1) -> c1
Reid Spencer3ed469c2006-11-02 20:25:50 +00004465 if (isa<ConstantSDNode>(N0))
Bill Wendling6ce610f2009-01-30 22:23:15 +00004466 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00004467 // fold (zext (zext x)) -> (zext x)
Chris Lattner310b5782006-05-06 23:06:26 +00004468 // fold (zext (aext x)) -> (zext x)
4469 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004470 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT,
4471 N0.getOperand(0));
Chris Lattner6007b842006-09-21 06:00:20 +00004472
Chandler Carruthf103b3d2012-01-11 08:41:08 +00004473 // fold (zext (truncate x)) -> (zext x) or
4474 // (zext (truncate x)) -> (truncate x)
4475 // This is valid when the truncated bits of x are already zero.
4476 // FIXME: We should extend this to work for vectors too.
Rafael Espindoladecbc432012-04-09 16:06:03 +00004477 SDValue Op;
4478 APInt KnownZero;
4479 if (!VT.isVector() && isTruncateOf(DAG, N0, Op, KnownZero)) {
4480 APInt TruncatedBits =
4481 (Op.getValueSizeInBits() == N0.getValueSizeInBits()) ?
4482 APInt(Op.getValueSizeInBits(), 0) :
4483 APInt::getBitsSet(Op.getValueSizeInBits(),
4484 N0.getValueSizeInBits(),
4485 std::min(Op.getValueSizeInBits(),
4486 VT.getSizeInBits()));
Rafael Espindola26c8dcc2012-04-04 12:51:34 +00004487 if (TruncatedBits == (KnownZero & TruncatedBits)) {
Chandler Carruthf103b3d2012-01-11 08:41:08 +00004488 if (VT.bitsGT(Op.getValueType()))
4489 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, Op);
4490 if (VT.bitsLT(Op.getValueType()))
4491 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, Op);
4492
4493 return Op;
4494 }
4495 }
4496
Evan Chengc88138f2007-03-22 01:54:19 +00004497 // fold (zext (truncate (load x))) -> (zext (smaller load x))
4498 // fold (zext (truncate (srl (load x), c))) -> (zext (small load (x+c/n)))
Dale Johannesen2041a0e2007-03-30 21:38:07 +00004499 if (N0.getOpcode() == ISD::TRUNCATE) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004500 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
4501 if (NarrowLoad.getNode()) {
Dale Johannesen61734eb2010-05-25 17:50:03 +00004502 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4503 if (NarrowLoad.getNode() != N0.getNode()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004504 CombineTo(N0.getNode(), NarrowLoad);
Dale Johannesen61734eb2010-05-25 17:50:03 +00004505 // CombineTo deleted the truncate, if needed, but not what's under it.
4506 AddToWorkList(oye);
4507 }
Eli Friedmane545d382011-04-16 23:25:34 +00004508 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng0b063de2007-03-23 02:16:52 +00004509 }
Evan Chengc88138f2007-03-22 01:54:19 +00004510 }
4511
Chris Lattner6007b842006-09-21 06:00:20 +00004512 // fold (zext (truncate x)) -> (and x, mask)
4513 if (N0.getOpcode() == ISD::TRUNCATE &&
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004514 (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT))) {
Dan Gohman394d6292010-11-03 01:47:46 +00004515
4516 // fold (zext (truncate (load x))) -> (zext (smaller load x))
4517 // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
4518 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
4519 if (NarrowLoad.getNode()) {
4520 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4521 if (NarrowLoad.getNode() != N0.getNode()) {
4522 CombineTo(N0.getNode(), NarrowLoad);
4523 // CombineTo deleted the truncate, if needed, but not what's under it.
4524 AddToWorkList(oye);
4525 }
4526 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4527 }
4528
Dan Gohman475871a2008-07-27 21:46:04 +00004529 SDValue Op = N0.getOperand(0);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004530 if (Op.getValueType().bitsLT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004531 Op = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, Op);
Elena Demikhovsky1da58672012-04-22 09:39:03 +00004532 AddToWorkList(Op.getNode());
Duncan Sands8e4eb092008-06-08 20:54:56 +00004533 } else if (Op.getValueType().bitsGT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004534 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, Op);
Elena Demikhovsky1da58672012-04-22 09:39:03 +00004535 AddToWorkList(Op.getNode());
Chris Lattner6007b842006-09-21 06:00:20 +00004536 }
Dan Gohman87862e72009-12-11 21:31:27 +00004537 return DAG.getZeroExtendInReg(Op, N->getDebugLoc(),
4538 N0.getValueType().getScalarType());
Chris Lattner6007b842006-09-21 06:00:20 +00004539 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004540
Dan Gohman97121ba2009-04-08 00:15:30 +00004541 // Fold (zext (and (trunc x), cst)) -> (and x, cst),
4542 // if either of the casts is not free.
Chris Lattner111c2282006-09-21 06:14:31 +00004543 if (N0.getOpcode() == ISD::AND &&
4544 N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
Dan Gohman97121ba2009-04-08 00:15:30 +00004545 N0.getOperand(1).getOpcode() == ISD::Constant &&
4546 (!TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
4547 N0.getValueType()) ||
4548 !TLI.isZExtFree(N0.getValueType(), VT))) {
Dan Gohman475871a2008-07-27 21:46:04 +00004549 SDValue X = N0.getOperand(0).getOperand(0);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004550 if (X.getValueType().bitsLT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004551 X = DAG.getNode(ISD::ANY_EXTEND, X.getDebugLoc(), VT, X);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004552 } else if (X.getValueType().bitsGT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004553 X = DAG.getNode(ISD::TRUNCATE, X.getDebugLoc(), VT, X);
Chris Lattner111c2282006-09-21 06:14:31 +00004554 }
Dan Gohman220a8232008-03-03 23:51:38 +00004555 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00004556 Mask = Mask.zext(VT.getSizeInBits());
Bill Wendling6ce610f2009-01-30 22:23:15 +00004557 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
4558 X, DAG.getConstant(Mask, VT));
Chris Lattner111c2282006-09-21 06:14:31 +00004559 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004560
Evan Cheng110dec22005-12-14 02:19:23 +00004561 // fold (zext (load x)) -> (zext (truncate (zextload x)))
Nadav Rotemed9b9342011-02-20 12:37:50 +00004562 // None of the supported targets knows how to perform load and vector_zext
Nadav Rotemfcd96192011-02-27 07:40:43 +00004563 // on vectors in one instruction. We only perform this transformation on
4564 // scalars.
Nadav Rotemed9b9342011-02-20 12:37:50 +00004565 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004566 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00004567 TLI.isLoadExtLegal(ISD::ZEXTLOAD, N0.getValueType()))) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004568 bool DoXform = true;
4569 SmallVector<SDNode*, 4> SetCCs;
4570 if (!N0.hasOneUse())
4571 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ZERO_EXTEND, SetCCs, TLI);
4572 if (DoXform) {
4573 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00004574 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling6ce610f2009-01-30 22:23:15 +00004575 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004576 LN0->getBasePtr(), LN0->getPointerInfo(),
Duncan Sands25cf2272008-11-24 14:53:14 +00004577 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00004578 LN0->isVolatile(), LN0->isNonTemporal(),
4579 LN0->getAlignment());
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004580 CombineTo(N, ExtLoad);
Bill Wendling6ce610f2009-01-30 22:23:15 +00004581 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4582 N0.getValueType(), ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00004583 CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1));
Bill Wendling6ce610f2009-01-30 22:23:15 +00004584
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004585 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4586 ISD::ZERO_EXTEND);
Dan Gohman475871a2008-07-27 21:46:04 +00004587 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004588 }
Evan Cheng110dec22005-12-14 02:19:23 +00004589 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004590
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004591 // fold (zext (and/or/xor (load x), cst)) ->
4592 // (and/or/xor (zextload x), (zext cst))
4593 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
4594 N0.getOpcode() == ISD::XOR) &&
4595 isa<LoadSDNode>(N0.getOperand(0)) &&
4596 N0.getOperand(1).getOpcode() == ISD::Constant &&
4597 TLI.isLoadExtLegal(ISD::ZEXTLOAD, N0.getValueType()) &&
4598 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
4599 LoadSDNode *LN0 = cast<LoadSDNode>(N0.getOperand(0));
4600 if (LN0->getExtensionType() != ISD::SEXTLOAD) {
4601 bool DoXform = true;
4602 SmallVector<SDNode*, 4> SetCCs;
4603 if (!N0.hasOneUse())
4604 DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::ZERO_EXTEND,
4605 SetCCs, TLI);
4606 if (DoXform) {
4607 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, LN0->getDebugLoc(), VT,
4608 LN0->getChain(), LN0->getBasePtr(),
4609 LN0->getPointerInfo(),
4610 LN0->getMemoryVT(),
4611 LN0->isVolatile(),
4612 LN0->isNonTemporal(),
4613 LN0->getAlignment());
4614 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
4615 Mask = Mask.zext(VT.getSizeInBits());
4616 SDValue And = DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
4617 ExtLoad, DAG.getConstant(Mask, VT));
4618 SDValue Trunc = DAG.getNode(ISD::TRUNCATE,
4619 N0.getOperand(0).getDebugLoc(),
4620 N0.getOperand(0).getValueType(), ExtLoad);
4621 CombineTo(N, And);
4622 CombineTo(N0.getOperand(0).getNode(), Trunc, ExtLoad.getValue(1));
4623 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4624 ISD::ZERO_EXTEND);
4625 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4626 }
4627 }
4628 }
4629
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004630 // fold (zext (zextload x)) -> (zext (truncate (zextload x)))
4631 // fold (zext ( extload x)) -> (zext (truncate (zextload x)))
Gabor Greifba36cb52008-08-28 21:40:38 +00004632 if ((ISD::isZEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) &&
4633 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
Evan Cheng466685d2006-10-09 20:57:25 +00004634 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00004635 EVT MemVT = LN0->getMemoryVT();
Duncan Sands25cf2272008-11-24 14:53:14 +00004636 if ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00004637 TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT)) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00004638 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling6ce610f2009-01-30 22:23:15 +00004639 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004640 LN0->getBasePtr(), LN0->getPointerInfo(),
4641 MemVT,
David Greene1e559442010-02-15 17:00:31 +00004642 LN0->isVolatile(), LN0->isNonTemporal(),
4643 LN0->getAlignment());
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004644 CombineTo(N, ExtLoad);
Gabor Greif12632d22008-08-30 19:29:20 +00004645 CombineTo(N0.getNode(),
Bill Wendling6ce610f2009-01-30 22:23:15 +00004646 DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(), N0.getValueType(),
4647 ExtLoad),
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004648 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00004649 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004650 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004651 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004652
Chris Lattner20a35c32007-04-11 05:32:27 +00004653 if (N0.getOpcode() == ISD::SETCC) {
Evan Cheng0a942db2010-05-19 01:08:17 +00004654 if (!LegalOperations && VT.isVector()) {
4655 // zext(setcc) -> (and (vsetcc), (1, 1, ...) for vectors.
4656 // Only do this before legalize for now.
4657 EVT N0VT = N0.getOperand(0).getValueType();
4658 EVT EltVT = VT.getVectorElementType();
4659 SmallVector<SDValue,8> OneOps(VT.getVectorNumElements(),
4660 DAG.getConstant(1, EltVT));
Dan Gohman71dc7c92011-05-17 22:20:36 +00004661 if (VT.getSizeInBits() == N0VT.getSizeInBits())
Evan Cheng0a942db2010-05-19 01:08:17 +00004662 // We know that the # elements of the results is the same as the
4663 // # elements of the compare (and the # elements of the compare result
4664 // for that matter). Check to see that they are the same size. If so,
4665 // we know that the element size of the sext'd result matches the
4666 // element size of the compare operands.
4667 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
Duncan Sands28b77e92011-09-06 19:07:46 +00004668 DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
Evan Cheng0a942db2010-05-19 01:08:17 +00004669 N0.getOperand(1),
4670 cast<CondCodeSDNode>(N0.getOperand(2))->get()),
4671 DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT,
4672 &OneOps[0], OneOps.size()));
Dan Gohman71dc7c92011-05-17 22:20:36 +00004673
4674 // If the desired elements are smaller or larger than the source
4675 // elements we can use a matching integer vector type and then
4676 // truncate/sign extend
4677 EVT MatchingElementType =
4678 EVT::getIntegerVT(*DAG.getContext(),
4679 N0VT.getScalarType().getSizeInBits());
4680 EVT MatchingVectorType =
4681 EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
4682 N0VT.getVectorNumElements());
4683 SDValue VsetCC =
Duncan Sands28b77e92011-09-06 19:07:46 +00004684 DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
Dan Gohman71dc7c92011-05-17 22:20:36 +00004685 N0.getOperand(1),
4686 cast<CondCodeSDNode>(N0.getOperand(2))->get());
4687 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
4688 DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT),
4689 DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT,
4690 &OneOps[0], OneOps.size()));
Evan Cheng0a942db2010-05-19 01:08:17 +00004691 }
4692
4693 // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
Scott Michelfdc40a02009-02-17 22:15:04 +00004694 SDValue SCC =
Bill Wendling836ca7d2009-01-30 23:59:18 +00004695 SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1),
Chris Lattner20a35c32007-04-11 05:32:27 +00004696 DAG.getConstant(1, VT), DAG.getConstant(0, VT),
Chris Lattner1eba01e2007-04-11 06:50:51 +00004697 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
Gabor Greifba36cb52008-08-28 21:40:38 +00004698 if (SCC.getNode()) return SCC;
Chris Lattner20a35c32007-04-11 05:32:27 +00004699 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004700
Evan Cheng9818c042009-12-15 03:00:32 +00004701 // (zext (shl (zext x), cst)) -> (shl (zext x), cst)
Evan Cheng99b653c2009-12-15 00:41:36 +00004702 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
Evan Cheng9818c042009-12-15 03:00:32 +00004703 isa<ConstantSDNode>(N0.getOperand(1)) &&
Evan Cheng99b653c2009-12-15 00:41:36 +00004704 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND &&
4705 N0.hasOneUse()) {
Chris Lattnere0751182011-02-13 19:09:16 +00004706 SDValue ShAmt = N0.getOperand(1);
4707 unsigned ShAmtVal = cast<ConstantSDNode>(ShAmt)->getZExtValue();
Evan Cheng9818c042009-12-15 03:00:32 +00004708 if (N0.getOpcode() == ISD::SHL) {
Chris Lattnere0751182011-02-13 19:09:16 +00004709 SDValue InnerZExt = N0.getOperand(0);
Evan Cheng9818c042009-12-15 03:00:32 +00004710 // If the original shl may be shifting out bits, do not perform this
4711 // transformation.
Chris Lattnere0751182011-02-13 19:09:16 +00004712 unsigned KnownZeroBits = InnerZExt.getValueType().getSizeInBits() -
4713 InnerZExt.getOperand(0).getValueType().getSizeInBits();
4714 if (ShAmtVal > KnownZeroBits)
Evan Cheng9818c042009-12-15 03:00:32 +00004715 return SDValue();
4716 }
Chris Lattnere0751182011-02-13 19:09:16 +00004717
4718 DebugLoc DL = N->getDebugLoc();
Owen Anderson95771af2011-02-25 21:41:48 +00004719
4720 // Ensure that the shift amount is wide enough for the shifted value.
Chris Lattnere0751182011-02-13 19:09:16 +00004721 if (VT.getSizeInBits() >= 256)
4722 ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt);
Owen Anderson95771af2011-02-25 21:41:48 +00004723
Chris Lattnere0751182011-02-13 19:09:16 +00004724 return DAG.getNode(N0.getOpcode(), DL, VT,
4725 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)),
4726 ShAmt);
Evan Cheng99b653c2009-12-15 00:41:36 +00004727 }
4728
Evan Chengb3a3d5e2010-04-28 07:10:39 +00004729 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00004730}
4731
Dan Gohman475871a2008-07-27 21:46:04 +00004732SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
4733 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004734 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00004735
Chris Lattner5ffc0662006-05-05 05:58:59 +00004736 // fold (aext c1) -> c1
Chris Lattner310b5782006-05-06 23:06:26 +00004737 if (isa<ConstantSDNode>(N0))
Bill Wendlingfc4b6772009-02-01 11:19:36 +00004738 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, N0);
Chris Lattner5ffc0662006-05-05 05:58:59 +00004739 // fold (aext (aext x)) -> (aext x)
4740 // fold (aext (zext x)) -> (zext x)
4741 // fold (aext (sext x)) -> (sext x)
4742 if (N0.getOpcode() == ISD::ANY_EXTEND ||
4743 N0.getOpcode() == ISD::ZERO_EXTEND ||
4744 N0.getOpcode() == ISD::SIGN_EXTEND)
Bill Wendling683c9572009-01-30 22:27:33 +00004745 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT, N0.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00004746
Evan Chengc88138f2007-03-22 01:54:19 +00004747 // fold (aext (truncate (load x))) -> (aext (smaller load x))
4748 // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n)))
4749 if (N0.getOpcode() == ISD::TRUNCATE) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004750 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
4751 if (NarrowLoad.getNode()) {
Dale Johannesen86234c32010-05-25 18:47:23 +00004752 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4753 if (NarrowLoad.getNode() != N0.getNode()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004754 CombineTo(N0.getNode(), NarrowLoad);
Dale Johannesen86234c32010-05-25 18:47:23 +00004755 // CombineTo deleted the truncate, if needed, but not what's under it.
4756 AddToWorkList(oye);
4757 }
Eli Friedmane545d382011-04-16 23:25:34 +00004758 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng0b063de2007-03-23 02:16:52 +00004759 }
Evan Chengc88138f2007-03-22 01:54:19 +00004760 }
4761
Chris Lattner84750582006-09-20 06:29:17 +00004762 // fold (aext (truncate x))
4763 if (N0.getOpcode() == ISD::TRUNCATE) {
Dan Gohman475871a2008-07-27 21:46:04 +00004764 SDValue TruncOp = N0.getOperand(0);
Chris Lattner84750582006-09-20 06:29:17 +00004765 if (TruncOp.getValueType() == VT)
4766 return TruncOp; // x iff x size == zext size.
Duncan Sands8e4eb092008-06-08 20:54:56 +00004767 if (TruncOp.getValueType().bitsGT(VT))
Bill Wendling683c9572009-01-30 22:27:33 +00004768 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, TruncOp);
4769 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, TruncOp);
Chris Lattner84750582006-09-20 06:29:17 +00004770 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004771
Dan Gohman97121ba2009-04-08 00:15:30 +00004772 // Fold (aext (and (trunc x), cst)) -> (and x, cst)
4773 // if the trunc is not free.
Chris Lattner0e4b9222006-09-21 06:40:43 +00004774 if (N0.getOpcode() == ISD::AND &&
4775 N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
Dan Gohman97121ba2009-04-08 00:15:30 +00004776 N0.getOperand(1).getOpcode() == ISD::Constant &&
4777 !TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
4778 N0.getValueType())) {
Dan Gohman475871a2008-07-27 21:46:04 +00004779 SDValue X = N0.getOperand(0).getOperand(0);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004780 if (X.getValueType().bitsLT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004781 X = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, X);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004782 } else if (X.getValueType().bitsGT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004783 X = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, X);
Chris Lattner0e4b9222006-09-21 06:40:43 +00004784 }
Dan Gohman220a8232008-03-03 23:51:38 +00004785 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00004786 Mask = Mask.zext(VT.getSizeInBits());
Bill Wendling683c9572009-01-30 22:27:33 +00004787 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
4788 X, DAG.getConstant(Mask, VT));
Chris Lattner0e4b9222006-09-21 06:40:43 +00004789 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004790
Chris Lattner5ffc0662006-05-05 05:58:59 +00004791 // fold (aext (load x)) -> (aext (truncate (extload x)))
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004792 // None of the supported targets knows how to perform load and any_ext
Nadav Rotemfcd96192011-02-27 07:40:43 +00004793 // on vectors in one instruction. We only perform this transformation on
4794 // scalars.
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004795 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004796 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00004797 TLI.isLoadExtLegal(ISD::EXTLOAD, N0.getValueType()))) {
Dan Gohman57fc82d2009-04-09 03:51:29 +00004798 bool DoXform = true;
4799 SmallVector<SDNode*, 4> SetCCs;
4800 if (!N0.hasOneUse())
4801 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ANY_EXTEND, SetCCs, TLI);
4802 if (DoXform) {
4803 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00004804 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, N->getDebugLoc(), VT,
Dan Gohman57fc82d2009-04-09 03:51:29 +00004805 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004806 LN0->getBasePtr(), LN0->getPointerInfo(),
Dan Gohman57fc82d2009-04-09 03:51:29 +00004807 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00004808 LN0->isVolatile(), LN0->isNonTemporal(),
4809 LN0->getAlignment());
Dan Gohman57fc82d2009-04-09 03:51:29 +00004810 CombineTo(N, ExtLoad);
4811 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4812 N0.getValueType(), ExtLoad);
4813 CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1));
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004814 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4815 ISD::ANY_EXTEND);
Dan Gohman57fc82d2009-04-09 03:51:29 +00004816 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4817 }
Chris Lattner5ffc0662006-05-05 05:58:59 +00004818 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004819
Chris Lattner5ffc0662006-05-05 05:58:59 +00004820 // fold (aext (zextload x)) -> (aext (truncate (zextload x)))
4821 // fold (aext (sextload x)) -> (aext (truncate (sextload x)))
4822 // fold (aext ( extload x)) -> (aext (truncate (extload x)))
Evan Cheng83060c52007-03-07 08:07:03 +00004823 if (N0.getOpcode() == ISD::LOAD &&
Gabor Greifba36cb52008-08-28 21:40:38 +00004824 !ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
Evan Cheng466685d2006-10-09 20:57:25 +00004825 N0.hasOneUse()) {
4826 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00004827 EVT MemVT = LN0->getMemoryVT();
Stuart Hastingsa9011292011-02-16 16:23:55 +00004828 SDValue ExtLoad = DAG.getExtLoad(LN0->getExtensionType(), N->getDebugLoc(),
4829 VT, LN0->getChain(), LN0->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004830 LN0->getPointerInfo(), MemVT,
David Greene1e559442010-02-15 17:00:31 +00004831 LN0->isVolatile(), LN0->isNonTemporal(),
4832 LN0->getAlignment());
Chris Lattner5ffc0662006-05-05 05:58:59 +00004833 CombineTo(N, ExtLoad);
Evan Cheng45299662008-08-29 23:20:46 +00004834 CombineTo(N0.getNode(),
Bill Wendling683c9572009-01-30 22:27:33 +00004835 DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4836 N0.getValueType(), ExtLoad),
Chris Lattner5ffc0662006-05-05 05:58:59 +00004837 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00004838 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattner5ffc0662006-05-05 05:58:59 +00004839 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004840
Chris Lattner20a35c32007-04-11 05:32:27 +00004841 if (N0.getOpcode() == ISD::SETCC) {
Evan Cheng0a942db2010-05-19 01:08:17 +00004842 // aext(setcc) -> sext_in_reg(vsetcc) for vectors.
4843 // Only do this before legalize for now.
4844 if (VT.isVector() && !LegalOperations) {
4845 EVT N0VT = N0.getOperand(0).getValueType();
4846 // We know that the # elements of the results is the same as the
4847 // # elements of the compare (and the # elements of the compare result
4848 // for that matter). Check to see that they are the same size. If so,
4849 // we know that the element size of the sext'd result matches the
4850 // element size of the compare operands.
4851 if (VT.getSizeInBits() == N0VT.getSizeInBits())
Duncan Sands28b77e92011-09-06 19:07:46 +00004852 return DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
Duncan Sands34727662010-07-12 08:16:59 +00004853 N0.getOperand(1),
4854 cast<CondCodeSDNode>(N0.getOperand(2))->get());
Evan Cheng0a942db2010-05-19 01:08:17 +00004855 // If the desired elements are smaller or larger than the source
4856 // elements we can use a matching integer vector type and then
4857 // truncate/sign extend
4858 else {
Duncan Sands34727662010-07-12 08:16:59 +00004859 EVT MatchingElementType =
4860 EVT::getIntegerVT(*DAG.getContext(),
4861 N0VT.getScalarType().getSizeInBits());
4862 EVT MatchingVectorType =
4863 EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
4864 N0VT.getVectorNumElements());
4865 SDValue VsetCC =
Duncan Sands28b77e92011-09-06 19:07:46 +00004866 DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
Duncan Sands34727662010-07-12 08:16:59 +00004867 N0.getOperand(1),
4868 cast<CondCodeSDNode>(N0.getOperand(2))->get());
4869 return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT);
Evan Cheng0a942db2010-05-19 01:08:17 +00004870 }
4871 }
4872
4873 // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
Scott Michelfdc40a02009-02-17 22:15:04 +00004874 SDValue SCC =
Bill Wendling836ca7d2009-01-30 23:59:18 +00004875 SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1),
Chris Lattner1eba01e2007-04-11 06:50:51 +00004876 DAG.getConstant(1, VT), DAG.getConstant(0, VT),
Chris Lattnerc24bbad2007-04-11 16:51:53 +00004877 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
Gabor Greifba36cb52008-08-28 21:40:38 +00004878 if (SCC.getNode())
Chris Lattnerc56a81d2007-04-11 06:43:25 +00004879 return SCC;
Chris Lattner20a35c32007-04-11 05:32:27 +00004880 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004881
Evan Chengb3a3d5e2010-04-28 07:10:39 +00004882 return SDValue();
Chris Lattner5ffc0662006-05-05 05:58:59 +00004883}
4884
Chris Lattner2b4c2792007-10-13 06:35:54 +00004885/// GetDemandedBits - See if the specified operand can be simplified with the
4886/// knowledge that only the bits specified by Mask are used. If so, return the
Dan Gohman475871a2008-07-27 21:46:04 +00004887/// simpler operand, otherwise return a null SDValue.
4888SDValue DAGCombiner::GetDemandedBits(SDValue V, const APInt &Mask) {
Chris Lattner2b4c2792007-10-13 06:35:54 +00004889 switch (V.getOpcode()) {
4890 default: break;
Lang Hames5207bf22011-11-08 18:56:23 +00004891 case ISD::Constant: {
4892 const ConstantSDNode *CV = cast<ConstantSDNode>(V.getNode());
4893 assert(CV != 0 && "Const value should be ConstSDNode.");
4894 const APInt &CVal = CV->getAPIntValue();
4895 APInt NewVal = CVal & Mask;
4896 if (NewVal != CVal) {
4897 return DAG.getConstant(NewVal, V.getValueType());
4898 }
4899 break;
4900 }
Chris Lattner2b4c2792007-10-13 06:35:54 +00004901 case ISD::OR:
4902 case ISD::XOR:
4903 // If the LHS or RHS don't contribute bits to the or, drop them.
4904 if (DAG.MaskedValueIsZero(V.getOperand(0), Mask))
4905 return V.getOperand(1);
4906 if (DAG.MaskedValueIsZero(V.getOperand(1), Mask))
4907 return V.getOperand(0);
4908 break;
Chris Lattnere33544c2007-10-13 06:58:48 +00004909 case ISD::SRL:
4910 // Only look at single-use SRLs.
Gabor Greifba36cb52008-08-28 21:40:38 +00004911 if (!V.getNode()->hasOneUse())
Chris Lattnere33544c2007-10-13 06:58:48 +00004912 break;
4913 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
4914 // See if we can recursively simplify the LHS.
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00004915 unsigned Amt = RHSC->getZExtValue();
Bill Wendling8509c902009-01-30 22:33:24 +00004916
Dan Gohmancc91d632009-01-03 19:22:06 +00004917 // Watch out for shift count overflow though.
4918 if (Amt >= Mask.getBitWidth()) break;
Dan Gohman2e68b6f2008-02-25 21:11:39 +00004919 APInt NewMask = Mask << Amt;
Dan Gohman475871a2008-07-27 21:46:04 +00004920 SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask);
Bill Wendling8509c902009-01-30 22:33:24 +00004921 if (SimplifyLHS.getNode())
Scott Michelfdc40a02009-02-17 22:15:04 +00004922 return DAG.getNode(ISD::SRL, V.getDebugLoc(), V.getValueType(),
Chris Lattnere33544c2007-10-13 06:58:48 +00004923 SimplifyLHS, V.getOperand(1));
Chris Lattnere33544c2007-10-13 06:58:48 +00004924 }
Chris Lattner2b4c2792007-10-13 06:35:54 +00004925 }
Dan Gohman475871a2008-07-27 21:46:04 +00004926 return SDValue();
Chris Lattner2b4c2792007-10-13 06:35:54 +00004927}
4928
Evan Chengc88138f2007-03-22 01:54:19 +00004929/// ReduceLoadWidth - If the result of a wider load is shifted to right of N
4930/// bits and then truncated to a narrower type and where N is a multiple
4931/// of number of bits of the narrower type, transform it to a narrower load
4932/// from address + N / num of bits of new type. If the result is to be
4933/// extended, also fold the extension to form a extending load.
Dan Gohman475871a2008-07-27 21:46:04 +00004934SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
Evan Chengc88138f2007-03-22 01:54:19 +00004935 unsigned Opc = N->getOpcode();
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004936
Evan Chengc88138f2007-03-22 01:54:19 +00004937 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
Dan Gohman475871a2008-07-27 21:46:04 +00004938 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004939 EVT VT = N->getValueType(0);
4940 EVT ExtVT = VT;
Evan Chengc88138f2007-03-22 01:54:19 +00004941
Dan Gohman7f8613e2008-08-14 20:04:46 +00004942 // This transformation isn't valid for vector loads.
4943 if (VT.isVector())
4944 return SDValue();
4945
Dan Gohmand1996362010-01-09 02:13:55 +00004946 // Special case: SIGN_EXTEND_INREG is basically truncating to ExtVT then
Evan Chenge177e302007-03-23 22:13:36 +00004947 // extended to VT.
Evan Chengc88138f2007-03-22 01:54:19 +00004948 if (Opc == ISD::SIGN_EXTEND_INREG) {
4949 ExtType = ISD::SEXTLOAD;
Owen Andersone50ed302009-08-10 22:56:29 +00004950 ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT();
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004951 } else if (Opc == ISD::SRL) {
Chris Lattner90b03642010-12-21 18:05:22 +00004952 // Another special-case: SRL is basically zero-extending a narrower value.
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004953 ExtType = ISD::ZEXTLOAD;
4954 N0 = SDValue(N, 0);
4955 ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4956 if (!N01) return SDValue();
4957 ExtVT = EVT::getIntegerVT(*DAG.getContext(),
4958 VT.getSizeInBits() - N01->getZExtValue());
Evan Chengc88138f2007-03-22 01:54:19 +00004959 }
Richard Osborne4e3740e2011-01-31 17:41:44 +00004960 if (LegalOperations && !TLI.isLoadExtLegal(ExtType, ExtVT))
4961 return SDValue();
Evan Chengc88138f2007-03-22 01:54:19 +00004962
Owen Andersone50ed302009-08-10 22:56:29 +00004963 unsigned EVTBits = ExtVT.getSizeInBits();
Owen Anderson95771af2011-02-25 21:41:48 +00004964
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00004965 // Do not generate loads of non-round integer types since these can
4966 // be expensive (and would be wrong if the type is not byte sized).
4967 if (!ExtVT.isRound())
4968 return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00004969
Evan Chengc88138f2007-03-22 01:54:19 +00004970 unsigned ShAmt = 0;
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00004971 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
Evan Chengc88138f2007-03-22 01:54:19 +00004972 if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00004973 ShAmt = N01->getZExtValue();
Evan Chengc88138f2007-03-22 01:54:19 +00004974 // Is the shift amount a multiple of size of VT?
4975 if ((ShAmt & (EVTBits-1)) == 0) {
4976 N0 = N0.getOperand(0);
Eli Friedmand68eea22009-08-19 08:46:10 +00004977 // Is the load width a multiple of size of VT?
4978 if ((N0.getValueType().getSizeInBits() & (EVTBits-1)) != 0)
Dan Gohman475871a2008-07-27 21:46:04 +00004979 return SDValue();
Evan Chengc88138f2007-03-22 01:54:19 +00004980 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004981
Chris Lattnercbf68df2010-12-22 08:02:57 +00004982 // At this point, we must have a load or else we can't do the transform.
4983 if (!isa<LoadSDNode>(N0)) return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00004984
Chris Lattner2831a192010-10-01 05:36:09 +00004985 // If the shift amount is larger than the input type then we're not
4986 // accessing any of the loaded bytes. If the load was a zextload/extload
4987 // then the result of the shift+trunc is zero/undef (handled elsewhere).
4988 // If the load was a sextload then the result is a splat of the sign bit
4989 // of the extended byte. This is not worth optimizing for.
Chris Lattnercbf68df2010-12-22 08:02:57 +00004990 if (ShAmt >= cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits())
Chris Lattner2831a192010-10-01 05:36:09 +00004991 return SDValue();
Evan Chengc88138f2007-03-22 01:54:19 +00004992 }
4993 }
4994
Dan Gohman394d6292010-11-03 01:47:46 +00004995 // If the load is shifted left (and the result isn't shifted back right),
4996 // we can fold the truncate through the shift.
4997 unsigned ShLeftAmt = 0;
4998 if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
Chris Lattner4c32bc22010-12-22 07:36:50 +00004999 ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) {
Dan Gohman394d6292010-11-03 01:47:46 +00005000 if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
5001 ShLeftAmt = N01->getZExtValue();
5002 N0 = N0.getOperand(0);
5003 }
5004 }
Owen Anderson95771af2011-02-25 21:41:48 +00005005
Chris Lattner4c32bc22010-12-22 07:36:50 +00005006 // If we haven't found a load, we can't narrow it. Don't transform one with
5007 // multiple uses, this would require adding a new load.
5008 if (!isa<LoadSDNode>(N0) || !N0.hasOneUse() ||
5009 // Don't change the width of a volatile load.
5010 cast<LoadSDNode>(N0)->isVolatile())
5011 return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00005012
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00005013 // Verify that we are actually reducing a load width here.
5014 if (cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits() < EVTBits)
Chris Lattner4c32bc22010-12-22 07:36:50 +00005015 return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00005016
Chris Lattner4c32bc22010-12-22 07:36:50 +00005017 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
5018 EVT PtrType = N0.getOperand(1).getValueType();
Bill Wendling8509c902009-01-30 22:33:24 +00005019
Chris Lattner4c32bc22010-12-22 07:36:50 +00005020 // For big endian targets, we need to adjust the offset to the pointer to
5021 // load the correct bytes.
5022 if (TLI.isBigEndian()) {
5023 unsigned LVTStoreBits = LN0->getMemoryVT().getStoreSizeInBits();
5024 unsigned EVTStoreBits = ExtVT.getStoreSizeInBits();
5025 ShAmt = LVTStoreBits - EVTStoreBits - ShAmt;
Evan Chengc88138f2007-03-22 01:54:19 +00005026 }
5027
Chris Lattner4c32bc22010-12-22 07:36:50 +00005028 uint64_t PtrOff = ShAmt / 8;
5029 unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff);
5030 SDValue NewPtr = DAG.getNode(ISD::ADD, LN0->getDebugLoc(),
5031 PtrType, LN0->getBasePtr(),
5032 DAG.getConstant(PtrOff, PtrType));
5033 AddToWorkList(NewPtr.getNode());
5034
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00005035 SDValue Load;
5036 if (ExtType == ISD::NON_EXTLOAD)
5037 Load = DAG.getLoad(VT, N0.getDebugLoc(), LN0->getChain(), NewPtr,
5038 LN0->getPointerInfo().getWithOffset(PtrOff),
Pete Cooperd752e0f2011-11-08 18:42:53 +00005039 LN0->isVolatile(), LN0->isNonTemporal(),
5040 LN0->isInvariant(), NewAlign);
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00005041 else
Stuart Hastingsa9011292011-02-16 16:23:55 +00005042 Load = DAG.getExtLoad(ExtType, N0.getDebugLoc(), VT, LN0->getChain(),NewPtr,
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00005043 LN0->getPointerInfo().getWithOffset(PtrOff),
5044 ExtVT, LN0->isVolatile(), LN0->isNonTemporal(),
5045 NewAlign);
Chris Lattner4c32bc22010-12-22 07:36:50 +00005046
5047 // Replace the old load's chain with the new load's chain.
5048 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00005049 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
Chris Lattner4c32bc22010-12-22 07:36:50 +00005050
5051 // Shift the result left, if we've swallowed a left shift.
5052 SDValue Result = Load;
5053 if (ShLeftAmt != 0) {
Owen Anderson95771af2011-02-25 21:41:48 +00005054 EVT ShImmTy = getShiftAmountTy(Result.getValueType());
Chris Lattner4c32bc22010-12-22 07:36:50 +00005055 if (!isUIntN(ShImmTy.getSizeInBits(), ShLeftAmt))
5056 ShImmTy = VT;
5057 Result = DAG.getNode(ISD::SHL, N0.getDebugLoc(), VT,
5058 Result, DAG.getConstant(ShLeftAmt, ShImmTy));
5059 }
5060
5061 // Return the new loaded value.
5062 return Result;
Evan Chengc88138f2007-03-22 01:54:19 +00005063}
5064
Dan Gohman475871a2008-07-27 21:46:04 +00005065SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
5066 SDValue N0 = N->getOperand(0);
5067 SDValue N1 = N->getOperand(1);
Owen Andersone50ed302009-08-10 22:56:29 +00005068 EVT VT = N->getValueType(0);
5069 EVT EVT = cast<VTSDNode>(N1)->getVT();
Dan Gohman87862e72009-12-11 21:31:27 +00005070 unsigned VTBits = VT.getScalarType().getSizeInBits();
Dan Gohmand1996362010-01-09 02:13:55 +00005071 unsigned EVTBits = EVT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00005072
Nate Begeman1d4d4142005-09-01 00:19:25 +00005073 // fold (sext_in_reg c1) -> c1
Chris Lattnereaeda562006-05-08 20:59:41 +00005074 if (isa<ConstantSDNode>(N0) || N0.getOpcode() == ISD::UNDEF)
Bill Wendling8509c902009-01-30 22:33:24 +00005075 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00005076
Chris Lattner541a24f2006-05-06 22:43:44 +00005077 // If the input is already sign extended, just drop the extension.
Dan Gohman87862e72009-12-11 21:31:27 +00005078 if (DAG.ComputeNumSignBits(N0) >= VTBits-EVTBits+1)
Chris Lattneree4ea922006-05-06 09:30:03 +00005079 return N0;
Scott Michelfdc40a02009-02-17 22:15:04 +00005080
Nate Begeman646d7e22005-09-02 21:18:40 +00005081 // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
5082 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
Duncan Sands8e4eb092008-06-08 20:54:56 +00005083 EVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT())) {
Bill Wendling8509c902009-01-30 22:33:24 +00005084 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT,
5085 N0.getOperand(0), N1);
Nate Begeman646d7e22005-09-02 21:18:40 +00005086 }
Chris Lattner4b37e872006-05-08 21:18:59 +00005087
Dan Gohman75dcf082008-07-31 00:50:31 +00005088 // fold (sext_in_reg (sext x)) -> (sext x)
5089 // fold (sext_in_reg (aext x)) -> (sext x)
5090 // if x is small enough.
5091 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
5092 SDValue N00 = N0.getOperand(0);
Evan Cheng003d7c42010-04-16 22:26:19 +00005093 if (N00.getValueType().getScalarType().getSizeInBits() <= EVTBits &&
5094 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
Bill Wendling8509c902009-01-30 22:33:24 +00005095 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, N00, N1);
Dan Gohman75dcf082008-07-31 00:50:31 +00005096 }
5097
Chris Lattner95a5e052007-04-17 19:03:21 +00005098 // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero.
Dan Gohman2e68b6f2008-02-25 21:11:39 +00005099 if (DAG.MaskedValueIsZero(N0, APInt::getBitsSet(VTBits, EVTBits-1, EVTBits)))
Bill Wendlingfc4b6772009-02-01 11:19:36 +00005100 return DAG.getZeroExtendInReg(N0, N->getDebugLoc(), EVT);
Scott Michelfdc40a02009-02-17 22:15:04 +00005101
Chris Lattner95a5e052007-04-17 19:03:21 +00005102 // fold operands of sext_in_reg based on knowledge that the top bits are not
5103 // demanded.
Dan Gohman475871a2008-07-27 21:46:04 +00005104 if (SimplifyDemandedBits(SDValue(N, 0)))
5105 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005106
Evan Chengc88138f2007-03-22 01:54:19 +00005107 // fold (sext_in_reg (load x)) -> (smaller sextload x)
5108 // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
Dan Gohman475871a2008-07-27 21:46:04 +00005109 SDValue NarrowLoad = ReduceLoadWidth(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005110 if (NarrowLoad.getNode())
Evan Chengc88138f2007-03-22 01:54:19 +00005111 return NarrowLoad;
5112
Bill Wendling8509c902009-01-30 22:33:24 +00005113 // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24)
5114 // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible.
Chris Lattner4b37e872006-05-08 21:18:59 +00005115 // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above.
5116 if (N0.getOpcode() == ISD::SRL) {
5117 if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
Dan Gohman87862e72009-12-11 21:31:27 +00005118 if (ShAmt->getZExtValue()+EVTBits <= VTBits) {
Chris Lattner4b37e872006-05-08 21:18:59 +00005119 // We can turn this into an SRA iff the input to the SRL is already sign
5120 // extended enough.
Dan Gohmanea859be2007-06-22 14:59:07 +00005121 unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
Dan Gohman87862e72009-12-11 21:31:27 +00005122 if (VTBits-(ShAmt->getZExtValue()+EVTBits) < InSignBits)
Bill Wendling8509c902009-01-30 22:33:24 +00005123 return DAG.getNode(ISD::SRA, N->getDebugLoc(), VT,
5124 N0.getOperand(0), N0.getOperand(1));
Chris Lattner4b37e872006-05-08 21:18:59 +00005125 }
5126 }
Evan Chengc88138f2007-03-22 01:54:19 +00005127
Nate Begemanded49632005-10-13 03:11:28 +00005128 // fold (sext_inreg (extload x)) -> (sextload x)
Scott Michelfdc40a02009-02-17 22:15:04 +00005129 if (ISD::isEXTLoad(N0.getNode()) &&
Gabor Greifba36cb52008-08-28 21:40:38 +00005130 ISD::isUNINDEXEDLoad(N0.getNode()) &&
Dan Gohmanb625f2f2008-01-30 00:15:11 +00005131 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00005132 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00005133 TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT))) {
Evan Cheng466685d2006-10-09 20:57:25 +00005134 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00005135 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling8509c902009-01-30 22:33:24 +00005136 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00005137 LN0->getBasePtr(), LN0->getPointerInfo(),
5138 EVT,
David Greene1e559442010-02-15 17:00:31 +00005139 LN0->isVolatile(), LN0->isNonTemporal(),
5140 LN0->getAlignment());
Chris Lattnerd4771842005-12-14 19:25:30 +00005141 CombineTo(N, ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00005142 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00005143 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00005144 }
5145 // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
Gabor Greifba36cb52008-08-28 21:40:38 +00005146 if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
Evan Cheng83060c52007-03-07 08:07:03 +00005147 N0.hasOneUse() &&
Dan Gohmanb625f2f2008-01-30 00:15:11 +00005148 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00005149 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00005150 TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT))) {
Evan Cheng466685d2006-10-09 20:57:25 +00005151 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00005152 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling8509c902009-01-30 22:33:24 +00005153 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00005154 LN0->getBasePtr(), LN0->getPointerInfo(),
5155 EVT,
David Greene1e559442010-02-15 17:00:31 +00005156 LN0->isVolatile(), LN0->isNonTemporal(),
5157 LN0->getAlignment());
Chris Lattnerd4771842005-12-14 19:25:30 +00005158 CombineTo(N, ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00005159 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00005160 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00005161 }
Evan Cheng9568e5c2011-06-21 06:01:08 +00005162
5163 // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
5164 if (EVTBits <= 16 && N0.getOpcode() == ISD::OR) {
5165 SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
5166 N0.getOperand(1), false);
5167 if (BSwap.getNode() != 0)
5168 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT,
5169 BSwap, N1);
5170 }
5171
Dan Gohman475871a2008-07-27 21:46:04 +00005172 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005173}
5174
Dan Gohman475871a2008-07-27 21:46:04 +00005175SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
5176 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00005177 EVT VT = N->getValueType(0);
Nadav Rotem7e413e9c2012-02-03 13:18:25 +00005178 bool isLE = TLI.isLittleEndian();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005179
5180 // noop truncate
5181 if (N0.getValueType() == N->getValueType(0))
Nate Begeman83e75ec2005-09-06 04:43:02 +00005182 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00005183 // fold (truncate c1) -> c1
Chris Lattner310b5782006-05-06 23:06:26 +00005184 if (isa<ConstantSDNode>(N0))
Bill Wendling67a67682009-01-30 22:44:24 +00005185 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00005186 // fold (truncate (truncate x)) -> (truncate x)
5187 if (N0.getOpcode() == ISD::TRUNCATE)
Bill Wendling67a67682009-01-30 22:44:24 +00005188 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, N0.getOperand(0));
Nate Begeman1d4d4142005-09-01 00:19:25 +00005189 // fold (truncate (ext x)) -> (ext x) or (truncate x) or x
Chris Lattner7f893c02010-04-07 18:13:33 +00005190 if (N0.getOpcode() == ISD::ZERO_EXTEND ||
5191 N0.getOpcode() == ISD::SIGN_EXTEND ||
Chris Lattnerb72773b2006-05-05 22:56:26 +00005192 N0.getOpcode() == ISD::ANY_EXTEND) {
Duncan Sands8e4eb092008-06-08 20:54:56 +00005193 if (N0.getOperand(0).getValueType().bitsLT(VT))
Nate Begeman1d4d4142005-09-01 00:19:25 +00005194 // if the source is smaller than the dest, we still need an extend
Bill Wendling67a67682009-01-30 22:44:24 +00005195 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
5196 N0.getOperand(0));
Duncan Sands8e4eb092008-06-08 20:54:56 +00005197 else if (N0.getOperand(0).getValueType().bitsGT(VT))
Nate Begeman1d4d4142005-09-01 00:19:25 +00005198 // if the source is larger than the dest, than we just need the truncate
Bill Wendling67a67682009-01-30 22:44:24 +00005199 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, N0.getOperand(0));
Nate Begeman1d4d4142005-09-01 00:19:25 +00005200 else
5201 // if the source and dest are the same type, we can drop both the extend
Evan Chengd40d03e2010-01-06 19:38:29 +00005202 // and the truncate.
Nate Begeman83e75ec2005-09-06 04:43:02 +00005203 return N0.getOperand(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00005204 }
Evan Cheng007b69e2007-03-21 20:14:05 +00005205
Nadav Rotemcc870a82012-02-05 11:39:23 +00005206 // Fold extract-and-trunc into a narrow extract. For example:
5207 // i64 x = EXTRACT_VECTOR_ELT(v2i64 val, i32 1)
5208 // i32 y = TRUNCATE(i64 x)
5209 // -- becomes --
5210 // v16i8 b = BITCAST (v2i64 val)
5211 // i8 x = EXTRACT_VECTOR_ELT(v16i8 b, i32 8)
5212 //
5213 // Note: We only run this optimization after type legalization (which often
Nadav Rotem7e413e9c2012-02-03 13:18:25 +00005214 // creates this pattern) and before operation legalization after which
5215 // we need to be more careful about the vector instructions that we generate.
5216 if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5217 LegalTypes && !LegalOperations && N0->hasOneUse()) {
5218
5219 EVT VecTy = N0.getOperand(0).getValueType();
5220 EVT ExTy = N0.getValueType();
5221 EVT TrTy = N->getValueType(0);
5222
5223 unsigned NumElem = VecTy.getVectorNumElements();
5224 unsigned SizeRatio = ExTy.getSizeInBits()/TrTy.getSizeInBits();
5225
5226 EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, SizeRatio * NumElem);
5227 assert(NVT.getSizeInBits() == VecTy.getSizeInBits() && "Invalid Size");
5228
5229 SDValue EltNo = N0->getOperand(1);
5230 if (isa<ConstantSDNode>(EltNo) && isTypeLegal(NVT)) {
5231 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
Jim Grosbacha249f7d2012-05-08 20:56:07 +00005232 EVT IndexTy = N0->getOperand(1).getValueType();
Nadav Rotem7e413e9c2012-02-03 13:18:25 +00005233 int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
5234
5235 SDValue V = DAG.getNode(ISD::BITCAST, N->getDebugLoc(),
5236 NVT, N0.getOperand(0));
5237
5238 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT,
5239 N->getDebugLoc(), TrTy, V,
Jim Grosbacha249f7d2012-05-08 20:56:07 +00005240 DAG.getConstant(Index, IndexTy));
Nadav Rotem7e413e9c2012-02-03 13:18:25 +00005241 }
5242 }
5243
Chris Lattner2b4c2792007-10-13 06:35:54 +00005244 // See if we can simplify the input to this truncate through knowledge that
Nadav Rotem8c20ec52011-02-24 21:01:34 +00005245 // only the low bits are being used.
5246 // For example "trunc (or (shl x, 8), y)" // -> trunc y
Nadav Rotemfcd96192011-02-27 07:40:43 +00005247 // Currently we only perform this optimization on scalars because vectors
Nadav Rotem8c20ec52011-02-24 21:01:34 +00005248 // may have different active low bits.
5249 if (!VT.isVector()) {
5250 SDValue Shorter =
5251 GetDemandedBits(N0, APInt::getLowBitsSet(N0.getValueSizeInBits(),
5252 VT.getSizeInBits()));
5253 if (Shorter.getNode())
5254 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, Shorter);
5255 }
Nate Begeman3df4d522005-10-12 20:40:40 +00005256 // fold (truncate (load x)) -> (smaller load x)
Evan Cheng007b69e2007-03-21 20:14:05 +00005257 // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
Dan Gohman4e39e9d2010-06-24 14:30:44 +00005258 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) {
5259 SDValue Reduced = ReduceLoadWidth(N);
5260 if (Reduced.getNode())
5261 return Reduced;
5262 }
5263
5264 // Simplify the operands using demanded-bits information.
5265 if (!VT.isVector() &&
5266 SimplifyDemandedBits(SDValue(N, 0)))
5267 return SDValue(N, 0);
5268
Evan Chenge5b51ac2010-04-17 06:13:15 +00005269 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005270}
5271
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005272static SDNode *getBuildPairElt(SDNode *N, unsigned i) {
Dan Gohman475871a2008-07-27 21:46:04 +00005273 SDValue Elt = N->getOperand(i);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005274 if (Elt.getOpcode() != ISD::MERGE_VALUES)
Gabor Greifba36cb52008-08-28 21:40:38 +00005275 return Elt.getNode();
5276 return Elt.getOperand(Elt.getResNo()).getNode();
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005277}
5278
5279/// CombineConsecutiveLoads - build_pair (load, load) -> load
Scott Michelfdc40a02009-02-17 22:15:04 +00005280/// if load locations are consecutive.
Owen Andersone50ed302009-08-10 22:56:29 +00005281SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005282 assert(N->getOpcode() == ISD::BUILD_PAIR);
5283
Nate Begemanabc01992009-06-05 21:37:30 +00005284 LoadSDNode *LD1 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 0));
5285 LoadSDNode *LD2 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 1));
Chris Lattnerfa459012010-09-21 16:08:50 +00005286 if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !LD1->hasOneUse() ||
5287 LD1->getPointerInfo().getAddrSpace() !=
5288 LD2->getPointerInfo().getAddrSpace())
Dan Gohman475871a2008-07-27 21:46:04 +00005289 return SDValue();
Owen Andersone50ed302009-08-10 22:56:29 +00005290 EVT LD1VT = LD1->getValueType(0);
Bill Wendling67a67682009-01-30 22:44:24 +00005291
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005292 if (ISD::isNON_EXTLoad(LD2) &&
5293 LD2->hasOneUse() &&
Duncan Sandsd4b9c172008-06-13 19:07:40 +00005294 // If both are volatile this would reduce the number of volatile loads.
5295 // If one is volatile it might be ok, but play conservative and bail out.
Nate Begemanabc01992009-06-05 21:37:30 +00005296 !LD1->isVolatile() &&
5297 !LD2->isVolatile() &&
Evan Cheng64fa4a92009-12-09 01:36:00 +00005298 DAG.isConsecutiveLoad(LD2, LD1, LD1VT.getSizeInBits()/8, 1)) {
Nate Begemanabc01992009-06-05 21:37:30 +00005299 unsigned Align = LD1->getAlignment();
Dan Gohman6448d912008-09-04 15:39:15 +00005300 unsigned NewAlign = TLI.getTargetData()->
Owen Anderson23b9b192009-08-12 00:36:31 +00005301 getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
Bill Wendling67a67682009-01-30 22:44:24 +00005302
Duncan Sandsd4b9c172008-06-13 19:07:40 +00005303 if (NewAlign <= Align &&
Duncan Sands25cf2272008-11-24 14:53:14 +00005304 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)))
Nate Begemanabc01992009-06-05 21:37:30 +00005305 return DAG.getLoad(VT, N->getDebugLoc(), LD1->getChain(),
Chris Lattnerfa459012010-09-21 16:08:50 +00005306 LD1->getBasePtr(), LD1->getPointerInfo(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00005307 false, false, false, Align);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005308 }
Bill Wendling67a67682009-01-30 22:44:24 +00005309
Dan Gohman475871a2008-07-27 21:46:04 +00005310 return SDValue();
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005311}
5312
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005313SDValue DAGCombiner::visitBITCAST(SDNode *N) {
Dan Gohman475871a2008-07-27 21:46:04 +00005314 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00005315 EVT VT = N->getValueType(0);
Chris Lattner94683772005-12-23 05:30:37 +00005316
Dan Gohman7f321562007-06-25 16:23:39 +00005317 // If the input is a BUILD_VECTOR with all constant elements, fold this now.
5318 // Only do this before legalize, since afterward the target may be depending
5319 // on the bitconvert.
5320 // First check to see if this is all constant.
Duncan Sands25cf2272008-11-24 14:53:14 +00005321 if (!LegalTypes &&
Gabor Greifba36cb52008-08-28 21:40:38 +00005322 N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNode()->hasOneUse() &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00005323 VT.isVector()) {
Dan Gohman7f321562007-06-25 16:23:39 +00005324 bool isSimple = true;
5325 for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i)
5326 if (N0.getOperand(i).getOpcode() != ISD::UNDEF &&
5327 N0.getOperand(i).getOpcode() != ISD::Constant &&
5328 N0.getOperand(i).getOpcode() != ISD::ConstantFP) {
Scott Michelfdc40a02009-02-17 22:15:04 +00005329 isSimple = false;
Dan Gohman7f321562007-06-25 16:23:39 +00005330 break;
5331 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005332
Owen Andersone50ed302009-08-10 22:56:29 +00005333 EVT DestEltVT = N->getValueType(0).getVectorElementType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00005334 assert(!DestEltVT.isVector() &&
Dan Gohman7f321562007-06-25 16:23:39 +00005335 "Element type of vector ValueType must not be vector!");
Bill Wendling67a67682009-01-30 22:44:24 +00005336 if (isSimple)
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005337 return ConstantFoldBITCASTofBUILD_VECTOR(N0.getNode(), DestEltVT);
Dan Gohman7f321562007-06-25 16:23:39 +00005338 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005339
Dan Gohman3dd168d2008-09-05 01:58:21 +00005340 // If the input is a constant, let getNode fold it.
Chris Lattner94683772005-12-23 05:30:37 +00005341 if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005342 SDValue Res = DAG.getNode(ISD::BITCAST, N->getDebugLoc(), VT, N0);
Dan Gohmana407ca12009-08-10 23:15:10 +00005343 if (Res.getNode() != N) {
5344 if (!LegalOperations ||
5345 TLI.isOperationLegal(Res.getNode()->getOpcode(), VT))
5346 return Res;
5347
5348 // Folding it resulted in an illegal node, and it's too late to
5349 // do that. Clean up the old node and forego the transformation.
5350 // Ideally this won't happen very often, because instcombine
5351 // and the earlier dagcombine runs (where illegal nodes are
5352 // permitted) should have folded most of them already.
5353 DAG.DeleteNode(Res.getNode());
5354 }
Chris Lattner94683772005-12-23 05:30:37 +00005355 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005356
Bill Wendling67a67682009-01-30 22:44:24 +00005357 // (conv (conv x, t1), t2) -> (conv x, t2)
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005358 if (N0.getOpcode() == ISD::BITCAST)
5359 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005360 N0.getOperand(0));
Chris Lattner6258fb22006-04-02 02:53:43 +00005361
Chris Lattner57104102005-12-23 05:44:41 +00005362 // fold (conv (load x)) -> (load (conv*)x)
Evan Cheng513da432007-10-06 08:19:55 +00005363 // If the resultant load doesn't need a higher alignment than the original!
Gabor Greifba36cb52008-08-28 21:40:38 +00005364 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
Duncan Sandsd4b9c172008-06-13 19:07:40 +00005365 // Do not change the width of a volatile load.
5366 !cast<LoadSDNode>(N0)->isVolatile() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00005367 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT))) {
Evan Cheng466685d2006-10-09 20:57:25 +00005368 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman6448d912008-09-04 15:39:15 +00005369 unsigned Align = TLI.getTargetData()->
Owen Anderson23b9b192009-08-12 00:36:31 +00005370 getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
Evan Cheng59d5b682007-05-07 21:27:48 +00005371 unsigned OrigAlign = LN0->getAlignment();
Bill Wendling67a67682009-01-30 22:44:24 +00005372
Evan Cheng59d5b682007-05-07 21:27:48 +00005373 if (Align <= OrigAlign) {
Bill Wendling67a67682009-01-30 22:44:24 +00005374 SDValue Load = DAG.getLoad(VT, N->getDebugLoc(), LN0->getChain(),
Chris Lattnerfa459012010-09-21 16:08:50 +00005375 LN0->getBasePtr(), LN0->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00005376 LN0->isVolatile(), LN0->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00005377 LN0->isInvariant(), OrigAlign);
Evan Cheng59d5b682007-05-07 21:27:48 +00005378 AddToWorkList(N);
Gabor Greif12632d22008-08-30 19:29:20 +00005379 CombineTo(N0.getNode(),
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005380 DAG.getNode(ISD::BITCAST, N0.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005381 N0.getValueType(), Load),
Evan Cheng59d5b682007-05-07 21:27:48 +00005382 Load.getValue(1));
5383 return Load;
5384 }
Chris Lattner57104102005-12-23 05:44:41 +00005385 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00005386
Bill Wendling67a67682009-01-30 22:44:24 +00005387 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
5388 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
Chris Lattner3bd39d42008-01-27 17:42:27 +00005389 // This often reduces constant pool loads.
Owen Anderson29f60f32012-04-02 22:10:29 +00005390 if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(VT)) ||
5391 (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(VT))) &&
Gabor Greifba36cb52008-08-28 21:40:38 +00005392 N0.getNode()->hasOneUse() && VT.isInteger() && !VT.isVector()) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005393 SDValue NewConv = DAG.getNode(ISD::BITCAST, N0.getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005394 N0.getOperand(0));
Gabor Greifba36cb52008-08-28 21:40:38 +00005395 AddToWorkList(NewConv.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00005396
Duncan Sands83ec4b62008-06-06 12:08:01 +00005397 APInt SignBit = APInt::getSignBit(VT.getSizeInBits());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005398 if (N0.getOpcode() == ISD::FNEG)
Bill Wendling67a67682009-01-30 22:44:24 +00005399 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT,
5400 NewConv, DAG.getConstant(SignBit, VT));
Chris Lattner3bd39d42008-01-27 17:42:27 +00005401 assert(N0.getOpcode() == ISD::FABS);
Bill Wendling67a67682009-01-30 22:44:24 +00005402 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
5403 NewConv, DAG.getConstant(~SignBit, VT));
Chris Lattner3bd39d42008-01-27 17:42:27 +00005404 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005405
Bill Wendling67a67682009-01-30 22:44:24 +00005406 // fold (bitconvert (fcopysign cst, x)) ->
5407 // (or (and (bitconvert x), sign), (and cst, (not sign)))
5408 // Note that we don't handle (copysign x, cst) because this can always be
5409 // folded to an fneg or fabs.
Gabor Greifba36cb52008-08-28 21:40:38 +00005410 if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse() &&
Chris Lattnerf32aac32008-01-27 23:32:17 +00005411 isa<ConstantFPSDNode>(N0.getOperand(0)) &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00005412 VT.isInteger() && !VT.isVector()) {
5413 unsigned OrigXWidth = N0.getOperand(1).getValueType().getSizeInBits();
Owen Anderson23b9b192009-08-12 00:36:31 +00005414 EVT IntXVT = EVT::getIntegerVT(*DAG.getContext(), OrigXWidth);
Chris Lattner2392ae72010-04-15 04:48:01 +00005415 if (isTypeLegal(IntXVT)) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005416 SDValue X = DAG.getNode(ISD::BITCAST, N0.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005417 IntXVT, N0.getOperand(1));
Duncan Sands25cf2272008-11-24 14:53:14 +00005418 AddToWorkList(X.getNode());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005419
Duncan Sands25cf2272008-11-24 14:53:14 +00005420 // If X has a different width than the result/lhs, sext it or truncate it.
5421 unsigned VTWidth = VT.getSizeInBits();
5422 if (OrigXWidth < VTWidth) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00005423 X = DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, X);
Duncan Sands25cf2272008-11-24 14:53:14 +00005424 AddToWorkList(X.getNode());
5425 } else if (OrigXWidth > VTWidth) {
5426 // To get the sign bit in the right place, we have to shift it right
5427 // before truncating.
Bill Wendling9729c5a2009-01-31 03:12:48 +00005428 X = DAG.getNode(ISD::SRL, X.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005429 X.getValueType(), X,
Duncan Sands25cf2272008-11-24 14:53:14 +00005430 DAG.getConstant(OrigXWidth-VTWidth, X.getValueType()));
5431 AddToWorkList(X.getNode());
Bill Wendling9729c5a2009-01-31 03:12:48 +00005432 X = DAG.getNode(ISD::TRUNCATE, X.getDebugLoc(), VT, X);
Duncan Sands25cf2272008-11-24 14:53:14 +00005433 AddToWorkList(X.getNode());
5434 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005435
Duncan Sands25cf2272008-11-24 14:53:14 +00005436 APInt SignBit = APInt::getSignBit(VT.getSizeInBits());
Bill Wendling9729c5a2009-01-31 03:12:48 +00005437 X = DAG.getNode(ISD::AND, X.getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005438 X, DAG.getConstant(SignBit, VT));
Duncan Sands25cf2272008-11-24 14:53:14 +00005439 AddToWorkList(X.getNode());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005440
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005441 SDValue Cst = DAG.getNode(ISD::BITCAST, N0.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005442 VT, N0.getOperand(0));
Bill Wendling9729c5a2009-01-31 03:12:48 +00005443 Cst = DAG.getNode(ISD::AND, Cst.getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005444 Cst, DAG.getConstant(~SignBit, VT));
Duncan Sands25cf2272008-11-24 14:53:14 +00005445 AddToWorkList(Cst.getNode());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005446
Bill Wendling67a67682009-01-30 22:44:24 +00005447 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, X, Cst);
Duncan Sands25cf2272008-11-24 14:53:14 +00005448 }
Chris Lattner3bd39d42008-01-27 17:42:27 +00005449 }
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005450
Scott Michelfdc40a02009-02-17 22:15:04 +00005451 // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive.
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005452 if (N0.getOpcode() == ISD::BUILD_PAIR) {
Gabor Greifba36cb52008-08-28 21:40:38 +00005453 SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT);
5454 if (CombineLD.getNode())
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005455 return CombineLD;
5456 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005457
Dan Gohman475871a2008-07-27 21:46:04 +00005458 return SDValue();
Chris Lattner94683772005-12-23 05:30:37 +00005459}
5460
Dan Gohman475871a2008-07-27 21:46:04 +00005461SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
Owen Andersone50ed302009-08-10 22:56:29 +00005462 EVT VT = N->getValueType(0);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005463 return CombineConsecutiveLoads(N, VT);
5464}
5465
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005466/// ConstantFoldBITCASTofBUILD_VECTOR - We know that BV is a build_vector
Scott Michelfdc40a02009-02-17 22:15:04 +00005467/// node with Constant, ConstantFP or Undef operands. DstEltVT indicates the
Chris Lattner6258fb22006-04-02 02:53:43 +00005468/// destination element value type.
Dan Gohman475871a2008-07-27 21:46:04 +00005469SDValue DAGCombiner::
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005470ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
Owen Andersone50ed302009-08-10 22:56:29 +00005471 EVT SrcEltVT = BV->getValueType(0).getVectorElementType();
Scott Michelfdc40a02009-02-17 22:15:04 +00005472
Chris Lattner6258fb22006-04-02 02:53:43 +00005473 // If this is already the right type, we're done.
Dan Gohman475871a2008-07-27 21:46:04 +00005474 if (SrcEltVT == DstEltVT) return SDValue(BV, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005475
Duncan Sands83ec4b62008-06-06 12:08:01 +00005476 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
5477 unsigned DstBitSize = DstEltVT.getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00005478
Chris Lattner6258fb22006-04-02 02:53:43 +00005479 // If this is a conversion of N elements of one type to N elements of another
5480 // type, convert each element. This handles FP<->INT cases.
5481 if (SrcBitSize == DstBitSize) {
Nate Begemane0efc212010-07-27 18:02:18 +00005482 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
5483 BV->getValueType(0).getVectorNumElements());
5484
5485 // Due to the FP element handling below calling this routine recursively,
5486 // we can end up with a scalar-to-vector node here.
5487 if (BV->getOpcode() == ISD::SCALAR_TO_VECTOR)
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005488 return DAG.getNode(ISD::SCALAR_TO_VECTOR, BV->getDebugLoc(), VT,
5489 DAG.getNode(ISD::BITCAST, BV->getDebugLoc(),
Nate Begemane0efc212010-07-27 18:02:18 +00005490 DstEltVT, BV->getOperand(0)));
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005491
Dan Gohman475871a2008-07-27 21:46:04 +00005492 SmallVector<SDValue, 8> Ops;
Dan Gohman7f321562007-06-25 16:23:39 +00005493 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
Bob Wilsonb1303d02009-04-13 22:05:19 +00005494 SDValue Op = BV->getOperand(i);
5495 // If the vector element type is not legal, the BUILD_VECTOR operands
5496 // are promoted and implicitly truncated. Make that explicit here.
Bob Wilsonc8851652009-04-20 17:27:09 +00005497 if (Op.getValueType() != SrcEltVT)
5498 Op = DAG.getNode(ISD::TRUNCATE, BV->getDebugLoc(), SrcEltVT, Op);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005499 Ops.push_back(DAG.getNode(ISD::BITCAST, BV->getDebugLoc(),
Bob Wilsonb1303d02009-04-13 22:05:19 +00005500 DstEltVT, Op));
Gabor Greifba36cb52008-08-28 21:40:38 +00005501 AddToWorkList(Ops.back().getNode());
Chris Lattner3e104b12006-04-08 04:15:24 +00005502 }
Evan Chenga87008d2009-02-25 22:49:59 +00005503 return DAG.getNode(ISD::BUILD_VECTOR, BV->getDebugLoc(), VT,
5504 &Ops[0], Ops.size());
Chris Lattner6258fb22006-04-02 02:53:43 +00005505 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005506
Chris Lattner6258fb22006-04-02 02:53:43 +00005507 // Otherwise, we're growing or shrinking the elements. To avoid having to
5508 // handle annoying details of growing/shrinking FP values, we convert them to
5509 // int first.
Duncan Sands83ec4b62008-06-06 12:08:01 +00005510 if (SrcEltVT.isFloatingPoint()) {
Chris Lattner6258fb22006-04-02 02:53:43 +00005511 // Convert the input float vector to a int vector where the elements are the
5512 // same sizes.
Owen Anderson825b72b2009-08-11 20:47:22 +00005513 assert((SrcEltVT == MVT::f32 || SrcEltVT == MVT::f64) && "Unknown FP VT!");
Owen Anderson23b9b192009-08-12 00:36:31 +00005514 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltVT.getSizeInBits());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005515 BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
Chris Lattner6258fb22006-04-02 02:53:43 +00005516 SrcEltVT = IntVT;
5517 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005518
Chris Lattner6258fb22006-04-02 02:53:43 +00005519 // Now we know the input is an integer vector. If the output is a FP type,
5520 // convert to integer first, then to FP of the right size.
Duncan Sands83ec4b62008-06-06 12:08:01 +00005521 if (DstEltVT.isFloatingPoint()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00005522 assert((DstEltVT == MVT::f32 || DstEltVT == MVT::f64) && "Unknown FP VT!");
Owen Anderson23b9b192009-08-12 00:36:31 +00005523 EVT TmpVT = EVT::getIntegerVT(*DAG.getContext(), DstEltVT.getSizeInBits());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005524 SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
Scott Michelfdc40a02009-02-17 22:15:04 +00005525
Chris Lattner6258fb22006-04-02 02:53:43 +00005526 // Next, convert to FP elements of the same size.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005527 return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
Chris Lattner6258fb22006-04-02 02:53:43 +00005528 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005529
Chris Lattner6258fb22006-04-02 02:53:43 +00005530 // Okay, we know the src/dst types are both integers of differing types.
5531 // Handling growing first.
Duncan Sands83ec4b62008-06-06 12:08:01 +00005532 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
Chris Lattner6258fb22006-04-02 02:53:43 +00005533 if (SrcBitSize < DstBitSize) {
5534 unsigned NumInputsPerOutput = DstBitSize/SrcBitSize;
Scott Michelfdc40a02009-02-17 22:15:04 +00005535
Dan Gohman475871a2008-07-27 21:46:04 +00005536 SmallVector<SDValue, 8> Ops;
Dan Gohman7f321562007-06-25 16:23:39 +00005537 for (unsigned i = 0, e = BV->getNumOperands(); i != e;
Chris Lattner6258fb22006-04-02 02:53:43 +00005538 i += NumInputsPerOutput) {
5539 bool isLE = TLI.isLittleEndian();
Dan Gohman220a8232008-03-03 23:51:38 +00005540 APInt NewBits = APInt(DstBitSize, 0);
Chris Lattner6258fb22006-04-02 02:53:43 +00005541 bool EltIsUndef = true;
5542 for (unsigned j = 0; j != NumInputsPerOutput; ++j) {
5543 // Shift the previously computed bits over.
5544 NewBits <<= SrcBitSize;
Dan Gohman475871a2008-07-27 21:46:04 +00005545 SDValue Op = BV->getOperand(i+ (isLE ? (NumInputsPerOutput-j-1) : j));
Chris Lattner6258fb22006-04-02 02:53:43 +00005546 if (Op.getOpcode() == ISD::UNDEF) continue;
5547 EltIsUndef = false;
Scott Michelfdc40a02009-02-17 22:15:04 +00005548
Jay Foad40f8f622010-12-07 08:25:19 +00005549 NewBits |= cast<ConstantSDNode>(Op)->getAPIntValue().
Dan Gohman58c25872010-04-12 02:24:01 +00005550 zextOrTrunc(SrcBitSize).zext(DstBitSize);
Chris Lattner6258fb22006-04-02 02:53:43 +00005551 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005552
Chris Lattner6258fb22006-04-02 02:53:43 +00005553 if (EltIsUndef)
Dale Johannesene8d72302009-02-06 23:05:02 +00005554 Ops.push_back(DAG.getUNDEF(DstEltVT));
Chris Lattner6258fb22006-04-02 02:53:43 +00005555 else
5556 Ops.push_back(DAG.getConstant(NewBits, DstEltVT));
5557 }
5558
Owen Anderson23b9b192009-08-12 00:36:31 +00005559 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size());
Evan Chenga87008d2009-02-25 22:49:59 +00005560 return DAG.getNode(ISD::BUILD_VECTOR, BV->getDebugLoc(), VT,
5561 &Ops[0], Ops.size());
Chris Lattner6258fb22006-04-02 02:53:43 +00005562 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005563
Chris Lattner6258fb22006-04-02 02:53:43 +00005564 // Finally, this must be the case where we are shrinking elements: each input
5565 // turns into multiple outputs.
Evan Chengefec7512008-02-18 23:04:32 +00005566 bool isS2V = ISD::isScalarToVector(BV);
Chris Lattner6258fb22006-04-02 02:53:43 +00005567 unsigned NumOutputsPerInput = SrcBitSize/DstBitSize;
Owen Anderson23b9b192009-08-12 00:36:31 +00005568 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
5569 NumOutputsPerInput*BV->getNumOperands());
Dan Gohman475871a2008-07-27 21:46:04 +00005570 SmallVector<SDValue, 8> Ops;
Bill Wendlingb0162f52009-01-30 22:53:48 +00005571
Dan Gohman7f321562007-06-25 16:23:39 +00005572 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
Chris Lattner6258fb22006-04-02 02:53:43 +00005573 if (BV->getOperand(i).getOpcode() == ISD::UNDEF) {
5574 for (unsigned j = 0; j != NumOutputsPerInput; ++j)
Dale Johannesene8d72302009-02-06 23:05:02 +00005575 Ops.push_back(DAG.getUNDEF(DstEltVT));
Chris Lattner6258fb22006-04-02 02:53:43 +00005576 continue;
5577 }
Bill Wendlingb0162f52009-01-30 22:53:48 +00005578
Jay Foad40f8f622010-12-07 08:25:19 +00005579 APInt OpVal = cast<ConstantSDNode>(BV->getOperand(i))->
5580 getAPIntValue().zextOrTrunc(SrcBitSize);
Bill Wendlingb0162f52009-01-30 22:53:48 +00005581
Chris Lattner6258fb22006-04-02 02:53:43 +00005582 for (unsigned j = 0; j != NumOutputsPerInput; ++j) {
Jay Foad40f8f622010-12-07 08:25:19 +00005583 APInt ThisVal = OpVal.trunc(DstBitSize);
Chris Lattner6258fb22006-04-02 02:53:43 +00005584 Ops.push_back(DAG.getConstant(ThisVal, DstEltVT));
Jay Foad40f8f622010-12-07 08:25:19 +00005585 if (isS2V && i == 0 && j == 0 && ThisVal.zext(SrcBitSize) == OpVal)
Evan Chengefec7512008-02-18 23:04:32 +00005586 // Simply turn this into a SCALAR_TO_VECTOR of the new type.
Bill Wendlingb0162f52009-01-30 22:53:48 +00005587 return DAG.getNode(ISD::SCALAR_TO_VECTOR, BV->getDebugLoc(), VT,
5588 Ops[0]);
Dan Gohman220a8232008-03-03 23:51:38 +00005589 OpVal = OpVal.lshr(DstBitSize);
Chris Lattner6258fb22006-04-02 02:53:43 +00005590 }
5591
5592 // For big endian targets, swap the order of the pieces of each element.
Duncan Sands0753fc12008-02-11 10:37:04 +00005593 if (TLI.isBigEndian())
Chris Lattner6258fb22006-04-02 02:53:43 +00005594 std::reverse(Ops.end()-NumOutputsPerInput, Ops.end());
5595 }
Bill Wendlingb0162f52009-01-30 22:53:48 +00005596
Evan Chenga87008d2009-02-25 22:49:59 +00005597 return DAG.getNode(ISD::BUILD_VECTOR, BV->getDebugLoc(), VT,
5598 &Ops[0], Ops.size());
Chris Lattner6258fb22006-04-02 02:53:43 +00005599}
5600
Dan Gohman475871a2008-07-27 21:46:04 +00005601SDValue DAGCombiner::visitFADD(SDNode *N) {
5602 SDValue N0 = N->getOperand(0);
5603 SDValue N1 = N->getOperand(1);
Nate Begemana0e221d2005-10-18 00:28:13 +00005604 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5605 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005606 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005607
Dan Gohman7f321562007-06-25 16:23:39 +00005608 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00005609 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005610 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005611 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00005612 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005613
Lang Hames01806942012-06-14 20:37:15 +00005614 // fold (fadd c1, c2) -> c1 + c2
Owen Anderson825b72b2009-08-11 20:47:22 +00005615 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005616 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0, N1);
Nate Begemana0e221d2005-10-18 00:28:13 +00005617 // canonicalize constant to RHS
5618 if (N0CFP && !N1CFP)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005619 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N1, N0);
5620 // fold (fadd A, 0) -> A
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005621 if (DAG.getTarget().Options.UnsafeFPMath && N1CFP &&
5622 N1CFP->getValueAPF().isZero())
Dan Gohman760f86f2009-01-22 21:58:43 +00005623 return N0;
Bill Wendlingb0162f52009-01-30 22:53:48 +00005624 // fold (fadd A, (fneg B)) -> (fsub A, B)
Owen Andersonafd3d562012-03-06 00:29:31 +00005625 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
5626 isNegatibleForFree(N1, LegalOperations, TLI, &DAG.getTarget().Options) == 2)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005627 return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N0,
Duncan Sands25cf2272008-11-24 14:53:14 +00005628 GetNegatedExpression(N1, DAG, LegalOperations));
Bill Wendlingb0162f52009-01-30 22:53:48 +00005629 // fold (fadd (fneg A), B) -> (fsub B, A)
Owen Andersonafd3d562012-03-06 00:29:31 +00005630 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
5631 isNegatibleForFree(N0, LegalOperations, TLI, &DAG.getTarget().Options) == 2)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005632 return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N1,
Duncan Sands25cf2272008-11-24 14:53:14 +00005633 GetNegatedExpression(N0, DAG, LegalOperations));
Scott Michelfdc40a02009-02-17 22:15:04 +00005634
Chris Lattnerddae4bd2007-01-08 23:04:05 +00005635 // If allowed, fold (fadd (fadd x, c1), c2) -> (fadd x, (fadd c1, c2))
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005636 if (DAG.getTarget().Options.UnsafeFPMath && N1CFP &&
5637 N0.getOpcode() == ISD::FADD && N0.getNode()->hasOneUse() &&
5638 isa<ConstantFPSDNode>(N0.getOperand(1)))
Bill Wendlingb0162f52009-01-30 22:53:48 +00005639 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0.getOperand(0),
Bill Wendlingfc4b6772009-02-01 11:19:36 +00005640 DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
5641 N0.getOperand(1), N1));
Scott Michelfdc40a02009-02-17 22:15:04 +00005642
Lang Hamesd693caf2012-06-19 22:51:23 +00005643 // FADD -> FMA combines:
5644 if ((DAG.getTarget().Options.AllowExcessFPPrecision ||
5645 DAG.getTarget().Options.UnsafeFPMath) &&
5646 DAG.getTarget().getTargetLowering()->isFMAFasterThanMulAndAdd(VT) &&
5647 TLI.isOperationLegal(ISD::FMA, VT)) {
5648
5649 // fold (fadd (fmul x, y), z) -> (fma x, y, z)
5650 if (N0.getOpcode() == ISD::FMUL && N0->hasOneUse()) {
5651 return DAG.getNode(ISD::FMA, N->getDebugLoc(), VT,
5652 N0.getOperand(0), N0.getOperand(1), N1);
5653 }
5654
5655 // fold (fadd x, (fmul y, z)) -> (fma x, y, z)
5656 // Note: Commutes FADD operands.
5657 if (N1.getOpcode() == ISD::FMUL && N1->hasOneUse()) {
5658 return DAG.getNode(ISD::FMA, N->getDebugLoc(), VT,
5659 N1.getOperand(0), N1.getOperand(1), N0);
5660 }
5661 }
5662
Dan Gohman475871a2008-07-27 21:46:04 +00005663 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005664}
5665
Dan Gohman475871a2008-07-27 21:46:04 +00005666SDValue DAGCombiner::visitFSUB(SDNode *N) {
5667 SDValue N0 = N->getOperand(0);
5668 SDValue N1 = N->getOperand(1);
Nate Begemana0e221d2005-10-18 00:28:13 +00005669 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5670 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005671 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005672
Dan Gohman7f321562007-06-25 16:23:39 +00005673 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00005674 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005675 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005676 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00005677 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005678
Nate Begemana0e221d2005-10-18 00:28:13 +00005679 // fold (fsub c1, c2) -> c1-c2
Owen Anderson825b72b2009-08-11 20:47:22 +00005680 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlingfc4b6772009-02-01 11:19:36 +00005681 return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N0, N1);
Bill Wendlingb0162f52009-01-30 22:53:48 +00005682 // fold (fsub A, 0) -> A
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005683 if (DAG.getTarget().Options.UnsafeFPMath &&
5684 N1CFP && N1CFP->getValueAPF().isZero())
Dan Gohmana90c8e62009-01-23 19:10:37 +00005685 return N0;
Bill Wendlingb0162f52009-01-30 22:53:48 +00005686 // fold (fsub 0, B) -> -B
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005687 if (DAG.getTarget().Options.UnsafeFPMath &&
5688 N0CFP && N0CFP->getValueAPF().isZero()) {
Owen Andersonafd3d562012-03-06 00:29:31 +00005689 if (isNegatibleForFree(N1, LegalOperations, TLI, &DAG.getTarget().Options))
Duncan Sands25cf2272008-11-24 14:53:14 +00005690 return GetNegatedExpression(N1, DAG, LegalOperations);
Dan Gohman760f86f2009-01-22 21:58:43 +00005691 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
Bill Wendlingb0162f52009-01-30 22:53:48 +00005692 return DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT, N1);
Dan Gohman23ff1822007-07-02 15:48:56 +00005693 }
Bill Wendlingb0162f52009-01-30 22:53:48 +00005694 // fold (fsub A, (fneg B)) -> (fadd A, B)
Owen Andersonafd3d562012-03-06 00:29:31 +00005695 if (isNegatibleForFree(N1, LegalOperations, TLI, &DAG.getTarget().Options))
Bill Wendlingb0162f52009-01-30 22:53:48 +00005696 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0,
Duncan Sands25cf2272008-11-24 14:53:14 +00005697 GetNegatedExpression(N1, DAG, LegalOperations));
Scott Michelfdc40a02009-02-17 22:15:04 +00005698
Bill Wendling5a894342012-03-15 05:12:00 +00005699 // If 'unsafe math' is enabled, fold
Owen Anderson713e9532012-05-07 20:51:25 +00005700 // (fsub x, x) -> 0.0 &
Bill Wendling5a894342012-03-15 05:12:00 +00005701 // (fsub x, (fadd x, y)) -> (fneg y) &
5702 // (fsub x, (fadd y, x)) -> (fneg y)
5703 if (DAG.getTarget().Options.UnsafeFPMath) {
Owen Anderson713e9532012-05-07 20:51:25 +00005704 if (N0 == N1)
5705 return DAG.getConstantFP(0.0f, VT);
5706
Bill Wendling5a894342012-03-15 05:12:00 +00005707 if (N1.getOpcode() == ISD::FADD) {
5708 SDValue N10 = N1->getOperand(0);
5709 SDValue N11 = N1->getOperand(1);
5710
5711 if (N10 == N0 && isNegatibleForFree(N11, LegalOperations, TLI,
5712 &DAG.getTarget().Options))
5713 return GetNegatedExpression(N11, DAG, LegalOperations);
5714 else if (N11 == N0 && isNegatibleForFree(N10, LegalOperations, TLI,
5715 &DAG.getTarget().Options))
5716 return GetNegatedExpression(N10, DAG, LegalOperations);
5717 }
5718 }
5719
Lang Hamesd693caf2012-06-19 22:51:23 +00005720 // FSUB -> FMA combines:
5721 if ((DAG.getTarget().Options.AllowExcessFPPrecision ||
5722 DAG.getTarget().Options.UnsafeFPMath) &&
5723 DAG.getTarget().getTargetLowering()->isFMAFasterThanMulAndAdd(VT) &&
5724 TLI.isOperationLegal(ISD::FMA, VT)) {
5725
5726 // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
5727 if (N0.getOpcode() == ISD::FMUL && N0->hasOneUse()) {
5728 return DAG.getNode(ISD::FMA, N->getDebugLoc(), VT,
5729 N0.getOperand(0), N0.getOperand(1),
5730 DAG.getNode(ISD::FNEG, N1->getDebugLoc(), VT, N1));
5731 }
5732
5733 // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
5734 // Note: Commutes FSUB operands.
5735 if (N1.getOpcode() == ISD::FMUL && N1->hasOneUse()) {
5736 return DAG.getNode(ISD::FMA, N->getDebugLoc(), VT,
5737 DAG.getNode(ISD::FNEG, N1->getDebugLoc(), VT,
5738 N1.getOperand(0)),
5739 N1.getOperand(1), N0);
5740 }
5741 }
5742
Dan Gohman475871a2008-07-27 21:46:04 +00005743 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005744}
5745
Dan Gohman475871a2008-07-27 21:46:04 +00005746SDValue DAGCombiner::visitFMUL(SDNode *N) {
5747 SDValue N0 = N->getOperand(0);
5748 SDValue N1 = N->getOperand(1);
Nate Begeman11af4ea2005-10-17 20:40:11 +00005749 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5750 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005751 EVT VT = N->getValueType(0);
Owen Andersonafd3d562012-03-06 00:29:31 +00005752 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Chris Lattner01b3d732005-09-28 22:28:18 +00005753
Dan Gohman7f321562007-06-25 16:23:39 +00005754 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00005755 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005756 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005757 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00005758 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005759
Nate Begeman11af4ea2005-10-17 20:40:11 +00005760 // fold (fmul c1, c2) -> c1*c2
Owen Anderson825b72b2009-08-11 20:47:22 +00005761 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005762 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N0, N1);
Nate Begeman11af4ea2005-10-17 20:40:11 +00005763 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00005764 if (N0CFP && !N1CFP)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005765 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N1, N0);
5766 // fold (fmul A, 0) -> 0
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005767 if (DAG.getTarget().Options.UnsafeFPMath &&
5768 N1CFP && N1CFP->getValueAPF().isZero())
Dan Gohman760f86f2009-01-22 21:58:43 +00005769 return N1;
Dan Gohman77b81fe2009-06-04 17:12:12 +00005770 // fold (fmul A, 0) -> 0, vector edition.
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005771 if (DAG.getTarget().Options.UnsafeFPMath &&
5772 ISD::isBuildVectorAllZeros(N1.getNode()))
Dan Gohman77b81fe2009-06-04 17:12:12 +00005773 return N1;
Owen Anderson363e4b92012-05-02 21:32:35 +00005774 // fold (fmul A, 1.0) -> A
5775 if (N1CFP && N1CFP->isExactlyValue(1.0))
5776 return N0;
Nate Begeman11af4ea2005-10-17 20:40:11 +00005777 // fold (fmul X, 2.0) -> (fadd X, X)
5778 if (N1CFP && N1CFP->isExactlyValue(+2.0))
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005779 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0, N0);
Dan Gohmaneb1fedc2009-08-10 16:50:32 +00005780 // fold (fmul X, -1.0) -> (fneg X)
Chris Lattner29446522007-05-14 22:04:50 +00005781 if (N1CFP && N1CFP->isExactlyValue(-1.0))
Dan Gohman760f86f2009-01-22 21:58:43 +00005782 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005783 return DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005784
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005785 // fold (fmul (fneg X), (fneg Y)) -> (fmul X, Y)
Owen Andersonafd3d562012-03-06 00:29:31 +00005786 if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, TLI,
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005787 &DAG.getTarget().Options)) {
Owen Andersonafd3d562012-03-06 00:29:31 +00005788 if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, TLI,
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005789 &DAG.getTarget().Options)) {
Chris Lattner29446522007-05-14 22:04:50 +00005790 // Both can be negated for free, check to see if at least one is cheaper
5791 // negated.
5792 if (LHSNeg == 2 || RHSNeg == 2)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005793 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
Duncan Sands25cf2272008-11-24 14:53:14 +00005794 GetNegatedExpression(N0, DAG, LegalOperations),
5795 GetNegatedExpression(N1, DAG, LegalOperations));
Chris Lattner29446522007-05-14 22:04:50 +00005796 }
5797 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005798
Chris Lattnerddae4bd2007-01-08 23:04:05 +00005799 // If allowed, fold (fmul (fmul x, c1), c2) -> (fmul x, (fmul c1, c2))
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005800 if (DAG.getTarget().Options.UnsafeFPMath &&
5801 N1CFP && N0.getOpcode() == ISD::FMUL &&
Gabor Greifba36cb52008-08-28 21:40:38 +00005802 N0.getNode()->hasOneUse() && isa<ConstantFPSDNode>(N0.getOperand(1)))
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005803 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N0.getOperand(0),
Scott Michelfdc40a02009-02-17 22:15:04 +00005804 DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
Dale Johannesende064702009-02-06 21:50:26 +00005805 N0.getOperand(1), N1));
Scott Michelfdc40a02009-02-17 22:15:04 +00005806
Dan Gohman475871a2008-07-27 21:46:04 +00005807 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005808}
5809
Owen Anderson062c0a52012-05-02 22:17:40 +00005810SDValue DAGCombiner::visitFMA(SDNode *N) {
5811 SDValue N0 = N->getOperand(0);
5812 SDValue N1 = N->getOperand(1);
5813 SDValue N2 = N->getOperand(2);
5814 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5815 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
5816 EVT VT = N->getValueType(0);
5817
5818 if (N0CFP && N0CFP->isExactlyValue(1.0))
5819 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N1, N2);
5820 if (N1CFP && N1CFP->isExactlyValue(1.0))
5821 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0, N2);
5822
Owen Anderson85ef6f42012-05-30 18:50:39 +00005823 // Canonicalize (fma c, x, y) -> (fma x, c, y)
Owen Andersonf917d202012-05-30 18:54:50 +00005824 if (N0CFP && !N1CFP)
Owen Anderson85ef6f42012-05-30 18:50:39 +00005825 return DAG.getNode(ISD::FMA, N->getDebugLoc(), VT, N1, N0, N2);
5826
Owen Anderson062c0a52012-05-02 22:17:40 +00005827 return SDValue();
5828}
5829
Dan Gohman475871a2008-07-27 21:46:04 +00005830SDValue DAGCombiner::visitFDIV(SDNode *N) {
5831 SDValue N0 = N->getOperand(0);
5832 SDValue N1 = N->getOperand(1);
Nate Begemana148d982006-01-18 22:35:16 +00005833 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5834 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005835 EVT VT = N->getValueType(0);
Owen Andersonafd3d562012-03-06 00:29:31 +00005836 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Chris Lattner01b3d732005-09-28 22:28:18 +00005837
Dan Gohman7f321562007-06-25 16:23:39 +00005838 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00005839 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005840 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005841 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00005842 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005843
Nate Begemana148d982006-01-18 22:35:16 +00005844 // fold (fdiv c1, c2) -> c1/c2
Owen Anderson825b72b2009-08-11 20:47:22 +00005845 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005846 return DAG.getNode(ISD::FDIV, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00005847
Duncan Sands3ef3fcf2012-04-08 18:08:12 +00005848 // fold (fdiv X, c2) -> fmul X, 1/c2 if losing precision is acceptable.
5849 if (N1CFP && VT != MVT::ppcf128 && DAG.getTarget().Options.UnsafeFPMath) {
Duncan Sands961d6662012-04-07 20:04:00 +00005850 // Compute the reciprocal 1.0 / c2.
5851 APFloat N1APF = N1CFP->getValueAPF();
5852 APFloat Recip(N1APF.getSemantics(), 1); // 1.0
5853 APFloat::opStatus st = Recip.divide(N1APF, APFloat::rmNearestTiesToEven);
Duncan Sands507bb7a2012-04-10 20:35:27 +00005854 // Only do the transform if the reciprocal is a legal fp immediate that
5855 // isn't too nasty (eg NaN, denormal, ...).
5856 if ((st == APFloat::opOK || st == APFloat::opInexact) && // Not too nasty
Anton Korobeynikov999821c2012-04-10 13:22:49 +00005857 (!LegalOperations ||
5858 // FIXME: custom lowering of ConstantFP might fail (see e.g. ARM
5859 // backend)... we should handle this gracefully after Legalize.
5860 // TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT) ||
5861 TLI.isOperationLegal(llvm::ISD::ConstantFP, VT) ||
5862 TLI.isFPImmLegal(Recip, VT)))
Duncan Sands961d6662012-04-07 20:04:00 +00005863 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N0,
5864 DAG.getConstantFP(Recip, VT));
5865 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005866
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005867 // (fdiv (fneg X), (fneg Y)) -> (fdiv X, Y)
Owen Andersonafd3d562012-03-06 00:29:31 +00005868 if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, TLI,
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005869 &DAG.getTarget().Options)) {
Owen Andersonafd3d562012-03-06 00:29:31 +00005870 if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, TLI,
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005871 &DAG.getTarget().Options)) {
Chris Lattner29446522007-05-14 22:04:50 +00005872 // Both can be negated for free, check to see if at least one is cheaper
5873 // negated.
5874 if (LHSNeg == 2 || RHSNeg == 2)
Scott Michelfdc40a02009-02-17 22:15:04 +00005875 return DAG.getNode(ISD::FDIV, N->getDebugLoc(), VT,
Duncan Sands25cf2272008-11-24 14:53:14 +00005876 GetNegatedExpression(N0, DAG, LegalOperations),
5877 GetNegatedExpression(N1, DAG, LegalOperations));
Chris Lattner29446522007-05-14 22:04:50 +00005878 }
5879 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005880
Dan Gohman475871a2008-07-27 21:46:04 +00005881 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005882}
5883
Dan Gohman475871a2008-07-27 21:46:04 +00005884SDValue DAGCombiner::visitFREM(SDNode *N) {
5885 SDValue N0 = N->getOperand(0);
5886 SDValue N1 = N->getOperand(1);
Nate Begemana148d982006-01-18 22:35:16 +00005887 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5888 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005889 EVT VT = N->getValueType(0);
Chris Lattner01b3d732005-09-28 22:28:18 +00005890
Nate Begemana148d982006-01-18 22:35:16 +00005891 // fold (frem c1, c2) -> fmod(c1,c2)
Owen Anderson825b72b2009-08-11 20:47:22 +00005892 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005893 return DAG.getNode(ISD::FREM, N->getDebugLoc(), VT, N0, N1);
Dan Gohman7f321562007-06-25 16:23:39 +00005894
Dan Gohman475871a2008-07-27 21:46:04 +00005895 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005896}
5897
Dan Gohman475871a2008-07-27 21:46:04 +00005898SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
5899 SDValue N0 = N->getOperand(0);
5900 SDValue N1 = N->getOperand(1);
Chris Lattner12d83032006-03-05 05:30:57 +00005901 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5902 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005903 EVT VT = N->getValueType(0);
Chris Lattner12d83032006-03-05 05:30:57 +00005904
Owen Anderson825b72b2009-08-11 20:47:22 +00005905 if (N0CFP && N1CFP && VT != MVT::ppcf128) // Constant fold
Bill Wendlingfc4b6772009-02-01 11:19:36 +00005906 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00005907
Chris Lattner12d83032006-03-05 05:30:57 +00005908 if (N1CFP) {
Dale Johannesene6c17422007-08-26 01:18:27 +00005909 const APFloat& V = N1CFP->getValueAPF();
Chris Lattner12d83032006-03-05 05:30:57 +00005910 // copysign(x, c1) -> fabs(x) iff ispos(c1)
5911 // copysign(x, c1) -> fneg(fabs(x)) iff isneg(c1)
Dan Gohman760f86f2009-01-22 21:58:43 +00005912 if (!V.isNegative()) {
5913 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005914 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0);
Dan Gohman760f86f2009-01-22 21:58:43 +00005915 } else {
5916 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005917 return DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT,
Bill Wendling9729c5a2009-01-31 03:12:48 +00005918 DAG.getNode(ISD::FABS, N0.getDebugLoc(), VT, N0));
Dan Gohman760f86f2009-01-22 21:58:43 +00005919 }
Chris Lattner12d83032006-03-05 05:30:57 +00005920 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005921
Chris Lattner12d83032006-03-05 05:30:57 +00005922 // copysign(fabs(x), y) -> copysign(x, y)
5923 // copysign(fneg(x), y) -> copysign(x, y)
5924 // copysign(copysign(x,z), y) -> copysign(x, y)
5925 if (N0.getOpcode() == ISD::FABS || N0.getOpcode() == ISD::FNEG ||
5926 N0.getOpcode() == ISD::FCOPYSIGN)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005927 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
5928 N0.getOperand(0), N1);
Chris Lattner12d83032006-03-05 05:30:57 +00005929
5930 // copysign(x, abs(y)) -> abs(x)
5931 if (N1.getOpcode() == ISD::FABS)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005932 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005933
Chris Lattner12d83032006-03-05 05:30:57 +00005934 // copysign(x, copysign(y,z)) -> copysign(x, z)
5935 if (N1.getOpcode() == ISD::FCOPYSIGN)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005936 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
5937 N0, N1.getOperand(1));
Scott Michelfdc40a02009-02-17 22:15:04 +00005938
Chris Lattner12d83032006-03-05 05:30:57 +00005939 // copysign(x, fp_extend(y)) -> copysign(x, y)
5940 // copysign(x, fp_round(y)) -> copysign(x, y)
5941 if (N1.getOpcode() == ISD::FP_EXTEND || N1.getOpcode() == ISD::FP_ROUND)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005942 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
5943 N0, N1.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00005944
Dan Gohman475871a2008-07-27 21:46:04 +00005945 return SDValue();
Chris Lattner12d83032006-03-05 05:30:57 +00005946}
5947
Dan Gohman475871a2008-07-27 21:46:04 +00005948SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
5949 SDValue N0 = N->getOperand(0);
Nate Begeman646d7e22005-09-02 21:18:40 +00005950 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00005951 EVT VT = N->getValueType(0);
5952 EVT OpVT = N0.getValueType();
Chris Lattnercda88752008-06-26 00:16:49 +00005953
Nate Begeman1d4d4142005-09-01 00:19:25 +00005954 // fold (sint_to_fp c1) -> c1fp
Stuart Hastings7e334182011-03-02 19:36:30 +00005955 if (N0C && OpVT != MVT::ppcf128 &&
5956 // ...but only if the target supports immediate floating-point values
Eli Friedman50185242011-11-12 00:35:34 +00005957 (!LegalOperations ||
Evan Cheng9568e5c2011-06-21 06:01:08 +00005958 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT)))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005959 return DAG.getNode(ISD::SINT_TO_FP, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005960
Chris Lattnercda88752008-06-26 00:16:49 +00005961 // If the input is a legal type, and SINT_TO_FP is not legal on this target,
5962 // but UINT_TO_FP is legal on this target, try to convert.
Dan Gohmanf560ffa2009-01-28 17:46:25 +00005963 if (!TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT) &&
5964 TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00005965 // If the sign bit is known to be zero, we can change this to UINT_TO_FP.
Chris Lattnercda88752008-06-26 00:16:49 +00005966 if (DAG.SignBitIsZero(N0))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005967 return DAG.getNode(ISD::UINT_TO_FP, N->getDebugLoc(), VT, N0);
Chris Lattnercda88752008-06-26 00:16:49 +00005968 }
Bill Wendling0225a1d2009-01-30 23:15:49 +00005969
Dan Gohman475871a2008-07-27 21:46:04 +00005970 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005971}
5972
Dan Gohman475871a2008-07-27 21:46:04 +00005973SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
5974 SDValue N0 = N->getOperand(0);
Nate Begeman646d7e22005-09-02 21:18:40 +00005975 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00005976 EVT VT = N->getValueType(0);
5977 EVT OpVT = N0.getValueType();
Nate Begemana148d982006-01-18 22:35:16 +00005978
Nate Begeman1d4d4142005-09-01 00:19:25 +00005979 // fold (uint_to_fp c1) -> c1fp
Stuart Hastings7e334182011-03-02 19:36:30 +00005980 if (N0C && OpVT != MVT::ppcf128 &&
5981 // ...but only if the target supports immediate floating-point values
Eli Friedman50185242011-11-12 00:35:34 +00005982 (!LegalOperations ||
Evan Cheng9568e5c2011-06-21 06:01:08 +00005983 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT)))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005984 return DAG.getNode(ISD::UINT_TO_FP, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005985
Chris Lattnercda88752008-06-26 00:16:49 +00005986 // If the input is a legal type, and UINT_TO_FP is not legal on this target,
5987 // but SINT_TO_FP is legal on this target, try to convert.
Dan Gohmanf560ffa2009-01-28 17:46:25 +00005988 if (!TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT) &&
5989 TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00005990 // If the sign bit is known to be zero, we can change this to SINT_TO_FP.
Chris Lattnercda88752008-06-26 00:16:49 +00005991 if (DAG.SignBitIsZero(N0))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005992 return DAG.getNode(ISD::SINT_TO_FP, N->getDebugLoc(), VT, N0);
Chris Lattnercda88752008-06-26 00:16:49 +00005993 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005994
Dan Gohman475871a2008-07-27 21:46:04 +00005995 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005996}
5997
Dan Gohman475871a2008-07-27 21:46:04 +00005998SDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) {
5999 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00006000 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006001 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006002
Nate Begeman1d4d4142005-09-01 00:19:25 +00006003 // fold (fp_to_sint c1fp) -> c1
Nate Begeman646d7e22005-09-02 21:18:40 +00006004 if (N0CFP)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006005 return DAG.getNode(ISD::FP_TO_SINT, N->getDebugLoc(), VT, N0);
6006
Dan Gohman475871a2008-07-27 21:46:04 +00006007 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006008}
6009
Dan Gohman475871a2008-07-27 21:46:04 +00006010SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) {
6011 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00006012 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006013 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006014
Nate Begeman1d4d4142005-09-01 00:19:25 +00006015 // fold (fp_to_uint c1fp) -> c1
Owen Anderson825b72b2009-08-11 20:47:22 +00006016 if (N0CFP && VT != MVT::ppcf128)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006017 return DAG.getNode(ISD::FP_TO_UINT, N->getDebugLoc(), VT, N0);
6018
Dan Gohman475871a2008-07-27 21:46:04 +00006019 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006020}
6021
Dan Gohman475871a2008-07-27 21:46:04 +00006022SDValue DAGCombiner::visitFP_ROUND(SDNode *N) {
6023 SDValue N0 = N->getOperand(0);
6024 SDValue N1 = N->getOperand(1);
Nate Begemana148d982006-01-18 22:35:16 +00006025 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006026 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006027
Nate Begeman1d4d4142005-09-01 00:19:25 +00006028 // fold (fp_round c1fp) -> c1fp
Owen Anderson825b72b2009-08-11 20:47:22 +00006029 if (N0CFP && N0.getValueType() != MVT::ppcf128)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006030 return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00006031
Chris Lattner79dbea52006-03-13 06:26:26 +00006032 // fold (fp_round (fp_extend x)) -> x
6033 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType())
6034 return N0.getOperand(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006035
Chris Lattner0aa5e6f2008-01-24 06:45:35 +00006036 // fold (fp_round (fp_round x)) -> (fp_round x)
6037 if (N0.getOpcode() == ISD::FP_ROUND) {
6038 // This is a value preserving truncation if both round's are.
6039 bool IsTrunc = N->getConstantOperandVal(1) == 1 &&
Gabor Greifba36cb52008-08-28 21:40:38 +00006040 N0.getNode()->getConstantOperandVal(1) == 1;
Bill Wendling0225a1d2009-01-30 23:15:49 +00006041 return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), VT, N0.getOperand(0),
Chris Lattner0aa5e6f2008-01-24 06:45:35 +00006042 DAG.getIntPtrConstant(IsTrunc));
6043 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006044
Chris Lattner79dbea52006-03-13 06:26:26 +00006045 // fold (fp_round (copysign X, Y)) -> (copysign (fp_round X), Y)
Gabor Greifba36cb52008-08-28 21:40:38 +00006046 if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse()) {
Bill Wendling0225a1d2009-01-30 23:15:49 +00006047 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, N0.getDebugLoc(), VT,
6048 N0.getOperand(0), N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00006049 AddToWorkList(Tmp.getNode());
Bill Wendling0225a1d2009-01-30 23:15:49 +00006050 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
6051 Tmp, N0.getOperand(1));
Chris Lattner79dbea52006-03-13 06:26:26 +00006052 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006053
Dan Gohman475871a2008-07-27 21:46:04 +00006054 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006055}
6056
Dan Gohman475871a2008-07-27 21:46:04 +00006057SDValue DAGCombiner::visitFP_ROUND_INREG(SDNode *N) {
6058 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00006059 EVT VT = N->getValueType(0);
6060 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
Nate Begeman646d7e22005-09-02 21:18:40 +00006061 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006062
Nate Begeman1d4d4142005-09-01 00:19:25 +00006063 // fold (fp_round_inreg c1fp) -> c1fp
Chris Lattner2392ae72010-04-15 04:48:01 +00006064 if (N0CFP && isTypeLegal(EVT)) {
Dan Gohman4fbd7962008-09-12 18:08:03 +00006065 SDValue Round = DAG.getConstantFP(*N0CFP->getConstantFPValue(), EVT);
Bill Wendling0225a1d2009-01-30 23:15:49 +00006066 return DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), VT, Round);
Nate Begeman1d4d4142005-09-01 00:19:25 +00006067 }
Bill Wendling0225a1d2009-01-30 23:15:49 +00006068
Dan Gohman475871a2008-07-27 21:46:04 +00006069 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006070}
6071
Dan Gohman475871a2008-07-27 21:46:04 +00006072SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
6073 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00006074 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006075 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006076
Chris Lattner5938bef2007-12-29 06:55:23 +00006077 // If this is fp_round(fpextend), don't fold it, allow ourselves to be folded.
Scott Michelfdc40a02009-02-17 22:15:04 +00006078 if (N->hasOneUse() &&
Dan Gohmane7852d02009-01-26 04:35:06 +00006079 N->use_begin()->getOpcode() == ISD::FP_ROUND)
Dan Gohman475871a2008-07-27 21:46:04 +00006080 return SDValue();
Chris Lattner0bd48932008-01-17 07:00:52 +00006081
Nate Begeman1d4d4142005-09-01 00:19:25 +00006082 // fold (fp_extend c1fp) -> c1fp
Owen Anderson825b72b2009-08-11 20:47:22 +00006083 if (N0CFP && VT != MVT::ppcf128)
Bill Wendling0225a1d2009-01-30 23:15:49 +00006084 return DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), VT, N0);
Chris Lattner0bd48932008-01-17 07:00:52 +00006085
6086 // Turn fp_extend(fp_round(X, 1)) -> x since the fp_round doesn't affect the
6087 // value of X.
Gabor Greif12632d22008-08-30 19:29:20 +00006088 if (N0.getOpcode() == ISD::FP_ROUND
6089 && N0.getNode()->getConstantOperandVal(1) == 1) {
Dan Gohman475871a2008-07-27 21:46:04 +00006090 SDValue In = N0.getOperand(0);
Chris Lattner0bd48932008-01-17 07:00:52 +00006091 if (In.getValueType() == VT) return In;
Duncan Sands8e4eb092008-06-08 20:54:56 +00006092 if (VT.bitsLT(In.getValueType()))
Bill Wendling0225a1d2009-01-30 23:15:49 +00006093 return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), VT,
6094 In, N0.getOperand(1));
6095 return DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), VT, In);
Chris Lattner0bd48932008-01-17 07:00:52 +00006096 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006097
Chris Lattner0bd48932008-01-17 07:00:52 +00006098 // fold (fpext (load x)) -> (fpext (fptrunc (extload x)))
Gabor Greifba36cb52008-08-28 21:40:38 +00006099 if (ISD::isNON_EXTLoad(N0.getNode()) && N0.hasOneUse() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00006100 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00006101 TLI.isLoadExtLegal(ISD::EXTLOAD, N0.getValueType()))) {
Evan Cheng466685d2006-10-09 20:57:25 +00006102 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00006103 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, N->getDebugLoc(), VT,
Bill Wendling0225a1d2009-01-30 23:15:49 +00006104 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00006105 LN0->getBasePtr(), LN0->getPointerInfo(),
Duncan Sands25cf2272008-11-24 14:53:14 +00006106 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00006107 LN0->isVolatile(), LN0->isNonTemporal(),
6108 LN0->getAlignment());
Chris Lattnere564dbb2006-05-05 21:34:35 +00006109 CombineTo(N, ExtLoad);
Bill Wendling0225a1d2009-01-30 23:15:49 +00006110 CombineTo(N0.getNode(),
6111 DAG.getNode(ISD::FP_ROUND, N0.getDebugLoc(),
6112 N0.getValueType(), ExtLoad, DAG.getIntPtrConstant(1)),
Chris Lattnere564dbb2006-05-05 21:34:35 +00006113 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00006114 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattnere564dbb2006-05-05 21:34:35 +00006115 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00006116
Dan Gohman475871a2008-07-27 21:46:04 +00006117 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006118}
6119
Dan Gohman475871a2008-07-27 21:46:04 +00006120SDValue DAGCombiner::visitFNEG(SDNode *N) {
6121 SDValue N0 = N->getOperand(0);
Anton Korobeynikov2bcf60a2009-10-20 21:37:45 +00006122 EVT VT = N->getValueType(0);
Nate Begemana148d982006-01-18 22:35:16 +00006123
Owen Andersonafd3d562012-03-06 00:29:31 +00006124 if (isNegatibleForFree(N0, LegalOperations, DAG.getTargetLoweringInfo(),
6125 &DAG.getTarget().Options))
Duncan Sands25cf2272008-11-24 14:53:14 +00006126 return GetNegatedExpression(N0, DAG, LegalOperations);
Dan Gohman23ff1822007-07-02 15:48:56 +00006127
Chris Lattner3bd39d42008-01-27 17:42:27 +00006128 // Transform fneg(bitconvert(x)) -> bitconvert(x^sign) to avoid loading
6129 // constant pool values.
Owen Anderson29f60f32012-04-02 22:10:29 +00006130 if (!TLI.isFNegFree(VT) && N0.getOpcode() == ISD::BITCAST &&
Anton Korobeynikov2bcf60a2009-10-20 21:37:45 +00006131 !VT.isVector() &&
6132 N0.getNode()->hasOneUse() &&
6133 N0.getOperand(0).getValueType().isInteger()) {
Dan Gohman475871a2008-07-27 21:46:04 +00006134 SDValue Int = N0.getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00006135 EVT IntVT = Int.getValueType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00006136 if (IntVT.isInteger() && !IntVT.isVector()) {
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00006137 Int = DAG.getNode(ISD::XOR, N0.getDebugLoc(), IntVT, Int,
6138 DAG.getConstant(APInt::getSignBit(IntVT.getSizeInBits()), IntVT));
Gabor Greifba36cb52008-08-28 21:40:38 +00006139 AddToWorkList(Int.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006140 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(),
Anton Korobeynikov2bcf60a2009-10-20 21:37:45 +00006141 VT, Int);
Chris Lattner3bd39d42008-01-27 17:42:27 +00006142 }
6143 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006144
Dan Gohman475871a2008-07-27 21:46:04 +00006145 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006146}
6147
Dan Gohman475871a2008-07-27 21:46:04 +00006148SDValue DAGCombiner::visitFABS(SDNode *N) {
6149 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00006150 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00006151 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00006152
Nate Begeman1d4d4142005-09-01 00:19:25 +00006153 // fold (fabs c1) -> fabs(c1)
Owen Anderson825b72b2009-08-11 20:47:22 +00006154 if (N0CFP && VT != MVT::ppcf128)
Bill Wendlingc0debad2009-01-30 23:27:35 +00006155 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00006156 // fold (fabs (fabs x)) -> (fabs x)
Chris Lattner12d83032006-03-05 05:30:57 +00006157 if (N0.getOpcode() == ISD::FABS)
Nate Begeman83e75ec2005-09-06 04:43:02 +00006158 return N->getOperand(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00006159 // fold (fabs (fneg x)) -> (fabs x)
Chris Lattner12d83032006-03-05 05:30:57 +00006160 // fold (fabs (fcopysign x, y)) -> (fabs x)
6161 if (N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FCOPYSIGN)
Bill Wendlingc0debad2009-01-30 23:27:35 +00006162 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00006163
Chris Lattner3bd39d42008-01-27 17:42:27 +00006164 // Transform fabs(bitconvert(x)) -> bitconvert(x&~sign) to avoid loading
6165 // constant pool values.
Owen Anderson29f60f32012-04-02 22:10:29 +00006166 if (!TLI.isFAbsFree(VT) &&
6167 N0.getOpcode() == ISD::BITCAST && N0.getNode()->hasOneUse() &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00006168 N0.getOperand(0).getValueType().isInteger() &&
6169 !N0.getOperand(0).getValueType().isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00006170 SDValue Int = N0.getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00006171 EVT IntVT = Int.getValueType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00006172 if (IntVT.isInteger() && !IntVT.isVector()) {
Scott Michelfdc40a02009-02-17 22:15:04 +00006173 Int = DAG.getNode(ISD::AND, N0.getDebugLoc(), IntVT, Int,
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00006174 DAG.getConstant(~APInt::getSignBit(IntVT.getSizeInBits()), IntVT));
Gabor Greifba36cb52008-08-28 21:40:38 +00006175 AddToWorkList(Int.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006176 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(),
Bill Wendlingc0debad2009-01-30 23:27:35 +00006177 N->getValueType(0), Int);
Chris Lattner3bd39d42008-01-27 17:42:27 +00006178 }
6179 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006180
Dan Gohman475871a2008-07-27 21:46:04 +00006181 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00006182}
6183
Dan Gohman475871a2008-07-27 21:46:04 +00006184SDValue DAGCombiner::visitBRCOND(SDNode *N) {
6185 SDValue Chain = N->getOperand(0);
6186 SDValue N1 = N->getOperand(1);
6187 SDValue N2 = N->getOperand(2);
Scott Michelfdc40a02009-02-17 22:15:04 +00006188
Dan Gohmane0f06c72009-11-17 00:47:23 +00006189 // If N is a constant we could fold this into a fallthrough or unconditional
6190 // branch. However that doesn't happen very often in normal code, because
6191 // Instcombine/SimplifyCFG should have handled the available opportunities.
6192 // If we did this folding here, it would be necessary to update the
6193 // MachineBasicBlock CFG, which is awkward.
6194
Nate Begeman750ac1b2006-02-01 07:19:44 +00006195 // fold a brcond with a setcc condition into a BR_CC node if BR_CC is legal
6196 // on the target.
Scott Michelfdc40a02009-02-17 22:15:04 +00006197 if (N1.getOpcode() == ISD::SETCC &&
Owen Anderson825b72b2009-08-11 20:47:22 +00006198 TLI.isOperationLegalOrCustom(ISD::BR_CC, MVT::Other)) {
6199 return DAG.getNode(ISD::BR_CC, N->getDebugLoc(), MVT::Other,
Bill Wendlingc0debad2009-01-30 23:27:35 +00006200 Chain, N1.getOperand(2),
Nate Begeman750ac1b2006-02-01 07:19:44 +00006201 N1.getOperand(0), N1.getOperand(1), N2);
6202 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006203
Evan Cheng2a135ae2010-10-04 22:41:01 +00006204 if ((N1.hasOneUse() && N1.getOpcode() == ISD::SRL) ||
6205 ((N1.getOpcode() == ISD::TRUNCATE && N1.hasOneUse()) &&
6206 (N1.getOperand(0).hasOneUse() &&
6207 N1.getOperand(0).getOpcode() == ISD::SRL))) {
6208 SDNode *Trunc = 0;
6209 if (N1.getOpcode() == ISD::TRUNCATE) {
6210 // Look pass the truncate.
6211 Trunc = N1.getNode();
6212 N1 = N1.getOperand(0);
6213 }
Evan Chengd40d03e2010-01-06 19:38:29 +00006214
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006215 // Match this pattern so that we can generate simpler code:
6216 //
6217 // %a = ...
6218 // %b = and i32 %a, 2
6219 // %c = srl i32 %b, 1
6220 // brcond i32 %c ...
6221 //
6222 // into
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006223 //
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006224 // %a = ...
Evan Chengd40d03e2010-01-06 19:38:29 +00006225 // %b = and i32 %a, 2
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006226 // %c = setcc eq %b, 0
6227 // brcond %c ...
6228 //
6229 // This applies only when the AND constant value has one bit set and the
6230 // SRL constant is equal to the log2 of the AND constant. The back-end is
6231 // smart enough to convert the result into a TEST/JMP sequence.
6232 SDValue Op0 = N1.getOperand(0);
6233 SDValue Op1 = N1.getOperand(1);
6234
6235 if (Op0.getOpcode() == ISD::AND &&
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006236 Op1.getOpcode() == ISD::Constant) {
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006237 SDValue AndOp1 = Op0.getOperand(1);
6238
6239 if (AndOp1.getOpcode() == ISD::Constant) {
6240 const APInt &AndConst = cast<ConstantSDNode>(AndOp1)->getAPIntValue();
6241
6242 if (AndConst.isPowerOf2() &&
6243 cast<ConstantSDNode>(Op1)->getAPIntValue()==AndConst.logBase2()) {
6244 SDValue SetCC =
6245 DAG.getSetCC(N->getDebugLoc(),
6246 TLI.getSetCCResultType(Op0.getValueType()),
6247 Op0, DAG.getConstant(0, Op0.getValueType()),
6248 ISD::SETNE);
6249
Evan Chengd40d03e2010-01-06 19:38:29 +00006250 SDValue NewBRCond = DAG.getNode(ISD::BRCOND, N->getDebugLoc(),
6251 MVT::Other, Chain, SetCC, N2);
6252 // Don't add the new BRCond into the worklist or else SimplifySelectCC
6253 // will convert it back to (X & C1) >> C2.
6254 CombineTo(N, NewBRCond, false);
6255 // Truncate is dead.
6256 if (Trunc) {
6257 removeFromWorkList(Trunc);
6258 DAG.DeleteNode(Trunc);
6259 }
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006260 // Replace the uses of SRL with SETCC
Evan Cheng2c755ba2010-02-27 07:36:59 +00006261 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006262 DAG.ReplaceAllUsesOfValueWith(N1, SetCC);
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006263 removeFromWorkList(N1.getNode());
6264 DAG.DeleteNode(N1.getNode());
Evan Chengd40d03e2010-01-06 19:38:29 +00006265 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006266 }
6267 }
6268 }
Evan Cheng2a135ae2010-10-04 22:41:01 +00006269
6270 if (Trunc)
6271 // Restore N1 if the above transformation doesn't match.
6272 N1 = N->getOperand(1);
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006273 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006274
Evan Cheng2c755ba2010-02-27 07:36:59 +00006275 // Transform br(xor(x, y)) -> br(x != y)
6276 // Transform br(xor(xor(x,y), 1)) -> br (x == y)
6277 if (N1.hasOneUse() && N1.getOpcode() == ISD::XOR) {
6278 SDNode *TheXor = N1.getNode();
6279 SDValue Op0 = TheXor->getOperand(0);
6280 SDValue Op1 = TheXor->getOperand(1);
6281 if (Op0.getOpcode() == Op1.getOpcode()) {
6282 // Avoid missing important xor optimizations.
6283 SDValue Tmp = visitXOR(TheXor);
Bill Wendling86c5abb2010-04-20 01:25:01 +00006284 if (Tmp.getNode() && Tmp.getNode() != TheXor) {
Evan Cheng2c755ba2010-02-27 07:36:59 +00006285 DEBUG(dbgs() << "\nReplacing.8 ";
6286 TheXor->dump(&DAG);
6287 dbgs() << "\nWith: ";
6288 Tmp.getNode()->dump(&DAG);
6289 dbgs() << '\n');
6290 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006291 DAG.ReplaceAllUsesOfValueWith(N1, Tmp);
Evan Cheng2c755ba2010-02-27 07:36:59 +00006292 removeFromWorkList(TheXor);
6293 DAG.DeleteNode(TheXor);
6294 return DAG.getNode(ISD::BRCOND, N->getDebugLoc(),
6295 MVT::Other, Chain, Tmp, N2);
6296 }
6297 }
6298
6299 if (Op0.getOpcode() != ISD::SETCC && Op1.getOpcode() != ISD::SETCC) {
6300 bool Equal = false;
6301 if (ConstantSDNode *RHSCI = dyn_cast<ConstantSDNode>(Op0))
6302 if (RHSCI->getAPIntValue() == 1 && Op0.hasOneUse() &&
6303 Op0.getOpcode() == ISD::XOR) {
6304 TheXor = Op0.getNode();
6305 Equal = true;
6306 }
6307
Evan Cheng2a135ae2010-10-04 22:41:01 +00006308 EVT SetCCVT = N1.getValueType();
Evan Cheng2c755ba2010-02-27 07:36:59 +00006309 if (LegalTypes)
6310 SetCCVT = TLI.getSetCCResultType(SetCCVT);
6311 SDValue SetCC = DAG.getSetCC(TheXor->getDebugLoc(),
6312 SetCCVT,
6313 Op0, Op1,
6314 Equal ? ISD::SETEQ : ISD::SETNE);
6315 // Replace the uses of XOR with SETCC
6316 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006317 DAG.ReplaceAllUsesOfValueWith(N1, SetCC);
Evan Cheng2a135ae2010-10-04 22:41:01 +00006318 removeFromWorkList(N1.getNode());
6319 DAG.DeleteNode(N1.getNode());
Evan Cheng2c755ba2010-02-27 07:36:59 +00006320 return DAG.getNode(ISD::BRCOND, N->getDebugLoc(),
6321 MVT::Other, Chain, SetCC, N2);
6322 }
6323 }
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00006324
Dan Gohman475871a2008-07-27 21:46:04 +00006325 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00006326}
6327
Chris Lattner3ea0b472005-10-05 06:47:48 +00006328// Operand List for BR_CC: Chain, CondCC, CondLHS, CondRHS, DestBB.
6329//
Dan Gohman475871a2008-07-27 21:46:04 +00006330SDValue DAGCombiner::visitBR_CC(SDNode *N) {
Chris Lattner3ea0b472005-10-05 06:47:48 +00006331 CondCodeSDNode *CC = cast<CondCodeSDNode>(N->getOperand(1));
Dan Gohman475871a2008-07-27 21:46:04 +00006332 SDValue CondLHS = N->getOperand(2), CondRHS = N->getOperand(3);
Scott Michelfdc40a02009-02-17 22:15:04 +00006333
Dan Gohmane0f06c72009-11-17 00:47:23 +00006334 // If N is a constant we could fold this into a fallthrough or unconditional
6335 // branch. However that doesn't happen very often in normal code, because
6336 // Instcombine/SimplifyCFG should have handled the available opportunities.
6337 // If we did this folding here, it would be necessary to update the
6338 // MachineBasicBlock CFG, which is awkward.
6339
Duncan Sands8eab8a22008-06-09 11:32:28 +00006340 // Use SimplifySetCC to simplify SETCC's.
Duncan Sands5480c042009-01-01 15:52:00 +00006341 SDValue Simp = SimplifySetCC(TLI.getSetCCResultType(CondLHS.getValueType()),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00006342 CondLHS, CondRHS, CC->get(), N->getDebugLoc(),
6343 false);
Gabor Greifba36cb52008-08-28 21:40:38 +00006344 if (Simp.getNode()) AddToWorkList(Simp.getNode());
Chris Lattner30f73e72006-10-14 03:52:46 +00006345
Nate Begemane17daeb2005-10-05 21:43:42 +00006346 // fold to a simpler setcc
Gabor Greifba36cb52008-08-28 21:40:38 +00006347 if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC)
Owen Anderson825b72b2009-08-11 20:47:22 +00006348 return DAG.getNode(ISD::BR_CC, N->getDebugLoc(), MVT::Other,
Bill Wendlingc0debad2009-01-30 23:27:35 +00006349 N->getOperand(0), Simp.getOperand(2),
6350 Simp.getOperand(0), Simp.getOperand(1),
6351 N->getOperand(4));
6352
Dan Gohman475871a2008-07-27 21:46:04 +00006353 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00006354}
6355
Evan Chengc4b527a2012-01-13 01:37:24 +00006356/// canFoldInAddressingMode - Return true if 'Use' is a load or a store that
6357/// uses N as its base pointer and that N may be folded in the load / store
Evan Cheng03be3622012-03-06 23:33:32 +00006358/// addressing mode.
Evan Chengc4b527a2012-01-13 01:37:24 +00006359static bool canFoldInAddressingMode(SDNode *N, SDNode *Use,
6360 SelectionDAG &DAG,
6361 const TargetLowering &TLI) {
6362 EVT VT;
6363 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Use)) {
6364 if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
6365 return false;
6366 VT = Use->getValueType(0);
6367 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Use)) {
6368 if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
6369 return false;
6370 VT = ST->getValue().getValueType();
6371 } else
6372 return false;
6373
6374 TargetLowering::AddrMode AM;
6375 if (N->getOpcode() == ISD::ADD) {
6376 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
6377 if (Offset)
Evan Cheng03be3622012-03-06 23:33:32 +00006378 // [reg +/- imm]
Evan Chengc4b527a2012-01-13 01:37:24 +00006379 AM.BaseOffs = Offset->getSExtValue();
6380 else
Evan Cheng03be3622012-03-06 23:33:32 +00006381 // [reg +/- reg]
6382 AM.Scale = 1;
Evan Chengc4b527a2012-01-13 01:37:24 +00006383 } else if (N->getOpcode() == ISD::SUB) {
6384 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
6385 if (Offset)
Evan Cheng03be3622012-03-06 23:33:32 +00006386 // [reg +/- imm]
Evan Chengc4b527a2012-01-13 01:37:24 +00006387 AM.BaseOffs = -Offset->getSExtValue();
6388 else
Evan Cheng03be3622012-03-06 23:33:32 +00006389 // [reg +/- reg]
6390 AM.Scale = 1;
Evan Chengc4b527a2012-01-13 01:37:24 +00006391 } else
6392 return false;
6393
6394 return TLI.isLegalAddressingMode(AM, VT.getTypeForEVT(*DAG.getContext()));
6395}
6396
Duncan Sandsec87aa82008-06-15 20:12:31 +00006397/// CombineToPreIndexedLoadStore - Try turning a load / store into a
6398/// pre-indexed load / store when the base pointer is an add or subtract
Chris Lattner448f2192006-11-11 00:39:41 +00006399/// and it has other uses besides the load / store. After the
6400/// transformation, the new indexed load / store has effectively folded
6401/// the add / subtract in and all of its other uses are redirected to the
6402/// new load / store.
6403bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) {
Eli Friedman50185242011-11-12 00:35:34 +00006404 if (Level < AfterLegalizeDAG)
Chris Lattner448f2192006-11-11 00:39:41 +00006405 return false;
6406
6407 bool isLoad = true;
Dan Gohman475871a2008-07-27 21:46:04 +00006408 SDValue Ptr;
Owen Andersone50ed302009-08-10 22:56:29 +00006409 EVT VT;
Chris Lattner448f2192006-11-11 00:39:41 +00006410 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00006411 if (LD->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00006412 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006413 VT = LD->getMemoryVT();
Evan Cheng83060c52007-03-07 08:07:03 +00006414 if (!TLI.isIndexedLoadLegal(ISD::PRE_INC, VT) &&
Chris Lattner448f2192006-11-11 00:39:41 +00006415 !TLI.isIndexedLoadLegal(ISD::PRE_DEC, VT))
6416 return false;
6417 Ptr = LD->getBasePtr();
6418 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00006419 if (ST->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00006420 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006421 VT = ST->getMemoryVT();
Chris Lattner448f2192006-11-11 00:39:41 +00006422 if (!TLI.isIndexedStoreLegal(ISD::PRE_INC, VT) &&
6423 !TLI.isIndexedStoreLegal(ISD::PRE_DEC, VT))
6424 return false;
6425 Ptr = ST->getBasePtr();
6426 isLoad = false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00006427 } else {
Chris Lattner448f2192006-11-11 00:39:41 +00006428 return false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00006429 }
Chris Lattner448f2192006-11-11 00:39:41 +00006430
Chris Lattner9f1794e2006-11-11 00:56:29 +00006431 // If the pointer is not an add/sub, or if it doesn't have multiple uses, bail
6432 // out. There is no reason to make this a preinc/predec.
6433 if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) ||
Gabor Greifba36cb52008-08-28 21:40:38 +00006434 Ptr.getNode()->hasOneUse())
Chris Lattner9f1794e2006-11-11 00:56:29 +00006435 return false;
Chris Lattner448f2192006-11-11 00:39:41 +00006436
Chris Lattner9f1794e2006-11-11 00:56:29 +00006437 // Ask the target to do addressing mode selection.
Dan Gohman475871a2008-07-27 21:46:04 +00006438 SDValue BasePtr;
6439 SDValue Offset;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006440 ISD::MemIndexedMode AM = ISD::UNINDEXED;
6441 if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG))
6442 return false;
Evan Chenga7d4a042007-05-03 23:52:19 +00006443 // Don't create a indexed load / store with zero offset.
6444 if (isa<ConstantSDNode>(Offset) &&
Dan Gohman002e5d02008-03-13 22:13:53 +00006445 cast<ConstantSDNode>(Offset)->isNullValue())
Evan Chenga7d4a042007-05-03 23:52:19 +00006446 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +00006447
Chris Lattner41e53fd2006-11-11 01:00:15 +00006448 // Try turning it into a pre-indexed load / store except when:
Evan Chengc843abe2007-05-24 02:35:39 +00006449 // 1) The new base ptr is a frame index.
6450 // 2) If N is a store and the new base ptr is either the same as or is a
Chris Lattner9f1794e2006-11-11 00:56:29 +00006451 // predecessor of the value being stored.
Evan Chengc843abe2007-05-24 02:35:39 +00006452 // 3) Another use of old base ptr is a predecessor of N. If ptr is folded
Chris Lattner9f1794e2006-11-11 00:56:29 +00006453 // that would create a cycle.
Evan Chengc843abe2007-05-24 02:35:39 +00006454 // 4) All uses are load / store ops that use it as old base ptr.
Chris Lattner448f2192006-11-11 00:39:41 +00006455
Chris Lattner41e53fd2006-11-11 01:00:15 +00006456 // Check #1. Preinc'ing a frame index would require copying the stack pointer
6457 // (plus the implicit offset) to a register to preinc anyway.
Evan Chengcaab1292009-05-06 18:25:01 +00006458 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
Chris Lattner41e53fd2006-11-11 01:00:15 +00006459 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +00006460
Chris Lattner41e53fd2006-11-11 01:00:15 +00006461 // Check #2.
Chris Lattner9f1794e2006-11-11 00:56:29 +00006462 if (!isLoad) {
Dan Gohman475871a2008-07-27 21:46:04 +00006463 SDValue Val = cast<StoreSDNode>(N)->getValue();
Gabor Greifba36cb52008-08-28 21:40:38 +00006464 if (Val == BasePtr || BasePtr.getNode()->isPredecessorOf(Val.getNode()))
Chris Lattner9f1794e2006-11-11 00:56:29 +00006465 return false;
Chris Lattner448f2192006-11-11 00:39:41 +00006466 }
Chris Lattner9f1794e2006-11-11 00:56:29 +00006467
Evan Chengc843abe2007-05-24 02:35:39 +00006468 // Now check for #3 and #4.
Chris Lattner9f1794e2006-11-11 00:56:29 +00006469 bool RealUse = false;
Lang Hames944520f2011-07-07 04:31:51 +00006470
6471 // Caches for hasPredecessorHelper
6472 SmallPtrSet<const SDNode *, 32> Visited;
6473 SmallVector<const SDNode *, 16> Worklist;
6474
Gabor Greifba36cb52008-08-28 21:40:38 +00006475 for (SDNode::use_iterator I = Ptr.getNode()->use_begin(),
6476 E = Ptr.getNode()->use_end(); I != E; ++I) {
Dan Gohman89684502008-07-27 20:43:25 +00006477 SDNode *Use = *I;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006478 if (Use == N)
6479 continue;
Lang Hames944520f2011-07-07 04:31:51 +00006480 if (N->hasPredecessorHelper(Use, Visited, Worklist))
Chris Lattner9f1794e2006-11-11 00:56:29 +00006481 return false;
6482
Evan Chengc4b527a2012-01-13 01:37:24 +00006483 // If Ptr may be folded in addressing mode of other use, then it's
6484 // not profitable to do this transformation.
6485 if (!canFoldInAddressingMode(Ptr.getNode(), Use, DAG, TLI))
Chris Lattner9f1794e2006-11-11 00:56:29 +00006486 RealUse = true;
6487 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006488
Chris Lattner9f1794e2006-11-11 00:56:29 +00006489 if (!RealUse)
6490 return false;
6491
Dan Gohman475871a2008-07-27 21:46:04 +00006492 SDValue Result;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006493 if (isLoad)
Bill Wendlingc0debad2009-01-30 23:27:35 +00006494 Result = DAG.getIndexedLoad(SDValue(N,0), N->getDebugLoc(),
6495 BasePtr, Offset, AM);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006496 else
Bill Wendlingc0debad2009-01-30 23:27:35 +00006497 Result = DAG.getIndexedStore(SDValue(N,0), N->getDebugLoc(),
6498 BasePtr, Offset, AM);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006499 ++PreIndexedNodes;
6500 ++NodesCombined;
David Greenef1090292010-01-05 01:25:00 +00006501 DEBUG(dbgs() << "\nReplacing.4 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006502 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006503 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006504 Result.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006505 dbgs() << '\n');
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006506 WorkListRemover DeadNodes(*this);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006507 if (isLoad) {
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006508 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0));
6509 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2));
Chris Lattner9f1794e2006-11-11 00:56:29 +00006510 } else {
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006511 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1));
Chris Lattner9f1794e2006-11-11 00:56:29 +00006512 }
6513
Chris Lattner9f1794e2006-11-11 00:56:29 +00006514 // Finally, since the node is now dead, remove it from the graph.
6515 DAG.DeleteNode(N);
6516
6517 // Replace the uses of Ptr with uses of the updated base value.
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006518 DAG.ReplaceAllUsesOfValueWith(Ptr, Result.getValue(isLoad ? 1 : 0));
Gabor Greifba36cb52008-08-28 21:40:38 +00006519 removeFromWorkList(Ptr.getNode());
6520 DAG.DeleteNode(Ptr.getNode());
Chris Lattner9f1794e2006-11-11 00:56:29 +00006521
6522 return true;
Chris Lattner448f2192006-11-11 00:39:41 +00006523}
6524
Duncan Sandsec87aa82008-06-15 20:12:31 +00006525/// CombineToPostIndexedLoadStore - Try to combine a load / store with a
Chris Lattner448f2192006-11-11 00:39:41 +00006526/// add / sub of the base pointer node into a post-indexed load / store.
6527/// The transformation folded the add / subtract into the new indexed
6528/// load / store effectively and all of its uses are redirected to the
6529/// new load / store.
6530bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) {
Eli Friedman50185242011-11-12 00:35:34 +00006531 if (Level < AfterLegalizeDAG)
Chris Lattner448f2192006-11-11 00:39:41 +00006532 return false;
6533
6534 bool isLoad = true;
Dan Gohman475871a2008-07-27 21:46:04 +00006535 SDValue Ptr;
Owen Andersone50ed302009-08-10 22:56:29 +00006536 EVT VT;
Chris Lattner448f2192006-11-11 00:39:41 +00006537 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00006538 if (LD->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00006539 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006540 VT = LD->getMemoryVT();
Chris Lattner448f2192006-11-11 00:39:41 +00006541 if (!TLI.isIndexedLoadLegal(ISD::POST_INC, VT) &&
6542 !TLI.isIndexedLoadLegal(ISD::POST_DEC, VT))
6543 return false;
6544 Ptr = LD->getBasePtr();
6545 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00006546 if (ST->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00006547 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006548 VT = ST->getMemoryVT();
Chris Lattner448f2192006-11-11 00:39:41 +00006549 if (!TLI.isIndexedStoreLegal(ISD::POST_INC, VT) &&
6550 !TLI.isIndexedStoreLegal(ISD::POST_DEC, VT))
6551 return false;
6552 Ptr = ST->getBasePtr();
6553 isLoad = false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00006554 } else {
Chris Lattner448f2192006-11-11 00:39:41 +00006555 return false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00006556 }
Chris Lattner448f2192006-11-11 00:39:41 +00006557
Gabor Greifba36cb52008-08-28 21:40:38 +00006558 if (Ptr.getNode()->hasOneUse())
Chris Lattner9f1794e2006-11-11 00:56:29 +00006559 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +00006560
Gabor Greifba36cb52008-08-28 21:40:38 +00006561 for (SDNode::use_iterator I = Ptr.getNode()->use_begin(),
6562 E = Ptr.getNode()->use_end(); I != E; ++I) {
Dan Gohman89684502008-07-27 20:43:25 +00006563 SDNode *Op = *I;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006564 if (Op == N ||
6565 (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB))
6566 continue;
6567
Dan Gohman475871a2008-07-27 21:46:04 +00006568 SDValue BasePtr;
6569 SDValue Offset;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006570 ISD::MemIndexedMode AM = ISD::UNINDEXED;
6571 if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) {
Evan Chenga7d4a042007-05-03 23:52:19 +00006572 // Don't create a indexed load / store with zero offset.
6573 if (isa<ConstantSDNode>(Offset) &&
Dan Gohman002e5d02008-03-13 22:13:53 +00006574 cast<ConstantSDNode>(Offset)->isNullValue())
Evan Chenga7d4a042007-05-03 23:52:19 +00006575 continue;
Chris Lattner448f2192006-11-11 00:39:41 +00006576
Chris Lattner9f1794e2006-11-11 00:56:29 +00006577 // Try turning it into a post-indexed load / store except when
Evan Chengc4b527a2012-01-13 01:37:24 +00006578 // 1) All uses are load / store ops that use it as base ptr (and
6579 // it may be folded as addressing mmode).
Chris Lattner9f1794e2006-11-11 00:56:29 +00006580 // 2) Op must be independent of N, i.e. Op is neither a predecessor
6581 // nor a successor of N. Otherwise, if Op is folded that would
6582 // create a cycle.
6583
Evan Chengcaab1292009-05-06 18:25:01 +00006584 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
6585 continue;
6586
Chris Lattner9f1794e2006-11-11 00:56:29 +00006587 // Check for #1.
6588 bool TryNext = false;
Gabor Greifba36cb52008-08-28 21:40:38 +00006589 for (SDNode::use_iterator II = BasePtr.getNode()->use_begin(),
6590 EE = BasePtr.getNode()->use_end(); II != EE; ++II) {
Dan Gohman89684502008-07-27 20:43:25 +00006591 SDNode *Use = *II;
Gabor Greifba36cb52008-08-28 21:40:38 +00006592 if (Use == Ptr.getNode())
Chris Lattner448f2192006-11-11 00:39:41 +00006593 continue;
6594
Chris Lattner9f1794e2006-11-11 00:56:29 +00006595 // If all the uses are load / store addresses, then don't do the
6596 // transformation.
6597 if (Use->getOpcode() == ISD::ADD || Use->getOpcode() == ISD::SUB){
6598 bool RealUse = false;
6599 for (SDNode::use_iterator III = Use->use_begin(),
6600 EEE = Use->use_end(); III != EEE; ++III) {
Dan Gohman89684502008-07-27 20:43:25 +00006601 SDNode *UseUse = *III;
Evan Chengc4b527a2012-01-13 01:37:24 +00006602 if (!canFoldInAddressingMode(Use, UseUse, DAG, TLI))
Chris Lattner9f1794e2006-11-11 00:56:29 +00006603 RealUse = true;
6604 }
Chris Lattner448f2192006-11-11 00:39:41 +00006605
Chris Lattner9f1794e2006-11-11 00:56:29 +00006606 if (!RealUse) {
6607 TryNext = true;
6608 break;
Chris Lattner448f2192006-11-11 00:39:41 +00006609 }
6610 }
Chris Lattner9f1794e2006-11-11 00:56:29 +00006611 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006612
Chris Lattner9f1794e2006-11-11 00:56:29 +00006613 if (TryNext)
6614 continue;
Chris Lattner448f2192006-11-11 00:39:41 +00006615
Chris Lattner9f1794e2006-11-11 00:56:29 +00006616 // Check for #2
Evan Cheng917be682008-03-04 00:41:45 +00006617 if (!Op->isPredecessorOf(N) && !N->isPredecessorOf(Op)) {
Dan Gohman475871a2008-07-27 21:46:04 +00006618 SDValue Result = isLoad
Bill Wendlingc0debad2009-01-30 23:27:35 +00006619 ? DAG.getIndexedLoad(SDValue(N,0), N->getDebugLoc(),
6620 BasePtr, Offset, AM)
6621 : DAG.getIndexedStore(SDValue(N,0), N->getDebugLoc(),
6622 BasePtr, Offset, AM);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006623 ++PostIndexedNodes;
6624 ++NodesCombined;
David Greenef1090292010-01-05 01:25:00 +00006625 DEBUG(dbgs() << "\nReplacing.5 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006626 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006627 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006628 Result.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006629 dbgs() << '\n');
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006630 WorkListRemover DeadNodes(*this);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006631 if (isLoad) {
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006632 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0));
6633 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2));
Chris Lattner9f1794e2006-11-11 00:56:29 +00006634 } else {
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006635 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1));
Chris Lattner448f2192006-11-11 00:39:41 +00006636 }
Chris Lattner9f1794e2006-11-11 00:56:29 +00006637
Chris Lattner9f1794e2006-11-11 00:56:29 +00006638 // Finally, since the node is now dead, remove it from the graph.
6639 DAG.DeleteNode(N);
6640
6641 // Replace the uses of Use with uses of the updated base value.
Dan Gohman475871a2008-07-27 21:46:04 +00006642 DAG.ReplaceAllUsesOfValueWith(SDValue(Op, 0),
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006643 Result.getValue(isLoad ? 1 : 0));
Chris Lattner9f1794e2006-11-11 00:56:29 +00006644 removeFromWorkList(Op);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006645 DAG.DeleteNode(Op);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006646 return true;
Chris Lattner448f2192006-11-11 00:39:41 +00006647 }
6648 }
6649 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006650
Chris Lattner448f2192006-11-11 00:39:41 +00006651 return false;
6652}
6653
Dan Gohman475871a2008-07-27 21:46:04 +00006654SDValue DAGCombiner::visitLOAD(SDNode *N) {
Evan Cheng466685d2006-10-09 20:57:25 +00006655 LoadSDNode *LD = cast<LoadSDNode>(N);
Dan Gohman475871a2008-07-27 21:46:04 +00006656 SDValue Chain = LD->getChain();
6657 SDValue Ptr = LD->getBasePtr();
Scott Michelfdc40a02009-02-17 22:15:04 +00006658
Evan Cheng45a7ca92007-05-01 00:38:21 +00006659 // If load is not volatile and there are no uses of the loaded value (and
6660 // the updated indexed value in case of indexed loads), change uses of the
6661 // chain value into uses of the chain input (i.e. delete the dead load).
6662 if (!LD->isVolatile()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00006663 if (N->getValueType(1) == MVT::Other) {
Evan Cheng498f5592007-05-01 08:53:39 +00006664 // Unindexed loads.
Craig Topper704e1a02012-01-07 18:31:09 +00006665 if (!N->hasAnyUseOfValue(0)) {
Evan Cheng02c42852008-01-16 23:11:54 +00006666 // It's not safe to use the two value CombineTo variant here. e.g.
6667 // v1, chain2 = load chain1, loc
6668 // v2, chain3 = load chain2, loc
6669 // v3 = add v2, c
Chris Lattner125991a2008-01-24 07:57:06 +00006670 // Now we replace use of chain2 with chain1. This makes the second load
6671 // isomorphic to the one we are deleting, and thus makes this load live.
David Greenef1090292010-01-05 01:25:00 +00006672 DEBUG(dbgs() << "\nReplacing.6 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006673 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006674 dbgs() << "\nWith chain: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006675 Chain.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006676 dbgs() << "\n");
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006677 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006678 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain);
Bill Wendlingc0debad2009-01-30 23:27:35 +00006679
Chris Lattner125991a2008-01-24 07:57:06 +00006680 if (N->use_empty()) {
6681 removeFromWorkList(N);
6682 DAG.DeleteNode(N);
6683 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006684
Dan Gohman475871a2008-07-27 21:46:04 +00006685 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng02c42852008-01-16 23:11:54 +00006686 }
Evan Cheng498f5592007-05-01 08:53:39 +00006687 } else {
6688 // Indexed loads.
Owen Anderson825b72b2009-08-11 20:47:22 +00006689 assert(N->getValueType(2) == MVT::Other && "Malformed indexed loads?");
Craig Topper704e1a02012-01-07 18:31:09 +00006690 if (!N->hasAnyUseOfValue(0) && !N->hasAnyUseOfValue(1)) {
Dale Johannesene8d72302009-02-06 23:05:02 +00006691 SDValue Undef = DAG.getUNDEF(N->getValueType(0));
Evan Cheng2c755ba2010-02-27 07:36:59 +00006692 DEBUG(dbgs() << "\nReplacing.7 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006693 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006694 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006695 Undef.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006696 dbgs() << " and 2 other values\n");
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006697 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006698 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Undef);
Dan Gohman475871a2008-07-27 21:46:04 +00006699 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1),
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00006700 DAG.getUNDEF(N->getValueType(1)));
6701 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 2), Chain);
Evan Cheng02c42852008-01-16 23:11:54 +00006702 removeFromWorkList(N);
Evan Cheng02c42852008-01-16 23:11:54 +00006703 DAG.DeleteNode(N);
Dan Gohman475871a2008-07-27 21:46:04 +00006704 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng45a7ca92007-05-01 00:38:21 +00006705 }
Evan Cheng45a7ca92007-05-01 00:38:21 +00006706 }
6707 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006708
Chris Lattner01a22022005-10-10 22:04:48 +00006709 // If this load is directly stored, replace the load value with the stored
6710 // value.
6711 // TODO: Handle store large -> read small portion.
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006712 // TODO: Handle TRUNCSTORE/LOADEXT
Evan Cheng9ef82ce2011-03-11 00:48:56 +00006713 if (ISD::isNormalLoad(N) && !LD->isVolatile()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00006714 if (ISD::isNON_TRUNCStore(Chain.getNode())) {
Evan Cheng8b2794a2006-10-13 21:14:26 +00006715 StoreSDNode *PrevST = cast<StoreSDNode>(Chain);
6716 if (PrevST->getBasePtr() == Ptr &&
6717 PrevST->getValue().getValueType() == N->getValueType(0))
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006718 return CombineTo(N, Chain.getOperand(1), Chain);
Evan Cheng8b2794a2006-10-13 21:14:26 +00006719 }
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006720 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006721
Evan Cheng255f20f2010-04-01 06:04:33 +00006722 // Try to infer better alignment information than the load already has.
6723 if (OptLevel != CodeGenOpt::None && LD->isUnindexed()) {
Evan Chenged1c0c72011-11-28 22:37:34 +00006724 if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
6725 if (Align > LD->getAlignment())
6726 return DAG.getExtLoad(LD->getExtensionType(), N->getDebugLoc(),
6727 LD->getValueType(0),
6728 Chain, Ptr, LD->getPointerInfo(),
6729 LD->getMemoryVT(),
6730 LD->isVolatile(), LD->isNonTemporal(), Align);
Evan Cheng255f20f2010-04-01 06:04:33 +00006731 }
6732 }
6733
Jim Laskey7ca56af2006-10-11 13:47:09 +00006734 if (CombinerAA) {
Jim Laskey279f0532006-09-25 16:29:54 +00006735 // Walk up chain skipping non-aliasing memory nodes.
Dan Gohman475871a2008-07-27 21:46:04 +00006736 SDValue BetterChain = FindBetterChain(N, Chain);
Scott Michelfdc40a02009-02-17 22:15:04 +00006737
Jim Laskey6ff23e52006-10-04 16:53:27 +00006738 // If there is a better chain.
Jim Laskey279f0532006-09-25 16:29:54 +00006739 if (Chain != BetterChain) {
Dan Gohman475871a2008-07-27 21:46:04 +00006740 SDValue ReplLoad;
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006741
Jim Laskey279f0532006-09-25 16:29:54 +00006742 // Replace the chain to void dependency.
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006743 if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
Bill Wendlingc0debad2009-01-30 23:27:35 +00006744 ReplLoad = DAG.getLoad(N->getValueType(0), LD->getDebugLoc(),
Chris Lattnerfa459012010-09-21 16:08:50 +00006745 BetterChain, Ptr, LD->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00006746 LD->isVolatile(), LD->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00006747 LD->isInvariant(), LD->getAlignment());
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006748 } else {
Stuart Hastingsa9011292011-02-16 16:23:55 +00006749 ReplLoad = DAG.getExtLoad(LD->getExtensionType(), LD->getDebugLoc(),
6750 LD->getValueType(0),
Chris Lattnerfa459012010-09-21 16:08:50 +00006751 BetterChain, Ptr, LD->getPointerInfo(),
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006752 LD->getMemoryVT(),
Scott Michelfdc40a02009-02-17 22:15:04 +00006753 LD->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00006754 LD->isNonTemporal(),
Christopher Lamb95c218a2007-04-22 23:15:30 +00006755 LD->getAlignment());
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006756 }
Jim Laskey279f0532006-09-25 16:29:54 +00006757
Jim Laskey6ff23e52006-10-04 16:53:27 +00006758 // Create token factor to keep old chain connected.
Bill Wendlingc0debad2009-01-30 23:27:35 +00006759 SDValue Token = DAG.getNode(ISD::TokenFactor, N->getDebugLoc(),
Owen Anderson825b72b2009-08-11 20:47:22 +00006760 MVT::Other, Chain, ReplLoad.getValue(1));
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006761
Nate Begemanb6aef5c2009-09-15 00:18:30 +00006762 // Make sure the new and old chains are cleaned up.
6763 AddToWorkList(Token.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006764
Jim Laskey274062c2006-10-13 23:32:28 +00006765 // Replace uses with load result and token factor. Don't add users
6766 // to work list.
6767 return CombineTo(N, ReplLoad.getValue(0), Token, false);
Jim Laskey279f0532006-09-25 16:29:54 +00006768 }
6769 }
6770
Evan Cheng7fc033a2006-11-03 03:06:21 +00006771 // Try transforming N to an indexed load.
Evan Chengbbd6f6e2006-11-07 09:03:05 +00006772 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
Dan Gohman475871a2008-07-27 21:46:04 +00006773 return SDValue(N, 0);
Evan Cheng7fc033a2006-11-03 03:06:21 +00006774
Dan Gohman475871a2008-07-27 21:46:04 +00006775 return SDValue();
Chris Lattner01a22022005-10-10 22:04:48 +00006776}
6777
Chris Lattner2392ae72010-04-15 04:48:01 +00006778/// CheckForMaskedLoad - Check to see if V is (and load (ptr), imm), where the
6779/// load is having specific bytes cleared out. If so, return the byte size
6780/// being masked out and the shift amount.
6781static std::pair<unsigned, unsigned>
6782CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) {
6783 std::pair<unsigned, unsigned> Result(0, 0);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006784
Chris Lattner2392ae72010-04-15 04:48:01 +00006785 // Check for the structure we're looking for.
6786 if (V->getOpcode() != ISD::AND ||
6787 !isa<ConstantSDNode>(V->getOperand(1)) ||
6788 !ISD::isNormalLoad(V->getOperand(0).getNode()))
6789 return Result;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006790
Chris Lattnere6987582010-04-15 06:10:49 +00006791 // Check the chain and pointer.
Chris Lattner2392ae72010-04-15 04:48:01 +00006792 LoadSDNode *LD = cast<LoadSDNode>(V->getOperand(0));
Chris Lattnere6987582010-04-15 06:10:49 +00006793 if (LD->getBasePtr() != Ptr) return Result; // Not from same pointer.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006794
Chris Lattnere6987582010-04-15 06:10:49 +00006795 // The store should be chained directly to the load or be an operand of a
6796 // tokenfactor.
6797 if (LD == Chain.getNode())
6798 ; // ok.
6799 else if (Chain->getOpcode() != ISD::TokenFactor)
6800 return Result; // Fail.
6801 else {
6802 bool isOk = false;
6803 for (unsigned i = 0, e = Chain->getNumOperands(); i != e; ++i)
6804 if (Chain->getOperand(i).getNode() == LD) {
6805 isOk = true;
6806 break;
6807 }
6808 if (!isOk) return Result;
6809 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006810
Chris Lattner2392ae72010-04-15 04:48:01 +00006811 // This only handles simple types.
6812 if (V.getValueType() != MVT::i16 &&
6813 V.getValueType() != MVT::i32 &&
6814 V.getValueType() != MVT::i64)
6815 return Result;
6816
6817 // Check the constant mask. Invert it so that the bits being masked out are
6818 // 0 and the bits being kept are 1. Use getSExtValue so that leading bits
6819 // follow the sign bit for uniformity.
6820 uint64_t NotMask = ~cast<ConstantSDNode>(V->getOperand(1))->getSExtValue();
6821 unsigned NotMaskLZ = CountLeadingZeros_64(NotMask);
6822 if (NotMaskLZ & 7) return Result; // Must be multiple of a byte.
6823 unsigned NotMaskTZ = CountTrailingZeros_64(NotMask);
6824 if (NotMaskTZ & 7) return Result; // Must be multiple of a byte.
6825 if (NotMaskLZ == 64) return Result; // All zero mask.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006826
Chris Lattner2392ae72010-04-15 04:48:01 +00006827 // See if we have a continuous run of bits. If so, we have 0*1+0*
6828 if (CountTrailingOnes_64(NotMask >> NotMaskTZ)+NotMaskTZ+NotMaskLZ != 64)
6829 return Result;
6830
6831 // Adjust NotMaskLZ down to be from the actual size of the int instead of i64.
6832 if (V.getValueType() != MVT::i64 && NotMaskLZ)
6833 NotMaskLZ -= 64-V.getValueSizeInBits();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006834
Chris Lattner2392ae72010-04-15 04:48:01 +00006835 unsigned MaskedBytes = (V.getValueSizeInBits()-NotMaskLZ-NotMaskTZ)/8;
6836 switch (MaskedBytes) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006837 case 1:
6838 case 2:
Chris Lattner2392ae72010-04-15 04:48:01 +00006839 case 4: break;
6840 default: return Result; // All one mask, or 5-byte mask.
6841 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006842
Chris Lattner2392ae72010-04-15 04:48:01 +00006843 // Verify that the first bit starts at a multiple of mask so that the access
6844 // is aligned the same as the access width.
6845 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes) return Result;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006846
Chris Lattner2392ae72010-04-15 04:48:01 +00006847 Result.first = MaskedBytes;
6848 Result.second = NotMaskTZ/8;
6849 return Result;
6850}
6851
6852
6853/// ShrinkLoadReplaceStoreWithStore - Check to see if IVal is something that
6854/// provides a value as specified by MaskInfo. If so, replace the specified
6855/// store with a narrower store of truncated IVal.
6856static SDNode *
6857ShrinkLoadReplaceStoreWithStore(const std::pair<unsigned, unsigned> &MaskInfo,
6858 SDValue IVal, StoreSDNode *St,
6859 DAGCombiner *DC) {
6860 unsigned NumBytes = MaskInfo.first;
6861 unsigned ByteShift = MaskInfo.second;
6862 SelectionDAG &DAG = DC->getDAG();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006863
Chris Lattner2392ae72010-04-15 04:48:01 +00006864 // Check to see if IVal is all zeros in the part being masked in by the 'or'
6865 // that uses this. If not, this is not a replacement.
6866 APInt Mask = ~APInt::getBitsSet(IVal.getValueSizeInBits(),
6867 ByteShift*8, (ByteShift+NumBytes)*8);
6868 if (!DAG.MaskedValueIsZero(IVal, Mask)) return 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006869
Chris Lattner2392ae72010-04-15 04:48:01 +00006870 // Check that it is legal on the target to do this. It is legal if the new
6871 // VT we're shrinking to (i8/i16/i32) is legal or we're still before type
6872 // legalization.
6873 MVT VT = MVT::getIntegerVT(NumBytes*8);
6874 if (!DC->isTypeLegal(VT))
6875 return 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006876
Chris Lattner2392ae72010-04-15 04:48:01 +00006877 // Okay, we can do this! Replace the 'St' store with a store of IVal that is
6878 // shifted by ByteShift and truncated down to NumBytes.
6879 if (ByteShift)
6880 IVal = DAG.getNode(ISD::SRL, IVal->getDebugLoc(), IVal.getValueType(), IVal,
Owen Anderson95771af2011-02-25 21:41:48 +00006881 DAG.getConstant(ByteShift*8,
6882 DC->getShiftAmountTy(IVal.getValueType())));
Chris Lattner2392ae72010-04-15 04:48:01 +00006883
6884 // Figure out the offset for the store and the alignment of the access.
6885 unsigned StOffset;
6886 unsigned NewAlign = St->getAlignment();
6887
6888 if (DAG.getTargetLoweringInfo().isLittleEndian())
6889 StOffset = ByteShift;
6890 else
6891 StOffset = IVal.getValueType().getStoreSize() - ByteShift - NumBytes;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006892
Chris Lattner2392ae72010-04-15 04:48:01 +00006893 SDValue Ptr = St->getBasePtr();
6894 if (StOffset) {
6895 Ptr = DAG.getNode(ISD::ADD, IVal->getDebugLoc(), Ptr.getValueType(),
6896 Ptr, DAG.getConstant(StOffset, Ptr.getValueType()));
6897 NewAlign = MinAlign(NewAlign, StOffset);
6898 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006899
Chris Lattner2392ae72010-04-15 04:48:01 +00006900 // Truncate down to the new size.
6901 IVal = DAG.getNode(ISD::TRUNCATE, IVal->getDebugLoc(), VT, IVal);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006902
Chris Lattner2392ae72010-04-15 04:48:01 +00006903 ++OpsNarrowed;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006904 return DAG.getStore(St->getChain(), St->getDebugLoc(), IVal, Ptr,
Chris Lattner6229d0a2010-09-21 18:41:36 +00006905 St->getPointerInfo().getWithOffset(StOffset),
Chris Lattner2392ae72010-04-15 04:48:01 +00006906 false, false, NewAlign).getNode();
6907}
6908
Evan Cheng8b944d32009-05-28 00:35:15 +00006909
6910/// ReduceLoadOpStoreWidth - Look for sequence of load / op / store where op is
6911/// one of 'or', 'xor', and 'and' of immediates. If 'op' is only touching some
6912/// of the loaded bits, try narrowing the load and store if it would end up
6913/// being a win for performance or code size.
6914SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
6915 StoreSDNode *ST = cast<StoreSDNode>(N);
Evan Chengcdcecc02009-05-28 18:41:02 +00006916 if (ST->isVolatile())
6917 return SDValue();
6918
Evan Cheng8b944d32009-05-28 00:35:15 +00006919 SDValue Chain = ST->getChain();
6920 SDValue Value = ST->getValue();
6921 SDValue Ptr = ST->getBasePtr();
Owen Andersone50ed302009-08-10 22:56:29 +00006922 EVT VT = Value.getValueType();
Evan Cheng8b944d32009-05-28 00:35:15 +00006923
6924 if (ST->isTruncatingStore() || VT.isVector() || !Value.hasOneUse())
Evan Chengcdcecc02009-05-28 18:41:02 +00006925 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00006926
6927 unsigned Opc = Value.getOpcode();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006928
Chris Lattner2392ae72010-04-15 04:48:01 +00006929 // If this is "store (or X, Y), P" and X is "(and (load P), cst)", where cst
6930 // is a byte mask indicating a consecutive number of bytes, check to see if
6931 // Y is known to provide just those bytes. If so, we try to replace the
6932 // load + replace + store sequence with a single (narrower) store, which makes
6933 // the load dead.
6934 if (Opc == ISD::OR) {
6935 std::pair<unsigned, unsigned> MaskedLoad;
6936 MaskedLoad = CheckForMaskedLoad(Value.getOperand(0), Ptr, Chain);
6937 if (MaskedLoad.first)
6938 if (SDNode *NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad,
6939 Value.getOperand(1), ST,this))
6940 return SDValue(NewST, 0);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006941
Chris Lattner2392ae72010-04-15 04:48:01 +00006942 // Or is commutative, so try swapping X and Y.
6943 MaskedLoad = CheckForMaskedLoad(Value.getOperand(1), Ptr, Chain);
6944 if (MaskedLoad.first)
6945 if (SDNode *NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad,
6946 Value.getOperand(0), ST,this))
6947 return SDValue(NewST, 0);
6948 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006949
Evan Cheng8b944d32009-05-28 00:35:15 +00006950 if ((Opc != ISD::OR && Opc != ISD::XOR && Opc != ISD::AND) ||
6951 Value.getOperand(1).getOpcode() != ISD::Constant)
Evan Chengcdcecc02009-05-28 18:41:02 +00006952 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00006953
6954 SDValue N0 = Value.getOperand(0);
Dan Gohman24bde5b2010-09-02 21:18:42 +00006955 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
6956 Chain == SDValue(N0.getNode(), 1)) {
Evan Cheng8b944d32009-05-28 00:35:15 +00006957 LoadSDNode *LD = cast<LoadSDNode>(N0);
Chris Lattnerfa459012010-09-21 16:08:50 +00006958 if (LD->getBasePtr() != Ptr ||
6959 LD->getPointerInfo().getAddrSpace() !=
6960 ST->getPointerInfo().getAddrSpace())
Evan Chengcdcecc02009-05-28 18:41:02 +00006961 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00006962
6963 // Find the type to narrow it the load / op / store to.
6964 SDValue N1 = Value.getOperand(1);
6965 unsigned BitWidth = N1.getValueSizeInBits();
6966 APInt Imm = cast<ConstantSDNode>(N1)->getAPIntValue();
6967 if (Opc == ISD::AND)
6968 Imm ^= APInt::getAllOnesValue(BitWidth);
Evan Chengd3c76bb2009-05-28 23:52:18 +00006969 if (Imm == 0 || Imm.isAllOnesValue())
6970 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00006971 unsigned ShAmt = Imm.countTrailingZeros();
6972 unsigned MSB = BitWidth - Imm.countLeadingZeros() - 1;
6973 unsigned NewBW = NextPowerOf2(MSB - ShAmt);
Owen Anderson23b9b192009-08-12 00:36:31 +00006974 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);
Evan Cheng8b944d32009-05-28 00:35:15 +00006975 while (NewBW < BitWidth &&
Evan Chengcdcecc02009-05-28 18:41:02 +00006976 !(TLI.isOperationLegalOrCustom(Opc, NewVT) &&
Evan Cheng8b944d32009-05-28 00:35:15 +00006977 TLI.isNarrowingProfitable(VT, NewVT))) {
6978 NewBW = NextPowerOf2(NewBW);
Owen Anderson23b9b192009-08-12 00:36:31 +00006979 NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);
Evan Cheng8b944d32009-05-28 00:35:15 +00006980 }
Evan Chengcdcecc02009-05-28 18:41:02 +00006981 if (NewBW >= BitWidth)
6982 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00006983
6984 // If the lsb changed does not start at the type bitwidth boundary,
6985 // start at the previous one.
6986 if (ShAmt % NewBW)
6987 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW;
6988 APInt Mask = APInt::getBitsSet(BitWidth, ShAmt, ShAmt + NewBW);
6989 if ((Imm & Mask) == Imm) {
6990 APInt NewImm = (Imm & Mask).lshr(ShAmt).trunc(NewBW);
6991 if (Opc == ISD::AND)
6992 NewImm ^= APInt::getAllOnesValue(NewBW);
6993 uint64_t PtrOff = ShAmt / 8;
6994 // For big endian targets, we need to adjust the offset to the pointer to
6995 // load the correct bytes.
6996 if (TLI.isBigEndian())
Evan Chengcdcecc02009-05-28 18:41:02 +00006997 PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff;
Evan Cheng8b944d32009-05-28 00:35:15 +00006998
6999 unsigned NewAlign = MinAlign(LD->getAlignment(), PtrOff);
Chris Lattnerdb125cf2011-07-18 04:54:35 +00007000 Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext());
Chris Lattner2392ae72010-04-15 04:48:01 +00007001 if (NewAlign < TLI.getTargetData()->getABITypeAlignment(NewVTTy))
Evan Chengcdcecc02009-05-28 18:41:02 +00007002 return SDValue();
7003
Evan Cheng8b944d32009-05-28 00:35:15 +00007004 SDValue NewPtr = DAG.getNode(ISD::ADD, LD->getDebugLoc(),
7005 Ptr.getValueType(), Ptr,
7006 DAG.getConstant(PtrOff, Ptr.getValueType()));
7007 SDValue NewLD = DAG.getLoad(NewVT, N0.getDebugLoc(),
7008 LD->getChain(), NewPtr,
Chris Lattnerfa459012010-09-21 16:08:50 +00007009 LD->getPointerInfo().getWithOffset(PtrOff),
David Greene1e559442010-02-15 17:00:31 +00007010 LD->isVolatile(), LD->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00007011 LD->isInvariant(), NewAlign);
Evan Cheng8b944d32009-05-28 00:35:15 +00007012 SDValue NewVal = DAG.getNode(Opc, Value.getDebugLoc(), NewVT, NewLD,
7013 DAG.getConstant(NewImm, NewVT));
7014 SDValue NewST = DAG.getStore(Chain, N->getDebugLoc(),
7015 NewVal, NewPtr,
Chris Lattnerfa459012010-09-21 16:08:50 +00007016 ST->getPointerInfo().getWithOffset(PtrOff),
David Greene1e559442010-02-15 17:00:31 +00007017 false, false, NewAlign);
Evan Cheng8b944d32009-05-28 00:35:15 +00007018
7019 AddToWorkList(NewPtr.getNode());
7020 AddToWorkList(NewLD.getNode());
7021 AddToWorkList(NewVal.getNode());
7022 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00007023 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLD.getValue(1));
Evan Cheng8b944d32009-05-28 00:35:15 +00007024 ++OpsNarrowed;
7025 return NewST;
7026 }
7027 }
7028
Evan Chengcdcecc02009-05-28 18:41:02 +00007029 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00007030}
7031
Evan Cheng31959b12011-02-02 01:06:55 +00007032/// TransformFPLoadStorePair - For a given floating point load / store pair,
7033/// if the load value isn't used by any other operations, then consider
7034/// transforming the pair to integer load / store operations if the target
7035/// deems the transformation profitable.
7036SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
7037 StoreSDNode *ST = cast<StoreSDNode>(N);
7038 SDValue Chain = ST->getChain();
7039 SDValue Value = ST->getValue();
7040 if (ISD::isNormalStore(ST) && ISD::isNormalLoad(Value.getNode()) &&
7041 Value.hasOneUse() &&
7042 Chain == SDValue(Value.getNode(), 1)) {
7043 LoadSDNode *LD = cast<LoadSDNode>(Value);
7044 EVT VT = LD->getMemoryVT();
7045 if (!VT.isFloatingPoint() ||
7046 VT != ST->getMemoryVT() ||
7047 LD->isNonTemporal() ||
7048 ST->isNonTemporal() ||
7049 LD->getPointerInfo().getAddrSpace() != 0 ||
7050 ST->getPointerInfo().getAddrSpace() != 0)
7051 return SDValue();
7052
7053 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
7054 if (!TLI.isOperationLegal(ISD::LOAD, IntVT) ||
7055 !TLI.isOperationLegal(ISD::STORE, IntVT) ||
7056 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) ||
7057 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT))
7058 return SDValue();
7059
7060 unsigned LDAlign = LD->getAlignment();
7061 unsigned STAlign = ST->getAlignment();
Chris Lattnerdb125cf2011-07-18 04:54:35 +00007062 Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext());
Evan Cheng31959b12011-02-02 01:06:55 +00007063 unsigned ABIAlign = TLI.getTargetData()->getABITypeAlignment(IntVTTy);
7064 if (LDAlign < ABIAlign || STAlign < ABIAlign)
7065 return SDValue();
7066
7067 SDValue NewLD = DAG.getLoad(IntVT, Value.getDebugLoc(),
7068 LD->getChain(), LD->getBasePtr(),
7069 LD->getPointerInfo(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00007070 false, false, false, LDAlign);
Evan Cheng31959b12011-02-02 01:06:55 +00007071
7072 SDValue NewST = DAG.getStore(NewLD.getValue(1), N->getDebugLoc(),
7073 NewLD, ST->getBasePtr(),
7074 ST->getPointerInfo(),
7075 false, false, STAlign);
7076
7077 AddToWorkList(NewLD.getNode());
7078 AddToWorkList(NewST.getNode());
7079 WorkListRemover DeadNodes(*this);
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00007080 DAG.ReplaceAllUsesOfValueWith(Value.getValue(1), NewLD.getValue(1));
Evan Cheng31959b12011-02-02 01:06:55 +00007081 ++LdStFP2Int;
7082 return NewST;
7083 }
7084
7085 return SDValue();
7086}
7087
Dan Gohman475871a2008-07-27 21:46:04 +00007088SDValue DAGCombiner::visitSTORE(SDNode *N) {
Evan Cheng8b2794a2006-10-13 21:14:26 +00007089 StoreSDNode *ST = cast<StoreSDNode>(N);
Dan Gohman475871a2008-07-27 21:46:04 +00007090 SDValue Chain = ST->getChain();
7091 SDValue Value = ST->getValue();
7092 SDValue Ptr = ST->getBasePtr();
Scott Michelfdc40a02009-02-17 22:15:04 +00007093
Evan Cheng59d5b682007-05-07 21:27:48 +00007094 // If this is a store of a bit convert, store the input value if the
Evan Cheng2c4f9432007-05-09 21:49:47 +00007095 // resultant store does not need a higher alignment than the original.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007096 if (Value.getOpcode() == ISD::BITCAST && !ST->isTruncatingStore() &&
Chris Lattnerddf89562008-01-17 19:59:44 +00007097 ST->isUnindexed()) {
Dan Gohman1ba519b2009-02-20 23:29:13 +00007098 unsigned OrigAlign = ST->getAlignment();
Owen Andersone50ed302009-08-10 22:56:29 +00007099 EVT SVT = Value.getOperand(0).getValueType();
Dan Gohman1ba519b2009-02-20 23:29:13 +00007100 unsigned Align = TLI.getTargetData()->
Owen Anderson23b9b192009-08-12 00:36:31 +00007101 getABITypeAlignment(SVT.getTypeForEVT(*DAG.getContext()));
Duncan Sandsd4b9c172008-06-13 19:07:40 +00007102 if (Align <= OrigAlign &&
Duncan Sands25cf2272008-11-24 14:53:14 +00007103 ((!LegalOperations && !ST->isVolatile()) ||
Dan Gohmanf560ffa2009-01-28 17:46:25 +00007104 TLI.isOperationLegalOrCustom(ISD::STORE, SVT)))
Bill Wendlingc144a572009-01-30 23:36:47 +00007105 return DAG.getStore(Chain, N->getDebugLoc(), Value.getOperand(0),
Chris Lattner6229d0a2010-09-21 18:41:36 +00007106 Ptr, ST->getPointerInfo(), ST->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00007107 ST->isNonTemporal(), OrigAlign);
Jim Laskey279f0532006-09-25 16:29:54 +00007108 }
Owen Andersona34d9362011-04-14 17:30:49 +00007109
Chris Lattnerb3452ea2011-04-09 02:32:02 +00007110 // Turn 'store undef, Ptr' -> nothing.
7111 if (Value.getOpcode() == ISD::UNDEF && ST->isUnindexed())
7112 return Chain;
Duncan Sandsd4b9c172008-06-13 19:07:40 +00007113
Nate Begeman2cbba892006-12-11 02:23:46 +00007114 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
Nate Begeman2cbba892006-12-11 02:23:46 +00007115 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
Duncan Sandsd4b9c172008-06-13 19:07:40 +00007116 // NOTE: If the original store is volatile, this transform must not increase
7117 // the number of stores. For example, on x86-32 an f64 can be stored in one
7118 // processor operation but an i64 (which is not legal) requires two. So the
7119 // transform should not be done in this case.
Evan Cheng25ece662006-12-11 17:25:19 +00007120 if (Value.getOpcode() != ISD::TargetConstantFP) {
Dan Gohman475871a2008-07-27 21:46:04 +00007121 SDValue Tmp;
Owen Anderson825b72b2009-08-11 20:47:22 +00007122 switch (CFP->getValueType(0).getSimpleVT().SimpleTy) {
Torok Edwinc23197a2009-07-14 16:55:14 +00007123 default: llvm_unreachable("Unknown FP type");
Owen Anderson825b72b2009-08-11 20:47:22 +00007124 case MVT::f80: // We don't do this for these yet.
7125 case MVT::f128:
7126 case MVT::ppcf128:
Dale Johannesenc7b21d52007-09-18 18:36:59 +00007127 break;
Owen Anderson825b72b2009-08-11 20:47:22 +00007128 case MVT::f32:
Chris Lattner2392ae72010-04-15 04:48:01 +00007129 if ((isTypeLegal(MVT::i32) && !LegalOperations && !ST->isVolatile()) ||
Owen Anderson825b72b2009-08-11 20:47:22 +00007130 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) {
Dale Johannesen9d5f4562007-09-12 03:30:33 +00007131 Tmp = DAG.getConstant((uint32_t)CFP->getValueAPF().
Owen Anderson825b72b2009-08-11 20:47:22 +00007132 bitcastToAPInt().getZExtValue(), MVT::i32);
Bill Wendlingc144a572009-01-30 23:36:47 +00007133 return DAG.getStore(Chain, N->getDebugLoc(), Tmp,
Chris Lattner6229d0a2010-09-21 18:41:36 +00007134 Ptr, ST->getPointerInfo(), ST->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00007135 ST->isNonTemporal(), ST->getAlignment());
Chris Lattner62be1a72006-12-12 04:16:14 +00007136 }
7137 break;
Owen Anderson825b72b2009-08-11 20:47:22 +00007138 case MVT::f64:
Chris Lattner2392ae72010-04-15 04:48:01 +00007139 if ((TLI.isTypeLegal(MVT::i64) && !LegalOperations &&
Dan Gohmanf560ffa2009-01-28 17:46:25 +00007140 !ST->isVolatile()) ||
Owen Anderson825b72b2009-08-11 20:47:22 +00007141 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) {
Dale Johannesen7111b022008-10-09 18:53:47 +00007142 Tmp = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
Owen Anderson825b72b2009-08-11 20:47:22 +00007143 getZExtValue(), MVT::i64);
Bill Wendlingc144a572009-01-30 23:36:47 +00007144 return DAG.getStore(Chain, N->getDebugLoc(), Tmp,
Chris Lattner6229d0a2010-09-21 18:41:36 +00007145 Ptr, ST->getPointerInfo(), ST->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00007146 ST->isNonTemporal(), ST->getAlignment());
Chris Lattnerb3452ea2011-04-09 02:32:02 +00007147 }
Owen Andersona34d9362011-04-14 17:30:49 +00007148
Chris Lattnerb3452ea2011-04-09 02:32:02 +00007149 if (!ST->isVolatile() &&
7150 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) {
Duncan Sandsdc846502007-10-28 12:59:45 +00007151 // Many FP stores are not made apparent until after legalize, e.g. for
Chris Lattner62be1a72006-12-12 04:16:14 +00007152 // argument passing. Since this is so common, custom legalize the
7153 // 64-bit integer store into two 32-bit stores.
Dale Johannesen7111b022008-10-09 18:53:47 +00007154 uint64_t Val = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
Owen Anderson825b72b2009-08-11 20:47:22 +00007155 SDValue Lo = DAG.getConstant(Val & 0xFFFFFFFF, MVT::i32);
7156 SDValue Hi = DAG.getConstant(Val >> 32, MVT::i32);
Duncan Sands0753fc12008-02-11 10:37:04 +00007157 if (TLI.isBigEndian()) std::swap(Lo, Hi);
Chris Lattner62be1a72006-12-12 04:16:14 +00007158
Dan Gohmand6fd1bc2007-07-09 22:18:38 +00007159 unsigned Alignment = ST->getAlignment();
7160 bool isVolatile = ST->isVolatile();
David Greene1e559442010-02-15 17:00:31 +00007161 bool isNonTemporal = ST->isNonTemporal();
Dan Gohmand6fd1bc2007-07-09 22:18:38 +00007162
Bill Wendlingc144a572009-01-30 23:36:47 +00007163 SDValue St0 = DAG.getStore(Chain, ST->getDebugLoc(), Lo,
Chris Lattner6229d0a2010-09-21 18:41:36 +00007164 Ptr, ST->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00007165 isVolatile, isNonTemporal,
7166 ST->getAlignment());
Bill Wendlingc144a572009-01-30 23:36:47 +00007167 Ptr = DAG.getNode(ISD::ADD, N->getDebugLoc(), Ptr.getValueType(), Ptr,
Chris Lattner62be1a72006-12-12 04:16:14 +00007168 DAG.getConstant(4, Ptr.getValueType()));
Duncan Sandsdc846502007-10-28 12:59:45 +00007169 Alignment = MinAlign(Alignment, 4U);
Bill Wendlingc144a572009-01-30 23:36:47 +00007170 SDValue St1 = DAG.getStore(Chain, ST->getDebugLoc(), Hi,
Chris Lattner6229d0a2010-09-21 18:41:36 +00007171 Ptr, ST->getPointerInfo().getWithOffset(4),
7172 isVolatile, isNonTemporal,
David Greene1e559442010-02-15 17:00:31 +00007173 Alignment);
Owen Anderson825b72b2009-08-11 20:47:22 +00007174 return DAG.getNode(ISD::TokenFactor, N->getDebugLoc(), MVT::Other,
Bill Wendlingc144a572009-01-30 23:36:47 +00007175 St0, St1);
Chris Lattner62be1a72006-12-12 04:16:14 +00007176 }
Bill Wendlingc144a572009-01-30 23:36:47 +00007177
Chris Lattner62be1a72006-12-12 04:16:14 +00007178 break;
Evan Cheng25ece662006-12-11 17:25:19 +00007179 }
Nate Begeman2cbba892006-12-11 02:23:46 +00007180 }
Nate Begeman2cbba892006-12-11 02:23:46 +00007181 }
7182
Evan Cheng255f20f2010-04-01 06:04:33 +00007183 // Try to infer better alignment information than the store already has.
7184 if (OptLevel != CodeGenOpt::None && ST->isUnindexed()) {
Evan Chenged1c0c72011-11-28 22:37:34 +00007185 if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
7186 if (Align > ST->getAlignment())
7187 return DAG.getTruncStore(Chain, N->getDebugLoc(), Value,
7188 Ptr, ST->getPointerInfo(), ST->getMemoryVT(),
7189 ST->isVolatile(), ST->isNonTemporal(), Align);
Evan Cheng255f20f2010-04-01 06:04:33 +00007190 }
7191 }
7192
Evan Cheng31959b12011-02-02 01:06:55 +00007193 // Try transforming a pair floating point load / store ops to integer
7194 // load / store ops.
7195 SDValue NewST = TransformFPLoadStorePair(N);
7196 if (NewST.getNode())
7197 return NewST;
7198
Scott Michelfdc40a02009-02-17 22:15:04 +00007199 if (CombinerAA) {
Jim Laskey279f0532006-09-25 16:29:54 +00007200 // Walk up chain skipping non-aliasing memory nodes.
Dan Gohman475871a2008-07-27 21:46:04 +00007201 SDValue BetterChain = FindBetterChain(N, Chain);
Scott Michelfdc40a02009-02-17 22:15:04 +00007202
Jim Laskey6ff23e52006-10-04 16:53:27 +00007203 // If there is a better chain.
Jim Laskey279f0532006-09-25 16:29:54 +00007204 if (Chain != BetterChain) {
Dan Gohman475871a2008-07-27 21:46:04 +00007205 SDValue ReplStore;
Nate Begemanb6aef5c2009-09-15 00:18:30 +00007206
7207 // Replace the chain to avoid dependency.
Jim Laskeyd4edf2c2006-10-14 12:14:27 +00007208 if (ST->isTruncatingStore()) {
Bill Wendlingc144a572009-01-30 23:36:47 +00007209 ReplStore = DAG.getTruncStore(BetterChain, N->getDebugLoc(), Value, Ptr,
Chris Lattnerda2d8e12010-09-21 17:42:31 +00007210 ST->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00007211 ST->getMemoryVT(), ST->isVolatile(),
7212 ST->isNonTemporal(), ST->getAlignment());
Jim Laskeyd4edf2c2006-10-14 12:14:27 +00007213 } else {
Bill Wendlingc144a572009-01-30 23:36:47 +00007214 ReplStore = DAG.getStore(BetterChain, N->getDebugLoc(), Value, Ptr,
Chris Lattner6229d0a2010-09-21 18:41:36 +00007215 ST->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00007216 ST->isVolatile(), ST->isNonTemporal(),
7217 ST->getAlignment());
Jim Laskeyd4edf2c2006-10-14 12:14:27 +00007218 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007219
Jim Laskey279f0532006-09-25 16:29:54 +00007220 // Create token to keep both nodes around.
Bill Wendlingc144a572009-01-30 23:36:47 +00007221 SDValue Token = DAG.getNode(ISD::TokenFactor, N->getDebugLoc(),
Owen Anderson825b72b2009-08-11 20:47:22 +00007222 MVT::Other, Chain, ReplStore);
Bill Wendlingc144a572009-01-30 23:36:47 +00007223
Nate Begemanb6aef5c2009-09-15 00:18:30 +00007224 // Make sure the new and old chains are cleaned up.
7225 AddToWorkList(Token.getNode());
7226
Jim Laskey274062c2006-10-13 23:32:28 +00007227 // Don't add users to work list.
7228 return CombineTo(N, Token, false);
Jim Laskey279f0532006-09-25 16:29:54 +00007229 }
Jim Laskeyd1aed7a2006-09-21 16:28:59 +00007230 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007231
Evan Cheng33dbedc2006-11-05 09:31:14 +00007232 // Try transforming N to an indexed store.
Evan Chengbbd6f6e2006-11-07 09:03:05 +00007233 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
Dan Gohman475871a2008-07-27 21:46:04 +00007234 return SDValue(N, 0);
Evan Cheng33dbedc2006-11-05 09:31:14 +00007235
Chris Lattner3c872852007-12-29 06:26:16 +00007236 // FIXME: is there such a thing as a truncating indexed store?
Chris Lattnerddf89562008-01-17 19:59:44 +00007237 if (ST->isTruncatingStore() && ST->isUnindexed() &&
Nadav Rotembaff46f2011-06-15 11:19:12 +00007238 Value.getValueType().isInteger()) {
Chris Lattner2b4c2792007-10-13 06:35:54 +00007239 // See if we can simplify the input to this truncstore with knowledge that
7240 // only the low bits are being used. For example:
7241 // "truncstore (or (shl x, 8), y), i8" -> "truncstore y, i8"
Scott Michelfdc40a02009-02-17 22:15:04 +00007242 SDValue Shorter =
Dan Gohman2e68b6f2008-02-25 21:11:39 +00007243 GetDemandedBits(Value,
Nadav Rotembaff46f2011-06-15 11:19:12 +00007244 APInt::getLowBitsSet(
7245 Value.getValueType().getScalarType().getSizeInBits(),
7246 ST->getMemoryVT().getScalarType().getSizeInBits()));
Gabor Greifba36cb52008-08-28 21:40:38 +00007247 AddToWorkList(Value.getNode());
7248 if (Shorter.getNode())
Bill Wendlingc144a572009-01-30 23:36:47 +00007249 return DAG.getTruncStore(Chain, N->getDebugLoc(), Shorter,
Chris Lattnerda2d8e12010-09-21 17:42:31 +00007250 Ptr, ST->getPointerInfo(), ST->getMemoryVT(),
David Greene1e559442010-02-15 17:00:31 +00007251 ST->isVolatile(), ST->isNonTemporal(),
7252 ST->getAlignment());
Scott Michelfdc40a02009-02-17 22:15:04 +00007253
Chris Lattnere33544c2007-10-13 06:58:48 +00007254 // Otherwise, see if we can simplify the operation with
7255 // SimplifyDemandedBits, which only works if the value has a single use.
Dan Gohman7b8d4a92008-02-27 00:25:32 +00007256 if (SimplifyDemandedBits(Value,
Eric Christopher503a64d2010-12-09 04:48:06 +00007257 APInt::getLowBitsSet(
7258 Value.getValueType().getScalarType().getSizeInBits(),
7259 ST->getMemoryVT().getScalarType().getSizeInBits())))
Dan Gohman475871a2008-07-27 21:46:04 +00007260 return SDValue(N, 0);
Chris Lattner2b4c2792007-10-13 06:35:54 +00007261 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007262
Chris Lattner3c872852007-12-29 06:26:16 +00007263 // If this is a load followed by a store to the same location, then the store
7264 // is dead/noop.
7265 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Value)) {
Dan Gohmanb625f2f2008-01-30 00:15:11 +00007266 if (Ld->getBasePtr() == Ptr && ST->getMemoryVT() == Ld->getMemoryVT() &&
Chris Lattnerddf89562008-01-17 19:59:44 +00007267 ST->isUnindexed() && !ST->isVolatile() &&
Chris Lattner07649d92008-01-08 23:08:06 +00007268 // There can't be any side effects between the load and store, such as
7269 // a call or store.
Dan Gohman475871a2008-07-27 21:46:04 +00007270 Chain.reachesChainWithoutSideEffects(SDValue(Ld, 1))) {
Chris Lattner3c872852007-12-29 06:26:16 +00007271 // The store is dead, remove it.
7272 return Chain;
7273 }
7274 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00007275
Chris Lattnerddf89562008-01-17 19:59:44 +00007276 // If this is an FP_ROUND or TRUNC followed by a store, fold this into a
7277 // truncating store. We can do this even if this is already a truncstore.
7278 if ((Value.getOpcode() == ISD::FP_ROUND || Value.getOpcode() == ISD::TRUNCATE)
Gabor Greifba36cb52008-08-28 21:40:38 +00007279 && Value.getNode()->hasOneUse() && ST->isUnindexed() &&
Chris Lattnerddf89562008-01-17 19:59:44 +00007280 TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(),
Dan Gohmanb625f2f2008-01-30 00:15:11 +00007281 ST->getMemoryVT())) {
Bill Wendlingc144a572009-01-30 23:36:47 +00007282 return DAG.getTruncStore(Chain, N->getDebugLoc(), Value.getOperand(0),
Chris Lattnerda2d8e12010-09-21 17:42:31 +00007283 Ptr, ST->getPointerInfo(), ST->getMemoryVT(),
David Greene1e559442010-02-15 17:00:31 +00007284 ST->isVolatile(), ST->isNonTemporal(),
7285 ST->getAlignment());
Chris Lattnerddf89562008-01-17 19:59:44 +00007286 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00007287
Evan Cheng8b944d32009-05-28 00:35:15 +00007288 return ReduceLoadOpStoreWidth(N);
Chris Lattner87514ca2005-10-10 22:31:19 +00007289}
7290
Dan Gohman475871a2008-07-27 21:46:04 +00007291SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
7292 SDValue InVec = N->getOperand(0);
7293 SDValue InVal = N->getOperand(1);
7294 SDValue EltNo = N->getOperand(2);
Eli Friedman9db817f2011-09-09 21:04:06 +00007295 DebugLoc dl = N->getDebugLoc();
Scott Michelfdc40a02009-02-17 22:15:04 +00007296
Bob Wilson492fd452010-05-19 23:42:58 +00007297 // If the inserted element is an UNDEF, just use the input vector.
7298 if (InVal.getOpcode() == ISD::UNDEF)
7299 return InVec;
7300
Nadav Rotem609d54e2011-02-12 14:40:33 +00007301 EVT VT = InVec.getValueType();
7302
Owen Anderson95771af2011-02-25 21:41:48 +00007303 // If we can't generate a legal BUILD_VECTOR, exit
Nadav Rotem609d54e2011-02-12 14:40:33 +00007304 if (LegalOperations && !TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
7305 return SDValue();
7306
Eli Friedman9db817f2011-09-09 21:04:06 +00007307 // Check that we know which element is being inserted
7308 if (!isa<ConstantSDNode>(EltNo))
7309 return SDValue();
7310 unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00007311
Eli Friedman9db817f2011-09-09 21:04:06 +00007312 // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially
7313 // be converted to a BUILD_VECTOR). Fill in the Ops vector with the
7314 // vector elements.
7315 SmallVector<SDValue, 8> Ops;
7316 if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
7317 Ops.append(InVec.getNode()->op_begin(),
7318 InVec.getNode()->op_end());
7319 } else if (InVec.getOpcode() == ISD::UNDEF) {
7320 unsigned NElts = VT.getVectorNumElements();
7321 Ops.append(NElts, DAG.getUNDEF(InVal.getValueType()));
7322 } else {
7323 return SDValue();
Nate Begeman9008ca62009-04-27 18:41:29 +00007324 }
Eli Friedman9db817f2011-09-09 21:04:06 +00007325
7326 // Insert the element
7327 if (Elt < Ops.size()) {
7328 // All the operands of BUILD_VECTOR must have the same type;
7329 // we enforce that here.
7330 EVT OpVT = Ops[0].getValueType();
7331 if (InVal.getValueType() != OpVT)
7332 InVal = OpVT.bitsGT(InVal.getValueType()) ?
7333 DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) :
7334 DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal);
7335 Ops[Elt] = InVal;
7336 }
7337
7338 // Return the new vector
7339 return DAG.getNode(ISD::BUILD_VECTOR, dl,
7340 VT, &Ops[0], Ops.size());
Chris Lattnerca242442006-03-19 01:27:56 +00007341}
7342
Dan Gohman475871a2008-07-27 21:46:04 +00007343SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
Mon P Wang7ac9cdf2009-01-17 00:07:25 +00007344 // (vextract (scalar_to_vector val, 0) -> val
7345 SDValue InVec = N->getOperand(0);
Nadav Rotemba05c912012-01-17 21:44:01 +00007346 EVT VT = InVec.getValueType();
7347 EVT NVT = N->getValueType(0);
Mon P Wang7ac9cdf2009-01-17 00:07:25 +00007348
Duncan Sandsc356f332011-05-09 08:03:33 +00007349 if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR) {
7350 // Check if the result type doesn't match the inserted element type. A
7351 // SCALAR_TO_VECTOR may truncate the inserted element and the
7352 // EXTRACT_VECTOR_ELT may widen the extracted vector.
7353 SDValue InOp = InVec.getOperand(0);
Duncan Sandsc356f332011-05-09 08:03:33 +00007354 if (InOp.getValueType() != NVT) {
7355 assert(InOp.getValueType().isInteger() && NVT.isInteger());
7356 return DAG.getSExtOrTrunc(InOp, InVec.getDebugLoc(), NVT);
7357 }
7358 return InOp;
7359 }
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007360
Nadav Rotemba05c912012-01-17 21:44:01 +00007361 SDValue EltNo = N->getOperand(1);
7362 bool ConstEltNo = isa<ConstantSDNode>(EltNo);
7363
7364 // Transform: (EXTRACT_VECTOR_ELT( VECTOR_SHUFFLE )) -> EXTRACT_VECTOR_ELT.
7365 // We only perform this optimization before the op legalization phase because
7366 // we may introduce new vector instructions which are not backed by TD patterns.
7367 // For example on AVX, extracting elements from a wide vector without using
7368 // extract_subvector.
7369 if (InVec.getOpcode() == ISD::VECTOR_SHUFFLE
7370 && ConstEltNo && !LegalOperations) {
7371 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
7372 int NumElem = VT.getVectorNumElements();
7373 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(InVec);
7374 // Find the new index to extract from.
7375 int OrigElt = SVOp->getMaskElt(Elt);
7376
7377 // Extracting an undef index is undef.
7378 if (OrigElt == -1)
7379 return DAG.getUNDEF(NVT);
7380
7381 // Select the right vector half to extract from.
7382 if (OrigElt < NumElem) {
7383 InVec = InVec->getOperand(0);
7384 } else {
7385 InVec = InVec->getOperand(1);
7386 OrigElt -= NumElem;
7387 }
7388
Jim Grosbacha249f7d2012-05-08 20:56:07 +00007389 EVT IndexTy = N->getOperand(1).getValueType();
Nadav Rotemba05c912012-01-17 21:44:01 +00007390 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), NVT,
Jim Grosbacha249f7d2012-05-08 20:56:07 +00007391 InVec, DAG.getConstant(OrigElt, IndexTy));
Nadav Rotemba05c912012-01-17 21:44:01 +00007392 }
7393
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007394 // Perform only after legalization to ensure build_vector / vector_shuffle
7395 // optimizations have already been done.
Duncan Sands25cf2272008-11-24 14:53:14 +00007396 if (!LegalOperations) return SDValue();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007397
Mon P Wang7ac9cdf2009-01-17 00:07:25 +00007398 // (vextract (v4f32 load $addr), c) -> (f32 load $addr+c*size)
7399 // (vextract (v4f32 s2v (f32 load $addr)), c) -> (f32 load $addr+c*size)
7400 // (vextract (v4f32 shuffle (load $addr), <1,u,u,u>), 0) -> (f32 load $addr)
Evan Cheng513da432007-10-06 08:19:55 +00007401
Nadav Rotemba05c912012-01-17 21:44:01 +00007402 if (ConstEltNo) {
Eric Christophercaebdd42010-11-03 09:36:40 +00007403 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
Evan Cheng513da432007-10-06 08:19:55 +00007404 bool NewLoad = false;
Mon P Wanga60b5232008-12-11 00:26:16 +00007405 bool BCNumEltsChanged = false;
Owen Andersone50ed302009-08-10 22:56:29 +00007406 EVT ExtVT = VT.getVectorElementType();
7407 EVT LVT = ExtVT;
Bill Wendlingc144a572009-01-30 23:36:47 +00007408
Evan Cheng84387ea2012-03-13 22:00:52 +00007409 // If the result of load has to be truncated, then it's not necessarily
7410 // profitable.
Evan Chenga03d3662012-03-13 22:16:11 +00007411 if (NVT.bitsLT(LVT) && !TLI.isTruncateFree(LVT, NVT))
Evan Cheng84387ea2012-03-13 22:00:52 +00007412 return SDValue();
7413
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007414 if (InVec.getOpcode() == ISD::BITCAST) {
Eli Friedmand6e25602011-12-26 22:49:32 +00007415 // Don't duplicate a load with other uses.
7416 if (!InVec.hasOneUse())
7417 return SDValue();
7418
Owen Andersone50ed302009-08-10 22:56:29 +00007419 EVT BCVT = InVec.getOperand(0).getValueType();
7420 if (!BCVT.isVector() || ExtVT.bitsGT(BCVT.getVectorElementType()))
Dan Gohman475871a2008-07-27 21:46:04 +00007421 return SDValue();
Mon P Wanga60b5232008-12-11 00:26:16 +00007422 if (VT.getVectorNumElements() != BCVT.getVectorNumElements())
7423 BCNumEltsChanged = true;
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007424 InVec = InVec.getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00007425 ExtVT = BCVT.getVectorElementType();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007426 NewLoad = true;
7427 }
Evan Cheng513da432007-10-06 08:19:55 +00007428
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007429 LoadSDNode *LN0 = NULL;
Nate Begeman5a5ca152009-04-29 05:20:52 +00007430 const ShuffleVectorSDNode *SVN = NULL;
Bill Wendlingc144a572009-01-30 23:36:47 +00007431 if (ISD::isNormalLoad(InVec.getNode())) {
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007432 LN0 = cast<LoadSDNode>(InVec);
Bill Wendlingc144a572009-01-30 23:36:47 +00007433 } else if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR &&
Owen Andersone50ed302009-08-10 22:56:29 +00007434 InVec.getOperand(0).getValueType() == ExtVT &&
Bill Wendlingc144a572009-01-30 23:36:47 +00007435 ISD::isNormalLoad(InVec.getOperand(0).getNode())) {
Eli Friedmand6e25602011-12-26 22:49:32 +00007436 // Don't duplicate a load with other uses.
7437 if (!InVec.hasOneUse())
7438 return SDValue();
7439
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007440 LN0 = cast<LoadSDNode>(InVec.getOperand(0));
Nate Begeman5a5ca152009-04-29 05:20:52 +00007441 } else if ((SVN = dyn_cast<ShuffleVectorSDNode>(InVec))) {
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007442 // (vextract (vector_shuffle (load $addr), v2, <1, u, u, u>), 1)
7443 // =>
7444 // (load $addr+1*size)
Scott Michelfdc40a02009-02-17 22:15:04 +00007445
Eli Friedmand6e25602011-12-26 22:49:32 +00007446 // Don't duplicate a load with other uses.
7447 if (!InVec.hasOneUse())
7448 return SDValue();
7449
Mon P Wanga60b5232008-12-11 00:26:16 +00007450 // If the bit convert changed the number of elements, it is unsafe
7451 // to examine the mask.
7452 if (BCNumEltsChanged)
7453 return SDValue();
Nate Begeman5a5ca152009-04-29 05:20:52 +00007454
7455 // Select the input vector, guarding against out of range extract vector.
7456 unsigned NumElems = VT.getVectorNumElements();
Eric Christophercaebdd42010-11-03 09:36:40 +00007457 int Idx = (Elt > (int)NumElems) ? -1 : SVN->getMaskElt(Elt);
Nate Begeman5a5ca152009-04-29 05:20:52 +00007458 InVec = (Idx < (int)NumElems) ? InVec.getOperand(0) : InVec.getOperand(1);
7459
Eli Friedmand6e25602011-12-26 22:49:32 +00007460 if (InVec.getOpcode() == ISD::BITCAST) {
7461 // Don't duplicate a load with other uses.
7462 if (!InVec.hasOneUse())
7463 return SDValue();
7464
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007465 InVec = InVec.getOperand(0);
Eli Friedmand6e25602011-12-26 22:49:32 +00007466 }
Gabor Greifba36cb52008-08-28 21:40:38 +00007467 if (ISD::isNormalLoad(InVec.getNode())) {
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007468 LN0 = cast<LoadSDNode>(InVec);
Ted Kremenekd0e88f32010-04-08 18:49:30 +00007469 Elt = (Idx < (int)NumElems) ? Idx : Idx - (int)NumElems;
Evan Cheng513da432007-10-06 08:19:55 +00007470 }
7471 }
Bill Wendlingc144a572009-01-30 23:36:47 +00007472
Eli Friedmand6e25602011-12-26 22:49:32 +00007473 // Make sure we found a non-volatile load and the extractelement is
7474 // the only use.
Nadav Rotem42febc62011-05-11 14:40:50 +00007475 if (!LN0 || !LN0->hasNUsesOfValue(1,0) || LN0->isVolatile())
Dan Gohman475871a2008-07-27 21:46:04 +00007476 return SDValue();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007477
Eric Christopherd81f17a2010-11-03 20:44:42 +00007478 // If Idx was -1 above, Elt is going to be -1, so just return undef.
7479 if (Elt == -1)
Eli Friedmaned4b4272011-07-25 22:25:42 +00007480 return DAG.getUNDEF(LVT);
Eric Christopherd81f17a2010-11-03 20:44:42 +00007481
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007482 unsigned Align = LN0->getAlignment();
7483 if (NewLoad) {
7484 // Check the resultant load doesn't need a higher alignment than the
7485 // original load.
Bill Wendlingc144a572009-01-30 23:36:47 +00007486 unsigned NewAlign =
Eric Christopher503a64d2010-12-09 04:48:06 +00007487 TLI.getTargetData()
7488 ->getABITypeAlignment(LVT.getTypeForEVT(*DAG.getContext()));
Bill Wendlingc144a572009-01-30 23:36:47 +00007489
Dan Gohmanf560ffa2009-01-28 17:46:25 +00007490 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, LVT))
Dan Gohman475871a2008-07-27 21:46:04 +00007491 return SDValue();
Bill Wendlingc144a572009-01-30 23:36:47 +00007492
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007493 Align = NewAlign;
7494 }
7495
Dan Gohman475871a2008-07-27 21:46:04 +00007496 SDValue NewPtr = LN0->getBasePtr();
Chris Lattnerfa459012010-09-21 16:08:50 +00007497 unsigned PtrOff = 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007498
Eric Christopherd81f17a2010-11-03 20:44:42 +00007499 if (Elt) {
Chris Lattnerfa459012010-09-21 16:08:50 +00007500 PtrOff = LVT.getSizeInBits() * Elt / 8;
Owen Andersone50ed302009-08-10 22:56:29 +00007501 EVT PtrType = NewPtr.getValueType();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007502 if (TLI.isBigEndian())
Duncan Sands83ec4b62008-06-06 12:08:01 +00007503 PtrOff = VT.getSizeInBits() / 8 - PtrOff;
Bill Wendlingc144a572009-01-30 23:36:47 +00007504 NewPtr = DAG.getNode(ISD::ADD, N->getDebugLoc(), PtrType, NewPtr,
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007505 DAG.getConstant(PtrOff, PtrType));
7506 }
Bill Wendlingc144a572009-01-30 23:36:47 +00007507
Eli Friedman4db4add2011-11-16 23:50:22 +00007508 // The replacement we need to do here is a little tricky: we need to
7509 // replace an extractelement of a load with a load.
7510 // Use ReplaceAllUsesOfValuesWith to do the replacement.
Eli Friedmand6e25602011-12-26 22:49:32 +00007511 // Note that this replacement assumes that the extractvalue is the only
7512 // use of the load; that's okay because we don't want to perform this
7513 // transformation in other cases anyway.
Evan Cheng84387ea2012-03-13 22:00:52 +00007514 SDValue Load;
Evan Chenga03d3662012-03-13 22:16:11 +00007515 SDValue Chain;
Evan Cheng84387ea2012-03-13 22:00:52 +00007516 if (NVT.bitsGT(LVT)) {
7517 // If the result type of vextract is wider than the load, then issue an
7518 // extending load instead.
7519 ISD::LoadExtType ExtType = TLI.isLoadExtLegal(ISD::ZEXTLOAD, LVT)
7520 ? ISD::ZEXTLOAD : ISD::EXTLOAD;
7521 Load = DAG.getExtLoad(ExtType, N->getDebugLoc(), NVT, LN0->getChain(),
7522 NewPtr, LN0->getPointerInfo().getWithOffset(PtrOff),
7523 LVT, LN0->isVolatile(), LN0->isNonTemporal(),Align);
Evan Chenga03d3662012-03-13 22:16:11 +00007524 Chain = Load.getValue(1);
7525 } else {
Evan Cheng84387ea2012-03-13 22:00:52 +00007526 Load = DAG.getLoad(LVT, N->getDebugLoc(), LN0->getChain(), NewPtr,
7527 LN0->getPointerInfo().getWithOffset(PtrOff),
7528 LN0->isVolatile(), LN0->isNonTemporal(),
7529 LN0->isInvariant(), Align);
Evan Chenga03d3662012-03-13 22:16:11 +00007530 Chain = Load.getValue(1);
7531 if (NVT.bitsLT(LVT))
7532 Load = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), NVT, Load);
7533 else
7534 Load = DAG.getNode(ISD::BITCAST, N->getDebugLoc(), NVT, Load);
7535 }
Eli Friedman4db4add2011-11-16 23:50:22 +00007536 WorkListRemover DeadNodes(*this);
7537 SDValue From[] = { SDValue(N, 0), SDValue(LN0,1) };
Evan Chenga03d3662012-03-13 22:16:11 +00007538 SDValue To[] = { Load, Chain };
Jakob Stoklund Olesenbc7d4482012-04-20 22:08:46 +00007539 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
Eli Friedman4db4add2011-11-16 23:50:22 +00007540 // Since we're explcitly calling ReplaceAllUses, add the new node to the
7541 // worklist explicitly as well.
7542 AddToWorkList(Load.getNode());
Craig Topper0c9da212012-03-20 05:28:39 +00007543 AddUsersToWorkList(Load.getNode()); // Add users too
Eli Friedman4db4add2011-11-16 23:50:22 +00007544 // Make sure to revisit this node to clean it up; it will usually be dead.
7545 AddToWorkList(N);
7546 return SDValue(N, 0);
Evan Cheng513da432007-10-06 08:19:55 +00007547 }
Bill Wendlingc144a572009-01-30 23:36:47 +00007548
Dan Gohman475871a2008-07-27 21:46:04 +00007549 return SDValue();
Evan Cheng513da432007-10-06 08:19:55 +00007550}
Evan Cheng513da432007-10-06 08:19:55 +00007551
Dan Gohman475871a2008-07-27 21:46:04 +00007552SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
Dan Gohman7f321562007-06-25 16:23:39 +00007553 unsigned NumInScalars = N->getNumOperands();
Nadav Rotemb00418a2011-10-29 21:23:04 +00007554 DebugLoc dl = N->getDebugLoc();
Owen Andersone50ed302009-08-10 22:56:29 +00007555 EVT VT = N->getValueType(0);
Nadav Rotemb00418a2011-10-29 21:23:04 +00007556 // Check to see if this is a BUILD_VECTOR of a bunch of values
7557 // which come from any_extend or zero_extend nodes. If so, we can create
7558 // a new BUILD_VECTOR using bit-casts which may enable other BUILD_VECTOR
Nadav Rotemf47368b2011-10-31 20:08:25 +00007559 // optimizations. We do not handle sign-extend because we can't fill the sign
7560 // using shuffles.
Nadav Rotemb00418a2011-10-29 21:23:04 +00007561 EVT SourceType = MVT::Other;
Craig Topperd3b58892012-01-17 09:09:48 +00007562 bool AllAnyExt = true;
7563 bool AllUndef = true;
7564 for (unsigned i = 0; i != NumInScalars; ++i) {
Nadav Rotemb00418a2011-10-29 21:23:04 +00007565 SDValue In = N->getOperand(i);
7566 // Ignore undef inputs.
7567 if (In.getOpcode() == ISD::UNDEF) continue;
Craig Topperd3b58892012-01-17 09:09:48 +00007568 AllUndef = false;
Nadav Rotemb00418a2011-10-29 21:23:04 +00007569
7570 bool AnyExt = In.getOpcode() == ISD::ANY_EXTEND;
7571 bool ZeroExt = In.getOpcode() == ISD::ZERO_EXTEND;
7572
Nadav Rotemf47368b2011-10-31 20:08:25 +00007573 // Abort if the element is not an extension.
Nadav Rotemb00418a2011-10-29 21:23:04 +00007574 if (!ZeroExt && !AnyExt) {
Nadav Rotemf47368b2011-10-31 20:08:25 +00007575 SourceType = MVT::Other;
Nadav Rotemb00418a2011-10-29 21:23:04 +00007576 break;
7577 }
7578
7579 // The input is a ZeroExt or AnyExt. Check the original type.
7580 EVT InTy = In.getOperand(0).getValueType();
7581
7582 // Check that all of the widened source types are the same.
7583 if (SourceType == MVT::Other)
Nadav Rotemf47368b2011-10-31 20:08:25 +00007584 // First time.
Nadav Rotemb00418a2011-10-29 21:23:04 +00007585 SourceType = InTy;
7586 else if (InTy != SourceType) {
7587 // Multiple income types. Abort.
Nadav Rotemf47368b2011-10-31 20:08:25 +00007588 SourceType = MVT::Other;
Nadav Rotemb00418a2011-10-29 21:23:04 +00007589 break;
7590 }
7591
7592 // Check if all of the extends are ANY_EXTENDs.
Craig Topperd3b58892012-01-17 09:09:48 +00007593 AllAnyExt &= AnyExt;
Nadav Rotemb00418a2011-10-29 21:23:04 +00007594 }
7595
Craig Topperd3b58892012-01-17 09:09:48 +00007596 if (AllUndef)
7597 return DAG.getUNDEF(VT);
Nadav Rotemf47368b2011-10-31 20:08:25 +00007598
7599 // In order to have valid types, all of the inputs must be extended from the
7600 // same source type and all of the inputs must be any or zero extend.
7601 // Scalar sizes must be a power of two.
7602 EVT OutScalarTy = N->getValueType(0).getScalarType();
Nadav Rotem2ee746b2012-02-12 15:05:31 +00007603 bool ValidTypes = SourceType != MVT::Other &&
Nadav Rotemf47368b2011-10-31 20:08:25 +00007604 isPowerOf2_32(OutScalarTy.getSizeInBits()) &&
7605 isPowerOf2_32(SourceType.getSizeInBits());
7606
7607 // We perform this optimization post type-legalization because
7608 // the type-legalizer often scalarizes integer-promoted vectors.
7609 // Performing this optimization before may create bit-casts which
7610 // will be type-legalized to complex code sequences.
7611 // We perform this optimization only before the operation legalizer because we
7612 // may introduce illegal operations.
Nadav Rotem6431ff92012-03-15 08:49:06 +00007613 // Create a new simpler BUILD_VECTOR sequence which other optimizations can
7614 // turn into a single shuffle instruction.
Nadav Rotem2ee746b2012-02-12 15:05:31 +00007615 if ((Level == AfterLegalizeVectorOps || Level == AfterLegalizeTypes) &&
7616 ValidTypes) {
Nadav Rotemb00418a2011-10-29 21:23:04 +00007617 bool isLE = TLI.isLittleEndian();
Nadav Rotemf47368b2011-10-31 20:08:25 +00007618 unsigned ElemRatio = OutScalarTy.getSizeInBits()/SourceType.getSizeInBits();
Nadav Rotemb00418a2011-10-29 21:23:04 +00007619 assert(ElemRatio > 1 && "Invalid element size ratio");
Craig Topperd3b58892012-01-17 09:09:48 +00007620 SDValue Filler = AllAnyExt ? DAG.getUNDEF(SourceType):
Nadav Rotemf47368b2011-10-31 20:08:25 +00007621 DAG.getConstant(0, SourceType);
Nadav Rotemb00418a2011-10-29 21:23:04 +00007622
7623 unsigned NewBVElems = ElemRatio * N->getValueType(0).getVectorNumElements();
Benjamin Kramer50bf86e2011-10-30 08:39:55 +00007624 SmallVector<SDValue, 8> Ops(NewBVElems, Filler);
Nadav Rotemb00418a2011-10-29 21:23:04 +00007625
7626 // Populate the new build_vector
7627 for (unsigned i=0; i < N->getNumOperands(); ++i) {
7628 SDValue Cast = N->getOperand(i);
Benjamin Kramer50bf86e2011-10-30 08:39:55 +00007629 assert((Cast.getOpcode() == ISD::ANY_EXTEND ||
7630 Cast.getOpcode() == ISD::ZERO_EXTEND ||
7631 Cast.getOpcode() == ISD::UNDEF) && "Invalid cast opcode");
Nadav Rotemb00418a2011-10-29 21:23:04 +00007632 SDValue In;
7633 if (Cast.getOpcode() == ISD::UNDEF)
Nadav Rotemf47368b2011-10-31 20:08:25 +00007634 In = DAG.getUNDEF(SourceType);
Nadav Rotemb00418a2011-10-29 21:23:04 +00007635 else
7636 In = Cast->getOperand(0);
7637 unsigned Index = isLE ? (i * ElemRatio) :
7638 (i * ElemRatio + (ElemRatio - 1));
7639
7640 assert(Index < Ops.size() && "Invalid index");
7641 Ops[Index] = In;
7642 }
7643
7644 // The type of the new BUILD_VECTOR node.
Nadav Rotemf47368b2011-10-31 20:08:25 +00007645 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), SourceType, NewBVElems);
Nadav Rotemb00418a2011-10-29 21:23:04 +00007646 assert(VecVT.getSizeInBits() == N->getValueType(0).getSizeInBits() &&
7647 "Invalid vector size");
Nadav Rotemf47368b2011-10-31 20:08:25 +00007648 // Check if the new vector type is legal.
7649 if (!isTypeLegal(VecVT)) return SDValue();
Nadav Rotemb00418a2011-10-29 21:23:04 +00007650
7651 // Make the new BUILD_VECTOR.
7652 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
7653 VecVT, &Ops[0], Ops.size());
7654
Nadav Rotem6431ff92012-03-15 08:49:06 +00007655 // The new BUILD_VECTOR node has the potential to be further optimized.
7656 AddToWorkList(BV.getNode());
Nadav Rotemb00418a2011-10-29 21:23:04 +00007657 // Bitcast to the desired type.
7658 return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), BV);
7659 }
Chris Lattnerca242442006-03-19 01:27:56 +00007660
Dan Gohman7f321562007-06-25 16:23:39 +00007661 // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT
7662 // operations. If so, and if the EXTRACT_VECTOR_ELT vector inputs come from
7663 // at most two distinct vectors, turn this into a shuffle node.
Duncan Sands00294ca2012-03-19 15:35:44 +00007664
7665 // May only combine to shuffle after legalize if shuffle is legal.
7666 if (LegalOperations &&
7667 !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))
7668 return SDValue();
7669
Dan Gohman475871a2008-07-27 21:46:04 +00007670 SDValue VecIn1, VecIn2;
Chris Lattnerd7648c82006-03-28 20:28:38 +00007671 for (unsigned i = 0; i != NumInScalars; ++i) {
7672 // Ignore undef inputs.
7673 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00007674
Dan Gohman7f321562007-06-25 16:23:39 +00007675 // If this input is something other than a EXTRACT_VECTOR_ELT with a
Chris Lattnerd7648c82006-03-28 20:28:38 +00007676 // constant index, bail out.
Dan Gohman7f321562007-06-25 16:23:39 +00007677 if (N->getOperand(i).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
Chris Lattnerd7648c82006-03-28 20:28:38 +00007678 !isa<ConstantSDNode>(N->getOperand(i).getOperand(1))) {
Dan Gohman475871a2008-07-27 21:46:04 +00007679 VecIn1 = VecIn2 = SDValue(0, 0);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007680 break;
7681 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007682
Nadav Rotem2ee746b2012-02-12 15:05:31 +00007683 // We allow up to two distinct input vectors.
Dan Gohman475871a2008-07-27 21:46:04 +00007684 SDValue ExtractedFromVec = N->getOperand(i).getOperand(0);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007685 if (ExtractedFromVec == VecIn1 || ExtractedFromVec == VecIn2)
7686 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00007687
Gabor Greifba36cb52008-08-28 21:40:38 +00007688 if (VecIn1.getNode() == 0) {
Chris Lattnerd7648c82006-03-28 20:28:38 +00007689 VecIn1 = ExtractedFromVec;
Gabor Greifba36cb52008-08-28 21:40:38 +00007690 } else if (VecIn2.getNode() == 0) {
Chris Lattnerd7648c82006-03-28 20:28:38 +00007691 VecIn2 = ExtractedFromVec;
7692 } else {
7693 // Too many inputs.
Dan Gohman475871a2008-07-27 21:46:04 +00007694 VecIn1 = VecIn2 = SDValue(0, 0);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007695 break;
7696 }
7697 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007698
Nadav Rotem2ee746b2012-02-12 15:05:31 +00007699 // If everything is good, we can make a shuffle operation.
Gabor Greifba36cb52008-08-28 21:40:38 +00007700 if (VecIn1.getNode()) {
Nate Begeman9008ca62009-04-27 18:41:29 +00007701 SmallVector<int, 8> Mask;
Chris Lattnerd7648c82006-03-28 20:28:38 +00007702 for (unsigned i = 0; i != NumInScalars; ++i) {
7703 if (N->getOperand(i).getOpcode() == ISD::UNDEF) {
Nate Begeman9008ca62009-04-27 18:41:29 +00007704 Mask.push_back(-1);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007705 continue;
7706 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007707
Rafael Espindola15684b22009-04-24 12:40:33 +00007708 // If extracting from the first vector, just use the index directly.
Nate Begeman9008ca62009-04-27 18:41:29 +00007709 SDValue Extract = N->getOperand(i);
Mon P Wang93b74152009-03-17 06:33:10 +00007710 SDValue ExtVal = Extract.getOperand(1);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007711 if (Extract.getOperand(0) == VecIn1) {
Nate Begeman5a5ca152009-04-29 05:20:52 +00007712 unsigned ExtIndex = cast<ConstantSDNode>(ExtVal)->getZExtValue();
7713 if (ExtIndex > VT.getVectorNumElements())
7714 return SDValue();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007715
Nate Begeman5a5ca152009-04-29 05:20:52 +00007716 Mask.push_back(ExtIndex);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007717 continue;
7718 }
7719
7720 // Otherwise, use InIdx + VecSize
Mon P Wang93b74152009-03-17 06:33:10 +00007721 unsigned Idx = cast<ConstantSDNode>(ExtVal)->getZExtValue();
Nate Begeman9008ca62009-04-27 18:41:29 +00007722 Mask.push_back(Idx+NumInScalars);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007723 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007724
Nadav Rotem2ee746b2012-02-12 15:05:31 +00007725 // We can't generate a shuffle node with mismatched input and output types.
7726 // Attempt to transform a single input vector to the correct type.
7727 if ((VT != VecIn1.getValueType())) {
7728 // We don't support shuffeling between TWO values of different types.
7729 if (VecIn2.getNode() != 0)
7730 return SDValue();
7731
7732 // We only support widening of vectors which are half the size of the
7733 // output registers. For example XMM->YMM widening on X86 with AVX.
7734 if (VecIn1.getValueType().getSizeInBits()*2 != VT.getSizeInBits())
7735 return SDValue();
7736
7737 // Widen the input vector by adding undef values.
7738 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, N->getDebugLoc(), VT,
7739 VecIn1, DAG.getUNDEF(VecIn1.getValueType()));
7740 }
7741
7742 // If VecIn2 is unused then change it to undef.
7743 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
7744
Nadav Rotem0877fdf2012-02-13 12:42:26 +00007745 // Check that we were able to transform all incoming values to the same type.
7746 if (VecIn2.getValueType() != VecIn1.getValueType() ||
7747 VecIn1.getValueType() != VT)
7748 return SDValue();
7749
Nadav Rotem2ee746b2012-02-12 15:05:31 +00007750 // Only type-legal BUILD_VECTOR nodes are converted to shuffle nodes.
Nadav Rotem0877fdf2012-02-13 12:42:26 +00007751 if (!isTypeLegal(VT))
Duncan Sands25cf2272008-11-24 14:53:14 +00007752 return SDValue();
7753
Dan Gohman7f321562007-06-25 16:23:39 +00007754 // Return the new VECTOR_SHUFFLE node.
Nate Begeman9008ca62009-04-27 18:41:29 +00007755 SDValue Ops[2];
Chris Lattnerbd564bf2006-08-08 02:23:42 +00007756 Ops[0] = VecIn1;
Nadav Rotem2ee746b2012-02-12 15:05:31 +00007757 Ops[1] = VecIn2;
Nate Begeman9008ca62009-04-27 18:41:29 +00007758 return DAG.getVectorShuffle(VT, N->getDebugLoc(), Ops[0], Ops[1], &Mask[0]);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007759 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007760
Dan Gohman475871a2008-07-27 21:46:04 +00007761 return SDValue();
Chris Lattnerd7648c82006-03-28 20:28:38 +00007762}
7763
Dan Gohman475871a2008-07-27 21:46:04 +00007764SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
Dan Gohman7f321562007-06-25 16:23:39 +00007765 // TODO: Check to see if this is a CONCAT_VECTORS of a bunch of
7766 // EXTRACT_SUBVECTOR operations. If so, and if the EXTRACT_SUBVECTOR vector
7767 // inputs come from at most two distinct vectors, turn this into a shuffle
7768 // node.
7769
7770 // If we only have one input vector, we don't need to do any concatenation.
Bill Wendlingc144a572009-01-30 23:36:47 +00007771 if (N->getNumOperands() == 1)
Dan Gohman7f321562007-06-25 16:23:39 +00007772 return N->getOperand(0);
Dan Gohman7f321562007-06-25 16:23:39 +00007773
Dan Gohman475871a2008-07-27 21:46:04 +00007774 return SDValue();
Dan Gohman7f321562007-06-25 16:23:39 +00007775}
7776
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00007777SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) {
7778 EVT NVT = N->getValueType(0);
7779 SDValue V = N->getOperand(0);
7780
7781 if (V->getOpcode() == ISD::INSERT_SUBVECTOR) {
7782 // Handle only simple case where vector being inserted and vector
7783 // being extracted are of same type, and are half size of larger vectors.
7784 EVT BigVT = V->getOperand(0).getValueType();
7785 EVT SmallVT = V->getOperand(1).getValueType();
7786 if (NVT != SmallVT || NVT.getSizeInBits()*2 != BigVT.getSizeInBits())
7787 return SDValue();
7788
Eli Friedman26323442011-12-07 00:11:56 +00007789 // Only handle cases where both indexes are constants with the same type.
7790 ConstantSDNode *InsIdx = dyn_cast<ConstantSDNode>(N->getOperand(1));
7791 ConstantSDNode *ExtIdx = dyn_cast<ConstantSDNode>(V->getOperand(2));
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00007792
Eli Friedman26323442011-12-07 00:11:56 +00007793 if (InsIdx && ExtIdx &&
7794 InsIdx->getValueType(0).getSizeInBits() <= 64 &&
7795 ExtIdx->getValueType(0).getSizeInBits() <= 64) {
7796 // Combine:
7797 // (extract_subvec (insert_subvec V1, V2, InsIdx), ExtIdx)
7798 // Into:
7799 // indices are equal => V1
7800 // otherwise => (extract_subvec V1, ExtIdx)
7801 if (InsIdx->getZExtValue() == ExtIdx->getZExtValue())
7802 return V->getOperand(1);
7803 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), NVT,
7804 V->getOperand(0), N->getOperand(1));
7805 }
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00007806 }
7807
7808 return SDValue();
7809}
7810
Dan Gohman475871a2008-07-27 21:46:04 +00007811SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
Owen Andersone50ed302009-08-10 22:56:29 +00007812 EVT VT = N->getValueType(0);
Nate Begeman9008ca62009-04-27 18:41:29 +00007813 unsigned NumElts = VT.getVectorNumElements();
Chris Lattnerf1d0c622006-03-31 22:16:43 +00007814
Mon P Wangaeb06d22008-11-10 04:46:22 +00007815 SDValue N0 = N->getOperand(0);
Craig Topper481b79c2012-01-04 08:07:43 +00007816 SDValue N1 = N->getOperand(1);
Mon P Wangaeb06d22008-11-10 04:46:22 +00007817
Craig Topperae1bec52012-04-09 05:16:56 +00007818 assert(N0.getValueType() == VT && "Vector shuffle must be normalized in DAG");
Mon P Wangaeb06d22008-11-10 04:46:22 +00007819
Craig Topper481b79c2012-01-04 08:07:43 +00007820 // Canonicalize shuffle undef, undef -> undef
7821 if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF)
7822 return DAG.getUNDEF(VT);
7823
7824 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
7825
7826 // Canonicalize shuffle v, v -> v, undef
7827 if (N0 == N1) {
7828 SmallVector<int, 8> NewMask;
7829 for (unsigned i = 0; i != NumElts; ++i) {
7830 int Idx = SVN->getMaskElt(i);
7831 if (Idx >= (int)NumElts) Idx -= NumElts;
7832 NewMask.push_back(Idx);
7833 }
7834 return DAG.getVectorShuffle(VT, N->getDebugLoc(), N0, DAG.getUNDEF(VT),
7835 &NewMask[0]);
7836 }
7837
7838 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
7839 if (N0.getOpcode() == ISD::UNDEF) {
7840 SmallVector<int, 8> NewMask;
7841 for (unsigned i = 0; i != NumElts; ++i) {
7842 int Idx = SVN->getMaskElt(i);
Craig Topper4b206bd2012-04-09 05:55:33 +00007843 if (Idx >= 0) {
7844 if (Idx < (int)NumElts)
7845 Idx += NumElts;
7846 else
7847 Idx -= NumElts;
7848 }
7849 NewMask.push_back(Idx);
Craig Topper481b79c2012-01-04 08:07:43 +00007850 }
7851 return DAG.getVectorShuffle(VT, N->getDebugLoc(), N1, DAG.getUNDEF(VT),
7852 &NewMask[0]);
7853 }
7854
7855 // Remove references to rhs if it is undef
7856 if (N1.getOpcode() == ISD::UNDEF) {
7857 bool Changed = false;
7858 SmallVector<int, 8> NewMask;
7859 for (unsigned i = 0; i != NumElts; ++i) {
7860 int Idx = SVN->getMaskElt(i);
7861 if (Idx >= (int)NumElts) {
7862 Idx = -1;
7863 Changed = true;
7864 }
7865 NewMask.push_back(Idx);
7866 }
7867 if (Changed)
7868 return DAG.getVectorShuffle(VT, N->getDebugLoc(), N0, N1, &NewMask[0]);
7869 }
Evan Chenge7bec0d2006-07-20 22:44:41 +00007870
Bob Wilson0f1db1a2010-10-28 17:06:14 +00007871 // If it is a splat, check if the argument vector is another splat or a
7872 // build_vector with all scalar elements the same.
Bob Wilson0f1db1a2010-10-28 17:06:14 +00007873 if (SVN->isSplat() && SVN->getSplatIndex() < (int)NumElts) {
Gabor Greifba36cb52008-08-28 21:40:38 +00007874 SDNode *V = N0.getNode();
Evan Cheng917ec982006-07-21 08:25:53 +00007875
Dan Gohman7f321562007-06-25 16:23:39 +00007876 // If this is a bit convert that changes the element type of the vector but
Evan Cheng59569222006-10-16 22:49:37 +00007877 // not the number of vector elements, look through it. Be careful not to
7878 // look though conversions that change things like v4f32 to v2f64.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007879 if (V->getOpcode() == ISD::BITCAST) {
Dan Gohman475871a2008-07-27 21:46:04 +00007880 SDValue ConvInput = V->getOperand(0);
Evan Cheng29257862008-07-22 20:42:56 +00007881 if (ConvInput.getValueType().isVector() &&
7882 ConvInput.getValueType().getVectorNumElements() == NumElts)
Gabor Greifba36cb52008-08-28 21:40:38 +00007883 V = ConvInput.getNode();
Evan Cheng59569222006-10-16 22:49:37 +00007884 }
7885
Dan Gohman7f321562007-06-25 16:23:39 +00007886 if (V->getOpcode() == ISD::BUILD_VECTOR) {
Bob Wilson0f1db1a2010-10-28 17:06:14 +00007887 assert(V->getNumOperands() == NumElts &&
7888 "BUILD_VECTOR has wrong number of operands");
7889 SDValue Base;
7890 bool AllSame = true;
7891 for (unsigned i = 0; i != NumElts; ++i) {
7892 if (V->getOperand(i).getOpcode() != ISD::UNDEF) {
7893 Base = V->getOperand(i);
7894 break;
Evan Cheng917ec982006-07-21 08:25:53 +00007895 }
Evan Cheng917ec982006-07-21 08:25:53 +00007896 }
Bob Wilson0f1db1a2010-10-28 17:06:14 +00007897 // Splat of <u, u, u, u>, return <u, u, u, u>
7898 if (!Base.getNode())
7899 return N0;
7900 for (unsigned i = 0; i != NumElts; ++i) {
7901 if (V->getOperand(i) != Base) {
7902 AllSame = false;
7903 break;
7904 }
7905 }
7906 // Splat of <x, x, x, x>, return <x, x, x, x>
7907 if (AllSame)
7908 return N0;
Evan Cheng917ec982006-07-21 08:25:53 +00007909 }
7910 }
Nadav Rotem4ac90812012-04-01 19:31:22 +00007911
7912 // If this shuffle node is simply a swizzle of another shuffle node,
Nadav Rotemd16c8d02012-04-07 21:19:08 +00007913 // and it reverses the swizzle of the previous shuffle then we can
7914 // optimize shuffle(shuffle(x, undef), undef) -> x.
Nadav Rotem4ac90812012-04-01 19:31:22 +00007915 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG &&
7916 N1.getOpcode() == ISD::UNDEF) {
7917
Nadav Rotem4ac90812012-04-01 19:31:22 +00007918 ShuffleVectorSDNode *OtherSV = cast<ShuffleVectorSDNode>(N0);
7919
Nadav Rotemd16c8d02012-04-07 21:19:08 +00007920 // Shuffle nodes can only reverse shuffles with a single non-undef value.
7921 if (N0.getOperand(1).getOpcode() != ISD::UNDEF)
7922 return SDValue();
7923
Craig Topperae1bec52012-04-09 05:16:56 +00007924 // The incoming shuffle must be of the same type as the result of the
7925 // current shuffle.
7926 assert(OtherSV->getOperand(0).getValueType() == VT &&
7927 "Shuffle types don't match");
Nadav Rotem4ac90812012-04-01 19:31:22 +00007928
7929 for (unsigned i = 0; i != NumElts; ++i) {
7930 int Idx = SVN->getMaskElt(i);
Craig Topperae1bec52012-04-09 05:16:56 +00007931 assert(Idx < (int)NumElts && "Index references undef operand");
Nadav Rotem4ac90812012-04-01 19:31:22 +00007932 // Next, this index comes from the first value, which is the incoming
7933 // shuffle. Adopt the incoming index.
7934 if (Idx >= 0)
7935 Idx = OtherSV->getMaskElt(Idx);
7936
Nadav Rotemd16c8d02012-04-07 21:19:08 +00007937 // The combined shuffle must map each index to itself.
Craig Topperae1bec52012-04-09 05:16:56 +00007938 if (Idx >= 0 && (unsigned)Idx != i)
Nadav Rotemd16c8d02012-04-07 21:19:08 +00007939 return SDValue();
Nadav Rotem4ac90812012-04-01 19:31:22 +00007940 }
Nadav Rotemd16c8d02012-04-07 21:19:08 +00007941
7942 return OtherSV->getOperand(0);
Nadav Rotem4ac90812012-04-01 19:31:22 +00007943 }
7944
Dan Gohman475871a2008-07-27 21:46:04 +00007945 return SDValue();
Chris Lattnerf1d0c622006-03-31 22:16:43 +00007946}
7947
Jim Grosbach9a526492010-06-23 16:07:42 +00007948SDValue DAGCombiner::visitMEMBARRIER(SDNode* N) {
7949 if (!TLI.getShouldFoldAtomicFences())
7950 return SDValue();
7951
7952 SDValue atomic = N->getOperand(0);
7953 switch (atomic.getOpcode()) {
7954 case ISD::ATOMIC_CMP_SWAP:
7955 case ISD::ATOMIC_SWAP:
7956 case ISD::ATOMIC_LOAD_ADD:
7957 case ISD::ATOMIC_LOAD_SUB:
7958 case ISD::ATOMIC_LOAD_AND:
7959 case ISD::ATOMIC_LOAD_OR:
7960 case ISD::ATOMIC_LOAD_XOR:
7961 case ISD::ATOMIC_LOAD_NAND:
7962 case ISD::ATOMIC_LOAD_MIN:
7963 case ISD::ATOMIC_LOAD_MAX:
7964 case ISD::ATOMIC_LOAD_UMIN:
7965 case ISD::ATOMIC_LOAD_UMAX:
7966 break;
7967 default:
7968 return SDValue();
7969 }
7970
7971 SDValue fence = atomic.getOperand(0);
7972 if (fence.getOpcode() != ISD::MEMBARRIER)
7973 return SDValue();
7974
7975 switch (atomic.getOpcode()) {
7976 case ISD::ATOMIC_CMP_SWAP:
7977 return SDValue(DAG.UpdateNodeOperands(atomic.getNode(),
7978 fence.getOperand(0),
7979 atomic.getOperand(1), atomic.getOperand(2),
7980 atomic.getOperand(3)), atomic.getResNo());
7981 case ISD::ATOMIC_SWAP:
7982 case ISD::ATOMIC_LOAD_ADD:
7983 case ISD::ATOMIC_LOAD_SUB:
7984 case ISD::ATOMIC_LOAD_AND:
7985 case ISD::ATOMIC_LOAD_OR:
7986 case ISD::ATOMIC_LOAD_XOR:
7987 case ISD::ATOMIC_LOAD_NAND:
7988 case ISD::ATOMIC_LOAD_MIN:
7989 case ISD::ATOMIC_LOAD_MAX:
7990 case ISD::ATOMIC_LOAD_UMIN:
7991 case ISD::ATOMIC_LOAD_UMAX:
7992 return SDValue(DAG.UpdateNodeOperands(atomic.getNode(),
7993 fence.getOperand(0),
7994 atomic.getOperand(1), atomic.getOperand(2)),
7995 atomic.getResNo());
7996 default:
7997 return SDValue();
7998 }
7999}
8000
Evan Cheng44f1f092006-04-20 08:56:16 +00008001/// XformToShuffleWithZero - Returns a vector_shuffle if it able to transform
Dan Gohman7f321562007-06-25 16:23:39 +00008002/// an AND to a vector_shuffle with the destination vector and a zero vector.
8003/// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==>
Evan Cheng44f1f092006-04-20 08:56:16 +00008004/// vector_shuffle V, Zero, <0, 4, 2, 4>
Dan Gohman475871a2008-07-27 21:46:04 +00008005SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) {
Owen Andersone50ed302009-08-10 22:56:29 +00008006 EVT VT = N->getValueType(0);
Nate Begeman9008ca62009-04-27 18:41:29 +00008007 DebugLoc dl = N->getDebugLoc();
Dan Gohman475871a2008-07-27 21:46:04 +00008008 SDValue LHS = N->getOperand(0);
8009 SDValue RHS = N->getOperand(1);
Dan Gohman7f321562007-06-25 16:23:39 +00008010 if (N->getOpcode() == ISD::AND) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008011 if (RHS.getOpcode() == ISD::BITCAST)
Evan Cheng44f1f092006-04-20 08:56:16 +00008012 RHS = RHS.getOperand(0);
Dan Gohman7f321562007-06-25 16:23:39 +00008013 if (RHS.getOpcode() == ISD::BUILD_VECTOR) {
Nate Begeman9008ca62009-04-27 18:41:29 +00008014 SmallVector<int, 8> Indices;
8015 unsigned NumElts = RHS.getNumOperands();
Evan Cheng44f1f092006-04-20 08:56:16 +00008016 for (unsigned i = 0; i != NumElts; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00008017 SDValue Elt = RHS.getOperand(i);
Evan Cheng44f1f092006-04-20 08:56:16 +00008018 if (!isa<ConstantSDNode>(Elt))
Dan Gohman475871a2008-07-27 21:46:04 +00008019 return SDValue();
Craig Topperb7135e52012-04-09 05:59:53 +00008020
8021 if (cast<ConstantSDNode>(Elt)->isAllOnesValue())
Nate Begeman9008ca62009-04-27 18:41:29 +00008022 Indices.push_back(i);
Evan Cheng44f1f092006-04-20 08:56:16 +00008023 else if (cast<ConstantSDNode>(Elt)->isNullValue())
Nate Begeman9008ca62009-04-27 18:41:29 +00008024 Indices.push_back(NumElts);
Evan Cheng44f1f092006-04-20 08:56:16 +00008025 else
Dan Gohman475871a2008-07-27 21:46:04 +00008026 return SDValue();
Evan Cheng44f1f092006-04-20 08:56:16 +00008027 }
8028
8029 // Let's see if the target supports this vector_shuffle.
Owen Andersone50ed302009-08-10 22:56:29 +00008030 EVT RVT = RHS.getValueType();
Nate Begeman9008ca62009-04-27 18:41:29 +00008031 if (!TLI.isVectorClearMaskLegal(Indices, RVT))
Dan Gohman475871a2008-07-27 21:46:04 +00008032 return SDValue();
Evan Cheng44f1f092006-04-20 08:56:16 +00008033
Dan Gohman7f321562007-06-25 16:23:39 +00008034 // Return the new VECTOR_SHUFFLE node.
Dan Gohman8a55ce42009-09-23 21:02:20 +00008035 EVT EltVT = RVT.getVectorElementType();
Nate Begeman9008ca62009-04-27 18:41:29 +00008036 SmallVector<SDValue,8> ZeroOps(RVT.getVectorNumElements(),
Dan Gohman8a55ce42009-09-23 21:02:20 +00008037 DAG.getConstant(0, EltVT));
Nate Begeman9008ca62009-04-27 18:41:29 +00008038 SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
8039 RVT, &ZeroOps[0], ZeroOps.size());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008040 LHS = DAG.getNode(ISD::BITCAST, dl, RVT, LHS);
Nate Begeman9008ca62009-04-27 18:41:29 +00008041 SDValue Shuf = DAG.getVectorShuffle(RVT, dl, LHS, Zero, &Indices[0]);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008042 return DAG.getNode(ISD::BITCAST, dl, VT, Shuf);
Evan Cheng44f1f092006-04-20 08:56:16 +00008043 }
8044 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008045
Dan Gohman475871a2008-07-27 21:46:04 +00008046 return SDValue();
Evan Cheng44f1f092006-04-20 08:56:16 +00008047}
8048
Dan Gohman7f321562007-06-25 16:23:39 +00008049/// SimplifyVBinOp - Visit a binary vector operation, like ADD.
Dan Gohman475871a2008-07-27 21:46:04 +00008050SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) {
Dan Gohman7f321562007-06-25 16:23:39 +00008051 // After legalize, the target may be depending on adds and other
8052 // binary ops to provide legal ways to construct constants or other
8053 // things. Simplifying them may result in a loss of legality.
Duncan Sands25cf2272008-11-24 14:53:14 +00008054 if (LegalOperations) return SDValue();
Dan Gohman7f321562007-06-25 16:23:39 +00008055
Bob Wilsond7273432010-12-17 23:06:49 +00008056 assert(N->getValueType(0).isVector() &&
8057 "SimplifyVBinOp only works on vectors!");
Dan Gohman7f321562007-06-25 16:23:39 +00008058
Dan Gohman475871a2008-07-27 21:46:04 +00008059 SDValue LHS = N->getOperand(0);
8060 SDValue RHS = N->getOperand(1);
8061 SDValue Shuffle = XformToShuffleWithZero(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00008062 if (Shuffle.getNode()) return Shuffle;
Evan Cheng44f1f092006-04-20 08:56:16 +00008063
Dan Gohman7f321562007-06-25 16:23:39 +00008064 // If the LHS and RHS are BUILD_VECTOR nodes, see if we can constant fold
Chris Lattneredab1b92006-04-02 03:25:57 +00008065 // this operation.
Scott Michelfdc40a02009-02-17 22:15:04 +00008066 if (LHS.getOpcode() == ISD::BUILD_VECTOR &&
Dan Gohman7f321562007-06-25 16:23:39 +00008067 RHS.getOpcode() == ISD::BUILD_VECTOR) {
Dan Gohman475871a2008-07-27 21:46:04 +00008068 SmallVector<SDValue, 8> Ops;
Dan Gohman7f321562007-06-25 16:23:39 +00008069 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00008070 SDValue LHSOp = LHS.getOperand(i);
8071 SDValue RHSOp = RHS.getOperand(i);
Chris Lattneredab1b92006-04-02 03:25:57 +00008072 // If these two elements can't be folded, bail out.
8073 if ((LHSOp.getOpcode() != ISD::UNDEF &&
8074 LHSOp.getOpcode() != ISD::Constant &&
8075 LHSOp.getOpcode() != ISD::ConstantFP) ||
8076 (RHSOp.getOpcode() != ISD::UNDEF &&
8077 RHSOp.getOpcode() != ISD::Constant &&
8078 RHSOp.getOpcode() != ISD::ConstantFP))
8079 break;
Bill Wendling836ca7d2009-01-30 23:59:18 +00008080
Evan Cheng7b336a82006-05-31 06:08:35 +00008081 // Can't fold divide by zero.
Dan Gohman7f321562007-06-25 16:23:39 +00008082 if (N->getOpcode() == ISD::SDIV || N->getOpcode() == ISD::UDIV ||
8083 N->getOpcode() == ISD::FDIV) {
Evan Cheng7b336a82006-05-31 06:08:35 +00008084 if ((RHSOp.getOpcode() == ISD::Constant &&
Gabor Greifba36cb52008-08-28 21:40:38 +00008085 cast<ConstantSDNode>(RHSOp.getNode())->isNullValue()) ||
Evan Cheng7b336a82006-05-31 06:08:35 +00008086 (RHSOp.getOpcode() == ISD::ConstantFP &&
Gabor Greifba36cb52008-08-28 21:40:38 +00008087 cast<ConstantFPSDNode>(RHSOp.getNode())->getValueAPF().isZero()))
Evan Cheng7b336a82006-05-31 06:08:35 +00008088 break;
8089 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008090
Bob Wilsond7273432010-12-17 23:06:49 +00008091 EVT VT = LHSOp.getValueType();
Bob Wilsondb2b18f2011-10-18 17:34:47 +00008092 EVT RVT = RHSOp.getValueType();
8093 if (RVT != VT) {
8094 // Integer BUILD_VECTOR operands may have types larger than the element
8095 // size (e.g., when the element type is not legal). Prior to type
8096 // legalization, the types may not match between the two BUILD_VECTORS.
8097 // Truncate one of the operands to make them match.
8098 if (RVT.getSizeInBits() > VT.getSizeInBits()) {
8099 RHSOp = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, RHSOp);
8100 } else {
8101 LHSOp = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), RVT, LHSOp);
8102 VT = RVT;
8103 }
8104 }
Bob Wilsond7273432010-12-17 23:06:49 +00008105 SDValue FoldOp = DAG.getNode(N->getOpcode(), LHS.getDebugLoc(), VT,
Evan Chenga0839882010-05-18 00:03:40 +00008106 LHSOp, RHSOp);
8107 if (FoldOp.getOpcode() != ISD::UNDEF &&
8108 FoldOp.getOpcode() != ISD::Constant &&
8109 FoldOp.getOpcode() != ISD::ConstantFP)
8110 break;
8111 Ops.push_back(FoldOp);
8112 AddToWorkList(FoldOp.getNode());
Chris Lattneredab1b92006-04-02 03:25:57 +00008113 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008114
Bob Wilsond7273432010-12-17 23:06:49 +00008115 if (Ops.size() == LHS.getNumOperands())
8116 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
8117 LHS.getValueType(), &Ops[0], Ops.size());
Chris Lattneredab1b92006-04-02 03:25:57 +00008118 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008119
Dan Gohman475871a2008-07-27 21:46:04 +00008120 return SDValue();
Chris Lattneredab1b92006-04-02 03:25:57 +00008121}
8122
Bill Wendling836ca7d2009-01-30 23:59:18 +00008123SDValue DAGCombiner::SimplifySelect(DebugLoc DL, SDValue N0,
8124 SDValue N1, SDValue N2){
Nate Begemanf845b452005-10-08 00:29:44 +00008125 assert(N0.getOpcode() ==ISD::SETCC && "First argument must be a SetCC node!");
Scott Michelfdc40a02009-02-17 22:15:04 +00008126
Bill Wendling836ca7d2009-01-30 23:59:18 +00008127 SDValue SCC = SimplifySelectCC(DL, N0.getOperand(0), N0.getOperand(1), N1, N2,
Nate Begemanf845b452005-10-08 00:29:44 +00008128 cast<CondCodeSDNode>(N0.getOperand(2))->get());
Bill Wendling836ca7d2009-01-30 23:59:18 +00008129
Nate Begemanf845b452005-10-08 00:29:44 +00008130 // If we got a simplified select_cc node back from SimplifySelectCC, then
8131 // break it down into a new SETCC node, and a new SELECT node, and then return
8132 // the SELECT node, since we were called with a SELECT node.
Gabor Greifba36cb52008-08-28 21:40:38 +00008133 if (SCC.getNode()) {
Nate Begemanf845b452005-10-08 00:29:44 +00008134 // Check to see if we got a select_cc back (to turn into setcc/select).
8135 // Otherwise, just return whatever node we got back, like fabs.
8136 if (SCC.getOpcode() == ISD::SELECT_CC) {
Bill Wendling836ca7d2009-01-30 23:59:18 +00008137 SDValue SETCC = DAG.getNode(ISD::SETCC, N0.getDebugLoc(),
8138 N0.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +00008139 SCC.getOperand(0), SCC.getOperand(1),
Bill Wendling836ca7d2009-01-30 23:59:18 +00008140 SCC.getOperand(4));
Gabor Greifba36cb52008-08-28 21:40:38 +00008141 AddToWorkList(SETCC.getNode());
Bill Wendling836ca7d2009-01-30 23:59:18 +00008142 return DAG.getNode(ISD::SELECT, SCC.getDebugLoc(), SCC.getValueType(),
8143 SCC.getOperand(2), SCC.getOperand(3), SETCC);
Nate Begemanf845b452005-10-08 00:29:44 +00008144 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008145
Nate Begemanf845b452005-10-08 00:29:44 +00008146 return SCC;
8147 }
Dan Gohman475871a2008-07-27 21:46:04 +00008148 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00008149}
8150
Chris Lattner40c62d52005-10-18 06:04:22 +00008151/// SimplifySelectOps - Given a SELECT or a SELECT_CC node, where LHS and RHS
8152/// are the two values being selected between, see if we can simplify the
Chris Lattner729c6d12006-05-27 00:43:02 +00008153/// select. Callers of this should assume that TheSelect is deleted if this
8154/// returns true. As such, they should return the appropriate thing (e.g. the
8155/// node) back to the top-level of the DAG combiner loop to avoid it being
8156/// looked at.
Scott Michelfdc40a02009-02-17 22:15:04 +00008157bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
Dan Gohman475871a2008-07-27 21:46:04 +00008158 SDValue RHS) {
Scott Michelfdc40a02009-02-17 22:15:04 +00008159
Nadav Rotemf94fdb62011-02-11 19:57:47 +00008160 // Cannot simplify select with vector condition
8161 if (TheSelect->getOperand(0).getValueType().isVector()) return false;
8162
Chris Lattner40c62d52005-10-18 06:04:22 +00008163 // If this is a select from two identical things, try to pull the operation
8164 // through the select.
Chris Lattner18061612010-09-21 15:46:59 +00008165 if (LHS.getOpcode() != RHS.getOpcode() ||
8166 !LHS.hasOneUse() || !RHS.hasOneUse())
8167 return false;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008168
Chris Lattner18061612010-09-21 15:46:59 +00008169 // If this is a load and the token chain is identical, replace the select
8170 // of two loads with a load through a select of the address to load from.
8171 // This triggers in things like "select bool X, 10.0, 123.0" after the FP
8172 // constants have been dropped into the constant pool.
8173 if (LHS.getOpcode() == ISD::LOAD) {
8174 LoadSDNode *LLD = cast<LoadSDNode>(LHS);
8175 LoadSDNode *RLD = cast<LoadSDNode>(RHS);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008176
Chris Lattner18061612010-09-21 15:46:59 +00008177 // Token chains must be identical.
8178 if (LHS.getOperand(0) != RHS.getOperand(0) ||
Duncan Sandsd4b9c172008-06-13 19:07:40 +00008179 // Do not let this transformation reduce the number of volatile loads.
Chris Lattner18061612010-09-21 15:46:59 +00008180 LLD->isVolatile() || RLD->isVolatile() ||
8181 // If this is an EXTLOAD, the VT's must match.
8182 LLD->getMemoryVT() != RLD->getMemoryVT() ||
Duncan Sandsdcfd3a72010-11-18 20:05:18 +00008183 // If this is an EXTLOAD, the kind of extension must match.
8184 (LLD->getExtensionType() != RLD->getExtensionType() &&
8185 // The only exception is if one of the extensions is anyext.
8186 LLD->getExtensionType() != ISD::EXTLOAD &&
8187 RLD->getExtensionType() != ISD::EXTLOAD) ||
Dan Gohman75832d72009-10-31 14:14:04 +00008188 // FIXME: this discards src value information. This is
8189 // over-conservative. It would be beneficial to be able to remember
Mon P Wangfe240b12010-01-11 20:12:49 +00008190 // both potential memory locations. Since we are discarding
8191 // src value info, don't do the transformation if the memory
8192 // locations are not in the default address space.
Chris Lattner18061612010-09-21 15:46:59 +00008193 LLD->getPointerInfo().getAddrSpace() != 0 ||
8194 RLD->getPointerInfo().getAddrSpace() != 0)
8195 return false;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008196
Chris Lattnerf1658062010-09-21 15:58:55 +00008197 // Check that the select condition doesn't reach either load. If so,
8198 // folding this will induce a cycle into the DAG. If not, this is safe to
8199 // xform, so create a select of the addresses.
Chris Lattner18061612010-09-21 15:46:59 +00008200 SDValue Addr;
8201 if (TheSelect->getOpcode() == ISD::SELECT) {
Chris Lattnerf1658062010-09-21 15:58:55 +00008202 SDNode *CondNode = TheSelect->getOperand(0).getNode();
8203 if ((LLD->hasAnyUseOfValue(1) && LLD->isPredecessorOf(CondNode)) ||
8204 (RLD->hasAnyUseOfValue(1) && RLD->isPredecessorOf(CondNode)))
8205 return false;
8206 Addr = DAG.getNode(ISD::SELECT, TheSelect->getDebugLoc(),
8207 LLD->getBasePtr().getValueType(),
8208 TheSelect->getOperand(0), LLD->getBasePtr(),
8209 RLD->getBasePtr());
Chris Lattner18061612010-09-21 15:46:59 +00008210 } else { // Otherwise SELECT_CC
Chris Lattnerf1658062010-09-21 15:58:55 +00008211 SDNode *CondLHS = TheSelect->getOperand(0).getNode();
8212 SDNode *CondRHS = TheSelect->getOperand(1).getNode();
8213
8214 if ((LLD->hasAnyUseOfValue(1) &&
8215 (LLD->isPredecessorOf(CondLHS) || LLD->isPredecessorOf(CondRHS))) ||
Chris Lattner77d95212012-03-27 16:27:21 +00008216 (RLD->hasAnyUseOfValue(1) &&
8217 (RLD->isPredecessorOf(CondLHS) || RLD->isPredecessorOf(CondRHS))))
Chris Lattnerf1658062010-09-21 15:58:55 +00008218 return false;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008219
Chris Lattnerf1658062010-09-21 15:58:55 +00008220 Addr = DAG.getNode(ISD::SELECT_CC, TheSelect->getDebugLoc(),
8221 LLD->getBasePtr().getValueType(),
8222 TheSelect->getOperand(0),
8223 TheSelect->getOperand(1),
8224 LLD->getBasePtr(), RLD->getBasePtr(),
8225 TheSelect->getOperand(4));
Chris Lattner18061612010-09-21 15:46:59 +00008226 }
8227
Chris Lattnerf1658062010-09-21 15:58:55 +00008228 SDValue Load;
8229 if (LLD->getExtensionType() == ISD::NON_EXTLOAD) {
8230 Load = DAG.getLoad(TheSelect->getValueType(0),
8231 TheSelect->getDebugLoc(),
8232 // FIXME: Discards pointer info.
8233 LLD->getChain(), Addr, MachinePointerInfo(),
8234 LLD->isVolatile(), LLD->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00008235 LLD->isInvariant(), LLD->getAlignment());
Chris Lattnerf1658062010-09-21 15:58:55 +00008236 } else {
Duncan Sandsb9064bb2010-11-18 21:16:28 +00008237 Load = DAG.getExtLoad(LLD->getExtensionType() == ISD::EXTLOAD ?
8238 RLD->getExtensionType() : LLD->getExtensionType(),
Chris Lattnerf1658062010-09-21 15:58:55 +00008239 TheSelect->getDebugLoc(),
Stuart Hastingsa9011292011-02-16 16:23:55 +00008240 TheSelect->getValueType(0),
Chris Lattnerf1658062010-09-21 15:58:55 +00008241 // FIXME: Discards pointer info.
8242 LLD->getChain(), Addr, MachinePointerInfo(),
8243 LLD->getMemoryVT(), LLD->isVolatile(),
8244 LLD->isNonTemporal(), LLD->getAlignment());
Chris Lattner40c62d52005-10-18 06:04:22 +00008245 }
Chris Lattnerf1658062010-09-21 15:58:55 +00008246
8247 // Users of the select now use the result of the load.
8248 CombineTo(TheSelect, Load);
8249
8250 // Users of the old loads now use the new load's chain. We know the
8251 // old-load value is dead now.
8252 CombineTo(LHS.getNode(), Load.getValue(0), Load.getValue(1));
8253 CombineTo(RHS.getNode(), Load.getValue(0), Load.getValue(1));
8254 return true;
Chris Lattner40c62d52005-10-18 06:04:22 +00008255 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008256
Chris Lattner40c62d52005-10-18 06:04:22 +00008257 return false;
8258}
8259
Chris Lattner600fec32009-03-11 05:08:08 +00008260/// SimplifySelectCC - Simplify an expression of the form (N0 cond N1) ? N2 : N3
8261/// where 'cond' is the comparison specified by CC.
Scott Michelfdc40a02009-02-17 22:15:04 +00008262SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1,
Dan Gohman475871a2008-07-27 21:46:04 +00008263 SDValue N2, SDValue N3,
8264 ISD::CondCode CC, bool NotExtCompare) {
Chris Lattner600fec32009-03-11 05:08:08 +00008265 // (x ? y : y) -> y.
8266 if (N2 == N3) return N2;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008267
Owen Andersone50ed302009-08-10 22:56:29 +00008268 EVT VT = N2.getValueType();
Gabor Greifba36cb52008-08-28 21:40:38 +00008269 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
8270 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
8271 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00008272
8273 // Determine if the condition we're dealing with is constant
Duncan Sands5480c042009-01-01 15:52:00 +00008274 SDValue SCC = SimplifySetCC(TLI.getSetCCResultType(N0.getValueType()),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00008275 N0, N1, CC, DL, false);
Gabor Greifba36cb52008-08-28 21:40:38 +00008276 if (SCC.getNode()) AddToWorkList(SCC.getNode());
8277 ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00008278
8279 // fold select_cc true, x, y -> x
Dan Gohman002e5d02008-03-13 22:13:53 +00008280 if (SCCC && !SCCC->isNullValue())
Nate Begemanf845b452005-10-08 00:29:44 +00008281 return N2;
8282 // fold select_cc false, x, y -> y
Dan Gohman002e5d02008-03-13 22:13:53 +00008283 if (SCCC && SCCC->isNullValue())
Nate Begemanf845b452005-10-08 00:29:44 +00008284 return N3;
Scott Michelfdc40a02009-02-17 22:15:04 +00008285
Nate Begemanf845b452005-10-08 00:29:44 +00008286 // Check to see if we can simplify the select into an fabs node
8287 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) {
8288 // Allow either -0.0 or 0.0
Dale Johannesen87503a62007-08-25 22:10:57 +00008289 if (CFP->getValueAPF().isZero()) {
Nate Begemanf845b452005-10-08 00:29:44 +00008290 // select (setg[te] X, +/-0.0), X, fneg(X) -> fabs
8291 if ((CC == ISD::SETGE || CC == ISD::SETGT) &&
8292 N0 == N2 && N3.getOpcode() == ISD::FNEG &&
8293 N2 == N3.getOperand(0))
Bill Wendling836ca7d2009-01-30 23:59:18 +00008294 return DAG.getNode(ISD::FABS, DL, VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00008295
Nate Begemanf845b452005-10-08 00:29:44 +00008296 // select (setl[te] X, +/-0.0), fneg(X), X -> fabs
8297 if ((CC == ISD::SETLT || CC == ISD::SETLE) &&
8298 N0 == N3 && N2.getOpcode() == ISD::FNEG &&
8299 N2.getOperand(0) == N3)
Bill Wendling836ca7d2009-01-30 23:59:18 +00008300 return DAG.getNode(ISD::FABS, DL, VT, N3);
Nate Begemanf845b452005-10-08 00:29:44 +00008301 }
8302 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008303
Chris Lattner600fec32009-03-11 05:08:08 +00008304 // Turn "(a cond b) ? 1.0f : 2.0f" into "load (tmp + ((a cond b) ? 0 : 4)"
8305 // where "tmp" is a constant pool entry containing an array with 1.0 and 2.0
8306 // in it. This is a win when the constant is not otherwise available because
8307 // it replaces two constant pool loads with one. We only do this if the FP
8308 // type is known to be legal, because if it isn't, then we are before legalize
8309 // types an we want the other legalization to happen first (e.g. to avoid
Mon P Wang0b7a7862009-03-14 00:25:19 +00008310 // messing with soft float) and if the ConstantFP is not legal, because if
8311 // it is legal, we may not need to store the FP constant in a constant pool.
Chris Lattner600fec32009-03-11 05:08:08 +00008312 if (ConstantFPSDNode *TV = dyn_cast<ConstantFPSDNode>(N2))
8313 if (ConstantFPSDNode *FV = dyn_cast<ConstantFPSDNode>(N3)) {
8314 if (TLI.isTypeLegal(N2.getValueType()) &&
Mon P Wang0b7a7862009-03-14 00:25:19 +00008315 (TLI.getOperationAction(ISD::ConstantFP, N2.getValueType()) !=
8316 TargetLowering::Legal) &&
Chris Lattner600fec32009-03-11 05:08:08 +00008317 // If both constants have multiple uses, then we won't need to do an
8318 // extra load, they are likely around in registers for other users.
8319 (TV->hasOneUse() || FV->hasOneUse())) {
8320 Constant *Elts[] = {
8321 const_cast<ConstantFP*>(FV->getConstantFPValue()),
8322 const_cast<ConstantFP*>(TV->getConstantFPValue())
8323 };
Chris Lattnerdb125cf2011-07-18 04:54:35 +00008324 Type *FPTy = Elts[0]->getType();
Chris Lattner600fec32009-03-11 05:08:08 +00008325 const TargetData &TD = *TLI.getTargetData();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008326
Chris Lattner600fec32009-03-11 05:08:08 +00008327 // Create a ConstantArray of the two constants.
Jay Foad26701082011-06-22 09:24:39 +00008328 Constant *CA = ConstantArray::get(ArrayType::get(FPTy, 2), Elts);
Chris Lattner600fec32009-03-11 05:08:08 +00008329 SDValue CPIdx = DAG.getConstantPool(CA, TLI.getPointerTy(),
8330 TD.getPrefTypeAlignment(FPTy));
Evan Cheng1606e8e2009-03-13 07:51:59 +00008331 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
Chris Lattner600fec32009-03-11 05:08:08 +00008332
8333 // Get the offsets to the 0 and 1 element of the array so that we can
8334 // select between them.
8335 SDValue Zero = DAG.getIntPtrConstant(0);
Duncan Sands777d2302009-05-09 07:06:46 +00008336 unsigned EltSize = (unsigned)TD.getTypeAllocSize(Elts[0]->getType());
Chris Lattner600fec32009-03-11 05:08:08 +00008337 SDValue One = DAG.getIntPtrConstant(EltSize);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008338
Chris Lattner600fec32009-03-11 05:08:08 +00008339 SDValue Cond = DAG.getSetCC(DL,
8340 TLI.getSetCCResultType(N0.getValueType()),
8341 N0, N1, CC);
Dan Gohman7b316c92011-09-22 23:01:29 +00008342 AddToWorkList(Cond.getNode());
Chris Lattner600fec32009-03-11 05:08:08 +00008343 SDValue CstOffset = DAG.getNode(ISD::SELECT, DL, Zero.getValueType(),
8344 Cond, One, Zero);
Dan Gohman7b316c92011-09-22 23:01:29 +00008345 AddToWorkList(CstOffset.getNode());
Chris Lattner600fec32009-03-11 05:08:08 +00008346 CPIdx = DAG.getNode(ISD::ADD, DL, TLI.getPointerTy(), CPIdx,
8347 CstOffset);
Dan Gohman7b316c92011-09-22 23:01:29 +00008348 AddToWorkList(CPIdx.getNode());
Chris Lattner600fec32009-03-11 05:08:08 +00008349 return DAG.getLoad(TV->getValueType(0), DL, DAG.getEntryNode(), CPIdx,
Chris Lattner85ca1062010-09-21 07:32:19 +00008350 MachinePointerInfo::getConstantPool(), false,
Pete Cooperd752e0f2011-11-08 18:42:53 +00008351 false, false, Alignment);
Chris Lattner600fec32009-03-11 05:08:08 +00008352
8353 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008354 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008355
Nate Begemanf845b452005-10-08 00:29:44 +00008356 // Check to see if we can perform the "gzip trick", transforming
Bill Wendling836ca7d2009-01-30 23:59:18 +00008357 // (select_cc setlt X, 0, A, 0) -> (and (sra X, (sub size(X), 1), A)
Chris Lattnere3152e52006-09-20 06:41:35 +00008358 if (N1C && N3C && N3C->isNullValue() && CC == ISD::SETLT &&
Dan Gohman002e5d02008-03-13 22:13:53 +00008359 (N1C->isNullValue() || // (a < 0) ? b : 0
8360 (N1C->getAPIntValue() == 1 && N0 == N2))) { // (a < 1) ? a : 0
Owen Andersone50ed302009-08-10 22:56:29 +00008361 EVT XType = N0.getValueType();
8362 EVT AType = N2.getValueType();
Duncan Sands8e4eb092008-06-08 20:54:56 +00008363 if (XType.bitsGE(AType)) {
Nate Begemanf845b452005-10-08 00:29:44 +00008364 // and (sra X, size(X)-1, A) -> "and (srl X, C2), A" iff A is a
Nate Begeman07ed4172005-10-10 21:26:48 +00008365 // single-bit constant.
Dan Gohman002e5d02008-03-13 22:13:53 +00008366 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue()-1)) == 0)) {
8367 unsigned ShCtV = N2C->getAPIntValue().logBase2();
Duncan Sands83ec4b62008-06-06 12:08:01 +00008368 ShCtV = XType.getSizeInBits()-ShCtV-1;
Owen Anderson95771af2011-02-25 21:41:48 +00008369 SDValue ShCt = DAG.getConstant(ShCtV,
8370 getShiftAmountTy(N0.getValueType()));
Bill Wendling9729c5a2009-01-31 03:12:48 +00008371 SDValue Shift = DAG.getNode(ISD::SRL, N0.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00008372 XType, N0, ShCt);
Gabor Greifba36cb52008-08-28 21:40:38 +00008373 AddToWorkList(Shift.getNode());
Bill Wendling836ca7d2009-01-30 23:59:18 +00008374
Duncan Sands8e4eb092008-06-08 20:54:56 +00008375 if (XType.bitsGT(AType)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00008376 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
Gabor Greifba36cb52008-08-28 21:40:38 +00008377 AddToWorkList(Shift.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00008378 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008379
8380 return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
Nate Begemanf845b452005-10-08 00:29:44 +00008381 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008382
Bill Wendling9729c5a2009-01-31 03:12:48 +00008383 SDValue Shift = DAG.getNode(ISD::SRA, N0.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00008384 XType, N0,
8385 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00008386 getShiftAmountTy(N0.getValueType())));
Gabor Greifba36cb52008-08-28 21:40:38 +00008387 AddToWorkList(Shift.getNode());
Bill Wendling836ca7d2009-01-30 23:59:18 +00008388
Duncan Sands8e4eb092008-06-08 20:54:56 +00008389 if (XType.bitsGT(AType)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00008390 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
Gabor Greifba36cb52008-08-28 21:40:38 +00008391 AddToWorkList(Shift.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00008392 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008393
8394 return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
Nate Begemanf845b452005-10-08 00:29:44 +00008395 }
8396 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008397
Owen Andersoned1088a2010-09-22 22:58:22 +00008398 // fold (select_cc seteq (and x, y), 0, 0, A) -> (and (shr (shl x)) A)
8399 // where y is has a single bit set.
8400 // A plaintext description would be, we can turn the SELECT_CC into an AND
8401 // when the condition can be materialized as an all-ones register. Any
8402 // single bit-test can be materialized as an all-ones register with
8403 // shift-left and shift-right-arith.
8404 if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND &&
8405 N0->getValueType(0) == VT &&
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008406 N1C && N1C->isNullValue() &&
Owen Andersoned1088a2010-09-22 22:58:22 +00008407 N2C && N2C->isNullValue()) {
8408 SDValue AndLHS = N0->getOperand(0);
8409 ConstantSDNode *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->getOperand(1));
8410 if (ConstAndRHS && ConstAndRHS->getAPIntValue().countPopulation() == 1) {
8411 // Shift the tested bit over the sign bit.
8412 APInt AndMask = ConstAndRHS->getAPIntValue();
8413 SDValue ShlAmt =
Owen Anderson95771af2011-02-25 21:41:48 +00008414 DAG.getConstant(AndMask.countLeadingZeros(),
8415 getShiftAmountTy(AndLHS.getValueType()));
Owen Andersoned1088a2010-09-22 22:58:22 +00008416 SDValue Shl = DAG.getNode(ISD::SHL, N0.getDebugLoc(), VT, AndLHS, ShlAmt);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008417
Owen Andersoned1088a2010-09-22 22:58:22 +00008418 // Now arithmetic right shift it all the way over, so the result is either
8419 // all-ones, or zero.
8420 SDValue ShrAmt =
Owen Anderson95771af2011-02-25 21:41:48 +00008421 DAG.getConstant(AndMask.getBitWidth()-1,
8422 getShiftAmountTy(Shl.getValueType()));
Owen Andersoned1088a2010-09-22 22:58:22 +00008423 SDValue Shr = DAG.getNode(ISD::SRA, N0.getDebugLoc(), VT, Shl, ShrAmt);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008424
Owen Andersoned1088a2010-09-22 22:58:22 +00008425 return DAG.getNode(ISD::AND, DL, VT, Shr, N3);
8426 }
8427 }
8428
Nate Begeman07ed4172005-10-10 21:26:48 +00008429 // fold select C, 16, 0 -> shl C, 4
Dan Gohman002e5d02008-03-13 22:13:53 +00008430 if (N2C && N3C && N3C->isNullValue() && N2C->getAPIntValue().isPowerOf2() &&
Duncan Sands28b77e92011-09-06 19:07:46 +00008431 TLI.getBooleanContents(N0.getValueType().isVector()) ==
8432 TargetLowering::ZeroOrOneBooleanContent) {
Scott Michelfdc40a02009-02-17 22:15:04 +00008433
Chris Lattner1eba01e2007-04-11 06:50:51 +00008434 // If the caller doesn't want us to simplify this into a zext of a compare,
8435 // don't do it.
Dan Gohman002e5d02008-03-13 22:13:53 +00008436 if (NotExtCompare && N2C->getAPIntValue() == 1)
Dan Gohman475871a2008-07-27 21:46:04 +00008437 return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00008438
Nate Begeman07ed4172005-10-10 21:26:48 +00008439 // Get a SetCC of the condition
8440 // FIXME: Should probably make sure that setcc is legal if we ever have a
8441 // target where it isn't.
Dan Gohman475871a2008-07-27 21:46:04 +00008442 SDValue Temp, SCC;
Nate Begeman07ed4172005-10-10 21:26:48 +00008443 // cast from setcc result type to select result type
Duncan Sands25cf2272008-11-24 14:53:14 +00008444 if (LegalTypes) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00008445 SCC = DAG.getSetCC(DL, TLI.getSetCCResultType(N0.getValueType()),
Duncan Sands5480c042009-01-01 15:52:00 +00008446 N0, N1, CC);
Duncan Sands8e4eb092008-06-08 20:54:56 +00008447 if (N2.getValueType().bitsLT(SCC.getValueType()))
Bill Wendling9729c5a2009-01-31 03:12:48 +00008448 Temp = DAG.getZeroExtendInReg(SCC, N2.getDebugLoc(), N2.getValueType());
Chris Lattner555d8d62006-12-07 22:36:47 +00008449 else
Bill Wendling9729c5a2009-01-31 03:12:48 +00008450 Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00008451 N2.getValueType(), SCC);
Nate Begemanb0d04a72006-02-18 02:40:58 +00008452 } else {
Owen Anderson825b72b2009-08-11 20:47:22 +00008453 SCC = DAG.getSetCC(N0.getDebugLoc(), MVT::i1, N0, N1, CC);
Bill Wendling9729c5a2009-01-31 03:12:48 +00008454 Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00008455 N2.getValueType(), SCC);
Nate Begemanb0d04a72006-02-18 02:40:58 +00008456 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008457
Gabor Greifba36cb52008-08-28 21:40:38 +00008458 AddToWorkList(SCC.getNode());
8459 AddToWorkList(Temp.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00008460
Dan Gohman002e5d02008-03-13 22:13:53 +00008461 if (N2C->getAPIntValue() == 1)
Chris Lattnerc56a81d2007-04-11 06:43:25 +00008462 return Temp;
Bill Wendling836ca7d2009-01-30 23:59:18 +00008463
Nate Begeman07ed4172005-10-10 21:26:48 +00008464 // shl setcc result by log2 n2c
Bill Wendling836ca7d2009-01-30 23:59:18 +00008465 return DAG.getNode(ISD::SHL, DL, N2.getValueType(), Temp,
Dan Gohman002e5d02008-03-13 22:13:53 +00008466 DAG.getConstant(N2C->getAPIntValue().logBase2(),
Owen Anderson95771af2011-02-25 21:41:48 +00008467 getShiftAmountTy(Temp.getValueType())));
Nate Begeman07ed4172005-10-10 21:26:48 +00008468 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008469
Nate Begemanf845b452005-10-08 00:29:44 +00008470 // Check to see if this is the equivalent of setcc
8471 // FIXME: Turn all of these into setcc if setcc if setcc is legal
8472 // otherwise, go ahead with the folds.
Dan Gohman002e5d02008-03-13 22:13:53 +00008473 if (0 && N3C && N3C->isNullValue() && N2C && (N2C->getAPIntValue() == 1ULL)) {
Owen Andersone50ed302009-08-10 22:56:29 +00008474 EVT XType = N0.getValueType();
Duncan Sands25cf2272008-11-24 14:53:14 +00008475 if (!LegalOperations ||
Duncan Sands5480c042009-01-01 15:52:00 +00008476 TLI.isOperationLegal(ISD::SETCC, TLI.getSetCCResultType(XType))) {
Bill Wendling836ca7d2009-01-30 23:59:18 +00008477 SDValue Res = DAG.getSetCC(DL, TLI.getSetCCResultType(XType), N0, N1, CC);
Nate Begemanf845b452005-10-08 00:29:44 +00008478 if (Res.getValueType() != VT)
Bill Wendling836ca7d2009-01-30 23:59:18 +00008479 Res = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Res);
Nate Begemanf845b452005-10-08 00:29:44 +00008480 return Res;
8481 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008482
Bill Wendling836ca7d2009-01-30 23:59:18 +00008483 // fold (seteq X, 0) -> (srl (ctlz X, log2(size(X))))
Scott Michelfdc40a02009-02-17 22:15:04 +00008484 if (N1C && N1C->isNullValue() && CC == ISD::SETEQ &&
Duncan Sands25cf2272008-11-24 14:53:14 +00008485 (!LegalOperations ||
Duncan Sands184a8762008-06-14 17:48:34 +00008486 TLI.isOperationLegal(ISD::CTLZ, XType))) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00008487 SDValue Ctlz = DAG.getNode(ISD::CTLZ, N0.getDebugLoc(), XType, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00008488 return DAG.getNode(ISD::SRL, DL, XType, Ctlz,
Duncan Sands83ec4b62008-06-06 12:08:01 +00008489 DAG.getConstant(Log2_32(XType.getSizeInBits()),
Owen Anderson95771af2011-02-25 21:41:48 +00008490 getShiftAmountTy(Ctlz.getValueType())));
Nate Begemanf845b452005-10-08 00:29:44 +00008491 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008492 // fold (setgt X, 0) -> (srl (and (-X, ~X), size(X)-1))
Scott Michelfdc40a02009-02-17 22:15:04 +00008493 if (N1C && N1C->isNullValue() && CC == ISD::SETGT) {
Bill Wendling836ca7d2009-01-30 23:59:18 +00008494 SDValue NegN0 = DAG.getNode(ISD::SUB, N0.getDebugLoc(),
8495 XType, DAG.getConstant(0, XType), N0);
Bill Wendling7581bfa2009-01-30 23:03:19 +00008496 SDValue NotN0 = DAG.getNOT(N0.getDebugLoc(), N0, XType);
Bill Wendling836ca7d2009-01-30 23:59:18 +00008497 return DAG.getNode(ISD::SRL, DL, XType,
Bill Wendlingfc4b6772009-02-01 11:19:36 +00008498 DAG.getNode(ISD::AND, DL, XType, NegN0, NotN0),
Duncan Sands83ec4b62008-06-06 12:08:01 +00008499 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00008500 getShiftAmountTy(XType)));
Nate Begemanf845b452005-10-08 00:29:44 +00008501 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00008502 // fold (setgt X, -1) -> (xor (srl (X, size(X)-1), 1))
Nate Begemanf845b452005-10-08 00:29:44 +00008503 if (N1C && N1C->isAllOnesValue() && CC == ISD::SETGT) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00008504 SDValue Sign = DAG.getNode(ISD::SRL, N0.getDebugLoc(), XType, N0,
Bill Wendling836ca7d2009-01-30 23:59:18 +00008505 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00008506 getShiftAmountTy(N0.getValueType())));
Bill Wendling836ca7d2009-01-30 23:59:18 +00008507 return DAG.getNode(ISD::XOR, DL, XType, Sign, DAG.getConstant(1, XType));
Nate Begemanf845b452005-10-08 00:29:44 +00008508 }
8509 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008510
Benjamin Kramercde51102010-07-08 12:09:56 +00008511 // Check to see if this is an integer abs.
8512 // select_cc setg[te] X, 0, X, -X ->
8513 // select_cc setgt X, -1, X, -X ->
8514 // select_cc setl[te] X, 0, -X, X ->
8515 // select_cc setlt X, 1, -X, X ->
Nate Begemanf845b452005-10-08 00:29:44 +00008516 // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
Benjamin Kramercde51102010-07-08 12:09:56 +00008517 if (N1C) {
8518 ConstantSDNode *SubC = NULL;
8519 if (((N1C->isNullValue() && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
8520 (N1C->isAllOnesValue() && CC == ISD::SETGT)) &&
8521 N0 == N2 && N3.getOpcode() == ISD::SUB && N0 == N3.getOperand(1))
8522 SubC = dyn_cast<ConstantSDNode>(N3.getOperand(0));
8523 else if (((N1C->isNullValue() && (CC == ISD::SETLT || CC == ISD::SETLE)) ||
8524 (N1C->isOne() && CC == ISD::SETLT)) &&
8525 N0 == N3 && N2.getOpcode() == ISD::SUB && N0 == N2.getOperand(1))
8526 SubC = dyn_cast<ConstantSDNode>(N2.getOperand(0));
8527
Owen Andersone50ed302009-08-10 22:56:29 +00008528 EVT XType = N0.getValueType();
Benjamin Kramercde51102010-07-08 12:09:56 +00008529 if (SubC && SubC->isNullValue() && XType.isInteger()) {
8530 SDValue Shift = DAG.getNode(ISD::SRA, N0.getDebugLoc(), XType,
8531 N0,
8532 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00008533 getShiftAmountTy(N0.getValueType())));
Benjamin Kramercde51102010-07-08 12:09:56 +00008534 SDValue Add = DAG.getNode(ISD::ADD, N0.getDebugLoc(),
8535 XType, N0, Shift);
8536 AddToWorkList(Shift.getNode());
8537 AddToWorkList(Add.getNode());
8538 return DAG.getNode(ISD::XOR, DL, XType, Add, Shift);
Nate Begemanf845b452005-10-08 00:29:44 +00008539 }
8540 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008541
Dan Gohman475871a2008-07-27 21:46:04 +00008542 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00008543}
8544
Evan Chengfa1eb272007-02-08 22:13:59 +00008545/// SimplifySetCC - This is a stub for TargetLowering::SimplifySetCC.
Owen Andersone50ed302009-08-10 22:56:29 +00008546SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0,
Dan Gohman475871a2008-07-27 21:46:04 +00008547 SDValue N1, ISD::CondCode Cond,
Dale Johannesenff97d4f2009-02-03 00:47:48 +00008548 DebugLoc DL, bool foldBooleans) {
Scott Michelfdc40a02009-02-17 22:15:04 +00008549 TargetLowering::DAGCombinerInfo
Jakob Stoklund Olesen78d12642009-07-24 18:22:59 +00008550 DagCombineInfo(DAG, !LegalTypes, !LegalOperations, false, this);
Dale Johannesenff97d4f2009-02-03 00:47:48 +00008551 return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL);
Nate Begeman452d7beb2005-09-16 00:54:12 +00008552}
8553
Nate Begeman69575232005-10-20 02:15:44 +00008554/// BuildSDIVSequence - Given an ISD::SDIV node expressing a divide by constant,
8555/// return a DAG expression to select that will generate the same value by
8556/// multiplying by a magic number. See:
8557/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html>
Dan Gohman475871a2008-07-27 21:46:04 +00008558SDValue DAGCombiner::BuildSDIV(SDNode *N) {
Andrew Lenharth232c9102006-06-12 16:07:18 +00008559 std::vector<SDNode*> Built;
Richard Osborne19a4daf2011-11-07 17:09:05 +00008560 SDValue S = TLI.BuildSDIV(N, DAG, LegalOperations, &Built);
Andrew Lenharthdae9cbe2006-05-16 17:42:15 +00008561
Andrew Lenharth232c9102006-06-12 16:07:18 +00008562 for (std::vector<SDNode*>::iterator ii = Built.begin(), ee = Built.end();
Andrew Lenharthdae9cbe2006-05-16 17:42:15 +00008563 ii != ee; ++ii)
8564 AddToWorkList(*ii);
8565 return S;
Nate Begeman69575232005-10-20 02:15:44 +00008566}
8567
8568/// BuildUDIVSequence - Given an ISD::UDIV node expressing a divide by constant,
8569/// return a DAG expression to select that will generate the same value by
8570/// multiplying by a magic number. See:
8571/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html>
Dan Gohman475871a2008-07-27 21:46:04 +00008572SDValue DAGCombiner::BuildUDIV(SDNode *N) {
Andrew Lenharth232c9102006-06-12 16:07:18 +00008573 std::vector<SDNode*> Built;
Richard Osborne19a4daf2011-11-07 17:09:05 +00008574 SDValue S = TLI.BuildUDIV(N, DAG, LegalOperations, &Built);
Nate Begeman69575232005-10-20 02:15:44 +00008575
Andrew Lenharth232c9102006-06-12 16:07:18 +00008576 for (std::vector<SDNode*>::iterator ii = Built.begin(), ee = Built.end();
Andrew Lenharthdae9cbe2006-05-16 17:42:15 +00008577 ii != ee; ++ii)
8578 AddToWorkList(*ii);
8579 return S;
Nate Begeman69575232005-10-20 02:15:44 +00008580}
8581
Nate Begemancc66cdd2009-09-25 06:05:26 +00008582/// FindBaseOffset - Return true if base is a frame index, which is known not
Eric Christopher503a64d2010-12-09 04:48:06 +00008583// to alias with anything but itself. Provides base object and offset as
8584// results.
Nate Begemancc66cdd2009-09-25 06:05:26 +00008585static bool FindBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset,
Dan Gohman46510a72010-04-15 01:51:59 +00008586 const GlobalValue *&GV, void *&CV) {
Jim Laskey71382342006-10-07 23:37:56 +00008587 // Assume it is a primitive operation.
Nate Begemancc66cdd2009-09-25 06:05:26 +00008588 Base = Ptr; Offset = 0; GV = 0; CV = 0;
Scott Michelfdc40a02009-02-17 22:15:04 +00008589
Jim Laskey71382342006-10-07 23:37:56 +00008590 // If it's an adding a simple constant then integrate the offset.
8591 if (Base.getOpcode() == ISD::ADD) {
8592 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Base.getOperand(1))) {
8593 Base = Base.getOperand(0);
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00008594 Offset += C->getZExtValue();
Jim Laskey71382342006-10-07 23:37:56 +00008595 }
8596 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008597
Nate Begemancc66cdd2009-09-25 06:05:26 +00008598 // Return the underlying GlobalValue, and update the Offset. Return false
8599 // for GlobalAddressSDNode since the same GlobalAddress may be represented
8600 // by multiple nodes with different offsets.
8601 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Base)) {
8602 GV = G->getGlobal();
8603 Offset += G->getOffset();
8604 return false;
8605 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008606
Nate Begemancc66cdd2009-09-25 06:05:26 +00008607 // Return the underlying Constant value, and update the Offset. Return false
8608 // for ConstantSDNodes since the same constant pool entry may be represented
8609 // by multiple nodes with different offsets.
8610 if (ConstantPoolSDNode *C = dyn_cast<ConstantPoolSDNode>(Base)) {
8611 CV = C->isMachineConstantPoolEntry() ? (void *)C->getMachineCPVal()
8612 : (void *)C->getConstVal();
8613 Offset += C->getOffset();
8614 return false;
8615 }
Jim Laskey71382342006-10-07 23:37:56 +00008616 // If it's any of the following then it can't alias with anything but itself.
Nate Begemancc66cdd2009-09-25 06:05:26 +00008617 return isa<FrameIndexSDNode>(Base);
Jim Laskey71382342006-10-07 23:37:56 +00008618}
8619
8620/// isAlias - Return true if there is any possibility that the two addresses
8621/// overlap.
Dan Gohman475871a2008-07-27 21:46:04 +00008622bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1,
Jim Laskey096c22e2006-10-18 12:29:57 +00008623 const Value *SrcValue1, int SrcValueOffset1,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008624 unsigned SrcValueAlign1,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008625 const MDNode *TBAAInfo1,
Dan Gohman475871a2008-07-27 21:46:04 +00008626 SDValue Ptr2, int64_t Size2,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008627 const Value *SrcValue2, int SrcValueOffset2,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008628 unsigned SrcValueAlign2,
8629 const MDNode *TBAAInfo2) const {
Jim Laskey71382342006-10-07 23:37:56 +00008630 // If they are the same then they must be aliases.
8631 if (Ptr1 == Ptr2) return true;
Scott Michelfdc40a02009-02-17 22:15:04 +00008632
Jim Laskey71382342006-10-07 23:37:56 +00008633 // Gather base node and offset information.
Dan Gohman475871a2008-07-27 21:46:04 +00008634 SDValue Base1, Base2;
Jim Laskey71382342006-10-07 23:37:56 +00008635 int64_t Offset1, Offset2;
Dan Gohman46510a72010-04-15 01:51:59 +00008636 const GlobalValue *GV1, *GV2;
Nate Begemancc66cdd2009-09-25 06:05:26 +00008637 void *CV1, *CV2;
8638 bool isFrameIndex1 = FindBaseOffset(Ptr1, Base1, Offset1, GV1, CV1);
8639 bool isFrameIndex2 = FindBaseOffset(Ptr2, Base2, Offset2, GV2, CV2);
Scott Michelfdc40a02009-02-17 22:15:04 +00008640
Nate Begemancc66cdd2009-09-25 06:05:26 +00008641 // If they have a same base address then check to see if they overlap.
8642 if (Base1 == Base2 || (GV1 && (GV1 == GV2)) || (CV1 && (CV1 == CV2)))
Bill Wendling836ca7d2009-01-30 23:59:18 +00008643 return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1);
Scott Michelfdc40a02009-02-17 22:15:04 +00008644
Owen Anderson4a9f1502010-09-20 20:39:59 +00008645 // It is possible for different frame indices to alias each other, mostly
8646 // when tail call optimization reuses return address slots for arguments.
8647 // To catch this case, look up the actual index of frame indices to compute
8648 // the real alias relationship.
8649 if (isFrameIndex1 && isFrameIndex2) {
8650 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
8651 Offset1 += MFI->getObjectOffset(cast<FrameIndexSDNode>(Base1)->getIndex());
8652 Offset2 += MFI->getObjectOffset(cast<FrameIndexSDNode>(Base2)->getIndex());
8653 return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1);
8654 }
8655
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008656 // Otherwise, if we know what the bases are, and they aren't identical, then
Owen Anderson4a9f1502010-09-20 20:39:59 +00008657 // we know they cannot alias.
Nate Begemancc66cdd2009-09-25 06:05:26 +00008658 if ((isFrameIndex1 || CV1 || GV1) && (isFrameIndex2 || CV2 || GV2))
8659 return false;
Jim Laskey096c22e2006-10-18 12:29:57 +00008660
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008661 // If we know required SrcValue1 and SrcValue2 have relatively large alignment
8662 // compared to the size and offset of the access, we may be able to prove they
8663 // do not alias. This check is conservative for now to catch cases created by
8664 // splitting vector types.
8665 if ((SrcValueAlign1 == SrcValueAlign2) &&
8666 (SrcValueOffset1 != SrcValueOffset2) &&
8667 (Size1 == Size2) && (SrcValueAlign1 > Size1)) {
8668 int64_t OffAlign1 = SrcValueOffset1 % SrcValueAlign1;
8669 int64_t OffAlign2 = SrcValueOffset2 % SrcValueAlign1;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008670
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008671 // There is no overlap between these relatively aligned accesses of similar
8672 // size, return no alias.
8673 if ((OffAlign1 + Size1) <= OffAlign2 || (OffAlign2 + Size2) <= OffAlign1)
8674 return false;
8675 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008676
Jim Laskey07a27092006-10-18 19:08:31 +00008677 if (CombinerGlobalAA) {
8678 // Use alias analysis information.
Dan Gohmane9c8fa02007-08-27 16:32:11 +00008679 int64_t MinOffset = std::min(SrcValueOffset1, SrcValueOffset2);
8680 int64_t Overlap1 = Size1 + SrcValueOffset1 - MinOffset;
8681 int64_t Overlap2 = Size2 + SrcValueOffset2 - MinOffset;
Scott Michelfdc40a02009-02-17 22:15:04 +00008682 AliasAnalysis::AliasResult AAResult =
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008683 AA.alias(AliasAnalysis::Location(SrcValue1, Overlap1, TBAAInfo1),
8684 AliasAnalysis::Location(SrcValue2, Overlap2, TBAAInfo2));
Jim Laskey07a27092006-10-18 19:08:31 +00008685 if (AAResult == AliasAnalysis::NoAlias)
8686 return false;
8687 }
Jim Laskey096c22e2006-10-18 12:29:57 +00008688
8689 // Otherwise we have to assume they alias.
8690 return true;
Jim Laskey71382342006-10-07 23:37:56 +00008691}
8692
8693/// FindAliasInfo - Extracts the relevant alias information from the memory
8694/// node. Returns true if the operand was a load.
Jim Laskey7ca56af2006-10-11 13:47:09 +00008695bool DAGCombiner::FindAliasInfo(SDNode *N,
Benjamin Kramerae4746b2012-01-15 11:50:43 +00008696 SDValue &Ptr, int64_t &Size,
8697 const Value *&SrcValue,
8698 int &SrcValueOffset,
8699 unsigned &SrcValueAlign,
8700 const MDNode *&TBAAInfo) const {
8701 LSBaseSDNode *LS = cast<LSBaseSDNode>(N);
8702
8703 Ptr = LS->getBasePtr();
8704 Size = LS->getMemoryVT().getSizeInBits() >> 3;
8705 SrcValue = LS->getSrcValue();
8706 SrcValueOffset = LS->getSrcValueOffset();
8707 SrcValueAlign = LS->getOriginalAlignment();
8708 TBAAInfo = LS->getTBAAInfo();
8709 return isa<LoadSDNode>(LS);
Jim Laskey71382342006-10-07 23:37:56 +00008710}
8711
Jim Laskey6ff23e52006-10-04 16:53:27 +00008712/// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
8713/// looking for aliasing nodes and adding them to the Aliases vector.
Dan Gohman475871a2008-07-27 21:46:04 +00008714void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
8715 SmallVector<SDValue, 8> &Aliases) {
8716 SmallVector<SDValue, 8> Chains; // List of chains to visit.
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008717 SmallPtrSet<SDNode *, 16> Visited; // Visited node set.
Scott Michelfdc40a02009-02-17 22:15:04 +00008718
Jim Laskey279f0532006-09-25 16:29:54 +00008719 // Get alias information for node.
Dan Gohman475871a2008-07-27 21:46:04 +00008720 SDValue Ptr;
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008721 int64_t Size;
8722 const Value *SrcValue;
8723 int SrcValueOffset;
8724 unsigned SrcValueAlign;
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008725 const MDNode *SrcTBAAInfo;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008726 bool IsLoad = FindAliasInfo(N, Ptr, Size, SrcValue, SrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008727 SrcValueAlign, SrcTBAAInfo);
Jim Laskey279f0532006-09-25 16:29:54 +00008728
Jim Laskey6ff23e52006-10-04 16:53:27 +00008729 // Starting off.
Jim Laskeybc588b82006-10-05 15:07:25 +00008730 Chains.push_back(OriginalChain);
Nate Begeman677c89d2009-10-12 05:53:58 +00008731 unsigned Depth = 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008732
Jim Laskeybc588b82006-10-05 15:07:25 +00008733 // Look at each chain and determine if it is an alias. If so, add it to the
8734 // aliases list. If not, then continue up the chain looking for the next
Scott Michelfdc40a02009-02-17 22:15:04 +00008735 // candidate.
Jim Laskeybc588b82006-10-05 15:07:25 +00008736 while (!Chains.empty()) {
Dan Gohman475871a2008-07-27 21:46:04 +00008737 SDValue Chain = Chains.back();
Jim Laskeybc588b82006-10-05 15:07:25 +00008738 Chains.pop_back();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008739
8740 // For TokenFactor nodes, look at each operand and only continue up the
8741 // chain until we find two aliases. If we've seen two aliases, assume we'll
Nate Begeman677c89d2009-10-12 05:53:58 +00008742 // find more and revert to original chain since the xform is unlikely to be
8743 // profitable.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008744 //
8745 // FIXME: The depth check could be made to return the last non-aliasing
Nate Begeman677c89d2009-10-12 05:53:58 +00008746 // chain we found before we hit a tokenfactor rather than the original
8747 // chain.
8748 if (Depth > 6 || Aliases.size() == 2) {
8749 Aliases.clear();
8750 Aliases.push_back(OriginalChain);
8751 break;
8752 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008753
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008754 // Don't bother if we've been before.
8755 if (!Visited.insert(Chain.getNode()))
8756 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00008757
Jim Laskeybc588b82006-10-05 15:07:25 +00008758 switch (Chain.getOpcode()) {
8759 case ISD::EntryToken:
8760 // Entry token is ideal chain operand, but handled in FindBetterChain.
8761 break;
Scott Michelfdc40a02009-02-17 22:15:04 +00008762
Jim Laskeybc588b82006-10-05 15:07:25 +00008763 case ISD::LOAD:
8764 case ISD::STORE: {
8765 // Get alias information for Chain.
Dan Gohman475871a2008-07-27 21:46:04 +00008766 SDValue OpPtr;
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008767 int64_t OpSize;
8768 const Value *OpSrcValue;
8769 int OpSrcValueOffset;
8770 unsigned OpSrcValueAlign;
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008771 const MDNode *OpSrcTBAAInfo;
Gabor Greifba36cb52008-08-28 21:40:38 +00008772 bool IsOpLoad = FindAliasInfo(Chain.getNode(), OpPtr, OpSize,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008773 OpSrcValue, OpSrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008774 OpSrcValueAlign,
8775 OpSrcTBAAInfo);
Scott Michelfdc40a02009-02-17 22:15:04 +00008776
Jim Laskeybc588b82006-10-05 15:07:25 +00008777 // If chain is alias then stop here.
8778 if (!(IsLoad && IsOpLoad) &&
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008779 isAlias(Ptr, Size, SrcValue, SrcValueOffset, SrcValueAlign,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008780 SrcTBAAInfo,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008781 OpPtr, OpSize, OpSrcValue, OpSrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008782 OpSrcValueAlign, OpSrcTBAAInfo)) {
Jim Laskeybc588b82006-10-05 15:07:25 +00008783 Aliases.push_back(Chain);
8784 } else {
8785 // Look further up the chain.
Scott Michelfdc40a02009-02-17 22:15:04 +00008786 Chains.push_back(Chain.getOperand(0));
Nate Begeman677c89d2009-10-12 05:53:58 +00008787 ++Depth;
Jim Laskey279f0532006-09-25 16:29:54 +00008788 }
Jim Laskeybc588b82006-10-05 15:07:25 +00008789 break;
8790 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008791
Jim Laskeybc588b82006-10-05 15:07:25 +00008792 case ISD::TokenFactor:
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008793 // We have to check each of the operands of the token factor for "small"
8794 // token factors, so we queue them up. Adding the operands to the queue
8795 // (stack) in reverse order maintains the original order and increases the
8796 // likelihood that getNode will find a matching token factor (CSE.)
8797 if (Chain.getNumOperands() > 16) {
8798 Aliases.push_back(Chain);
8799 break;
8800 }
Jim Laskeybc588b82006-10-05 15:07:25 +00008801 for (unsigned n = Chain.getNumOperands(); n;)
8802 Chains.push_back(Chain.getOperand(--n));
Nate Begeman677c89d2009-10-12 05:53:58 +00008803 ++Depth;
Jim Laskeybc588b82006-10-05 15:07:25 +00008804 break;
Scott Michelfdc40a02009-02-17 22:15:04 +00008805
Jim Laskeybc588b82006-10-05 15:07:25 +00008806 default:
8807 // For all other instructions we will just have to take what we can get.
8808 Aliases.push_back(Chain);
8809 break;
Jim Laskey279f0532006-09-25 16:29:54 +00008810 }
8811 }
Jim Laskey6ff23e52006-10-04 16:53:27 +00008812}
8813
8814/// FindBetterChain - Walk up chain skipping non-aliasing memory nodes, looking
8815/// for a better chain (aliasing node.)
Dan Gohman475871a2008-07-27 21:46:04 +00008816SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) {
8817 SmallVector<SDValue, 8> Aliases; // Ops for replacing token factor.
Scott Michelfdc40a02009-02-17 22:15:04 +00008818
Jim Laskey6ff23e52006-10-04 16:53:27 +00008819 // Accumulate all the aliases to this node.
8820 GatherAllAliases(N, OldChain, Aliases);
Scott Michelfdc40a02009-02-17 22:15:04 +00008821
Dan Gohman71dc7c92011-05-17 22:20:36 +00008822 // If no operands then chain to entry token.
8823 if (Aliases.size() == 0)
Jim Laskey6ff23e52006-10-04 16:53:27 +00008824 return DAG.getEntryNode();
Dan Gohman71dc7c92011-05-17 22:20:36 +00008825
8826 // If a single operand then chain to it. We don't need to revisit it.
8827 if (Aliases.size() == 1)
Jim Laskey6ff23e52006-10-04 16:53:27 +00008828 return Aliases[0];
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008829
Jim Laskey6ff23e52006-10-04 16:53:27 +00008830 // Construct a custom tailored token factor.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008831 return DAG.getNode(ISD::TokenFactor, N->getDebugLoc(), MVT::Other,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008832 &Aliases[0], Aliases.size());
Jim Laskey279f0532006-09-25 16:29:54 +00008833}
8834
Nate Begeman1d4d4142005-09-01 00:19:25 +00008835// SelectionDAG::Combine - This is the entry point for the file.
8836//
Bill Wendlingbe8cc2a2009-04-29 00:15:41 +00008837void SelectionDAG::Combine(CombineLevel Level, AliasAnalysis &AA,
Bill Wendling98a366d2009-04-29 23:29:43 +00008838 CodeGenOpt::Level OptLevel) {
Nate Begeman1d4d4142005-09-01 00:19:25 +00008839 /// run - This is the main entry point to this class.
8840 ///
Bill Wendlingbe8cc2a2009-04-29 00:15:41 +00008841 DAGCombiner(*this, AA, OptLevel).Run(Level);
Nate Begeman1d4d4142005-09-01 00:19:25 +00008842}