blob: 5b2d86f511ddc13e9658934d2c6e371d9aa073c4 [file] [log] [blame]
Nate Begeman4ebd8052005-09-01 23:24:04 +00001//===-- DAGCombiner.cpp - Implement a DAG node combiner -------------------===//
Nate Begeman1d4d4142005-09-01 00:19:25 +00002//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner4ee451d2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Nate Begeman1d4d4142005-09-01 00:19:25 +00007//
8//===----------------------------------------------------------------------===//
9//
10// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
11// both before and after the DAG is legalized.
Scott Michelfdc40a02009-02-17 22:15:04 +000012//
Dan Gohman41287002009-04-25 17:09:45 +000013// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
14// primarily intended to handle simplification opportunities that are implicit
15// in the LLVM IR and exposed by the various codegen lowering phases.
16//
Nate Begeman1d4d4142005-09-01 00:19:25 +000017//===----------------------------------------------------------------------===//
18
19#define DEBUG_TYPE "dagcombine"
Nate Begeman1d4d4142005-09-01 00:19:25 +000020#include "llvm/CodeGen/SelectionDAG.h"
Chris Lattner600fec32009-03-11 05:08:08 +000021#include "llvm/DerivedTypes.h"
Owen Andersona90b3dc2009-07-15 21:51:10 +000022#include "llvm/LLVMContext.h"
Chris Lattner00161a62008-01-25 07:20:16 +000023#include "llvm/CodeGen/MachineFunction.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000025#include "llvm/Analysis/AliasAnalysis.h"
Evan Cheng59d5b682007-05-07 21:27:48 +000026#include "llvm/Target/TargetData.h"
Nate Begeman1d4d4142005-09-01 00:19:25 +000027#include "llvm/Target/TargetLowering.h"
Evan Cheng59d5b682007-05-07 21:27:48 +000028#include "llvm/Target/TargetMachine.h"
Chris Lattnerddae4bd2007-01-08 23:04:05 +000029#include "llvm/Target/TargetOptions.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000030#include "llvm/ADT/SmallPtrSet.h"
31#include "llvm/ADT/Statistic.h"
Jim Laskeyd1aed7a2006-09-21 16:28:59 +000032#include "llvm/Support/CommandLine.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000033#include "llvm/Support/Debug.h"
Torok Edwin7d696d82009-07-11 13:10:19 +000034#include "llvm/Support/ErrorHandling.h"
Chris Lattnerc76d4412007-05-16 06:37:59 +000035#include "llvm/Support/MathExtras.h"
Chris Lattnerbbbfa992009-08-23 06:35:02 +000036#include "llvm/Support/raw_ostream.h"
Chris Lattnera500fc62005-09-09 23:53:39 +000037#include <algorithm>
Nate Begeman1d4d4142005-09-01 00:19:25 +000038using namespace llvm;
39
Chris Lattnercd3245a2006-12-19 22:41:21 +000040STATISTIC(NodesCombined , "Number of dag nodes combined");
41STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
42STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
Evan Cheng8b944d32009-05-28 00:35:15 +000043STATISTIC(OpsNarrowed , "Number of load/op/store narrowed");
Evan Cheng31959b12011-02-02 01:06:55 +000044STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int");
Chris Lattnercd3245a2006-12-19 22:41:21 +000045
Nate Begeman1d4d4142005-09-01 00:19:25 +000046namespace {
Jim Laskey71382342006-10-07 23:37:56 +000047 static cl::opt<bool>
Owen Anderson0dcc8142010-09-19 21:01:26 +000048 CombinerAA("combiner-alias-analysis", cl::Hidden,
Jim Laskey26f7fa72006-10-17 19:33:52 +000049 cl::desc("Turn on alias analysis during testing"));
Jim Laskey3ad175b2006-10-12 15:22:24 +000050
Jim Laskey07a27092006-10-18 19:08:31 +000051 static cl::opt<bool>
52 CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
53 cl::desc("Include global information in alias analysis"));
54
Jim Laskeybc588b82006-10-05 15:07:25 +000055//------------------------------ DAGCombiner ---------------------------------//
56
Nick Lewycky6726b6d2009-10-25 06:33:48 +000057 class DAGCombiner {
Nate Begeman1d4d4142005-09-01 00:19:25 +000058 SelectionDAG &DAG;
Dan Gohman79ce2762009-01-15 19:20:50 +000059 const TargetLowering &TLI;
Duncan Sands25cf2272008-11-24 14:53:14 +000060 CombineLevel Level;
Bill Wendling98a366d2009-04-29 23:29:43 +000061 CodeGenOpt::Level OptLevel;
Duncan Sands25cf2272008-11-24 14:53:14 +000062 bool LegalOperations;
63 bool LegalTypes;
Nate Begeman1d4d4142005-09-01 00:19:25 +000064
65 // Worklist of all of the nodes that need to be simplified.
Evan Cheng17a568b2008-08-29 22:21:44 +000066 std::vector<SDNode*> WorkList;
Nate Begeman1d4d4142005-09-01 00:19:25 +000067
Jim Laskeyc7c3f112006-10-16 20:52:31 +000068 // AA - Used for DAG load/store alias analysis.
69 AliasAnalysis &AA;
70
Nate Begeman1d4d4142005-09-01 00:19:25 +000071 /// AddUsersToWorkList - When an instruction is simplified, add all users of
72 /// the instruction to the work lists because they might get more simplified
73 /// now.
74 ///
75 void AddUsersToWorkList(SDNode *N) {
76 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
Nate Begeman4ebd8052005-09-01 23:24:04 +000077 UI != UE; ++UI)
Dan Gohman89684502008-07-27 20:43:25 +000078 AddToWorkList(*UI);
Nate Begeman1d4d4142005-09-01 00:19:25 +000079 }
80
Dan Gohman389079b2007-10-08 17:57:15 +000081 /// visit - call the node-specific routine that knows how to fold each
82 /// particular type of node.
Dan Gohman475871a2008-07-27 21:46:04 +000083 SDValue visit(SDNode *N);
Dan Gohman389079b2007-10-08 17:57:15 +000084
Chris Lattner24664722006-03-01 04:53:38 +000085 public:
Jim Laskey6ff23e52006-10-04 16:53:27 +000086 /// AddToWorkList - Add to the work list making sure it's instance is at the
87 /// the back (next to be processed.)
Chris Lattner5750df92006-03-01 04:03:14 +000088 void AddToWorkList(SDNode *N) {
Jim Laskey6ff23e52006-10-04 16:53:27 +000089 removeFromWorkList(N);
Chris Lattner5750df92006-03-01 04:03:14 +000090 WorkList.push_back(N);
91 }
Jim Laskey6ff23e52006-10-04 16:53:27 +000092
Chris Lattnerf8dc0612008-02-03 06:49:24 +000093 /// removeFromWorkList - remove all instances of N from the worklist.
94 ///
95 void removeFromWorkList(SDNode *N) {
96 WorkList.erase(std::remove(WorkList.begin(), WorkList.end(), N),
97 WorkList.end());
Chris Lattner01a22022005-10-10 22:04:48 +000098 }
Scott Michelfdc40a02009-02-17 22:15:04 +000099
Dan Gohman475871a2008-07-27 21:46:04 +0000100 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
Evan Cheng0b0cd912009-03-28 05:57:29 +0000101 bool AddTo = true);
Scott Michelfdc40a02009-02-17 22:15:04 +0000102
Dan Gohman475871a2008-07-27 21:46:04 +0000103 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
Jim Laskey274062c2006-10-13 23:32:28 +0000104 return CombineTo(N, &Res, 1, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000105 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000106
Dan Gohman475871a2008-07-27 21:46:04 +0000107 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
Evan Cheng0b0cd912009-03-28 05:57:29 +0000108 bool AddTo = true) {
Dan Gohman475871a2008-07-27 21:46:04 +0000109 SDValue To[] = { Res0, Res1 };
Jim Laskey274062c2006-10-13 23:32:28 +0000110 return CombineTo(N, To, 2, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000111 }
Dan Gohmane5af2d32009-01-29 01:59:02 +0000112
113 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
Scott Michelfdc40a02009-02-17 22:15:04 +0000114
115 private:
116
Chris Lattner012f2412006-02-17 21:58:01 +0000117 /// SimplifyDemandedBits - Check the specified integer node value to see if
Chris Lattnerb2742f42006-03-01 19:55:35 +0000118 /// it can be simplified or if things it uses can be simplified by bit
Chris Lattner012f2412006-02-17 21:58:01 +0000119 /// propagation. If so, return true.
Dan Gohman475871a2008-07-27 21:46:04 +0000120 bool SimplifyDemandedBits(SDValue Op) {
Dan Gohman87862e72009-12-11 21:31:27 +0000121 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
122 APInt Demanded = APInt::getAllOnesValue(BitWidth);
Dan Gohman7b8d4a92008-02-27 00:25:32 +0000123 return SimplifyDemandedBits(Op, Demanded);
124 }
125
Dan Gohman475871a2008-07-27 21:46:04 +0000126 bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
Chris Lattner87514ca2005-10-10 22:31:19 +0000127
Chris Lattner448f2192006-11-11 00:39:41 +0000128 bool CombineToPreIndexedLoadStore(SDNode *N);
129 bool CombineToPostIndexedLoadStore(SDNode *N);
Scott Michelfdc40a02009-02-17 22:15:04 +0000130
Evan Cheng95c57ea2010-04-24 04:43:44 +0000131 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
132 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
133 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
134 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
Evan Cheng64b7bf72010-04-16 06:14:10 +0000135 SDValue PromoteIntBinOp(SDValue Op);
Evan Cheng07c4e102010-04-22 20:19:46 +0000136 SDValue PromoteIntShiftOp(SDValue Op);
Evan Cheng4c26e932010-04-19 19:29:22 +0000137 SDValue PromoteExtend(SDValue Op);
138 bool PromoteLoad(SDValue Op);
Scott Michelfdc40a02009-02-17 22:15:04 +0000139
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +0000140 void ExtendSetCCUses(SmallVector<SDNode*, 4> SetCCs,
141 SDValue Trunc, SDValue ExtLoad, DebugLoc DL,
142 ISD::NodeType ExtType);
143
Dan Gohman389079b2007-10-08 17:57:15 +0000144 /// combine - call the node-specific routine that knows how to fold each
145 /// particular type of node. If that doesn't do anything, try the
146 /// target-specific DAG combines.
Dan Gohman475871a2008-07-27 21:46:04 +0000147 SDValue combine(SDNode *N);
Nate Begeman1d4d4142005-09-01 00:19:25 +0000148
149 // Visitation implementation - Implement dag node combining for different
150 // node types. The semantics are as follows:
151 // Return Value:
Evan Cheng17a568b2008-08-29 22:21:44 +0000152 // SDValue.getNode() == 0 - No change was made
153 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
154 // otherwise - N should be replaced by the returned Operand.
Nate Begeman1d4d4142005-09-01 00:19:25 +0000155 //
Dan Gohman475871a2008-07-27 21:46:04 +0000156 SDValue visitTokenFactor(SDNode *N);
157 SDValue visitMERGE_VALUES(SDNode *N);
158 SDValue visitADD(SDNode *N);
159 SDValue visitSUB(SDNode *N);
160 SDValue visitADDC(SDNode *N);
161 SDValue visitADDE(SDNode *N);
162 SDValue visitMUL(SDNode *N);
163 SDValue visitSDIV(SDNode *N);
164 SDValue visitUDIV(SDNode *N);
165 SDValue visitSREM(SDNode *N);
166 SDValue visitUREM(SDNode *N);
167 SDValue visitMULHU(SDNode *N);
168 SDValue visitMULHS(SDNode *N);
169 SDValue visitSMUL_LOHI(SDNode *N);
170 SDValue visitUMUL_LOHI(SDNode *N);
Benjamin Kramerf55d26e2011-05-21 18:31:55 +0000171 SDValue visitSMULO(SDNode *N);
172 SDValue visitUMULO(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000173 SDValue visitSDIVREM(SDNode *N);
174 SDValue visitUDIVREM(SDNode *N);
175 SDValue visitAND(SDNode *N);
176 SDValue visitOR(SDNode *N);
177 SDValue visitXOR(SDNode *N);
178 SDValue SimplifyVBinOp(SDNode *N);
179 SDValue visitSHL(SDNode *N);
180 SDValue visitSRA(SDNode *N);
181 SDValue visitSRL(SDNode *N);
182 SDValue visitCTLZ(SDNode *N);
Chandler Carruth63974b22011-12-13 01:56:10 +0000183 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000184 SDValue visitCTTZ(SDNode *N);
Chandler Carruth63974b22011-12-13 01:56:10 +0000185 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000186 SDValue visitCTPOP(SDNode *N);
187 SDValue visitSELECT(SDNode *N);
188 SDValue visitSELECT_CC(SDNode *N);
189 SDValue visitSETCC(SDNode *N);
190 SDValue visitSIGN_EXTEND(SDNode *N);
191 SDValue visitZERO_EXTEND(SDNode *N);
192 SDValue visitANY_EXTEND(SDNode *N);
193 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
194 SDValue visitTRUNCATE(SDNode *N);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000195 SDValue visitBITCAST(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000196 SDValue visitBUILD_PAIR(SDNode *N);
197 SDValue visitFADD(SDNode *N);
198 SDValue visitFSUB(SDNode *N);
199 SDValue visitFMUL(SDNode *N);
200 SDValue visitFDIV(SDNode *N);
201 SDValue visitFREM(SDNode *N);
202 SDValue visitFCOPYSIGN(SDNode *N);
203 SDValue visitSINT_TO_FP(SDNode *N);
204 SDValue visitUINT_TO_FP(SDNode *N);
205 SDValue visitFP_TO_SINT(SDNode *N);
206 SDValue visitFP_TO_UINT(SDNode *N);
207 SDValue visitFP_ROUND(SDNode *N);
208 SDValue visitFP_ROUND_INREG(SDNode *N);
209 SDValue visitFP_EXTEND(SDNode *N);
210 SDValue visitFNEG(SDNode *N);
211 SDValue visitFABS(SDNode *N);
212 SDValue visitBRCOND(SDNode *N);
213 SDValue visitBR_CC(SDNode *N);
214 SDValue visitLOAD(SDNode *N);
215 SDValue visitSTORE(SDNode *N);
216 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
217 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
218 SDValue visitBUILD_VECTOR(SDNode *N);
219 SDValue visitCONCAT_VECTORS(SDNode *N);
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +0000220 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
Dan Gohman475871a2008-07-27 21:46:04 +0000221 SDValue visitVECTOR_SHUFFLE(SDNode *N);
Jim Grosbach9a526492010-06-23 16:07:42 +0000222 SDValue visitMEMBARRIER(SDNode *N);
Chris Lattner01a22022005-10-10 22:04:48 +0000223
Dan Gohman475871a2008-07-27 21:46:04 +0000224 SDValue XformToShuffleWithZero(SDNode *N);
Bill Wendling35247c32009-01-30 00:45:56 +0000225 SDValue ReassociateOps(unsigned Opc, DebugLoc DL, SDValue LHS, SDValue RHS);
Scott Michelfdc40a02009-02-17 22:15:04 +0000226
Dan Gohman475871a2008-07-27 21:46:04 +0000227 SDValue visitShiftByConstant(SDNode *N, unsigned Amt);
Chris Lattnere70da202007-12-06 07:33:36 +0000228
Dan Gohman475871a2008-07-27 21:46:04 +0000229 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
230 SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
Bill Wendling836ca7d2009-01-30 23:59:18 +0000231 SDValue SimplifySelect(DebugLoc DL, SDValue N0, SDValue N1, SDValue N2);
Scott Michelfdc40a02009-02-17 22:15:04 +0000232 SDValue SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1, SDValue N2,
233 SDValue N3, ISD::CondCode CC,
Bill Wendling836ca7d2009-01-30 23:59:18 +0000234 bool NotExtCompare = false);
Owen Andersone50ed302009-08-10 22:56:29 +0000235 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
Dale Johannesenff97d4f2009-02-03 00:47:48 +0000236 DebugLoc DL, bool foldBooleans = true);
Scott Michelfdc40a02009-02-17 22:15:04 +0000237 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
Chris Lattner5eee4272008-01-26 01:09:19 +0000238 unsigned HiOp);
Owen Andersone50ed302009-08-10 22:56:29 +0000239 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000240 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
Dan Gohman475871a2008-07-27 21:46:04 +0000241 SDValue BuildSDIV(SDNode *N);
242 SDValue BuildUDIV(SDNode *N);
Evan Cheng9568e5c2011-06-21 06:01:08 +0000243 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
244 bool DemandHighBits = true);
245 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
Bill Wendling317bd702009-01-30 21:14:50 +0000246 SDNode *MatchRotate(SDValue LHS, SDValue RHS, DebugLoc DL);
Dan Gohman475871a2008-07-27 21:46:04 +0000247 SDValue ReduceLoadWidth(SDNode *N);
Evan Cheng8b944d32009-05-28 00:35:15 +0000248 SDValue ReduceLoadOpStoreWidth(SDNode *N);
Evan Cheng31959b12011-02-02 01:06:55 +0000249 SDValue TransformFPLoadStorePair(SDNode *N);
Scott Michelfdc40a02009-02-17 22:15:04 +0000250
Dan Gohman475871a2008-07-27 21:46:04 +0000251 SDValue GetDemandedBits(SDValue V, const APInt &Mask);
Scott Michelfdc40a02009-02-17 22:15:04 +0000252
Jim Laskey6ff23e52006-10-04 16:53:27 +0000253 /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
254 /// looking for aliasing nodes and adding them to the Aliases vector.
Dan Gohman475871a2008-07-27 21:46:04 +0000255 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
256 SmallVector<SDValue, 8> &Aliases);
Jim Laskey6ff23e52006-10-04 16:53:27 +0000257
Jim Laskey096c22e2006-10-18 12:29:57 +0000258 /// isAlias - Return true if there is any possibility that the two addresses
259 /// overlap.
Dan Gohman475871a2008-07-27 21:46:04 +0000260 bool isAlias(SDValue Ptr1, int64_t Size1,
Jim Laskey096c22e2006-10-18 12:29:57 +0000261 const Value *SrcValue1, int SrcValueOffset1,
Nate Begemanb6aef5c2009-09-15 00:18:30 +0000262 unsigned SrcValueAlign1,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +0000263 const MDNode *TBAAInfo1,
Dan Gohman475871a2008-07-27 21:46:04 +0000264 SDValue Ptr2, int64_t Size2,
Nate Begemanb6aef5c2009-09-15 00:18:30 +0000265 const Value *SrcValue2, int SrcValueOffset2,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +0000266 unsigned SrcValueAlign2,
267 const MDNode *TBAAInfo2) const;
Scott Michelfdc40a02009-02-17 22:15:04 +0000268
Jim Laskey7ca56af2006-10-11 13:47:09 +0000269 /// FindAliasInfo - Extracts the relevant alias information from the memory
270 /// node. Returns true if the operand was a load.
271 bool FindAliasInfo(SDNode *N,
Dan Gohman475871a2008-07-27 21:46:04 +0000272 SDValue &Ptr, int64_t &Size,
Nate Begemanb6aef5c2009-09-15 00:18:30 +0000273 const Value *&SrcValue, int &SrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +0000274 unsigned &SrcValueAlignment,
275 const MDNode *&TBAAInfo) const;
Scott Michelfdc40a02009-02-17 22:15:04 +0000276
Jim Laskey279f0532006-09-25 16:29:54 +0000277 /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes,
Jim Laskey6ff23e52006-10-04 16:53:27 +0000278 /// looking for a better chain (aliasing node.)
Dan Gohman475871a2008-07-27 21:46:04 +0000279 SDValue FindBetterChain(SDNode *N, SDValue Chain);
Duncan Sands92abc622009-01-31 15:50:11 +0000280
Chris Lattner2392ae72010-04-15 04:48:01 +0000281 public:
Bill Wendling98a366d2009-04-29 23:29:43 +0000282 DAGCombiner(SelectionDAG &D, AliasAnalysis &A, CodeGenOpt::Level OL)
Eli Friedman50185242011-11-12 00:35:34 +0000283 : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
Chris Lattner2392ae72010-04-15 04:48:01 +0000284 OptLevel(OL), LegalOperations(false), LegalTypes(false), AA(A) {}
Scott Michelfdc40a02009-02-17 22:15:04 +0000285
Nate Begeman1d4d4142005-09-01 00:19:25 +0000286 /// Run - runs the dag combiner on all nodes in the work list
Duncan Sands25cf2272008-11-24 14:53:14 +0000287 void Run(CombineLevel AtLevel);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000288
Chris Lattner2392ae72010-04-15 04:48:01 +0000289 SelectionDAG &getDAG() const { return DAG; }
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000290
Chris Lattner2392ae72010-04-15 04:48:01 +0000291 /// getShiftAmountTy - Returns a type large enough to hold any valid
292 /// shift amount - before type legalization these can be huge.
Owen Anderson95771af2011-02-25 21:41:48 +0000293 EVT getShiftAmountTy(EVT LHSTy) {
294 return LegalTypes ? TLI.getShiftAmountTy(LHSTy) : TLI.getPointerTy();
Chris Lattner2392ae72010-04-15 04:48:01 +0000295 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000296
Chris Lattner2392ae72010-04-15 04:48:01 +0000297 /// isTypeLegal - This method returns true if we are running before type
298 /// legalization or if the specified VT is legal.
299 bool isTypeLegal(const EVT &VT) {
300 if (!LegalTypes) return true;
301 return TLI.isTypeLegal(VT);
302 }
Nate Begeman1d4d4142005-09-01 00:19:25 +0000303 };
304}
305
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000306
307namespace {
308/// WorkListRemover - This class is a DAGUpdateListener that removes any deleted
309/// nodes from the worklist.
Nick Lewycky6726b6d2009-10-25 06:33:48 +0000310class WorkListRemover : public SelectionDAG::DAGUpdateListener {
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000311 DAGCombiner &DC;
312public:
Dan Gohmanb5660dc2008-02-20 16:44:09 +0000313 explicit WorkListRemover(DAGCombiner &dc) : DC(dc) {}
Scott Michelfdc40a02009-02-17 22:15:04 +0000314
Duncan Sandsedfcf592008-06-11 11:42:12 +0000315 virtual void NodeDeleted(SDNode *N, SDNode *E) {
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000316 DC.removeFromWorkList(N);
317 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000318
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000319 virtual void NodeUpdated(SDNode *N) {
320 // Ignore updates.
321 }
322};
323}
324
Chris Lattner24664722006-03-01 04:53:38 +0000325//===----------------------------------------------------------------------===//
326// TargetLowering::DAGCombinerInfo implementation
327//===----------------------------------------------------------------------===//
328
329void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
330 ((DAGCombiner*)DC)->AddToWorkList(N);
331}
332
Cameron Zwariched3caf92011-04-02 02:40:26 +0000333void TargetLowering::DAGCombinerInfo::RemoveFromWorklist(SDNode *N) {
334 ((DAGCombiner*)DC)->removeFromWorkList(N);
335}
336
Dan Gohman475871a2008-07-27 21:46:04 +0000337SDValue TargetLowering::DAGCombinerInfo::
Evan Cheng0b0cd912009-03-28 05:57:29 +0000338CombineTo(SDNode *N, const std::vector<SDValue> &To, bool AddTo) {
339 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000340}
341
Dan Gohman475871a2008-07-27 21:46:04 +0000342SDValue TargetLowering::DAGCombinerInfo::
Evan Cheng0b0cd912009-03-28 05:57:29 +0000343CombineTo(SDNode *N, SDValue Res, bool AddTo) {
344 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000345}
346
347
Dan Gohman475871a2008-07-27 21:46:04 +0000348SDValue TargetLowering::DAGCombinerInfo::
Evan Cheng0b0cd912009-03-28 05:57:29 +0000349CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
350 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
Chris Lattner24664722006-03-01 04:53:38 +0000351}
352
Dan Gohmane5af2d32009-01-29 01:59:02 +0000353void TargetLowering::DAGCombinerInfo::
354CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
355 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
356}
Chris Lattner24664722006-03-01 04:53:38 +0000357
Chris Lattner24664722006-03-01 04:53:38 +0000358//===----------------------------------------------------------------------===//
Chris Lattner29446522007-05-14 22:04:50 +0000359// Helper Functions
360//===----------------------------------------------------------------------===//
361
362/// isNegatibleForFree - Return 1 if we can compute the negated form of the
363/// specified expression for the same cost as the expression itself, or 2 if we
364/// can compute the negated form more cheaply than the expression itself.
Duncan Sands25cf2272008-11-24 14:53:14 +0000365static char isNegatibleForFree(SDValue Op, bool LegalOperations,
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000366 const TargetOptions *Options,
Chris Lattner0254e702008-02-26 07:04:54 +0000367 unsigned Depth = 0) {
Dale Johannesendb44bf82007-10-16 23:38:29 +0000368 // No compile time optimizations on this type.
Owen Anderson825b72b2009-08-11 20:47:22 +0000369 if (Op.getValueType() == MVT::ppcf128)
Dale Johannesendb44bf82007-10-16 23:38:29 +0000370 return 0;
371
Chris Lattner29446522007-05-14 22:04:50 +0000372 // fneg is removable even if it has multiple uses.
373 if (Op.getOpcode() == ISD::FNEG) return 2;
Scott Michelfdc40a02009-02-17 22:15:04 +0000374
Chris Lattner29446522007-05-14 22:04:50 +0000375 // Don't allow anything with multiple uses.
376 if (!Op.hasOneUse()) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000377
Chris Lattner3adf9512007-05-25 02:19:06 +0000378 // Don't recurse exponentially.
379 if (Depth > 6) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000380
Chris Lattner29446522007-05-14 22:04:50 +0000381 switch (Op.getOpcode()) {
382 default: return false;
383 case ISD::ConstantFP:
Chris Lattner0254e702008-02-26 07:04:54 +0000384 // Don't invert constant FP values after legalize. The negated constant
385 // isn't necessarily legal.
Duncan Sands25cf2272008-11-24 14:53:14 +0000386 return LegalOperations ? 0 : 1;
Chris Lattner29446522007-05-14 22:04:50 +0000387 case ISD::FADD:
388 // FIXME: determine better conditions for this xform.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000389 if (!Options->UnsafeFPMath) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000390
Bill Wendlingd34470c2009-01-30 23:10:18 +0000391 // fold (fsub (fadd A, B)) -> (fsub (fneg A), B)
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000392 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, Options,
393 Depth + 1))
Chris Lattner29446522007-05-14 22:04:50 +0000394 return V;
Bill Wendlingd34470c2009-01-30 23:10:18 +0000395 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000396 return isNegatibleForFree(Op.getOperand(1), LegalOperations, Options,
397 Depth + 1);
Chris Lattner29446522007-05-14 22:04:50 +0000398 case ISD::FSUB:
Scott Michelfdc40a02009-02-17 22:15:04 +0000399 // We can't turn -(A-B) into B-A when we honor signed zeros.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000400 if (!Options->UnsafeFPMath) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000401
Bill Wendlingd34470c2009-01-30 23:10:18 +0000402 // fold (fneg (fsub A, B)) -> (fsub B, A)
Chris Lattner29446522007-05-14 22:04:50 +0000403 return 1;
Scott Michelfdc40a02009-02-17 22:15:04 +0000404
Chris Lattner29446522007-05-14 22:04:50 +0000405 case ISD::FMUL:
406 case ISD::FDIV:
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000407 if (Options->HonorSignDependentRoundingFPMath()) return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +0000408
Bill Wendlingd34470c2009-01-30 23:10:18 +0000409 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000410 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, Options,
411 Depth + 1))
Chris Lattner29446522007-05-14 22:04:50 +0000412 return V;
Scott Michelfdc40a02009-02-17 22:15:04 +0000413
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000414 return isNegatibleForFree(Op.getOperand(1), LegalOperations, Options,
415 Depth + 1);
Scott Michelfdc40a02009-02-17 22:15:04 +0000416
Chris Lattner29446522007-05-14 22:04:50 +0000417 case ISD::FP_EXTEND:
418 case ISD::FP_ROUND:
419 case ISD::FSIN:
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000420 return isNegatibleForFree(Op.getOperand(0), LegalOperations, Options,
421 Depth + 1);
Chris Lattner29446522007-05-14 22:04:50 +0000422 }
423}
424
425/// GetNegatedExpression - If isNegatibleForFree returns true, this function
426/// returns the newly negated expression.
Dan Gohman475871a2008-07-27 21:46:04 +0000427static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000428 bool LegalOperations, unsigned Depth = 0) {
Chris Lattner29446522007-05-14 22:04:50 +0000429 // fneg is removable even if it has multiple uses.
430 if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
Scott Michelfdc40a02009-02-17 22:15:04 +0000431
Chris Lattner29446522007-05-14 22:04:50 +0000432 // Don't allow anything with multiple uses.
433 assert(Op.hasOneUse() && "Unknown reuse!");
Scott Michelfdc40a02009-02-17 22:15:04 +0000434
Chris Lattner3adf9512007-05-25 02:19:06 +0000435 assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
Chris Lattner29446522007-05-14 22:04:50 +0000436 switch (Op.getOpcode()) {
Torok Edwinc23197a2009-07-14 16:55:14 +0000437 default: llvm_unreachable("Unknown code");
Dale Johannesenc4dd3c32007-08-31 23:34:27 +0000438 case ISD::ConstantFP: {
439 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
440 V.changeSign();
441 return DAG.getConstantFP(V, Op.getValueType());
442 }
Chris Lattner29446522007-05-14 22:04:50 +0000443 case ISD::FADD:
444 // FIXME: determine better conditions for this xform.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000445 assert(DAG.getTarget().Options.UnsafeFPMath);
Scott Michelfdc40a02009-02-17 22:15:04 +0000446
Bill Wendlingd34470c2009-01-30 23:10:18 +0000447 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000448 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
449 &DAG.getTarget().Options, Depth+1))
Bill Wendling35247c32009-01-30 00:45:56 +0000450 return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000451 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000452 LegalOperations, Depth+1),
Chris Lattner29446522007-05-14 22:04:50 +0000453 Op.getOperand(1));
Bill Wendlingd34470c2009-01-30 23:10:18 +0000454 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
Bill Wendling35247c32009-01-30 00:45:56 +0000455 return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000456 GetNegatedExpression(Op.getOperand(1), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000457 LegalOperations, Depth+1),
Chris Lattner29446522007-05-14 22:04:50 +0000458 Op.getOperand(0));
459 case ISD::FSUB:
Scott Michelfdc40a02009-02-17 22:15:04 +0000460 // We can't turn -(A-B) into B-A when we honor signed zeros.
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000461 assert(DAG.getTarget().Options.UnsafeFPMath);
Dan Gohman23ff1822007-07-02 15:48:56 +0000462
Bill Wendlingd34470c2009-01-30 23:10:18 +0000463 // fold (fneg (fsub 0, B)) -> B
Dan Gohman23ff1822007-07-02 15:48:56 +0000464 if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
Dale Johannesenc4dd3c32007-08-31 23:34:27 +0000465 if (N0CFP->getValueAPF().isZero())
Dan Gohman23ff1822007-07-02 15:48:56 +0000466 return Op.getOperand(1);
Scott Michelfdc40a02009-02-17 22:15:04 +0000467
Bill Wendlingd34470c2009-01-30 23:10:18 +0000468 // fold (fneg (fsub A, B)) -> (fsub B, A)
Bill Wendling35247c32009-01-30 00:45:56 +0000469 return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(),
470 Op.getOperand(1), Op.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +0000471
Chris Lattner29446522007-05-14 22:04:50 +0000472 case ISD::FMUL:
473 case ISD::FDIV:
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000474 assert(!DAG.getTarget().Options.HonorSignDependentRoundingFPMath());
Scott Michelfdc40a02009-02-17 22:15:04 +0000475
Bill Wendlingd34470c2009-01-30 23:10:18 +0000476 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
Nick Lewycky8a8d4792011-12-02 22:16:29 +0000477 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
478 &DAG.getTarget().Options, Depth+1))
Bill Wendling35247c32009-01-30 00:45:56 +0000479 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000480 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000481 LegalOperations, Depth+1),
Chris Lattner29446522007-05-14 22:04:50 +0000482 Op.getOperand(1));
Scott Michelfdc40a02009-02-17 22:15:04 +0000483
Bill Wendlingd34470c2009-01-30 23:10:18 +0000484 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
Bill Wendling35247c32009-01-30 00:45:56 +0000485 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(),
Chris Lattner29446522007-05-14 22:04:50 +0000486 Op.getOperand(0),
Chris Lattner0254e702008-02-26 07:04:54 +0000487 GetNegatedExpression(Op.getOperand(1), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000488 LegalOperations, Depth+1));
Scott Michelfdc40a02009-02-17 22:15:04 +0000489
Chris Lattner29446522007-05-14 22:04:50 +0000490 case ISD::FP_EXTEND:
Chris Lattner29446522007-05-14 22:04:50 +0000491 case ISD::FSIN:
Bill Wendling35247c32009-01-30 00:45:56 +0000492 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000493 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000494 LegalOperations, Depth+1));
Chris Lattner0bd48932008-01-17 07:00:52 +0000495 case ISD::FP_ROUND:
Bill Wendling35247c32009-01-30 00:45:56 +0000496 return DAG.getNode(ISD::FP_ROUND, Op.getDebugLoc(), Op.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +0000497 GetNegatedExpression(Op.getOperand(0), DAG,
Duncan Sands25cf2272008-11-24 14:53:14 +0000498 LegalOperations, Depth+1),
Chris Lattner0bd48932008-01-17 07:00:52 +0000499 Op.getOperand(1));
Chris Lattner29446522007-05-14 22:04:50 +0000500 }
501}
Chris Lattner24664722006-03-01 04:53:38 +0000502
503
Nate Begeman4ebd8052005-09-01 23:24:04 +0000504// isSetCCEquivalent - Return true if this node is a setcc, or is a select_cc
505// that selects between the values 1 and 0, making it equivalent to a setcc.
Scott Michelfdc40a02009-02-17 22:15:04 +0000506// Also, set the incoming LHS, RHS, and CC references to the appropriate
Nate Begeman646d7e22005-09-02 21:18:40 +0000507// nodes based on the type of node we are checking. This simplifies life a
508// bit for the callers.
Dan Gohman475871a2008-07-27 21:46:04 +0000509static bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
510 SDValue &CC) {
Nate Begeman646d7e22005-09-02 21:18:40 +0000511 if (N.getOpcode() == ISD::SETCC) {
512 LHS = N.getOperand(0);
513 RHS = N.getOperand(1);
514 CC = N.getOperand(2);
Nate Begeman4ebd8052005-09-01 23:24:04 +0000515 return true;
Nate Begeman646d7e22005-09-02 21:18:40 +0000516 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000517 if (N.getOpcode() == ISD::SELECT_CC &&
Nate Begeman1d4d4142005-09-01 00:19:25 +0000518 N.getOperand(2).getOpcode() == ISD::Constant &&
519 N.getOperand(3).getOpcode() == ISD::Constant &&
Dan Gohman002e5d02008-03-13 22:13:53 +0000520 cast<ConstantSDNode>(N.getOperand(2))->getAPIntValue() == 1 &&
Nate Begeman646d7e22005-09-02 21:18:40 +0000521 cast<ConstantSDNode>(N.getOperand(3))->isNullValue()) {
522 LHS = N.getOperand(0);
523 RHS = N.getOperand(1);
524 CC = N.getOperand(4);
Nate Begeman1d4d4142005-09-01 00:19:25 +0000525 return true;
Nate Begeman646d7e22005-09-02 21:18:40 +0000526 }
Nate Begeman1d4d4142005-09-01 00:19:25 +0000527 return false;
528}
529
Nate Begeman99801192005-09-07 23:25:52 +0000530// isOneUseSetCC - Return true if this is a SetCC-equivalent operation with only
531// one use. If this is true, it allows the users to invert the operation for
532// free when it is profitable to do so.
Dan Gohman475871a2008-07-27 21:46:04 +0000533static bool isOneUseSetCC(SDValue N) {
534 SDValue N0, N1, N2;
Gabor Greifba36cb52008-08-28 21:40:38 +0000535 if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
Nate Begeman4ebd8052005-09-01 23:24:04 +0000536 return true;
537 return false;
538}
539
Bill Wendling35247c32009-01-30 00:45:56 +0000540SDValue DAGCombiner::ReassociateOps(unsigned Opc, DebugLoc DL,
541 SDValue N0, SDValue N1) {
Owen Andersone50ed302009-08-10 22:56:29 +0000542 EVT VT = N0.getValueType();
Nate Begemancd4d58c2006-02-03 06:46:56 +0000543 if (N0.getOpcode() == Opc && isa<ConstantSDNode>(N0.getOperand(1))) {
544 if (isa<ConstantSDNode>(N1)) {
Bill Wendling35247c32009-01-30 00:45:56 +0000545 // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
Bill Wendling6af76182009-01-30 20:50:00 +0000546 SDValue OpNode =
547 DAG.FoldConstantArithmetic(Opc, VT,
548 cast<ConstantSDNode>(N0.getOperand(1)),
549 cast<ConstantSDNode>(N1));
Bill Wendlingd69c3142009-01-30 02:23:43 +0000550 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
Dan Gohman71dc7c92011-05-17 22:20:36 +0000551 }
552 if (N0.hasOneUse()) {
Bill Wendling35247c32009-01-30 00:45:56 +0000553 // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one use
554 SDValue OpNode = DAG.getNode(Opc, N0.getDebugLoc(), VT,
555 N0.getOperand(0), N1);
Gabor Greifba36cb52008-08-28 21:40:38 +0000556 AddToWorkList(OpNode.getNode());
Bill Wendling35247c32009-01-30 00:45:56 +0000557 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
Nate Begemancd4d58c2006-02-03 06:46:56 +0000558 }
559 }
Bill Wendling35247c32009-01-30 00:45:56 +0000560
Nate Begemancd4d58c2006-02-03 06:46:56 +0000561 if (N1.getOpcode() == Opc && isa<ConstantSDNode>(N1.getOperand(1))) {
562 if (isa<ConstantSDNode>(N0)) {
Bill Wendling35247c32009-01-30 00:45:56 +0000563 // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
Bill Wendling6af76182009-01-30 20:50:00 +0000564 SDValue OpNode =
565 DAG.FoldConstantArithmetic(Opc, VT,
566 cast<ConstantSDNode>(N1.getOperand(1)),
567 cast<ConstantSDNode>(N0));
Bill Wendlingd69c3142009-01-30 02:23:43 +0000568 return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
Dan Gohman71dc7c92011-05-17 22:20:36 +0000569 }
570 if (N1.hasOneUse()) {
Bill Wendling35247c32009-01-30 00:45:56 +0000571 // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one use
Bill Wendlingd69c3142009-01-30 02:23:43 +0000572 SDValue OpNode = DAG.getNode(Opc, N0.getDebugLoc(), VT,
Bill Wendling35247c32009-01-30 00:45:56 +0000573 N1.getOperand(0), N0);
Gabor Greifba36cb52008-08-28 21:40:38 +0000574 AddToWorkList(OpNode.getNode());
Bill Wendling35247c32009-01-30 00:45:56 +0000575 return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
Nate Begemancd4d58c2006-02-03 06:46:56 +0000576 }
577 }
Bill Wendling35247c32009-01-30 00:45:56 +0000578
Dan Gohman475871a2008-07-27 21:46:04 +0000579 return SDValue();
Nate Begemancd4d58c2006-02-03 06:46:56 +0000580}
581
Dan Gohman475871a2008-07-27 21:46:04 +0000582SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
583 bool AddTo) {
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000584 assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
585 ++NodesCombined;
David Greenef1090292010-01-05 01:25:00 +0000586 DEBUG(dbgs() << "\nReplacing.1 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000587 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000588 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000589 To[0].getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000590 dbgs() << " and " << NumTo-1 << " other values\n";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000591 for (unsigned i = 0, e = NumTo; i != e; ++i)
Jakob Stoklund Olesen9f0d4e62009-12-03 05:15:35 +0000592 assert((!To[i].getNode() ||
593 N->getValueType(i) == To[i].getValueType()) &&
Dan Gohman764fd0c2009-01-21 15:17:51 +0000594 "Cannot combine value to value of different type!"));
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000595 WorkListRemover DeadNodes(*this);
596 DAG.ReplaceAllUsesWith(N, To, &DeadNodes);
Scott Michelfdc40a02009-02-17 22:15:04 +0000597
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000598 if (AddTo) {
599 // Push the new nodes and any users onto the worklist
600 for (unsigned i = 0, e = NumTo; i != e; ++i) {
Chris Lattnerd1980a52009-03-12 06:52:53 +0000601 if (To[i].getNode()) {
602 AddToWorkList(To[i].getNode());
603 AddUsersToWorkList(To[i].getNode());
604 }
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000605 }
606 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000607
Dan Gohmandbe664a2009-01-19 21:44:21 +0000608 // Finally, if the node is now dead, remove it from the graph. The node
609 // may not be dead if the replacement process recursively simplified to
610 // something else needing this node.
611 if (N->use_empty()) {
612 // Nodes can be reintroduced into the worklist. Make sure we do not
613 // process a node that has been replaced.
614 removeFromWorkList(N);
Scott Michelfdc40a02009-02-17 22:15:04 +0000615
Dan Gohmandbe664a2009-01-19 21:44:21 +0000616 // Finally, since the node is now dead, remove it from the graph.
617 DAG.DeleteNode(N);
618 }
Dan Gohman475871a2008-07-27 21:46:04 +0000619 return SDValue(N, 0);
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000620}
621
Evan Chenge5b51ac2010-04-17 06:13:15 +0000622void DAGCombiner::
623CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
Scott Michelfdc40a02009-02-17 22:15:04 +0000624 // Replace all uses. If any nodes become isomorphic to other nodes and
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000625 // are deleted, make sure to remove them from our worklist.
626 WorkListRemover DeadNodes(*this);
627 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New, &DeadNodes);
Dan Gohmane5af2d32009-01-29 01:59:02 +0000628
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000629 // Push the new node and any (possibly new) users onto the worklist.
Gabor Greifba36cb52008-08-28 21:40:38 +0000630 AddToWorkList(TLO.New.getNode());
631 AddUsersToWorkList(TLO.New.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000632
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000633 // Finally, if the node is now dead, remove it from the graph. The node
634 // may not be dead if the replacement process recursively simplified to
635 // something else needing this node.
Gabor Greifba36cb52008-08-28 21:40:38 +0000636 if (TLO.Old.getNode()->use_empty()) {
637 removeFromWorkList(TLO.Old.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000638
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000639 // If the operands of this node are only used by the node, they will now
640 // be dead. Make sure to visit them first to delete dead nodes early.
Gabor Greifba36cb52008-08-28 21:40:38 +0000641 for (unsigned i = 0, e = TLO.Old.getNode()->getNumOperands(); i != e; ++i)
642 if (TLO.Old.getNode()->getOperand(i).getNode()->hasOneUse())
643 AddToWorkList(TLO.Old.getNode()->getOperand(i).getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000644
Gabor Greifba36cb52008-08-28 21:40:38 +0000645 DAG.DeleteNode(TLO.Old.getNode());
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000646 }
Dan Gohmane5af2d32009-01-29 01:59:02 +0000647}
648
649/// SimplifyDemandedBits - Check the specified integer node value to see if
650/// it can be simplified or if things it uses can be simplified by bit
651/// propagation. If so, return true.
652bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
Evan Chenge5b51ac2010-04-17 06:13:15 +0000653 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
Dan Gohmane5af2d32009-01-29 01:59:02 +0000654 APInt KnownZero, KnownOne;
655 if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
656 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +0000657
Dan Gohmane5af2d32009-01-29 01:59:02 +0000658 // Revisit the node.
659 AddToWorkList(Op.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000660
Dan Gohmane5af2d32009-01-29 01:59:02 +0000661 // Replace the old value with the new one.
662 ++NodesCombined;
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000663 DEBUG(dbgs() << "\nReplacing.2 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000664 TLO.Old.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000665 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000666 TLO.New.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +0000667 dbgs() << '\n');
Scott Michelfdc40a02009-02-17 22:15:04 +0000668
Dan Gohmane5af2d32009-01-29 01:59:02 +0000669 CommitTargetLoweringOpt(TLO);
Chris Lattnerf8dc0612008-02-03 06:49:24 +0000670 return true;
671}
672
Evan Cheng95c57ea2010-04-24 04:43:44 +0000673void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
674 DebugLoc dl = Load->getDebugLoc();
675 EVT VT = Load->getValueType(0);
676 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0));
Evan Cheng4c26e932010-04-19 19:29:22 +0000677
Evan Cheng95c57ea2010-04-24 04:43:44 +0000678 DEBUG(dbgs() << "\nReplacing.9 ";
679 Load->dump(&DAG);
680 dbgs() << "\nWith: ";
681 Trunc.getNode()->dump(&DAG);
682 dbgs() << '\n');
683 WorkListRemover DeadNodes(*this);
684 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc, &DeadNodes);
685 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1),
686 &DeadNodes);
687 removeFromWorkList(Load);
688 DAG.DeleteNode(Load);
Evan Chengac7eae52010-04-27 19:48:13 +0000689 AddToWorkList(Trunc.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000690}
691
692SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
693 Replace = false;
Evan Cheng4c26e932010-04-19 19:29:22 +0000694 DebugLoc dl = Op.getDebugLoc();
Evan Chenge5b51ac2010-04-17 06:13:15 +0000695 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
Evan Chengac7eae52010-04-27 19:48:13 +0000696 EVT MemVT = LD->getMemoryVT();
697 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
Owen Anderson95771af2011-02-25 21:41:48 +0000698 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
Eric Christopher503a64d2010-12-09 04:48:06 +0000699 : ISD::EXTLOAD)
Evan Chengac7eae52010-04-27 19:48:13 +0000700 : LD->getExtensionType();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000701 Replace = true;
Stuart Hastingsa9011292011-02-16 16:23:55 +0000702 return DAG.getExtLoad(ExtType, dl, PVT,
Evan Chenge5b51ac2010-04-17 06:13:15 +0000703 LD->getChain(), LD->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +0000704 LD->getPointerInfo(),
Evan Chengac7eae52010-04-27 19:48:13 +0000705 MemVT, LD->isVolatile(),
Evan Chenge5b51ac2010-04-17 06:13:15 +0000706 LD->isNonTemporal(), LD->getAlignment());
707 }
708
Evan Cheng4c26e932010-04-19 19:29:22 +0000709 unsigned Opc = Op.getOpcode();
Evan Chengcaf77402010-04-23 19:10:30 +0000710 switch (Opc) {
711 default: break;
712 case ISD::AssertSext:
Evan Cheng4c26e932010-04-19 19:29:22 +0000713 return DAG.getNode(ISD::AssertSext, dl, PVT,
Evan Cheng95c57ea2010-04-24 04:43:44 +0000714 SExtPromoteOperand(Op.getOperand(0), PVT),
Evan Cheng4c26e932010-04-19 19:29:22 +0000715 Op.getOperand(1));
Evan Chengcaf77402010-04-23 19:10:30 +0000716 case ISD::AssertZext:
Evan Cheng4c26e932010-04-19 19:29:22 +0000717 return DAG.getNode(ISD::AssertZext, dl, PVT,
Evan Cheng95c57ea2010-04-24 04:43:44 +0000718 ZExtPromoteOperand(Op.getOperand(0), PVT),
Evan Cheng4c26e932010-04-19 19:29:22 +0000719 Op.getOperand(1));
Evan Chengcaf77402010-04-23 19:10:30 +0000720 case ISD::Constant: {
721 unsigned ExtOpc =
Evan Cheng4c26e932010-04-19 19:29:22 +0000722 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
Evan Chengcaf77402010-04-23 19:10:30 +0000723 return DAG.getNode(ExtOpc, dl, PVT, Op);
Wesley Peckbf17cfa2010-11-23 03:31:01 +0000724 }
Evan Chengcaf77402010-04-23 19:10:30 +0000725 }
726
727 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
Evan Chenge5b51ac2010-04-17 06:13:15 +0000728 return SDValue();
Evan Chengcaf77402010-04-23 19:10:30 +0000729 return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op);
Evan Cheng64b7bf72010-04-16 06:14:10 +0000730}
731
Evan Cheng95c57ea2010-04-24 04:43:44 +0000732SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
Evan Chenge5b51ac2010-04-17 06:13:15 +0000733 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
734 return SDValue();
735 EVT OldVT = Op.getValueType();
736 DebugLoc dl = Op.getDebugLoc();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000737 bool Replace = false;
738 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
739 if (NewOp.getNode() == 0)
Evan Chenge5b51ac2010-04-17 06:13:15 +0000740 return SDValue();
Evan Chengac7eae52010-04-27 19:48:13 +0000741 AddToWorkList(NewOp.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000742
743 if (Replace)
744 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
745 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp,
Evan Chenge5b51ac2010-04-17 06:13:15 +0000746 DAG.getValueType(OldVT));
747}
748
Evan Cheng95c57ea2010-04-24 04:43:44 +0000749SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
Evan Chenge5b51ac2010-04-17 06:13:15 +0000750 EVT OldVT = Op.getValueType();
751 DebugLoc dl = Op.getDebugLoc();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000752 bool Replace = false;
753 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
754 if (NewOp.getNode() == 0)
Evan Chenge5b51ac2010-04-17 06:13:15 +0000755 return SDValue();
Evan Chengac7eae52010-04-27 19:48:13 +0000756 AddToWorkList(NewOp.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000757
758 if (Replace)
759 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
760 return DAG.getZeroExtendInReg(NewOp, dl, OldVT);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000761}
762
Evan Cheng64b7bf72010-04-16 06:14:10 +0000763/// PromoteIntBinOp - Promote the specified integer binary operation if the
764/// target indicates it is beneficial. e.g. On x86, it's usually better to
765/// promote i16 operations to i32 since i16 instructions are longer.
766SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
767 if (!LegalOperations)
768 return SDValue();
769
770 EVT VT = Op.getValueType();
771 if (VT.isVector() || !VT.isInteger())
772 return SDValue();
773
Evan Chenge5b51ac2010-04-17 06:13:15 +0000774 // If operation type is 'undesirable', e.g. i16 on x86, consider
775 // promoting it.
776 unsigned Opc = Op.getOpcode();
777 if (TLI.isTypeDesirableForOp(Opc, VT))
778 return SDValue();
779
Evan Cheng64b7bf72010-04-16 06:14:10 +0000780 EVT PVT = VT;
Evan Chenge5b51ac2010-04-17 06:13:15 +0000781 // Consult target whether it is a good idea to promote this operation and
782 // what's the right type to promote it to.
783 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
Evan Cheng64b7bf72010-04-16 06:14:10 +0000784 assert(PVT != VT && "Don't know what type to promote to!");
785
Evan Cheng95c57ea2010-04-24 04:43:44 +0000786 bool Replace0 = false;
787 SDValue N0 = Op.getOperand(0);
788 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
789 if (NN0.getNode() == 0)
Evan Cheng07c4e102010-04-22 20:19:46 +0000790 return SDValue();
791
Evan Cheng95c57ea2010-04-24 04:43:44 +0000792 bool Replace1 = false;
793 SDValue N1 = Op.getOperand(1);
Evan Chengaad753b2010-05-10 19:03:57 +0000794 SDValue NN1;
795 if (N0 == N1)
796 NN1 = NN0;
797 else {
798 NN1 = PromoteOperand(N1, PVT, Replace1);
799 if (NN1.getNode() == 0)
800 return SDValue();
801 }
Evan Cheng07c4e102010-04-22 20:19:46 +0000802
Evan Cheng95c57ea2010-04-24 04:43:44 +0000803 AddToWorkList(NN0.getNode());
Evan Chengaad753b2010-05-10 19:03:57 +0000804 if (NN1.getNode())
805 AddToWorkList(NN1.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000806
807 if (Replace0)
808 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
809 if (Replace1)
810 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
Evan Cheng07c4e102010-04-22 20:19:46 +0000811
Evan Chengac7eae52010-04-27 19:48:13 +0000812 DEBUG(dbgs() << "\nPromoting ";
813 Op.getNode()->dump(&DAG));
Evan Cheng07c4e102010-04-22 20:19:46 +0000814 DebugLoc dl = Op.getDebugLoc();
815 return DAG.getNode(ISD::TRUNCATE, dl, VT,
Evan Cheng95c57ea2010-04-24 04:43:44 +0000816 DAG.getNode(Opc, dl, PVT, NN0, NN1));
Evan Cheng07c4e102010-04-22 20:19:46 +0000817 }
818 return SDValue();
819}
820
821/// PromoteIntShiftOp - Promote the specified integer shift operation if the
822/// target indicates it is beneficial. e.g. On x86, it's usually better to
823/// promote i16 operations to i32 since i16 instructions are longer.
824SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
825 if (!LegalOperations)
826 return SDValue();
827
828 EVT VT = Op.getValueType();
829 if (VT.isVector() || !VT.isInteger())
830 return SDValue();
831
832 // If operation type is 'undesirable', e.g. i16 on x86, consider
833 // promoting it.
834 unsigned Opc = Op.getOpcode();
835 if (TLI.isTypeDesirableForOp(Opc, VT))
836 return SDValue();
837
838 EVT PVT = VT;
839 // Consult target whether it is a good idea to promote this operation and
840 // what's the right type to promote it to.
841 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
842 assert(PVT != VT && "Don't know what type to promote to!");
843
Evan Cheng95c57ea2010-04-24 04:43:44 +0000844 bool Replace = false;
Evan Chenge5b51ac2010-04-17 06:13:15 +0000845 SDValue N0 = Op.getOperand(0);
846 if (Opc == ISD::SRA)
Evan Cheng95c57ea2010-04-24 04:43:44 +0000847 N0 = SExtPromoteOperand(Op.getOperand(0), PVT);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000848 else if (Opc == ISD::SRL)
Evan Cheng95c57ea2010-04-24 04:43:44 +0000849 N0 = ZExtPromoteOperand(Op.getOperand(0), PVT);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000850 else
Evan Cheng95c57ea2010-04-24 04:43:44 +0000851 N0 = PromoteOperand(N0, PVT, Replace);
Evan Chenge5b51ac2010-04-17 06:13:15 +0000852 if (N0.getNode() == 0)
853 return SDValue();
Evan Cheng95c57ea2010-04-24 04:43:44 +0000854
Evan Chenge5b51ac2010-04-17 06:13:15 +0000855 AddToWorkList(N0.getNode());
Evan Cheng95c57ea2010-04-24 04:43:44 +0000856 if (Replace)
857 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
Evan Cheng64b7bf72010-04-16 06:14:10 +0000858
Evan Chengac7eae52010-04-27 19:48:13 +0000859 DEBUG(dbgs() << "\nPromoting ";
860 Op.getNode()->dump(&DAG));
Evan Cheng64b7bf72010-04-16 06:14:10 +0000861 DebugLoc dl = Op.getDebugLoc();
862 return DAG.getNode(ISD::TRUNCATE, dl, VT,
Evan Cheng07c4e102010-04-22 20:19:46 +0000863 DAG.getNode(Opc, dl, PVT, N0, Op.getOperand(1)));
Evan Cheng64b7bf72010-04-16 06:14:10 +0000864 }
865 return SDValue();
866}
867
Evan Cheng4c26e932010-04-19 19:29:22 +0000868SDValue DAGCombiner::PromoteExtend(SDValue Op) {
869 if (!LegalOperations)
870 return SDValue();
871
872 EVT VT = Op.getValueType();
873 if (VT.isVector() || !VT.isInteger())
874 return SDValue();
875
876 // If operation type is 'undesirable', e.g. i16 on x86, consider
877 // promoting it.
878 unsigned Opc = Op.getOpcode();
879 if (TLI.isTypeDesirableForOp(Opc, VT))
880 return SDValue();
881
882 EVT PVT = VT;
883 // Consult target whether it is a good idea to promote this operation and
884 // what's the right type to promote it to.
885 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
886 assert(PVT != VT && "Don't know what type to promote to!");
887 // fold (aext (aext x)) -> (aext x)
888 // fold (aext (zext x)) -> (zext x)
889 // fold (aext (sext x)) -> (sext x)
Evan Chengac7eae52010-04-27 19:48:13 +0000890 DEBUG(dbgs() << "\nPromoting ";
891 Op.getNode()->dump(&DAG));
Evan Cheng4c26e932010-04-19 19:29:22 +0000892 return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), VT, Op.getOperand(0));
893 }
894 return SDValue();
895}
896
897bool DAGCombiner::PromoteLoad(SDValue Op) {
898 if (!LegalOperations)
899 return false;
900
901 EVT VT = Op.getValueType();
902 if (VT.isVector() || !VT.isInteger())
903 return false;
904
905 // If operation type is 'undesirable', e.g. i16 on x86, consider
906 // promoting it.
907 unsigned Opc = Op.getOpcode();
908 if (TLI.isTypeDesirableForOp(Opc, VT))
909 return false;
910
911 EVT PVT = VT;
912 // Consult target whether it is a good idea to promote this operation and
913 // what's the right type to promote it to.
914 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
915 assert(PVT != VT && "Don't know what type to promote to!");
916
917 DebugLoc dl = Op.getDebugLoc();
918 SDNode *N = Op.getNode();
919 LoadSDNode *LD = cast<LoadSDNode>(N);
Evan Chengac7eae52010-04-27 19:48:13 +0000920 EVT MemVT = LD->getMemoryVT();
921 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
Owen Anderson95771af2011-02-25 21:41:48 +0000922 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
Eric Christopher503a64d2010-12-09 04:48:06 +0000923 : ISD::EXTLOAD)
Evan Chengac7eae52010-04-27 19:48:13 +0000924 : LD->getExtensionType();
Stuart Hastingsa9011292011-02-16 16:23:55 +0000925 SDValue NewLD = DAG.getExtLoad(ExtType, dl, PVT,
Evan Cheng4c26e932010-04-19 19:29:22 +0000926 LD->getChain(), LD->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +0000927 LD->getPointerInfo(),
Evan Chengac7eae52010-04-27 19:48:13 +0000928 MemVT, LD->isVolatile(),
Evan Cheng4c26e932010-04-19 19:29:22 +0000929 LD->isNonTemporal(), LD->getAlignment());
930 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD);
931
Evan Cheng95c57ea2010-04-24 04:43:44 +0000932 DEBUG(dbgs() << "\nPromoting ";
Evan Cheng4c26e932010-04-19 19:29:22 +0000933 N->dump(&DAG);
Evan Cheng95c57ea2010-04-24 04:43:44 +0000934 dbgs() << "\nTo: ";
Evan Cheng4c26e932010-04-19 19:29:22 +0000935 Result.getNode()->dump(&DAG);
936 dbgs() << '\n');
937 WorkListRemover DeadNodes(*this);
938 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result, &DeadNodes);
939 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1), &DeadNodes);
940 removeFromWorkList(N);
941 DAG.DeleteNode(N);
Evan Chengac7eae52010-04-27 19:48:13 +0000942 AddToWorkList(Result.getNode());
Evan Cheng4c26e932010-04-19 19:29:22 +0000943 return true;
944 }
945 return false;
946}
947
Evan Chenge5b51ac2010-04-17 06:13:15 +0000948
Chris Lattner29446522007-05-14 22:04:50 +0000949//===----------------------------------------------------------------------===//
950// Main DAG Combiner implementation
951//===----------------------------------------------------------------------===//
952
Duncan Sands25cf2272008-11-24 14:53:14 +0000953void DAGCombiner::Run(CombineLevel AtLevel) {
954 // set the instance variables, so that the various visit routines may use it.
955 Level = AtLevel;
Eli Friedman50185242011-11-12 00:35:34 +0000956 LegalOperations = Level >= AfterLegalizeVectorOps;
957 LegalTypes = Level >= AfterLegalizeTypes;
Nate Begeman4ebd8052005-09-01 23:24:04 +0000958
Evan Cheng17a568b2008-08-29 22:21:44 +0000959 // Add all the dag nodes to the worklist.
960 WorkList.reserve(DAG.allnodes_size());
961 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
962 E = DAG.allnodes_end(); I != E; ++I)
963 WorkList.push_back(I);
Duncan Sands25cf2272008-11-24 14:53:14 +0000964
Evan Cheng17a568b2008-08-29 22:21:44 +0000965 // Create a dummy node (which is not added to allnodes), that adds a reference
966 // to the root node, preventing it from being deleted, and tracking any
967 // changes of the root.
968 HandleSDNode Dummy(DAG.getRoot());
Scott Michelfdc40a02009-02-17 22:15:04 +0000969
Jim Laskey26f7fa72006-10-17 19:33:52 +0000970 // The root of the dag may dangle to deleted nodes until the dag combiner is
971 // done. Set it to null to avoid confusion.
Dan Gohman475871a2008-07-27 21:46:04 +0000972 DAG.setRoot(SDValue());
Scott Michelfdc40a02009-02-17 22:15:04 +0000973
Evan Cheng17a568b2008-08-29 22:21:44 +0000974 // while the worklist isn't empty, inspect the node on the end of it and
975 // try and combine it.
976 while (!WorkList.empty()) {
977 SDNode *N = WorkList.back();
978 WorkList.pop_back();
Scott Michelfdc40a02009-02-17 22:15:04 +0000979
Evan Cheng17a568b2008-08-29 22:21:44 +0000980 // If N has no uses, it is dead. Make sure to revisit all N's operands once
981 // N is deleted from the DAG, since they too may now be dead or may have a
982 // reduced number of uses, allowing other xforms.
983 if (N->use_empty() && N != &Dummy) {
984 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
985 AddToWorkList(N->getOperand(i).getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +0000986
Evan Cheng17a568b2008-08-29 22:21:44 +0000987 DAG.DeleteNode(N);
988 continue;
Nate Begeman1d4d4142005-09-01 00:19:25 +0000989 }
Scott Michelfdc40a02009-02-17 22:15:04 +0000990
Evan Cheng17a568b2008-08-29 22:21:44 +0000991 SDValue RV = combine(N);
Scott Michelfdc40a02009-02-17 22:15:04 +0000992
Evan Cheng17a568b2008-08-29 22:21:44 +0000993 if (RV.getNode() == 0)
994 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +0000995
Evan Cheng17a568b2008-08-29 22:21:44 +0000996 ++NodesCombined;
Scott Michelfdc40a02009-02-17 22:15:04 +0000997
Evan Cheng17a568b2008-08-29 22:21:44 +0000998 // If we get back the same node we passed in, rather than a new node or
999 // zero, we know that the node must have defined multiple values and
Scott Michelfdc40a02009-02-17 22:15:04 +00001000 // CombineTo was used. Since CombineTo takes care of the worklist
Evan Cheng17a568b2008-08-29 22:21:44 +00001001 // mechanics for us, we have no work to do in this case.
1002 if (RV.getNode() == N)
1003 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00001004
Evan Cheng17a568b2008-08-29 22:21:44 +00001005 assert(N->getOpcode() != ISD::DELETED_NODE &&
1006 RV.getNode()->getOpcode() != ISD::DELETED_NODE &&
1007 "Node was deleted but visit returned new node!");
Chris Lattner729c6d12006-05-27 00:43:02 +00001008
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001009 DEBUG(dbgs() << "\nReplacing.3 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00001010 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00001011 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00001012 RV.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00001013 dbgs() << '\n');
Eric Christopher7332e6e2011-07-14 01:12:15 +00001014
Devang Patel9728ea22011-05-23 22:04:42 +00001015 // Transfer debug value.
1016 DAG.TransferDbgValues(SDValue(N, 0), RV);
Evan Cheng17a568b2008-08-29 22:21:44 +00001017 WorkListRemover DeadNodes(*this);
1018 if (N->getNumValues() == RV.getNode()->getNumValues())
1019 DAG.ReplaceAllUsesWith(N, RV.getNode(), &DeadNodes);
1020 else {
1021 assert(N->getValueType(0) == RV.getValueType() &&
1022 N->getNumValues() == 1 && "Type mismatch");
1023 SDValue OpV = RV;
1024 DAG.ReplaceAllUsesWith(N, &OpV, &DeadNodes);
1025 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001026
Evan Cheng17a568b2008-08-29 22:21:44 +00001027 // Push the new node and any users onto the worklist
1028 AddToWorkList(RV.getNode());
1029 AddUsersToWorkList(RV.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00001030
Evan Cheng17a568b2008-08-29 22:21:44 +00001031 // Add any uses of the old node to the worklist in case this node is the
1032 // last one that uses them. They may become dead after this node is
1033 // deleted.
1034 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1035 AddToWorkList(N->getOperand(i).getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00001036
Dan Gohmandbe664a2009-01-19 21:44:21 +00001037 // Finally, if the node is now dead, remove it from the graph. The node
1038 // may not be dead if the replacement process recursively simplified to
1039 // something else needing this node.
1040 if (N->use_empty()) {
1041 // Nodes can be reintroduced into the worklist. Make sure we do not
1042 // process a node that has been replaced.
1043 removeFromWorkList(N);
Scott Michelfdc40a02009-02-17 22:15:04 +00001044
Dan Gohmandbe664a2009-01-19 21:44:21 +00001045 // Finally, since the node is now dead, remove it from the graph.
1046 DAG.DeleteNode(N);
1047 }
Evan Cheng17a568b2008-08-29 22:21:44 +00001048 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001049
Chris Lattner95038592005-10-05 06:35:28 +00001050 // If the root changed (e.g. it was a dead load, update the root).
1051 DAG.setRoot(Dummy.getValue());
Nate Begeman1d4d4142005-09-01 00:19:25 +00001052}
1053
Dan Gohman475871a2008-07-27 21:46:04 +00001054SDValue DAGCombiner::visit(SDNode *N) {
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001055 switch (N->getOpcode()) {
Nate Begeman1d4d4142005-09-01 00:19:25 +00001056 default: break;
Nate Begeman4942a962005-09-01 00:33:32 +00001057 case ISD::TokenFactor: return visitTokenFactor(N);
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001058 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001059 case ISD::ADD: return visitADD(N);
1060 case ISD::SUB: return visitSUB(N);
Chris Lattner91153682007-03-04 20:03:15 +00001061 case ISD::ADDC: return visitADDC(N);
1062 case ISD::ADDE: return visitADDE(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001063 case ISD::MUL: return visitMUL(N);
1064 case ISD::SDIV: return visitSDIV(N);
1065 case ISD::UDIV: return visitUDIV(N);
1066 case ISD::SREM: return visitSREM(N);
1067 case ISD::UREM: return visitUREM(N);
1068 case ISD::MULHU: return visitMULHU(N);
1069 case ISD::MULHS: return visitMULHS(N);
Dan Gohman389079b2007-10-08 17:57:15 +00001070 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1071 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
Benjamin Kramerf55d26e2011-05-21 18:31:55 +00001072 case ISD::SMULO: return visitSMULO(N);
1073 case ISD::UMULO: return visitUMULO(N);
Dan Gohman389079b2007-10-08 17:57:15 +00001074 case ISD::SDIVREM: return visitSDIVREM(N);
1075 case ISD::UDIVREM: return visitUDIVREM(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001076 case ISD::AND: return visitAND(N);
1077 case ISD::OR: return visitOR(N);
1078 case ISD::XOR: return visitXOR(N);
1079 case ISD::SHL: return visitSHL(N);
1080 case ISD::SRA: return visitSRA(N);
1081 case ISD::SRL: return visitSRL(N);
1082 case ISD::CTLZ: return visitCTLZ(N);
Chandler Carruth63974b22011-12-13 01:56:10 +00001083 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001084 case ISD::CTTZ: return visitCTTZ(N);
Chandler Carruth63974b22011-12-13 01:56:10 +00001085 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001086 case ISD::CTPOP: return visitCTPOP(N);
Nate Begeman452d7beb2005-09-16 00:54:12 +00001087 case ISD::SELECT: return visitSELECT(N);
1088 case ISD::SELECT_CC: return visitSELECT_CC(N);
1089 case ISD::SETCC: return visitSETCC(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001090 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1091 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
Chris Lattner5ffc0662006-05-05 05:58:59 +00001092 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001093 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1094 case ISD::TRUNCATE: return visitTRUNCATE(N);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001095 case ISD::BITCAST: return visitBITCAST(N);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00001096 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
Chris Lattner01b3d732005-09-28 22:28:18 +00001097 case ISD::FADD: return visitFADD(N);
1098 case ISD::FSUB: return visitFSUB(N);
1099 case ISD::FMUL: return visitFMUL(N);
1100 case ISD::FDIV: return visitFDIV(N);
1101 case ISD::FREM: return visitFREM(N);
Chris Lattner12d83032006-03-05 05:30:57 +00001102 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
Nate Begeman646d7e22005-09-02 21:18:40 +00001103 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1104 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1105 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1106 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1107 case ISD::FP_ROUND: return visitFP_ROUND(N);
1108 case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
1109 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1110 case ISD::FNEG: return visitFNEG(N);
1111 case ISD::FABS: return visitFABS(N);
Nate Begeman44728a72005-09-19 22:34:01 +00001112 case ISD::BRCOND: return visitBRCOND(N);
Nate Begeman44728a72005-09-19 22:34:01 +00001113 case ISD::BR_CC: return visitBR_CC(N);
Chris Lattner01a22022005-10-10 22:04:48 +00001114 case ISD::LOAD: return visitLOAD(N);
Chris Lattner87514ca2005-10-10 22:31:19 +00001115 case ISD::STORE: return visitSTORE(N);
Chris Lattnerca242442006-03-19 01:27:56 +00001116 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
Evan Cheng513da432007-10-06 08:19:55 +00001117 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
Dan Gohman7f321562007-06-25 16:23:39 +00001118 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1119 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00001120 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
Chris Lattner66445d32006-03-28 22:11:53 +00001121 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
Jim Grosbach9a526492010-06-23 16:07:42 +00001122 case ISD::MEMBARRIER: return visitMEMBARRIER(N);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001123 }
Dan Gohman475871a2008-07-27 21:46:04 +00001124 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001125}
1126
Dan Gohman475871a2008-07-27 21:46:04 +00001127SDValue DAGCombiner::combine(SDNode *N) {
Dan Gohman475871a2008-07-27 21:46:04 +00001128 SDValue RV = visit(N);
Dan Gohman389079b2007-10-08 17:57:15 +00001129
1130 // If nothing happened, try a target-specific DAG combine.
Gabor Greifba36cb52008-08-28 21:40:38 +00001131 if (RV.getNode() == 0) {
Dan Gohman389079b2007-10-08 17:57:15 +00001132 assert(N->getOpcode() != ISD::DELETED_NODE &&
1133 "Node was deleted but visit returned NULL!");
1134
1135 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1136 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1137
1138 // Expose the DAG combiner to the target combiner impls.
Scott Michelfdc40a02009-02-17 22:15:04 +00001139 TargetLowering::DAGCombinerInfo
Jakob Stoklund Olesen78d12642009-07-24 18:22:59 +00001140 DagCombineInfo(DAG, !LegalTypes, !LegalOperations, false, this);
Dan Gohman389079b2007-10-08 17:57:15 +00001141
1142 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1143 }
1144 }
1145
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001146 // If nothing happened still, try promoting the operation.
1147 if (RV.getNode() == 0) {
1148 switch (N->getOpcode()) {
1149 default: break;
1150 case ISD::ADD:
1151 case ISD::SUB:
1152 case ISD::MUL:
1153 case ISD::AND:
1154 case ISD::OR:
1155 case ISD::XOR:
1156 RV = PromoteIntBinOp(SDValue(N, 0));
1157 break;
1158 case ISD::SHL:
1159 case ISD::SRA:
1160 case ISD::SRL:
1161 RV = PromoteIntShiftOp(SDValue(N, 0));
1162 break;
1163 case ISD::SIGN_EXTEND:
1164 case ISD::ZERO_EXTEND:
1165 case ISD::ANY_EXTEND:
1166 RV = PromoteExtend(SDValue(N, 0));
1167 break;
1168 case ISD::LOAD:
1169 if (PromoteLoad(SDValue(N, 0)))
1170 RV = SDValue(N, 0);
1171 break;
1172 }
1173 }
1174
Scott Michelfdc40a02009-02-17 22:15:04 +00001175 // If N is a commutative binary node, try commuting it to enable more
Evan Cheng08b11732008-03-22 01:55:50 +00001176 // sdisel CSE.
Scott Michelfdc40a02009-02-17 22:15:04 +00001177 if (RV.getNode() == 0 &&
Evan Cheng08b11732008-03-22 01:55:50 +00001178 SelectionDAG::isCommutativeBinOp(N->getOpcode()) &&
1179 N->getNumValues() == 1) {
Dan Gohman475871a2008-07-27 21:46:04 +00001180 SDValue N0 = N->getOperand(0);
1181 SDValue N1 = N->getOperand(1);
Bill Wendling5c71acf2009-01-30 01:13:16 +00001182
Evan Cheng08b11732008-03-22 01:55:50 +00001183 // Constant operands are canonicalized to RHS.
1184 if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
Dan Gohman475871a2008-07-27 21:46:04 +00001185 SDValue Ops[] = { N1, N0 };
Evan Cheng08b11732008-03-22 01:55:50 +00001186 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(),
1187 Ops, 2);
Evan Chengea100462008-03-24 23:55:16 +00001188 if (CSENode)
Dan Gohman475871a2008-07-27 21:46:04 +00001189 return SDValue(CSENode, 0);
Evan Cheng08b11732008-03-22 01:55:50 +00001190 }
1191 }
1192
Dan Gohman389079b2007-10-08 17:57:15 +00001193 return RV;
Scott Michelfdc40a02009-02-17 22:15:04 +00001194}
Dan Gohman389079b2007-10-08 17:57:15 +00001195
Chris Lattner6270f682006-10-08 22:57:01 +00001196/// getInputChainForNode - Given a node, return its input chain if it has one,
1197/// otherwise return a null sd operand.
Dan Gohman475871a2008-07-27 21:46:04 +00001198static SDValue getInputChainForNode(SDNode *N) {
Chris Lattner6270f682006-10-08 22:57:01 +00001199 if (unsigned NumOps = N->getNumOperands()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00001200 if (N->getOperand(0).getValueType() == MVT::Other)
Chris Lattner6270f682006-10-08 22:57:01 +00001201 return N->getOperand(0);
Owen Anderson825b72b2009-08-11 20:47:22 +00001202 else if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
Chris Lattner6270f682006-10-08 22:57:01 +00001203 return N->getOperand(NumOps-1);
1204 for (unsigned i = 1; i < NumOps-1; ++i)
Owen Anderson825b72b2009-08-11 20:47:22 +00001205 if (N->getOperand(i).getValueType() == MVT::Other)
Chris Lattner6270f682006-10-08 22:57:01 +00001206 return N->getOperand(i);
1207 }
Bill Wendling5c71acf2009-01-30 01:13:16 +00001208 return SDValue();
Chris Lattner6270f682006-10-08 22:57:01 +00001209}
1210
Dan Gohman475871a2008-07-27 21:46:04 +00001211SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
Chris Lattner6270f682006-10-08 22:57:01 +00001212 // If N has two operands, where one has an input chain equal to the other,
1213 // the 'other' chain is redundant.
1214 if (N->getNumOperands() == 2) {
Gabor Greifba36cb52008-08-28 21:40:38 +00001215 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
Chris Lattner6270f682006-10-08 22:57:01 +00001216 return N->getOperand(0);
Gabor Greifba36cb52008-08-28 21:40:38 +00001217 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
Chris Lattner6270f682006-10-08 22:57:01 +00001218 return N->getOperand(1);
1219 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001220
Chris Lattnerc76d4412007-05-16 06:37:59 +00001221 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
Dan Gohman475871a2008-07-27 21:46:04 +00001222 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
Scott Michelfdc40a02009-02-17 22:15:04 +00001223 SmallPtrSet<SDNode*, 16> SeenOps;
Chris Lattnerc76d4412007-05-16 06:37:59 +00001224 bool Changed = false; // If we should replace this token factor.
Scott Michelfdc40a02009-02-17 22:15:04 +00001225
Jim Laskey6ff23e52006-10-04 16:53:27 +00001226 // Start out with this token factor.
Jim Laskey279f0532006-09-25 16:29:54 +00001227 TFs.push_back(N);
Scott Michelfdc40a02009-02-17 22:15:04 +00001228
Jim Laskey71382342006-10-07 23:37:56 +00001229 // Iterate through token factors. The TFs grows when new token factors are
Jim Laskeybc588b82006-10-05 15:07:25 +00001230 // encountered.
1231 for (unsigned i = 0; i < TFs.size(); ++i) {
1232 SDNode *TF = TFs[i];
Scott Michelfdc40a02009-02-17 22:15:04 +00001233
Jim Laskey6ff23e52006-10-04 16:53:27 +00001234 // Check each of the operands.
1235 for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00001236 SDValue Op = TF->getOperand(i);
Scott Michelfdc40a02009-02-17 22:15:04 +00001237
Jim Laskey6ff23e52006-10-04 16:53:27 +00001238 switch (Op.getOpcode()) {
1239 case ISD::EntryToken:
Jim Laskeybc588b82006-10-05 15:07:25 +00001240 // Entry tokens don't need to be added to the list. They are
1241 // rededundant.
1242 Changed = true;
Jim Laskey6ff23e52006-10-04 16:53:27 +00001243 break;
Scott Michelfdc40a02009-02-17 22:15:04 +00001244
Jim Laskey6ff23e52006-10-04 16:53:27 +00001245 case ISD::TokenFactor:
Nate Begemanb6aef5c2009-09-15 00:18:30 +00001246 if (Op.hasOneUse() &&
Gabor Greifba36cb52008-08-28 21:40:38 +00001247 std::find(TFs.begin(), TFs.end(), Op.getNode()) == TFs.end()) {
Jim Laskey6ff23e52006-10-04 16:53:27 +00001248 // Queue up for processing.
Gabor Greifba36cb52008-08-28 21:40:38 +00001249 TFs.push_back(Op.getNode());
Jim Laskey6ff23e52006-10-04 16:53:27 +00001250 // Clean up in case the token factor is removed.
Gabor Greifba36cb52008-08-28 21:40:38 +00001251 AddToWorkList(Op.getNode());
Jim Laskey6ff23e52006-10-04 16:53:27 +00001252 Changed = true;
1253 break;
Jim Laskey279f0532006-09-25 16:29:54 +00001254 }
Jim Laskey6ff23e52006-10-04 16:53:27 +00001255 // Fall thru
Scott Michelfdc40a02009-02-17 22:15:04 +00001256
Jim Laskey6ff23e52006-10-04 16:53:27 +00001257 default:
Chris Lattnerc76d4412007-05-16 06:37:59 +00001258 // Only add if it isn't already in the list.
Gabor Greifba36cb52008-08-28 21:40:38 +00001259 if (SeenOps.insert(Op.getNode()))
Jim Laskeybc588b82006-10-05 15:07:25 +00001260 Ops.push_back(Op);
Chris Lattnerc76d4412007-05-16 06:37:59 +00001261 else
1262 Changed = true;
Jim Laskey6ff23e52006-10-04 16:53:27 +00001263 break;
Jim Laskey279f0532006-09-25 16:29:54 +00001264 }
1265 }
Jim Laskey6ff23e52006-10-04 16:53:27 +00001266 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001267
Dan Gohman475871a2008-07-27 21:46:04 +00001268 SDValue Result;
Jim Laskey6ff23e52006-10-04 16:53:27 +00001269
1270 // If we've change things around then replace token factor.
1271 if (Changed) {
Dan Gohman30359592008-01-29 13:02:09 +00001272 if (Ops.empty()) {
Jim Laskey6ff23e52006-10-04 16:53:27 +00001273 // The entry token is the only possible outcome.
1274 Result = DAG.getEntryNode();
1275 } else {
1276 // New and improved token factor.
Bill Wendling5c71acf2009-01-30 01:13:16 +00001277 Result = DAG.getNode(ISD::TokenFactor, N->getDebugLoc(),
Owen Anderson825b72b2009-08-11 20:47:22 +00001278 MVT::Other, &Ops[0], Ops.size());
Nate Begemanded49632005-10-13 03:11:28 +00001279 }
Bill Wendling5c71acf2009-01-30 01:13:16 +00001280
Jim Laskey274062c2006-10-13 23:32:28 +00001281 // Don't add users to work list.
1282 return CombineTo(N, Result, false);
Nate Begemanded49632005-10-13 03:11:28 +00001283 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001284
Jim Laskey6ff23e52006-10-04 16:53:27 +00001285 return Result;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001286}
1287
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001288/// MERGE_VALUES can always be eliminated.
Dan Gohman475871a2008-07-27 21:46:04 +00001289SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001290 WorkListRemover DeadNodes(*this);
Dan Gohman00edf392009-08-10 23:43:19 +00001291 // Replacing results may cause a different MERGE_VALUES to suddenly
1292 // be CSE'd with N, and carry its uses with it. Iterate until no
1293 // uses remain, to ensure that the node can be safely deleted.
1294 do {
1295 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1296 DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i),
1297 &DeadNodes);
1298 } while (!N->use_empty());
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001299 removeFromWorkList(N);
1300 DAG.DeleteNode(N);
Dan Gohman475871a2008-07-27 21:46:04 +00001301 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattnerfec42eb2008-02-13 07:25:05 +00001302}
1303
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001304static
Bill Wendlingd69c3142009-01-30 02:23:43 +00001305SDValue combineShlAddConstant(DebugLoc DL, SDValue N0, SDValue N1,
1306 SelectionDAG &DAG) {
Owen Andersone50ed302009-08-10 22:56:29 +00001307 EVT VT = N0.getValueType();
Dan Gohman475871a2008-07-27 21:46:04 +00001308 SDValue N00 = N0.getOperand(0);
1309 SDValue N01 = N0.getOperand(1);
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001310 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N01);
Bill Wendlingd69c3142009-01-30 02:23:43 +00001311
Gabor Greifba36cb52008-08-28 21:40:38 +00001312 if (N01C && N00.getOpcode() == ISD::ADD && N00.getNode()->hasOneUse() &&
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001313 isa<ConstantSDNode>(N00.getOperand(1))) {
Bill Wendlingd69c3142009-01-30 02:23:43 +00001314 // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<<c2), )
1315 N0 = DAG.getNode(ISD::ADD, N0.getDebugLoc(), VT,
1316 DAG.getNode(ISD::SHL, N00.getDebugLoc(), VT,
1317 N00.getOperand(0), N01),
1318 DAG.getNode(ISD::SHL, N01.getDebugLoc(), VT,
1319 N00.getOperand(1), N01));
1320 return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001321 }
Bill Wendlingd69c3142009-01-30 02:23:43 +00001322
Dan Gohman475871a2008-07-27 21:46:04 +00001323 return SDValue();
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001324}
1325
Dan Gohman475871a2008-07-27 21:46:04 +00001326SDValue DAGCombiner::visitADD(SDNode *N) {
1327 SDValue N0 = N->getOperand(0);
1328 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00001329 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1330 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001331 EVT VT = N0.getValueType();
Dan Gohman7f321562007-06-25 16:23:39 +00001332
1333 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001334 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001335 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001336 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001337 }
Bill Wendling2476e5d2008-12-10 22:36:00 +00001338
Dan Gohman613e0d82007-07-03 14:03:57 +00001339 // fold (add x, undef) -> undef
Dan Gohman70fb1ae2007-07-10 15:19:29 +00001340 if (N0.getOpcode() == ISD::UNDEF)
1341 return N0;
1342 if (N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00001343 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001344 // fold (add c1, c2) -> c1+c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001345 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001346 return DAG.FoldConstantArithmetic(ISD::ADD, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00001347 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00001348 if (N0C && !N1C)
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001349 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001350 // fold (add x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00001351 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00001352 return N0;
Dan Gohman6520e202008-10-18 02:06:02 +00001353 // fold (add Sym, c) -> Sym+c
1354 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
Duncan Sands25cf2272008-11-24 14:53:14 +00001355 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA) && N1C &&
Dan Gohman6520e202008-10-18 02:06:02 +00001356 GA->getOpcode() == ISD::GlobalAddress)
Devang Patel0d881da2010-07-06 22:08:15 +00001357 return DAG.getGlobalAddress(GA->getGlobal(), N1C->getDebugLoc(), VT,
Dan Gohman6520e202008-10-18 02:06:02 +00001358 GA->getOffset() +
1359 (uint64_t)N1C->getSExtValue());
Chris Lattner4aafb4f2006-01-12 20:22:43 +00001360 // fold ((c1-A)+c2) -> (c1+c2)-A
1361 if (N1C && N0.getOpcode() == ISD::SUB)
1362 if (ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getOperand(0)))
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001363 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
Dan Gohman002e5d02008-03-13 22:13:53 +00001364 DAG.getConstant(N1C->getAPIntValue()+
1365 N0C->getAPIntValue(), VT),
Chris Lattner4aafb4f2006-01-12 20:22:43 +00001366 N0.getOperand(1));
Nate Begemancd4d58c2006-02-03 06:46:56 +00001367 // reassociate add
Bill Wendling35247c32009-01-30 00:45:56 +00001368 SDValue RADD = ReassociateOps(ISD::ADD, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001369 if (RADD.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00001370 return RADD;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001371 // fold ((0-A) + B) -> B-A
1372 if (N0.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N0.getOperand(0)) &&
1373 cast<ConstantSDNode>(N0.getOperand(0))->isNullValue())
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001374 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1, N0.getOperand(1));
Nate Begeman1d4d4142005-09-01 00:19:25 +00001375 // fold (A + (0-B)) -> A-B
1376 if (N1.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N1.getOperand(0)) &&
1377 cast<ConstantSDNode>(N1.getOperand(0))->isNullValue())
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001378 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, N1.getOperand(1));
Chris Lattner01b3d732005-09-28 22:28:18 +00001379 // fold (A+(B-A)) -> B
1380 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
Nate Begeman83e75ec2005-09-06 04:43:02 +00001381 return N1.getOperand(0);
Dale Johannesen56eca912008-11-27 00:43:21 +00001382 // fold ((B-A)+A) -> B
1383 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
1384 return N0.getOperand(0);
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001385 // fold (A+(B-(A+C))) to (B-C)
1386 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001387 N0 == N1.getOperand(1).getOperand(0))
1388 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1.getOperand(0),
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001389 N1.getOperand(1).getOperand(1));
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001390 // fold (A+(B-(C+A))) to (B-C)
1391 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001392 N0 == N1.getOperand(1).getOperand(1))
1393 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1.getOperand(0),
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001394 N1.getOperand(1).getOperand(0));
Dale Johannesen7c7bc722008-12-23 23:47:22 +00001395 // fold (A+((B-A)+or-C)) to (B+or-C)
Dale Johannesen34d79852008-12-02 18:40:40 +00001396 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
1397 N1.getOperand(0).getOpcode() == ISD::SUB &&
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001398 N0 == N1.getOperand(0).getOperand(1))
1399 return DAG.getNode(N1.getOpcode(), N->getDebugLoc(), VT,
1400 N1.getOperand(0).getOperand(0), N1.getOperand(1));
Dale Johannesen34d79852008-12-02 18:40:40 +00001401
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001402 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
1403 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
1404 SDValue N00 = N0.getOperand(0);
1405 SDValue N01 = N0.getOperand(1);
1406 SDValue N10 = N1.getOperand(0);
1407 SDValue N11 = N1.getOperand(1);
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001408
1409 if (isa<ConstantSDNode>(N00) || isa<ConstantSDNode>(N10))
1410 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1411 DAG.getNode(ISD::ADD, N0.getDebugLoc(), VT, N00, N10),
1412 DAG.getNode(ISD::ADD, N1.getDebugLoc(), VT, N01, N11));
Dale Johannesen221cd2f2008-12-02 01:30:54 +00001413 }
Chris Lattner947c2892006-03-13 06:51:27 +00001414
Dan Gohman475871a2008-07-27 21:46:04 +00001415 if (!VT.isVector() && SimplifyDemandedBits(SDValue(N, 0)))
1416 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001417
Chris Lattner947c2892006-03-13 06:51:27 +00001418 // fold (a+b) -> (a|b) iff a and b share no bits.
Duncan Sands83ec4b62008-06-06 12:08:01 +00001419 if (VT.isInteger() && !VT.isVector()) {
Dan Gohman948d8ea2008-02-20 16:33:30 +00001420 APInt LHSZero, LHSOne;
1421 APInt RHSZero, RHSOne;
Dan Gohman5b870af2010-03-02 02:14:38 +00001422 APInt Mask = APInt::getAllOnesValue(VT.getScalarType().getSizeInBits());
Dan Gohmanea859be2007-06-22 14:59:07 +00001423 DAG.ComputeMaskedBits(N0, Mask, LHSZero, LHSOne);
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001424
Dan Gohman948d8ea2008-02-20 16:33:30 +00001425 if (LHSZero.getBoolValue()) {
Dan Gohmanea859be2007-06-22 14:59:07 +00001426 DAG.ComputeMaskedBits(N1, Mask, RHSZero, RHSOne);
Scott Michelfdc40a02009-02-17 22:15:04 +00001427
Chris Lattner947c2892006-03-13 06:51:27 +00001428 // If all possibly-set bits on the LHS are clear on the RHS, return an OR.
1429 // If all possibly-set bits on the RHS are clear on the LHS, return an OR.
1430 if ((RHSZero & (~LHSZero & Mask)) == (~LHSZero & Mask) ||
1431 (LHSZero & (~RHSZero & Mask)) == (~RHSZero & Mask))
Bill Wendlingf4eb2262009-01-30 02:31:17 +00001432 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N1);
Chris Lattner947c2892006-03-13 06:51:27 +00001433 }
1434 }
Evan Cheng3ef554d2006-11-06 08:14:30 +00001435
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001436 // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<<c2), )
Gabor Greifba36cb52008-08-28 21:40:38 +00001437 if (N0.getOpcode() == ISD::SHL && N0.getNode()->hasOneUse()) {
Bill Wendlingd69c3142009-01-30 02:23:43 +00001438 SDValue Result = combineShlAddConstant(N->getDebugLoc(), N0, N1, DAG);
Gabor Greifba36cb52008-08-28 21:40:38 +00001439 if (Result.getNode()) return Result;
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001440 }
Gabor Greifba36cb52008-08-28 21:40:38 +00001441 if (N1.getOpcode() == ISD::SHL && N1.getNode()->hasOneUse()) {
Bill Wendlingd69c3142009-01-30 02:23:43 +00001442 SDValue Result = combineShlAddConstant(N->getDebugLoc(), N1, N0, DAG);
Gabor Greifba36cb52008-08-28 21:40:38 +00001443 if (Result.getNode()) return Result;
Evan Cheng42d7ccf2007-01-19 17:51:44 +00001444 }
1445
Dan Gohmancd9e1552010-01-19 23:30:49 +00001446 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
1447 if (N1.getOpcode() == ISD::SHL &&
1448 N1.getOperand(0).getOpcode() == ISD::SUB)
1449 if (ConstantSDNode *C =
1450 dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(0)))
1451 if (C->getAPIntValue() == 0)
1452 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0,
1453 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1454 N1.getOperand(0).getOperand(1),
1455 N1.getOperand(1)));
1456 if (N0.getOpcode() == ISD::SHL &&
1457 N0.getOperand(0).getOpcode() == ISD::SUB)
1458 if (ConstantSDNode *C =
1459 dyn_cast<ConstantSDNode>(N0.getOperand(0).getOperand(0)))
1460 if (C->getAPIntValue() == 0)
1461 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1,
1462 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1463 N0.getOperand(0).getOperand(1),
1464 N0.getOperand(1)));
1465
Owen Andersonbc146b02010-09-21 20:42:50 +00001466 if (N1.getOpcode() == ISD::AND) {
1467 SDValue AndOp0 = N1.getOperand(0);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001468 ConstantSDNode *AndOp1 = dyn_cast<ConstantSDNode>(N1->getOperand(1));
Owen Andersonbc146b02010-09-21 20:42:50 +00001469 unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
1470 unsigned DestBits = VT.getScalarType().getSizeInBits();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00001471
Owen Andersonbc146b02010-09-21 20:42:50 +00001472 // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
1473 // and similar xforms where the inner op is either ~0 or 0.
1474 if (NumSignBits == DestBits && AndOp1 && AndOp1->isOne()) {
1475 DebugLoc DL = N->getDebugLoc();
1476 return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), AndOp0);
1477 }
1478 }
1479
Benjamin Kramerf50125e2010-12-22 23:17:45 +00001480 // add (sext i1), X -> sub X, (zext i1)
1481 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
1482 N0.getOperand(0).getValueType() == MVT::i1 &&
1483 !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
1484 DebugLoc DL = N->getDebugLoc();
1485 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
1486 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
1487 }
1488
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001489 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001490}
1491
Dan Gohman475871a2008-07-27 21:46:04 +00001492SDValue DAGCombiner::visitADDC(SDNode *N) {
1493 SDValue N0 = N->getOperand(0);
1494 SDValue N1 = N->getOperand(1);
Chris Lattner91153682007-03-04 20:03:15 +00001495 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1496 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001497 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00001498
Chris Lattner91153682007-03-04 20:03:15 +00001499 // If the flag result is dead, turn this into an ADD.
1500 if (N->hasNUsesOfValue(0, 1))
Bill Wendling14036c02009-01-30 02:38:00 +00001501 return CombineTo(N, DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N1, N0),
Dale Johannesen874ae252009-06-02 03:12:52 +00001502 DAG.getNode(ISD::CARRY_FALSE,
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00001503 N->getDebugLoc(), MVT::Glue));
Scott Michelfdc40a02009-02-17 22:15:04 +00001504
Chris Lattner91153682007-03-04 20:03:15 +00001505 // canonicalize constant to RHS.
Dan Gohman0a4627d2008-06-23 15:29:14 +00001506 if (N0C && !N1C)
Bill Wendling14036c02009-01-30 02:38:00 +00001507 return DAG.getNode(ISD::ADDC, N->getDebugLoc(), N->getVTList(), N1, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001508
Chris Lattnerb6541762007-03-04 20:40:38 +00001509 // fold (addc x, 0) -> x + no carry out
1510 if (N1C && N1C->isNullValue())
Dale Johannesen874ae252009-06-02 03:12:52 +00001511 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00001512 N->getDebugLoc(), MVT::Glue));
Scott Michelfdc40a02009-02-17 22:15:04 +00001513
Dale Johannesen874ae252009-06-02 03:12:52 +00001514 // fold (addc a, b) -> (or a, b), CARRY_FALSE iff a and b share no bits.
Dan Gohman948d8ea2008-02-20 16:33:30 +00001515 APInt LHSZero, LHSOne;
1516 APInt RHSZero, RHSOne;
Dan Gohman5b870af2010-03-02 02:14:38 +00001517 APInt Mask = APInt::getAllOnesValue(VT.getScalarType().getSizeInBits());
Dan Gohmanea859be2007-06-22 14:59:07 +00001518 DAG.ComputeMaskedBits(N0, Mask, LHSZero, LHSOne);
Bill Wendling14036c02009-01-30 02:38:00 +00001519
Dan Gohman948d8ea2008-02-20 16:33:30 +00001520 if (LHSZero.getBoolValue()) {
Dan Gohmanea859be2007-06-22 14:59:07 +00001521 DAG.ComputeMaskedBits(N1, Mask, RHSZero, RHSOne);
Scott Michelfdc40a02009-02-17 22:15:04 +00001522
Chris Lattnerb6541762007-03-04 20:40:38 +00001523 // If all possibly-set bits on the LHS are clear on the RHS, return an OR.
1524 // If all possibly-set bits on the RHS are clear on the LHS, return an OR.
1525 if ((RHSZero & (~LHSZero & Mask)) == (~LHSZero & Mask) ||
1526 (LHSZero & (~RHSZero & Mask)) == (~RHSZero & Mask))
Bill Wendling14036c02009-01-30 02:38:00 +00001527 return CombineTo(N, DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N1),
Dale Johannesen874ae252009-06-02 03:12:52 +00001528 DAG.getNode(ISD::CARRY_FALSE,
Chris Lattnerf1b4eaf2010-12-21 02:38:05 +00001529 N->getDebugLoc(), MVT::Glue));
Chris Lattnerb6541762007-03-04 20:40:38 +00001530 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001531
Dan Gohman475871a2008-07-27 21:46:04 +00001532 return SDValue();
Chris Lattner91153682007-03-04 20:03:15 +00001533}
1534
Dan Gohman475871a2008-07-27 21:46:04 +00001535SDValue DAGCombiner::visitADDE(SDNode *N) {
1536 SDValue N0 = N->getOperand(0);
1537 SDValue N1 = N->getOperand(1);
1538 SDValue CarryIn = N->getOperand(2);
Chris Lattner91153682007-03-04 20:03:15 +00001539 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1540 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00001541
Chris Lattner91153682007-03-04 20:03:15 +00001542 // canonicalize constant to RHS
Dan Gohman0a4627d2008-06-23 15:29:14 +00001543 if (N0C && !N1C)
Bill Wendling14036c02009-01-30 02:38:00 +00001544 return DAG.getNode(ISD::ADDE, N->getDebugLoc(), N->getVTList(),
1545 N1, N0, CarryIn);
Scott Michelfdc40a02009-02-17 22:15:04 +00001546
Chris Lattnerb6541762007-03-04 20:40:38 +00001547 // fold (adde x, y, false) -> (addc x, y)
Dale Johannesen874ae252009-06-02 03:12:52 +00001548 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
1549 return DAG.getNode(ISD::ADDC, N->getDebugLoc(), N->getVTList(), N1, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001550
Dan Gohman475871a2008-07-27 21:46:04 +00001551 return SDValue();
Chris Lattner91153682007-03-04 20:03:15 +00001552}
1553
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001554// Since it may not be valid to emit a fold to zero for vector initializers
1555// check if we can before folding.
1556static SDValue tryFoldToZero(DebugLoc DL, const TargetLowering &TLI, EVT VT,
Owen Anderson95771af2011-02-25 21:41:48 +00001557 SelectionDAG &DAG, bool LegalOperations) {
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001558 if (!VT.isVector()) {
1559 return DAG.getConstant(0, VT);
Dan Gohman71dc7c92011-05-17 22:20:36 +00001560 }
1561 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) {
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001562 // Produce a vector of zeros.
1563 SDValue El = DAG.getConstant(0, VT.getVectorElementType());
1564 std::vector<SDValue> Ops(VT.getVectorNumElements(), El);
1565 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT,
1566 &Ops[0], Ops.size());
1567 }
1568 return SDValue();
1569}
1570
Dan Gohman475871a2008-07-27 21:46:04 +00001571SDValue DAGCombiner::visitSUB(SDNode *N) {
1572 SDValue N0 = N->getOperand(0);
1573 SDValue N1 = N->getOperand(1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001574 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode());
1575 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
Eric Christopher7332e6e2011-07-14 01:12:15 +00001576 ConstantSDNode *N1C1 = N1.getOpcode() != ISD::ADD ? 0 :
1577 dyn_cast<ConstantSDNode>(N1.getOperand(1).getNode());
Owen Andersone50ed302009-08-10 22:56:29 +00001578 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00001579
Dan Gohman7f321562007-06-25 16:23:39 +00001580 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001581 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001582 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001583 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001584 }
Bill Wendling2476e5d2008-12-10 22:36:00 +00001585
Chris Lattner854077d2005-10-17 01:07:11 +00001586 // fold (sub x, x) -> 0
Eric Christopher169e1552011-02-16 01:10:03 +00001587 // FIXME: Refactor this and xor and other similar operations together.
Eric Christopher7bccf6a2011-02-16 04:50:12 +00001588 if (N0 == N1)
1589 return tryFoldToZero(N->getDebugLoc(), TLI, VT, DAG, LegalOperations);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001590 // fold (sub c1, c2) -> c1-c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001591 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001592 return DAG.FoldConstantArithmetic(ISD::SUB, VT, N0C, N1C);
Chris Lattner05b57432005-10-11 06:07:15 +00001593 // fold (sub x, c) -> (add x, -c)
1594 if (N1C)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001595 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00001596 DAG.getConstant(-N1C->getAPIntValue(), VT));
Evan Cheng1ad0e8b2010-01-18 21:38:44 +00001597 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
1598 if (N0C && N0C->isAllOnesValue())
1599 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N1, N0);
Benjamin Kramer2c94b422011-01-29 12:34:05 +00001600 // fold A-(A-B) -> B
1601 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
1602 return N1.getOperand(1);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001603 // fold (A+B)-A -> B
Chris Lattner01b3d732005-09-28 22:28:18 +00001604 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
Nate Begeman83e75ec2005-09-06 04:43:02 +00001605 return N0.getOperand(1);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001606 // fold (A+B)-B -> A
Chris Lattner01b3d732005-09-28 22:28:18 +00001607 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
Scott Michelfdc40a02009-02-17 22:15:04 +00001608 return N0.getOperand(0);
Eric Christopher7332e6e2011-07-14 01:12:15 +00001609 // fold C2-(A+C1) -> (C2-C1)-A
1610 if (N1.getOpcode() == ISD::ADD && N0C && N1C1) {
1611 SDValue NewC = DAG.getConstant((N0C->getAPIntValue() - N1C1->getAPIntValue()), VT);
1612 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, NewC,
1613 N1.getOperand(0));
1614 }
Dale Johannesen7c7bc722008-12-23 23:47:22 +00001615 // fold ((A+(B+or-C))-B) -> A+or-C
Dale Johannesenfd3b7b72008-12-16 22:13:49 +00001616 if (N0.getOpcode() == ISD::ADD &&
Dale Johannesenf9cbc1f2008-12-23 23:01:27 +00001617 (N0.getOperand(1).getOpcode() == ISD::SUB ||
1618 N0.getOperand(1).getOpcode() == ISD::ADD) &&
Dale Johannesenfd3b7b72008-12-16 22:13:49 +00001619 N0.getOperand(1).getOperand(0) == N1)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001620 return DAG.getNode(N0.getOperand(1).getOpcode(), N->getDebugLoc(), VT,
1621 N0.getOperand(0), N0.getOperand(1).getOperand(1));
Dale Johannesenf9cbc1f2008-12-23 23:01:27 +00001622 // fold ((A+(C+B))-B) -> A+C
1623 if (N0.getOpcode() == ISD::ADD &&
1624 N0.getOperand(1).getOpcode() == ISD::ADD &&
1625 N0.getOperand(1).getOperand(1) == N1)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001626 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT,
1627 N0.getOperand(0), N0.getOperand(1).getOperand(0));
Dale Johannesen58e39b02008-12-23 01:59:54 +00001628 // fold ((A-(B-C))-C) -> A-B
1629 if (N0.getOpcode() == ISD::SUB &&
1630 N0.getOperand(1).getOpcode() == ISD::SUB &&
1631 N0.getOperand(1).getOperand(1) == N1)
Bill Wendlingb0702e02009-01-30 02:42:10 +00001632 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1633 N0.getOperand(0), N0.getOperand(1).getOperand(0));
Bill Wendlingb0702e02009-01-30 02:42:10 +00001634
Dan Gohman613e0d82007-07-03 14:03:57 +00001635 // If either operand of a sub is undef, the result is undef
Dan Gohman70fb1ae2007-07-10 15:19:29 +00001636 if (N0.getOpcode() == ISD::UNDEF)
1637 return N0;
1638 if (N1.getOpcode() == ISD::UNDEF)
1639 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001640
Dan Gohman6520e202008-10-18 02:06:02 +00001641 // If the relocation model supports it, consider symbol offsets.
1642 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
Duncan Sands25cf2272008-11-24 14:53:14 +00001643 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
Dan Gohman6520e202008-10-18 02:06:02 +00001644 // fold (sub Sym, c) -> Sym-c
1645 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
Devang Patel0d881da2010-07-06 22:08:15 +00001646 return DAG.getGlobalAddress(GA->getGlobal(), N1C->getDebugLoc(), VT,
Dan Gohman6520e202008-10-18 02:06:02 +00001647 GA->getOffset() -
1648 (uint64_t)N1C->getSExtValue());
1649 // fold (sub Sym+c1, Sym+c2) -> c1-c2
1650 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
1651 if (GA->getGlobal() == GB->getGlobal())
1652 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
1653 VT);
1654 }
1655
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001656 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001657}
1658
Dan Gohman475871a2008-07-27 21:46:04 +00001659SDValue DAGCombiner::visitMUL(SDNode *N) {
1660 SDValue N0 = N->getOperand(0);
1661 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00001662 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1663 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001664 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00001665
Dan Gohman7f321562007-06-25 16:23:39 +00001666 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001667 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001668 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001669 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001670 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001671
Dan Gohman613e0d82007-07-03 14:03:57 +00001672 // fold (mul x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00001673 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00001674 return DAG.getConstant(0, VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001675 // fold (mul c1, c2) -> c1*c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001676 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001677 return DAG.FoldConstantArithmetic(ISD::MUL, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00001678 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00001679 if (N0C && !N1C)
Bill Wendling9c8148a2009-01-30 02:45:56 +00001680 return DAG.getNode(ISD::MUL, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001681 // fold (mul x, 0) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00001682 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00001683 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001684 // fold (mul x, -1) -> 0-x
Nate Begeman646d7e22005-09-02 21:18:40 +00001685 if (N1C && N1C->isAllOnesValue())
Bill Wendling9c8148a2009-01-30 02:45:56 +00001686 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1687 DAG.getConstant(0, VT), N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001688 // fold (mul x, (1 << c)) -> x << c
Dan Gohman002e5d02008-03-13 22:13:53 +00001689 if (N1C && N1C->getAPIntValue().isPowerOf2())
Bill Wendling9c8148a2009-01-30 02:45:56 +00001690 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00001691 DAG.getConstant(N1C->getAPIntValue().logBase2(),
Owen Anderson95771af2011-02-25 21:41:48 +00001692 getShiftAmountTy(N0.getValueType())));
Chris Lattner3e6099b2005-10-30 06:41:49 +00001693 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
Chris Lattner66b8bc32009-03-09 20:22:18 +00001694 if (N1C && (-N1C->getAPIntValue()).isPowerOf2()) {
1695 unsigned Log2Val = (-N1C->getAPIntValue()).logBase2();
Scott Michelfdc40a02009-02-17 22:15:04 +00001696 // FIXME: If the input is something that is easily negated (e.g. a
Chris Lattner3e6099b2005-10-30 06:41:49 +00001697 // single-use add), we should put the negate there.
Bill Wendling9c8148a2009-01-30 02:45:56 +00001698 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1699 DAG.getConstant(0, VT),
Bill Wendling73e16b22009-01-30 02:49:26 +00001700 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0,
Owen Anderson95771af2011-02-25 21:41:48 +00001701 DAG.getConstant(Log2Val,
1702 getShiftAmountTy(N0.getValueType()))));
Chris Lattner66b8bc32009-03-09 20:22:18 +00001703 }
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001704 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
Bill Wendling73e16b22009-01-30 02:49:26 +00001705 if (N1C && N0.getOpcode() == ISD::SHL &&
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001706 isa<ConstantSDNode>(N0.getOperand(1))) {
Bill Wendling9c8148a2009-01-30 02:45:56 +00001707 SDValue C3 = DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1708 N1, N0.getOperand(1));
Gabor Greifba36cb52008-08-28 21:40:38 +00001709 AddToWorkList(C3.getNode());
Bill Wendling9c8148a2009-01-30 02:45:56 +00001710 return DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
1711 N0.getOperand(0), C3);
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001712 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001713
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001714 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
1715 // use.
1716 {
Dan Gohman475871a2008-07-27 21:46:04 +00001717 SDValue Sh(0,0), Y(0,0);
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001718 // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
1719 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
Gabor Greifba36cb52008-08-28 21:40:38 +00001720 N0.getNode()->hasOneUse()) {
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001721 Sh = N0; Y = N1;
Scott Michelfdc40a02009-02-17 22:15:04 +00001722 } else if (N1.getOpcode() == ISD::SHL &&
Gabor Greif12632d22008-08-30 19:29:20 +00001723 isa<ConstantSDNode>(N1.getOperand(1)) &&
1724 N1.getNode()->hasOneUse()) {
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001725 Sh = N1; Y = N0;
1726 }
Bill Wendling73e16b22009-01-30 02:49:26 +00001727
Gabor Greifba36cb52008-08-28 21:40:38 +00001728 if (Sh.getNode()) {
Bill Wendling9c8148a2009-01-30 02:45:56 +00001729 SDValue Mul = DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
1730 Sh.getOperand(0), Y);
1731 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT,
1732 Mul, Sh.getOperand(1));
Chris Lattner0b1a85f2006-03-01 03:44:24 +00001733 }
1734 }
Bill Wendling73e16b22009-01-30 02:49:26 +00001735
Chris Lattnera1deca32006-03-04 23:33:26 +00001736 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
Scott Michelfdc40a02009-02-17 22:15:04 +00001737 if (N1C && N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse() &&
Bill Wendling9c8148a2009-01-30 02:45:56 +00001738 isa<ConstantSDNode>(N0.getOperand(1)))
1739 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT,
1740 DAG.getNode(ISD::MUL, N0.getDebugLoc(), VT,
1741 N0.getOperand(0), N1),
1742 DAG.getNode(ISD::MUL, N1.getDebugLoc(), VT,
1743 N0.getOperand(1), N1));
Scott Michelfdc40a02009-02-17 22:15:04 +00001744
Nate Begemancd4d58c2006-02-03 06:46:56 +00001745 // reassociate mul
Bill Wendling35247c32009-01-30 00:45:56 +00001746 SDValue RMUL = ReassociateOps(ISD::MUL, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001747 if (RMUL.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00001748 return RMUL;
Dan Gohman7f321562007-06-25 16:23:39 +00001749
Evan Chengb3a3d5e2010-04-28 07:10:39 +00001750 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001751}
1752
Dan Gohman475871a2008-07-27 21:46:04 +00001753SDValue DAGCombiner::visitSDIV(SDNode *N) {
1754 SDValue N0 = N->getOperand(0);
1755 SDValue N1 = N->getOperand(1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001756 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode());
1757 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
Owen Andersone50ed302009-08-10 22:56:29 +00001758 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001759
Dan Gohman7f321562007-06-25 16:23:39 +00001760 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001761 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001762 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001763 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001764 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001765
Nate Begeman1d4d4142005-09-01 00:19:25 +00001766 // fold (sdiv c1, c2) -> c1/c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001767 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001768 return DAG.FoldConstantArithmetic(ISD::SDIV, VT, N0C, N1C);
Nate Begeman405e3ec2005-10-21 00:02:42 +00001769 // fold (sdiv X, 1) -> X
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001770 if (N1C && N1C->getAPIntValue() == 1LL)
Nate Begeman405e3ec2005-10-21 00:02:42 +00001771 return N0;
1772 // fold (sdiv X, -1) -> 0-X
1773 if (N1C && N1C->isAllOnesValue())
Bill Wendling944d34b2009-01-30 02:52:17 +00001774 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1775 DAG.getConstant(0, VT), N0);
Chris Lattner094c8fc2005-10-07 06:10:46 +00001776 // If we know the sign bits of both operands are zero, strength reduce to a
1777 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
Duncan Sands83ec4b62008-06-06 12:08:01 +00001778 if (!VT.isVector()) {
Dan Gohman2e68b6f2008-02-25 21:11:39 +00001779 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
Bill Wendling944d34b2009-01-30 02:52:17 +00001780 return DAG.getNode(ISD::UDIV, N->getDebugLoc(), N1.getValueType(),
1781 N0, N1);
Chris Lattnerf32aac32008-01-27 23:32:17 +00001782 }
Nate Begemancd6a6ed2006-02-17 07:26:20 +00001783 // fold (sdiv X, pow2) -> simple ops after legalize
Eli Friedman1c663fe2011-12-07 03:55:52 +00001784 if (N1C && !N1C->isNullValue() &&
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001785 (N1C->getAPIntValue().isPowerOf2() ||
1786 (-N1C->getAPIntValue()).isPowerOf2())) {
Nate Begeman405e3ec2005-10-21 00:02:42 +00001787 // If dividing by powers of two is cheap, then don't perform the following
1788 // fold.
1789 if (TLI.isPow2DivCheap())
Dan Gohman475871a2008-07-27 21:46:04 +00001790 return SDValue();
Bill Wendling944d34b2009-01-30 02:52:17 +00001791
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001792 unsigned lg2 = N1C->getAPIntValue().countTrailingZeros();
Bill Wendling944d34b2009-01-30 02:52:17 +00001793
Chris Lattner8f4880b2006-02-16 08:02:36 +00001794 // Splat the sign bit into the register
Bill Wendling944d34b2009-01-30 02:52:17 +00001795 SDValue SGN = DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, N0,
1796 DAG.getConstant(VT.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00001797 getShiftAmountTy(N0.getValueType())));
Gabor Greifba36cb52008-08-28 21:40:38 +00001798 AddToWorkList(SGN.getNode());
Bill Wendling944d34b2009-01-30 02:52:17 +00001799
Chris Lattner8f4880b2006-02-16 08:02:36 +00001800 // Add (N0 < 0) ? abs2 - 1 : 0;
Bill Wendling944d34b2009-01-30 02:52:17 +00001801 SDValue SRL = DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, SGN,
1802 DAG.getConstant(VT.getSizeInBits() - lg2,
Owen Anderson95771af2011-02-25 21:41:48 +00001803 getShiftAmountTy(SGN.getValueType())));
Bill Wendling944d34b2009-01-30 02:52:17 +00001804 SDValue ADD = DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N0, SRL);
Gabor Greifba36cb52008-08-28 21:40:38 +00001805 AddToWorkList(SRL.getNode());
1806 AddToWorkList(ADD.getNode()); // Divide by pow2
Bill Wendling944d34b2009-01-30 02:52:17 +00001807 SDValue SRA = DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, ADD,
Owen Anderson95771af2011-02-25 21:41:48 +00001808 DAG.getConstant(lg2, getShiftAmountTy(ADD.getValueType())));
Bill Wendling944d34b2009-01-30 02:52:17 +00001809
Nate Begeman405e3ec2005-10-21 00:02:42 +00001810 // If we're dividing by a positive value, we're done. Otherwise, we must
1811 // negate the result.
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001812 if (N1C->getAPIntValue().isNonNegative())
Nate Begeman405e3ec2005-10-21 00:02:42 +00001813 return SRA;
Bill Wendling944d34b2009-01-30 02:52:17 +00001814
Gabor Greifba36cb52008-08-28 21:40:38 +00001815 AddToWorkList(SRA.getNode());
Bill Wendling944d34b2009-01-30 02:52:17 +00001816 return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT,
1817 DAG.getConstant(0, VT), SRA);
Nate Begeman405e3ec2005-10-21 00:02:42 +00001818 }
Bill Wendling944d34b2009-01-30 02:52:17 +00001819
Nate Begeman69575232005-10-20 02:15:44 +00001820 // if integer divide is expensive and we satisfy the requirements, emit an
1821 // alternate sequence.
Eli Friedmanfd58cd72011-10-27 02:06:39 +00001822 if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001823 SDValue Op = BuildSDIV(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001824 if (Op.getNode()) return Op;
Nate Begeman69575232005-10-20 02:15:44 +00001825 }
Dan Gohman7f321562007-06-25 16:23:39 +00001826
Dan Gohman613e0d82007-07-03 14:03:57 +00001827 // undef / X -> 0
1828 if (N0.getOpcode() == ISD::UNDEF)
1829 return DAG.getConstant(0, VT);
1830 // X / undef -> undef
1831 if (N1.getOpcode() == ISD::UNDEF)
1832 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001833
Dan Gohman475871a2008-07-27 21:46:04 +00001834 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001835}
1836
Dan Gohman475871a2008-07-27 21:46:04 +00001837SDValue DAGCombiner::visitUDIV(SDNode *N) {
1838 SDValue N0 = N->getOperand(0);
1839 SDValue N1 = N->getOperand(1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001840 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode());
1841 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
Owen Andersone50ed302009-08-10 22:56:29 +00001842 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001843
Dan Gohman7f321562007-06-25 16:23:39 +00001844 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00001845 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001846 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001847 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00001848 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001849
Nate Begeman1d4d4142005-09-01 00:19:25 +00001850 // fold (udiv c1, c2) -> c1/c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001851 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001852 return DAG.FoldConstantArithmetic(ISD::UDIV, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00001853 // fold (udiv x, (1 << c)) -> x >>u c
Dan Gohman002e5d02008-03-13 22:13:53 +00001854 if (N1C && N1C->getAPIntValue().isPowerOf2())
Scott Michelfdc40a02009-02-17 22:15:04 +00001855 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00001856 DAG.getConstant(N1C->getAPIntValue().logBase2(),
Owen Anderson95771af2011-02-25 21:41:48 +00001857 getShiftAmountTy(N0.getValueType())));
Nate Begemanfb5e4bd2006-02-05 07:20:23 +00001858 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
1859 if (N1.getOpcode() == ISD::SHL) {
1860 if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00001861 if (SHC->getAPIntValue().isPowerOf2()) {
Owen Andersone50ed302009-08-10 22:56:29 +00001862 EVT ADDVT = N1.getOperand(1).getValueType();
Bill Wendling07d85142009-01-30 02:55:25 +00001863 SDValue Add = DAG.getNode(ISD::ADD, N->getDebugLoc(), ADDVT,
1864 N1.getOperand(1),
1865 DAG.getConstant(SHC->getAPIntValue()
1866 .logBase2(),
1867 ADDVT));
Gabor Greifba36cb52008-08-28 21:40:38 +00001868 AddToWorkList(Add.getNode());
Bill Wendling07d85142009-01-30 02:55:25 +00001869 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0, Add);
Nate Begemanfb5e4bd2006-02-05 07:20:23 +00001870 }
1871 }
1872 }
Nate Begeman69575232005-10-20 02:15:44 +00001873 // fold (udiv x, c) -> alternate
Dan Gohman002e5d02008-03-13 22:13:53 +00001874 if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001875 SDValue Op = BuildUDIV(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00001876 if (Op.getNode()) return Op;
Chris Lattnere9936d12005-10-22 18:50:15 +00001877 }
Dan Gohman7f321562007-06-25 16:23:39 +00001878
Dan Gohman613e0d82007-07-03 14:03:57 +00001879 // undef / X -> 0
1880 if (N0.getOpcode() == ISD::UNDEF)
1881 return DAG.getConstant(0, VT);
1882 // X / undef -> undef
1883 if (N1.getOpcode() == ISD::UNDEF)
1884 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001885
Dan Gohman475871a2008-07-27 21:46:04 +00001886 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001887}
1888
Dan Gohman475871a2008-07-27 21:46:04 +00001889SDValue DAGCombiner::visitSREM(SDNode *N) {
1890 SDValue N0 = N->getOperand(0);
1891 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00001892 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1893 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001894 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001895
Nate Begeman1d4d4142005-09-01 00:19:25 +00001896 // fold (srem c1, c2) -> c1%c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001897 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001898 return DAG.FoldConstantArithmetic(ISD::SREM, VT, N0C, N1C);
Nate Begeman07ed4172005-10-10 21:26:48 +00001899 // If we know the sign bits of both operands are zero, strength reduce to a
1900 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
Duncan Sands83ec4b62008-06-06 12:08:01 +00001901 if (!VT.isVector()) {
Dan Gohman2e68b6f2008-02-25 21:11:39 +00001902 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001903 return DAG.getNode(ISD::UREM, N->getDebugLoc(), VT, N0, N1);
Chris Lattneree339f42008-01-27 23:21:58 +00001904 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001905
Dan Gohman77003042007-11-26 23:46:11 +00001906 // If X/C can be simplified by the division-by-constant logic, lower
1907 // X%C to the equivalent of X-X/C*C.
Chris Lattner26d29902006-10-12 20:58:32 +00001908 if (N1C && !N1C->isNullValue()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001909 SDValue Div = DAG.getNode(ISD::SDIV, N->getDebugLoc(), VT, N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00001910 AddToWorkList(Div.getNode());
1911 SDValue OptimizedDiv = combine(Div.getNode());
1912 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001913 SDValue Mul = DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
1914 OptimizedDiv, N1);
1915 SDValue Sub = DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, Mul);
Gabor Greifba36cb52008-08-28 21:40:38 +00001916 AddToWorkList(Mul.getNode());
Dan Gohman77003042007-11-26 23:46:11 +00001917 return Sub;
1918 }
Chris Lattner26d29902006-10-12 20:58:32 +00001919 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001920
Dan Gohman613e0d82007-07-03 14:03:57 +00001921 // undef % X -> 0
1922 if (N0.getOpcode() == ISD::UNDEF)
1923 return DAG.getConstant(0, VT);
1924 // X % undef -> undef
1925 if (N1.getOpcode() == ISD::UNDEF)
1926 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001927
Dan Gohman475871a2008-07-27 21:46:04 +00001928 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001929}
1930
Dan Gohman475871a2008-07-27 21:46:04 +00001931SDValue DAGCombiner::visitUREM(SDNode *N) {
1932 SDValue N0 = N->getOperand(0);
1933 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00001934 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1935 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001936 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00001937
Nate Begeman1d4d4142005-09-01 00:19:25 +00001938 // fold (urem c1, c2) -> c1%c2
Nate Begeman646d7e22005-09-02 21:18:40 +00001939 if (N0C && N1C && !N1C->isNullValue())
Bill Wendlingf3cbca22008-09-24 10:25:02 +00001940 return DAG.FoldConstantArithmetic(ISD::UREM, VT, N0C, N1C);
Nate Begeman07ed4172005-10-10 21:26:48 +00001941 // fold (urem x, pow2) -> (and x, pow2-1)
Dan Gohman002e5d02008-03-13 22:13:53 +00001942 if (N1C && !N1C->isNullValue() && N1C->getAPIntValue().isPowerOf2())
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001943 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0,
Dan Gohman002e5d02008-03-13 22:13:53 +00001944 DAG.getConstant(N1C->getAPIntValue()-1,VT));
Nate Begemanc031e332006-02-05 07:36:48 +00001945 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
1946 if (N1.getOpcode() == ISD::SHL) {
1947 if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00001948 if (SHC->getAPIntValue().isPowerOf2()) {
Dan Gohman475871a2008-07-27 21:46:04 +00001949 SDValue Add =
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001950 DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N1,
Duncan Sands83ec4b62008-06-06 12:08:01 +00001951 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()),
Dan Gohman002e5d02008-03-13 22:13:53 +00001952 VT));
Gabor Greifba36cb52008-08-28 21:40:38 +00001953 AddToWorkList(Add.getNode());
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001954 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0, Add);
Nate Begemanc031e332006-02-05 07:36:48 +00001955 }
1956 }
1957 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001958
Dan Gohman77003042007-11-26 23:46:11 +00001959 // If X/C can be simplified by the division-by-constant logic, lower
1960 // X%C to the equivalent of X-X/C*C.
Chris Lattner26d29902006-10-12 20:58:32 +00001961 if (N1C && !N1C->isNullValue()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001962 SDValue Div = DAG.getNode(ISD::UDIV, N->getDebugLoc(), VT, N0, N1);
Dan Gohman942ca7f2008-09-08 16:59:01 +00001963 AddToWorkList(Div.getNode());
Gabor Greifba36cb52008-08-28 21:40:38 +00001964 SDValue OptimizedDiv = combine(Div.getNode());
1965 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) {
Bill Wendling6d3bf8c2009-01-30 02:57:00 +00001966 SDValue Mul = DAG.getNode(ISD::MUL, N->getDebugLoc(), VT,
1967 OptimizedDiv, N1);
1968 SDValue Sub = DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, Mul);
Gabor Greifba36cb52008-08-28 21:40:38 +00001969 AddToWorkList(Mul.getNode());
Dan Gohman77003042007-11-26 23:46:11 +00001970 return Sub;
1971 }
Chris Lattner26d29902006-10-12 20:58:32 +00001972 }
Scott Michelfdc40a02009-02-17 22:15:04 +00001973
Dan Gohman613e0d82007-07-03 14:03:57 +00001974 // undef % X -> 0
1975 if (N0.getOpcode() == ISD::UNDEF)
1976 return DAG.getConstant(0, VT);
1977 // X % undef -> undef
1978 if (N1.getOpcode() == ISD::UNDEF)
1979 return N1;
Dan Gohman7f321562007-06-25 16:23:39 +00001980
Dan Gohman475871a2008-07-27 21:46:04 +00001981 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00001982}
1983
Dan Gohman475871a2008-07-27 21:46:04 +00001984SDValue DAGCombiner::visitMULHS(SDNode *N) {
1985 SDValue N0 = N->getOperand(0);
1986 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00001987 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00001988 EVT VT = N->getValueType(0);
Chris Lattnerde1c3602010-12-13 08:39:01 +00001989 DebugLoc DL = N->getDebugLoc();
Scott Michelfdc40a02009-02-17 22:15:04 +00001990
Nate Begeman1d4d4142005-09-01 00:19:25 +00001991 // fold (mulhs x, 0) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00001992 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00001993 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00001994 // fold (mulhs x, 1) -> (sra x, size(x)-1)
Dan Gohman002e5d02008-03-13 22:13:53 +00001995 if (N1C && N1C->getAPIntValue() == 1)
Bill Wendling326411d2009-01-30 03:00:18 +00001996 return DAG.getNode(ISD::SRA, N->getDebugLoc(), N0.getValueType(), N0,
1997 DAG.getConstant(N0.getValueType().getSizeInBits() - 1,
Owen Anderson95771af2011-02-25 21:41:48 +00001998 getShiftAmountTy(N0.getValueType())));
Dan Gohman613e0d82007-07-03 14:03:57 +00001999 // fold (mulhs x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00002000 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00002001 return DAG.getConstant(0, VT);
Dan Gohman7f321562007-06-25 16:23:39 +00002002
Chris Lattnerde1c3602010-12-13 08:39:01 +00002003 // If the type twice as wide is legal, transform the mulhs to a wider multiply
2004 // plus a shift.
2005 if (VT.isSimple() && !VT.isVector()) {
2006 MVT Simple = VT.getSimpleVT();
2007 unsigned SimpleSize = Simple.getSizeInBits();
2008 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2009 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2010 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
2011 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
2012 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
Chris Lattner1a0fbe22010-12-15 05:51:39 +00002013 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
Owen Anderson95771af2011-02-25 21:41:48 +00002014 DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType())));
Chris Lattnerde1c3602010-12-13 08:39:01 +00002015 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
2016 }
2017 }
Owen Anderson95771af2011-02-25 21:41:48 +00002018
Dan Gohman475871a2008-07-27 21:46:04 +00002019 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002020}
2021
Dan Gohman475871a2008-07-27 21:46:04 +00002022SDValue DAGCombiner::visitMULHU(SDNode *N) {
2023 SDValue N0 = N->getOperand(0);
2024 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00002025 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002026 EVT VT = N->getValueType(0);
Chris Lattnerde1c3602010-12-13 08:39:01 +00002027 DebugLoc DL = N->getDebugLoc();
Scott Michelfdc40a02009-02-17 22:15:04 +00002028
Nate Begeman1d4d4142005-09-01 00:19:25 +00002029 // fold (mulhu x, 0) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00002030 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002031 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00002032 // fold (mulhu x, 1) -> 0
Dan Gohman002e5d02008-03-13 22:13:53 +00002033 if (N1C && N1C->getAPIntValue() == 1)
Nate Begeman83e75ec2005-09-06 04:43:02 +00002034 return DAG.getConstant(0, N0.getValueType());
Dan Gohman613e0d82007-07-03 14:03:57 +00002035 // fold (mulhu x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00002036 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00002037 return DAG.getConstant(0, VT);
Dan Gohman7f321562007-06-25 16:23:39 +00002038
Chris Lattnerde1c3602010-12-13 08:39:01 +00002039 // If the type twice as wide is legal, transform the mulhu to a wider multiply
2040 // plus a shift.
2041 if (VT.isSimple() && !VT.isVector()) {
2042 MVT Simple = VT.getSimpleVT();
2043 unsigned SimpleSize = Simple.getSizeInBits();
2044 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2045 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2046 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
2047 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
2048 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
2049 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
Owen Anderson95771af2011-02-25 21:41:48 +00002050 DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType())));
Chris Lattnerde1c3602010-12-13 08:39:01 +00002051 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
2052 }
2053 }
Owen Anderson95771af2011-02-25 21:41:48 +00002054
Dan Gohman475871a2008-07-27 21:46:04 +00002055 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002056}
2057
Dan Gohman389079b2007-10-08 17:57:15 +00002058/// SimplifyNodeWithTwoResults - Perform optimizations common to nodes that
2059/// compute two values. LoOp and HiOp give the opcodes for the two computations
2060/// that are being performed. Return true if a simplification was made.
2061///
Scott Michelfdc40a02009-02-17 22:15:04 +00002062SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
Dan Gohman475871a2008-07-27 21:46:04 +00002063 unsigned HiOp) {
Dan Gohman389079b2007-10-08 17:57:15 +00002064 // If the high half is not needed, just compute the low half.
Evan Cheng44711942007-11-08 09:25:29 +00002065 bool HiExists = N->hasAnyUseOfValue(1);
2066 if (!HiExists &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002067 (!LegalOperations ||
Dan Gohman389079b2007-10-08 17:57:15 +00002068 TLI.isOperationLegal(LoOp, N->getValueType(0)))) {
Bill Wendling826d1142009-01-30 03:08:40 +00002069 SDValue Res = DAG.getNode(LoOp, N->getDebugLoc(), N->getValueType(0),
2070 N->op_begin(), N->getNumOperands());
Chris Lattner5eee4272008-01-26 01:09:19 +00002071 return CombineTo(N, Res, Res);
Dan Gohman389079b2007-10-08 17:57:15 +00002072 }
2073
2074 // If the low half is not needed, just compute the high half.
Evan Cheng44711942007-11-08 09:25:29 +00002075 bool LoExists = N->hasAnyUseOfValue(0);
2076 if (!LoExists &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002077 (!LegalOperations ||
Dan Gohman389079b2007-10-08 17:57:15 +00002078 TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
Bill Wendling826d1142009-01-30 03:08:40 +00002079 SDValue Res = DAG.getNode(HiOp, N->getDebugLoc(), N->getValueType(1),
2080 N->op_begin(), N->getNumOperands());
Chris Lattner5eee4272008-01-26 01:09:19 +00002081 return CombineTo(N, Res, Res);
Dan Gohman389079b2007-10-08 17:57:15 +00002082 }
2083
Evan Cheng44711942007-11-08 09:25:29 +00002084 // If both halves are used, return as it is.
2085 if (LoExists && HiExists)
Dan Gohman475871a2008-07-27 21:46:04 +00002086 return SDValue();
Evan Cheng44711942007-11-08 09:25:29 +00002087
2088 // If the two computed results can be simplified separately, separate them.
Evan Cheng44711942007-11-08 09:25:29 +00002089 if (LoExists) {
Bill Wendling826d1142009-01-30 03:08:40 +00002090 SDValue Lo = DAG.getNode(LoOp, N->getDebugLoc(), N->getValueType(0),
2091 N->op_begin(), N->getNumOperands());
Gabor Greifba36cb52008-08-28 21:40:38 +00002092 AddToWorkList(Lo.getNode());
2093 SDValue LoOpt = combine(Lo.getNode());
2094 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002095 (!LegalOperations ||
Duncan Sandsd4b9c172008-06-13 19:07:40 +00002096 TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
Chris Lattner5eee4272008-01-26 01:09:19 +00002097 return CombineTo(N, LoOpt, LoOpt);
Dan Gohman389079b2007-10-08 17:57:15 +00002098 }
2099
Evan Cheng44711942007-11-08 09:25:29 +00002100 if (HiExists) {
Bill Wendling826d1142009-01-30 03:08:40 +00002101 SDValue Hi = DAG.getNode(HiOp, N->getDebugLoc(), N->getValueType(1),
Duncan Sands25cf2272008-11-24 14:53:14 +00002102 N->op_begin(), N->getNumOperands());
Gabor Greifba36cb52008-08-28 21:40:38 +00002103 AddToWorkList(Hi.getNode());
2104 SDValue HiOpt = combine(Hi.getNode());
2105 if (HiOpt.getNode() && HiOpt != Hi &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002106 (!LegalOperations ||
Duncan Sandsd4b9c172008-06-13 19:07:40 +00002107 TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
Chris Lattner5eee4272008-01-26 01:09:19 +00002108 return CombineTo(N, HiOpt, HiOpt);
Evan Cheng44711942007-11-08 09:25:29 +00002109 }
Bill Wendling826d1142009-01-30 03:08:40 +00002110
Dan Gohman475871a2008-07-27 21:46:04 +00002111 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002112}
2113
Dan Gohman475871a2008-07-27 21:46:04 +00002114SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
2115 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS);
Gabor Greifba36cb52008-08-28 21:40:38 +00002116 if (Res.getNode()) return Res;
Dan Gohman389079b2007-10-08 17:57:15 +00002117
Chris Lattner33e77d32010-12-15 06:04:19 +00002118 EVT VT = N->getValueType(0);
2119 DebugLoc DL = N->getDebugLoc();
2120
2121 // If the type twice as wide is legal, transform the mulhu to a wider multiply
2122 // plus a shift.
2123 if (VT.isSimple() && !VT.isVector()) {
2124 MVT Simple = VT.getSimpleVT();
2125 unsigned SimpleSize = Simple.getSizeInBits();
2126 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2127 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2128 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
2129 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
2130 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
2131 // Compute the high part as N1.
2132 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
Owen Anderson95771af2011-02-25 21:41:48 +00002133 DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType())));
Chris Lattner33e77d32010-12-15 06:04:19 +00002134 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
2135 // Compute the low part as N0.
2136 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
2137 return CombineTo(N, Lo, Hi);
2138 }
2139 }
Owen Anderson95771af2011-02-25 21:41:48 +00002140
Dan Gohman475871a2008-07-27 21:46:04 +00002141 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002142}
2143
Dan Gohman475871a2008-07-27 21:46:04 +00002144SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
2145 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU);
Gabor Greifba36cb52008-08-28 21:40:38 +00002146 if (Res.getNode()) return Res;
Dan Gohman389079b2007-10-08 17:57:15 +00002147
Chris Lattner33e77d32010-12-15 06:04:19 +00002148 EVT VT = N->getValueType(0);
2149 DebugLoc DL = N->getDebugLoc();
Owen Anderson95771af2011-02-25 21:41:48 +00002150
Chris Lattner33e77d32010-12-15 06:04:19 +00002151 // If the type twice as wide is legal, transform the mulhu to a wider multiply
2152 // plus a shift.
2153 if (VT.isSimple() && !VT.isVector()) {
2154 MVT Simple = VT.getSimpleVT();
2155 unsigned SimpleSize = Simple.getSizeInBits();
2156 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2157 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2158 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
2159 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
2160 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
2161 // Compute the high part as N1.
2162 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
Owen Anderson95771af2011-02-25 21:41:48 +00002163 DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType())));
Chris Lattner33e77d32010-12-15 06:04:19 +00002164 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
2165 // Compute the low part as N0.
2166 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
2167 return CombineTo(N, Lo, Hi);
2168 }
2169 }
Owen Anderson95771af2011-02-25 21:41:48 +00002170
Dan Gohman475871a2008-07-27 21:46:04 +00002171 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002172}
2173
Benjamin Kramerf55d26e2011-05-21 18:31:55 +00002174SDValue DAGCombiner::visitSMULO(SDNode *N) {
2175 // (smulo x, 2) -> (saddo x, x)
2176 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2177 if (C2->getAPIntValue() == 2)
2178 return DAG.getNode(ISD::SADDO, N->getDebugLoc(), N->getVTList(),
2179 N->getOperand(0), N->getOperand(0));
2180
2181 return SDValue();
2182}
2183
2184SDValue DAGCombiner::visitUMULO(SDNode *N) {
2185 // (umulo x, 2) -> (uaddo x, x)
2186 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2187 if (C2->getAPIntValue() == 2)
2188 return DAG.getNode(ISD::UADDO, N->getDebugLoc(), N->getVTList(),
2189 N->getOperand(0), N->getOperand(0));
2190
2191 return SDValue();
2192}
2193
Dan Gohman475871a2008-07-27 21:46:04 +00002194SDValue DAGCombiner::visitSDIVREM(SDNode *N) {
2195 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::SDIV, ISD::SREM);
Gabor Greifba36cb52008-08-28 21:40:38 +00002196 if (Res.getNode()) return Res;
Scott Michelfdc40a02009-02-17 22:15:04 +00002197
Dan Gohman475871a2008-07-27 21:46:04 +00002198 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002199}
2200
Dan Gohman475871a2008-07-27 21:46:04 +00002201SDValue DAGCombiner::visitUDIVREM(SDNode *N) {
2202 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::UDIV, ISD::UREM);
Gabor Greifba36cb52008-08-28 21:40:38 +00002203 if (Res.getNode()) return Res;
Scott Michelfdc40a02009-02-17 22:15:04 +00002204
Dan Gohman475871a2008-07-27 21:46:04 +00002205 return SDValue();
Dan Gohman389079b2007-10-08 17:57:15 +00002206}
2207
Chris Lattner35e5c142006-05-05 05:51:50 +00002208/// SimplifyBinOpWithSameOpcodeHands - If this is a binary operator with
2209/// two operands of the same opcode, try to simplify it.
Dan Gohman475871a2008-07-27 21:46:04 +00002210SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
2211 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
Owen Andersone50ed302009-08-10 22:56:29 +00002212 EVT VT = N0.getValueType();
Chris Lattner35e5c142006-05-05 05:51:50 +00002213 assert(N0.getOpcode() == N1.getOpcode() && "Bad input!");
Scott Michelfdc40a02009-02-17 22:15:04 +00002214
Dan Gohmanff00a552010-01-14 03:08:49 +00002215 // Bail early if none of these transforms apply.
2216 if (N0.getNode()->getNumOperands() == 0) return SDValue();
2217
Chris Lattner540121f2006-05-05 06:31:05 +00002218 // For each of OP in AND/OR/XOR:
2219 // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
2220 // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
2221 // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
Dan Gohman4e39e9d2010-06-24 14:30:44 +00002222 // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
Nate Begeman93e0ed32009-12-03 07:11:29 +00002223 //
2224 // do not sink logical op inside of a vector extend, since it may combine
2225 // into a vsetcc.
Evan Chengd40d03e2010-01-06 19:38:29 +00002226 EVT Op0VT = N0.getOperand(0).getValueType();
2227 if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
Dan Gohman97121ba2009-04-08 00:15:30 +00002228 N0.getOpcode() == ISD::SIGN_EXTEND ||
Evan Chenge5b51ac2010-04-17 06:13:15 +00002229 // Avoid infinite looping with PromoteIntBinOp.
2230 (N0.getOpcode() == ISD::ANY_EXTEND &&
2231 (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
Dan Gohman4e39e9d2010-06-24 14:30:44 +00002232 (N0.getOpcode() == ISD::TRUNCATE &&
2233 (!TLI.isZExtFree(VT, Op0VT) ||
2234 !TLI.isTruncateFree(Op0VT, VT)) &&
2235 TLI.isTypeLegal(Op0VT))) &&
Nate Begeman93e0ed32009-12-03 07:11:29 +00002236 !VT.isVector() &&
Evan Chengd40d03e2010-01-06 19:38:29 +00002237 Op0VT == N1.getOperand(0).getValueType() &&
2238 (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
Bill Wendlingb74c8672009-01-30 19:25:47 +00002239 SDValue ORNode = DAG.getNode(N->getOpcode(), N0.getDebugLoc(),
2240 N0.getOperand(0).getValueType(),
2241 N0.getOperand(0), N1.getOperand(0));
Gabor Greifba36cb52008-08-28 21:40:38 +00002242 AddToWorkList(ORNode.getNode());
Bill Wendlingb74c8672009-01-30 19:25:47 +00002243 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT, ORNode);
Chris Lattner35e5c142006-05-05 05:51:50 +00002244 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002245
Chris Lattnera3dc3f62006-05-05 06:10:43 +00002246 // For each of OP in SHL/SRL/SRA/AND...
2247 // fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
2248 // fold (or (OP x, z), (OP y, z)) -> (OP (or x, y), z)
2249 // fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
Chris Lattner35e5c142006-05-05 05:51:50 +00002250 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
Chris Lattnera3dc3f62006-05-05 06:10:43 +00002251 N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
Chris Lattner35e5c142006-05-05 05:51:50 +00002252 N0.getOperand(1) == N1.getOperand(1)) {
Bill Wendlingb74c8672009-01-30 19:25:47 +00002253 SDValue ORNode = DAG.getNode(N->getOpcode(), N0.getDebugLoc(),
2254 N0.getOperand(0).getValueType(),
2255 N0.getOperand(0), N1.getOperand(0));
Gabor Greifba36cb52008-08-28 21:40:38 +00002256 AddToWorkList(ORNode.getNode());
Bill Wendlingb74c8672009-01-30 19:25:47 +00002257 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
2258 ORNode, N0.getOperand(1));
Chris Lattner35e5c142006-05-05 05:51:50 +00002259 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002260
Dan Gohman475871a2008-07-27 21:46:04 +00002261 return SDValue();
Chris Lattner35e5c142006-05-05 05:51:50 +00002262}
2263
Dan Gohman475871a2008-07-27 21:46:04 +00002264SDValue DAGCombiner::visitAND(SDNode *N) {
2265 SDValue N0 = N->getOperand(0);
2266 SDValue N1 = N->getOperand(1);
2267 SDValue LL, LR, RL, RR, CC0, CC1;
Nate Begeman646d7e22005-09-02 21:18:40 +00002268 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2269 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002270 EVT VT = N1.getValueType();
Dan Gohman6900a392010-03-04 00:23:16 +00002271 unsigned BitWidth = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00002272
Dan Gohman7f321562007-06-25 16:23:39 +00002273 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00002274 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00002275 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002276 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00002277 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002278
Dan Gohman613e0d82007-07-03 14:03:57 +00002279 // fold (and x, undef) -> 0
Dan Gohmand595b5f2007-07-10 14:20:37 +00002280 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00002281 return DAG.getConstant(0, VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00002282 // fold (and c1, c2) -> c1&c2
Nate Begeman646d7e22005-09-02 21:18:40 +00002283 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00002284 return DAG.FoldConstantArithmetic(ISD::AND, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00002285 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00002286 if (N0C && !N1C)
Bill Wendlingfc4b6772009-02-01 11:19:36 +00002287 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00002288 // fold (and x, -1) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00002289 if (N1C && N1C->isAllOnesValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002290 return N0;
2291 // if (and x, c) is known to be zero, return 0
Dan Gohman475871a2008-07-27 21:46:04 +00002292 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002293 APInt::getAllOnesValue(BitWidth)))
Nate Begeman83e75ec2005-09-06 04:43:02 +00002294 return DAG.getConstant(0, VT);
Nate Begemancd4d58c2006-02-03 06:46:56 +00002295 // reassociate and
Bill Wendling35247c32009-01-30 00:45:56 +00002296 SDValue RAND = ReassociateOps(ISD::AND, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00002297 if (RAND.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00002298 return RAND;
Bill Wendling7d9f2b92010-03-03 00:35:56 +00002299 // fold (and (or x, C), D) -> D if (C & D) == D
Nate Begeman5dc7e862005-11-02 18:42:59 +00002300 if (N1C && N0.getOpcode() == ISD::OR)
Nate Begeman1d4d4142005-09-01 00:19:25 +00002301 if (ConstantSDNode *ORI = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
Dan Gohman002e5d02008-03-13 22:13:53 +00002302 if ((ORI->getAPIntValue() & N1C->getAPIntValue()) == N1C->getAPIntValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002303 return N1;
Chris Lattner3603cd62006-02-02 07:17:31 +00002304 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
2305 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
Dan Gohman475871a2008-07-27 21:46:04 +00002306 SDValue N0Op0 = N0.getOperand(0);
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002307 APInt Mask = ~N1C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00002308 Mask = Mask.trunc(N0Op0.getValueSizeInBits());
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002309 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
Bill Wendling2627a882009-01-30 20:43:18 +00002310 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(),
2311 N0.getValueType(), N0Op0);
Scott Michelfdc40a02009-02-17 22:15:04 +00002312
Chris Lattner1ec05d12006-03-01 21:47:21 +00002313 // Replace uses of the AND with uses of the Zero extend node.
2314 CombineTo(N, Zext);
Scott Michelfdc40a02009-02-17 22:15:04 +00002315
Chris Lattner3603cd62006-02-02 07:17:31 +00002316 // We actually want to replace all uses of the any_extend with the
2317 // zero_extend, to avoid duplicating things. This will later cause this
2318 // AND to be folded.
Gabor Greifba36cb52008-08-28 21:40:38 +00002319 CombineTo(N0.getNode(), Zext);
Dan Gohman475871a2008-07-27 21:46:04 +00002320 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattner3603cd62006-02-02 07:17:31 +00002321 }
2322 }
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002323 // fold (and (setcc x), (setcc y)) -> (setcc (and x, y))
2324 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
2325 ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
2326 ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
Scott Michelfdc40a02009-02-17 22:15:04 +00002327
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002328 if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00002329 LL.getValueType().isInteger()) {
Bill Wendling2627a882009-01-30 20:43:18 +00002330 // fold (and (seteq X, 0), (seteq Y, 0)) -> (seteq (or X, Y), 0)
Dan Gohman002e5d02008-03-13 22:13:53 +00002331 if (cast<ConstantSDNode>(LR)->isNullValue() && Op1 == ISD::SETEQ) {
Bill Wendling2627a882009-01-30 20:43:18 +00002332 SDValue ORNode = DAG.getNode(ISD::OR, N0.getDebugLoc(),
2333 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00002334 AddToWorkList(ORNode.getNode());
Bill Wendling2627a882009-01-30 20:43:18 +00002335 return DAG.getSetCC(N->getDebugLoc(), VT, ORNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002336 }
Bill Wendling2627a882009-01-30 20:43:18 +00002337 // fold (and (seteq X, -1), (seteq Y, -1)) -> (seteq (and X, Y), -1)
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002338 if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETEQ) {
Bill Wendling2627a882009-01-30 20:43:18 +00002339 SDValue ANDNode = DAG.getNode(ISD::AND, N0.getDebugLoc(),
2340 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00002341 AddToWorkList(ANDNode.getNode());
Bill Wendling2627a882009-01-30 20:43:18 +00002342 return DAG.getSetCC(N->getDebugLoc(), VT, ANDNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002343 }
Bill Wendling2627a882009-01-30 20:43:18 +00002344 // fold (and (setgt X, -1), (setgt Y, -1)) -> (setgt (or X, Y), -1)
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002345 if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETGT) {
Bill Wendling2627a882009-01-30 20:43:18 +00002346 SDValue ORNode = DAG.getNode(ISD::OR, N0.getDebugLoc(),
2347 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00002348 AddToWorkList(ORNode.getNode());
Bill Wendling2627a882009-01-30 20:43:18 +00002349 return DAG.getSetCC(N->getDebugLoc(), VT, ORNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002350 }
2351 }
2352 // canonicalize equivalent to ll == rl
2353 if (LL == RR && LR == RL) {
2354 Op1 = ISD::getSetCCSwappedOperands(Op1);
2355 std::swap(RL, RR);
2356 }
2357 if (LL == RL && LR == RR) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00002358 bool isInteger = LL.getValueType().isInteger();
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002359 ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger);
Chris Lattner6e1c6232008-10-28 07:11:07 +00002360 if (Result != ISD::SETCC_INVALID &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002361 (!LegalOperations || TLI.isCondCodeLegal(Result, LL.getValueType())))
Bill Wendling2627a882009-01-30 20:43:18 +00002362 return DAG.getSetCC(N->getDebugLoc(), N0.getValueType(),
2363 LL, LR, Result);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002364 }
2365 }
Chris Lattner35e5c142006-05-05 05:51:50 +00002366
Bill Wendling2627a882009-01-30 20:43:18 +00002367 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
Chris Lattner35e5c142006-05-05 05:51:50 +00002368 if (N0.getOpcode() == N1.getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00002369 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002370 if (Tmp.getNode()) return Tmp;
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002371 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002372
Nate Begemande996292006-02-03 22:24:05 +00002373 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
2374 // fold (and (sra)) -> (and (srl)) when possible.
Duncan Sands83ec4b62008-06-06 12:08:01 +00002375 if (!VT.isVector() &&
Dan Gohman475871a2008-07-27 21:46:04 +00002376 SimplifyDemandedBits(SDValue(N, 0)))
2377 return SDValue(N, 0);
Evan Chengd40d03e2010-01-06 19:38:29 +00002378
Nate Begemanded49632005-10-13 03:11:28 +00002379 // fold (zext_inreg (extload x)) -> (zextload x)
Gabor Greifba36cb52008-08-28 21:40:38 +00002380 if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
Evan Cheng466685d2006-10-09 20:57:25 +00002381 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00002382 EVT MemVT = LN0->getMemoryVT();
Nate Begemanbfd65a02005-10-13 18:34:58 +00002383 // If we zero all the possible extended bits, then we can turn this into
2384 // a zextload if we are running before legalize or the operation is legal.
Dan Gohman6900a392010-03-04 00:23:16 +00002385 unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits();
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002386 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
Dan Gohman6900a392010-03-04 00:23:16 +00002387 BitWidth - MemVT.getScalarType().getSizeInBits())) &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002388 ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00002389 TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT))) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00002390 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N0.getDebugLoc(), VT,
Bill Wendling2627a882009-01-30 20:43:18 +00002391 LN0->getChain(), LN0->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002392 LN0->getPointerInfo(), MemVT,
David Greene1e559442010-02-15 17:00:31 +00002393 LN0->isVolatile(), LN0->isNonTemporal(),
2394 LN0->getAlignment());
Chris Lattner5750df92006-03-01 04:03:14 +00002395 AddToWorkList(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002396 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00002397 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00002398 }
2399 }
2400 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
Gabor Greifba36cb52008-08-28 21:40:38 +00002401 if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
Evan Cheng83060c52007-03-07 08:07:03 +00002402 N0.hasOneUse()) {
Evan Cheng466685d2006-10-09 20:57:25 +00002403 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00002404 EVT MemVT = LN0->getMemoryVT();
Nate Begemanbfd65a02005-10-13 18:34:58 +00002405 // If we zero all the possible extended bits, then we can turn this into
2406 // a zextload if we are running before legalize or the operation is legal.
Dan Gohman6900a392010-03-04 00:23:16 +00002407 unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits();
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002408 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
Dan Gohman6900a392010-03-04 00:23:16 +00002409 BitWidth - MemVT.getScalarType().getSizeInBits())) &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002410 ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00002411 TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT))) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00002412 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N0.getDebugLoc(), VT,
Bill Wendling2627a882009-01-30 20:43:18 +00002413 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002414 LN0->getBasePtr(), LN0->getPointerInfo(),
2415 MemVT,
David Greene1e559442010-02-15 17:00:31 +00002416 LN0->isVolatile(), LN0->isNonTemporal(),
2417 LN0->getAlignment());
Chris Lattner5750df92006-03-01 04:03:14 +00002418 AddToWorkList(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002419 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00002420 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00002421 }
2422 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002423
Chris Lattner35a9f5a2006-02-28 06:49:37 +00002424 // fold (and (load x), 255) -> (zextload x, i8)
2425 // fold (and (extload x, i16), 255) -> (zextload x, i8)
Evan Chengd40d03e2010-01-06 19:38:29 +00002426 // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
2427 if (N1C && (N0.getOpcode() == ISD::LOAD ||
2428 (N0.getOpcode() == ISD::ANY_EXTEND &&
2429 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
2430 bool HasAnyExt = N0.getOpcode() == ISD::ANY_EXTEND;
2431 LoadSDNode *LN0 = HasAnyExt
2432 ? cast<LoadSDNode>(N0.getOperand(0))
2433 : cast<LoadSDNode>(N0);
Evan Cheng466685d2006-10-09 20:57:25 +00002434 if (LN0->getExtensionType() != ISD::SEXTLOAD &&
Chris Lattnerbd1fccf2010-01-07 21:59:23 +00002435 LN0->isUnindexed() && N0.hasOneUse() && LN0->hasOneUse()) {
Duncan Sands8eab8a22008-06-09 11:32:28 +00002436 uint32_t ActiveBits = N1C->getAPIntValue().getActiveBits();
Evan Chengd40d03e2010-01-06 19:38:29 +00002437 if (ActiveBits > 0 && APIntOps::isMask(ActiveBits, N1C->getAPIntValue())){
2438 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
2439 EVT LoadedVT = LN0->getMemoryVT();
Duncan Sands8eab8a22008-06-09 11:32:28 +00002440
Evan Chengd40d03e2010-01-06 19:38:29 +00002441 if (ExtVT == LoadedVT &&
2442 (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT))) {
Chris Lattneref7634c2010-01-07 21:53:27 +00002443 EVT LoadResultTy = HasAnyExt ? LN0->getValueType(0) : VT;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002444
2445 SDValue NewLoad =
Stuart Hastingsa9011292011-02-16 16:23:55 +00002446 DAG.getExtLoad(ISD::ZEXTLOAD, LN0->getDebugLoc(), LoadResultTy,
Chris Lattneref7634c2010-01-07 21:53:27 +00002447 LN0->getChain(), LN0->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002448 LN0->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00002449 ExtVT, LN0->isVolatile(), LN0->isNonTemporal(),
2450 LN0->getAlignment());
Chris Lattneref7634c2010-01-07 21:53:27 +00002451 AddToWorkList(N);
2452 CombineTo(LN0, NewLoad, NewLoad.getValue(1));
2453 return SDValue(N, 0); // Return N so it doesn't get rechecked!
2454 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002455
Chris Lattneref7634c2010-01-07 21:53:27 +00002456 // Do not change the width of a volatile load.
2457 // Do not generate loads of non-round integer types since these can
2458 // be expensive (and would be wrong if the type is not byte sized).
2459 if (!LN0->isVolatile() && LoadedVT.bitsGT(ExtVT) && ExtVT.isRound() &&
2460 (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT))) {
2461 EVT PtrType = LN0->getOperand(1).getValueType();
Bill Wendling2627a882009-01-30 20:43:18 +00002462
Chris Lattneref7634c2010-01-07 21:53:27 +00002463 unsigned Alignment = LN0->getAlignment();
2464 SDValue NewPtr = LN0->getBasePtr();
2465
2466 // For big endian targets, we need to add an offset to the pointer
2467 // to load the correct bytes. For little endian systems, we merely
2468 // need to read fewer bytes from the same pointer.
2469 if (TLI.isBigEndian()) {
Evan Chengd40d03e2010-01-06 19:38:29 +00002470 unsigned LVTStoreBytes = LoadedVT.getStoreSize();
2471 unsigned EVTStoreBytes = ExtVT.getStoreSize();
2472 unsigned PtrOff = LVTStoreBytes - EVTStoreBytes;
Chris Lattneref7634c2010-01-07 21:53:27 +00002473 NewPtr = DAG.getNode(ISD::ADD, LN0->getDebugLoc(), PtrType,
2474 NewPtr, DAG.getConstant(PtrOff, PtrType));
2475 Alignment = MinAlign(Alignment, PtrOff);
Evan Chengd40d03e2010-01-06 19:38:29 +00002476 }
Chris Lattneref7634c2010-01-07 21:53:27 +00002477
2478 AddToWorkList(NewPtr.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00002479
Chris Lattneref7634c2010-01-07 21:53:27 +00002480 EVT LoadResultTy = HasAnyExt ? LN0->getValueType(0) : VT;
2481 SDValue Load =
Stuart Hastingsa9011292011-02-16 16:23:55 +00002482 DAG.getExtLoad(ISD::ZEXTLOAD, LN0->getDebugLoc(), LoadResultTy,
Chris Lattneref7634c2010-01-07 21:53:27 +00002483 LN0->getChain(), NewPtr,
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00002484 LN0->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00002485 ExtVT, LN0->isVolatile(), LN0->isNonTemporal(),
2486 Alignment);
Chris Lattneref7634c2010-01-07 21:53:27 +00002487 AddToWorkList(N);
2488 CombineTo(LN0, Load, Load.getValue(1));
2489 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Duncan Sandsdc846502007-10-28 12:59:45 +00002490 }
Evan Cheng466685d2006-10-09 20:57:25 +00002491 }
Chris Lattner15045b62006-02-28 06:35:35 +00002492 }
2493 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002494
Evan Chengb3a3d5e2010-04-28 07:10:39 +00002495 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002496}
2497
Evan Cheng9568e5c2011-06-21 06:01:08 +00002498/// MatchBSwapHWord - Match (a >> 8) | (a << 8) as (bswap a) >> 16
2499///
2500SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
2501 bool DemandHighBits) {
2502 if (!LegalOperations)
2503 return SDValue();
2504
2505 EVT VT = N->getValueType(0);
2506 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
2507 return SDValue();
2508 if (!TLI.isOperationLegal(ISD::BSWAP, VT))
2509 return SDValue();
2510
2511 // Recognize (and (shl a, 8), 0xff), (and (srl a, 8), 0xff00)
2512 bool LookPassAnd0 = false;
2513 bool LookPassAnd1 = false;
2514 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
2515 std::swap(N0, N1);
2516 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
2517 std::swap(N0, N1);
2518 if (N0.getOpcode() == ISD::AND) {
2519 if (!N0.getNode()->hasOneUse())
2520 return SDValue();
2521 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2522 if (!N01C || N01C->getZExtValue() != 0xFF00)
2523 return SDValue();
2524 N0 = N0.getOperand(0);
2525 LookPassAnd0 = true;
2526 }
2527
2528 if (N1.getOpcode() == ISD::AND) {
2529 if (!N1.getNode()->hasOneUse())
2530 return SDValue();
2531 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
2532 if (!N11C || N11C->getZExtValue() != 0xFF)
2533 return SDValue();
2534 N1 = N1.getOperand(0);
2535 LookPassAnd1 = true;
2536 }
2537
2538 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
2539 std::swap(N0, N1);
2540 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
2541 return SDValue();
2542 if (!N0.getNode()->hasOneUse() ||
2543 !N1.getNode()->hasOneUse())
2544 return SDValue();
2545
2546 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2547 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
2548 if (!N01C || !N11C)
2549 return SDValue();
2550 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
2551 return SDValue();
2552
2553 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
2554 SDValue N00 = N0->getOperand(0);
2555 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
2556 if (!N00.getNode()->hasOneUse())
2557 return SDValue();
2558 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
2559 if (!N001C || N001C->getZExtValue() != 0xFF)
2560 return SDValue();
2561 N00 = N00.getOperand(0);
2562 LookPassAnd0 = true;
2563 }
2564
2565 SDValue N10 = N1->getOperand(0);
2566 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
2567 if (!N10.getNode()->hasOneUse())
2568 return SDValue();
2569 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
2570 if (!N101C || N101C->getZExtValue() != 0xFF00)
2571 return SDValue();
2572 N10 = N10.getOperand(0);
2573 LookPassAnd1 = true;
2574 }
2575
2576 if (N00 != N10)
2577 return SDValue();
2578
2579 // Make sure everything beyond the low halfword is zero since the SRL 16
2580 // will clear the top bits.
2581 unsigned OpSizeInBits = VT.getSizeInBits();
2582 if (DemandHighBits && OpSizeInBits > 16 &&
2583 (!LookPassAnd0 || !LookPassAnd1) &&
2584 !DAG.MaskedValueIsZero(N10, APInt::getHighBitsSet(OpSizeInBits, 16)))
2585 return SDValue();
Eric Christopher7332e6e2011-07-14 01:12:15 +00002586
Evan Cheng9568e5c2011-06-21 06:01:08 +00002587 SDValue Res = DAG.getNode(ISD::BSWAP, N->getDebugLoc(), VT, N00);
2588 if (OpSizeInBits > 16)
2589 Res = DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, Res,
2590 DAG.getConstant(OpSizeInBits-16, getShiftAmountTy(VT)));
2591 return Res;
2592}
2593
2594/// isBSwapHWordElement - Return true if the specified node is an element
2595/// that makes up a 32-bit packed halfword byteswap. i.e.
2596/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8)
2597static bool isBSwapHWordElement(SDValue N, SmallVector<SDNode*,4> &Parts) {
2598 if (!N.getNode()->hasOneUse())
2599 return false;
2600
2601 unsigned Opc = N.getOpcode();
2602 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
2603 return false;
2604
2605 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2606 if (!N1C)
2607 return false;
2608
2609 unsigned Num;
2610 switch (N1C->getZExtValue()) {
2611 default:
2612 return false;
2613 case 0xFF: Num = 0; break;
2614 case 0xFF00: Num = 1; break;
2615 case 0xFF0000: Num = 2; break;
2616 case 0xFF000000: Num = 3; break;
2617 }
2618
2619 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
2620 SDValue N0 = N.getOperand(0);
2621 if (Opc == ISD::AND) {
2622 if (Num == 0 || Num == 2) {
2623 // (x >> 8) & 0xff
2624 // (x >> 8) & 0xff0000
2625 if (N0.getOpcode() != ISD::SRL)
2626 return false;
2627 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2628 if (!C || C->getZExtValue() != 8)
2629 return false;
2630 } else {
2631 // (x << 8) & 0xff00
2632 // (x << 8) & 0xff000000
2633 if (N0.getOpcode() != ISD::SHL)
2634 return false;
2635 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
2636 if (!C || C->getZExtValue() != 8)
2637 return false;
2638 }
2639 } else if (Opc == ISD::SHL) {
2640 // (x & 0xff) << 8
2641 // (x & 0xff0000) << 8
2642 if (Num != 0 && Num != 2)
2643 return false;
2644 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2645 if (!C || C->getZExtValue() != 8)
2646 return false;
2647 } else { // Opc == ISD::SRL
2648 // (x & 0xff00) >> 8
2649 // (x & 0xff000000) >> 8
2650 if (Num != 1 && Num != 3)
2651 return false;
2652 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2653 if (!C || C->getZExtValue() != 8)
2654 return false;
2655 }
2656
2657 if (Parts[Num])
2658 return false;
2659
2660 Parts[Num] = N0.getOperand(0).getNode();
2661 return true;
2662}
2663
2664/// MatchBSwapHWord - Match a 32-bit packed halfword bswap. That is
2665/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8)
2666/// => (rotl (bswap x), 16)
2667SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
2668 if (!LegalOperations)
2669 return SDValue();
2670
2671 EVT VT = N->getValueType(0);
2672 if (VT != MVT::i32)
2673 return SDValue();
2674 if (!TLI.isOperationLegal(ISD::BSWAP, VT))
2675 return SDValue();
2676
2677 SmallVector<SDNode*,4> Parts(4, (SDNode*)0);
2678 // Look for either
2679 // (or (or (and), (and)), (or (and), (and)))
2680 // (or (or (or (and), (and)), (and)), (and))
2681 if (N0.getOpcode() != ISD::OR)
2682 return SDValue();
2683 SDValue N00 = N0.getOperand(0);
2684 SDValue N01 = N0.getOperand(1);
2685
2686 if (N1.getOpcode() == ISD::OR) {
2687 // (or (or (and), (and)), (or (and), (and)))
2688 SDValue N000 = N00.getOperand(0);
2689 if (!isBSwapHWordElement(N000, Parts))
2690 return SDValue();
2691
2692 SDValue N001 = N00.getOperand(1);
2693 if (!isBSwapHWordElement(N001, Parts))
2694 return SDValue();
2695 SDValue N010 = N01.getOperand(0);
2696 if (!isBSwapHWordElement(N010, Parts))
2697 return SDValue();
2698 SDValue N011 = N01.getOperand(1);
2699 if (!isBSwapHWordElement(N011, Parts))
2700 return SDValue();
2701 } else {
2702 // (or (or (or (and), (and)), (and)), (and))
2703 if (!isBSwapHWordElement(N1, Parts))
2704 return SDValue();
2705 if (!isBSwapHWordElement(N01, Parts))
2706 return SDValue();
2707 if (N00.getOpcode() != ISD::OR)
2708 return SDValue();
2709 SDValue N000 = N00.getOperand(0);
2710 if (!isBSwapHWordElement(N000, Parts))
2711 return SDValue();
2712 SDValue N001 = N00.getOperand(1);
2713 if (!isBSwapHWordElement(N001, Parts))
2714 return SDValue();
2715 }
2716
2717 // Make sure the parts are all coming from the same node.
2718 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
2719 return SDValue();
2720
2721 SDValue BSwap = DAG.getNode(ISD::BSWAP, N->getDebugLoc(), VT,
2722 SDValue(Parts[0],0));
2723
2724 // Result of the bswap should be rotated by 16. If it's not legal, than
2725 // do (x << 16) | (x >> 16).
2726 SDValue ShAmt = DAG.getConstant(16, getShiftAmountTy(VT));
2727 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
2728 return DAG.getNode(ISD::ROTL, N->getDebugLoc(), VT, BSwap, ShAmt);
2729 else if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
2730 return DAG.getNode(ISD::ROTR, N->getDebugLoc(), VT, BSwap, ShAmt);
2731 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT,
2732 DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, BSwap, ShAmt),
2733 DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, BSwap, ShAmt));
2734}
2735
Dan Gohman475871a2008-07-27 21:46:04 +00002736SDValue DAGCombiner::visitOR(SDNode *N) {
2737 SDValue N0 = N->getOperand(0);
2738 SDValue N1 = N->getOperand(1);
2739 SDValue LL, LR, RL, RR, CC0, CC1;
Nate Begeman646d7e22005-09-02 21:18:40 +00002740 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2741 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00002742 EVT VT = N1.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00002743
Dan Gohman7f321562007-06-25 16:23:39 +00002744 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00002745 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00002746 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002747 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00002748 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002749
Dan Gohman613e0d82007-07-03 14:03:57 +00002750 // fold (or x, undef) -> -1
Bob Wilson86749492010-06-28 23:40:25 +00002751 if (!LegalOperations &&
2752 (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)) {
Nate Begeman93e0ed32009-12-03 07:11:29 +00002753 EVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT;
2754 return DAG.getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
2755 }
Nate Begeman1d4d4142005-09-01 00:19:25 +00002756 // fold (or c1, c2) -> c1|c2
Nate Begeman646d7e22005-09-02 21:18:40 +00002757 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00002758 return DAG.FoldConstantArithmetic(ISD::OR, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00002759 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00002760 if (N0C && !N1C)
Bill Wendling09025642009-01-30 20:59:34 +00002761 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00002762 // fold (or x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00002763 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002764 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00002765 // fold (or x, -1) -> -1
Nate Begeman646d7e22005-09-02 21:18:40 +00002766 if (N1C && N1C->isAllOnesValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00002767 return N1;
2768 // fold (or x, c) -> c iff (x & ~c) == 0
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002769 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
Nate Begeman83e75ec2005-09-06 04:43:02 +00002770 return N1;
Evan Cheng9568e5c2011-06-21 06:01:08 +00002771
2772 // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
2773 SDValue BSwap = MatchBSwapHWord(N, N0, N1);
2774 if (BSwap.getNode() != 0)
2775 return BSwap;
2776 BSwap = MatchBSwapHWordLow(N, N0, N1);
2777 if (BSwap.getNode() != 0)
2778 return BSwap;
2779
Nate Begemancd4d58c2006-02-03 06:46:56 +00002780 // reassociate or
Bill Wendling35247c32009-01-30 00:45:56 +00002781 SDValue ROR = ReassociateOps(ISD::OR, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00002782 if (ROR.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00002783 return ROR;
2784 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
Bill Wendling7d9f2b92010-03-03 00:35:56 +00002785 // iff (c1 & c2) == 0.
Gabor Greifba36cb52008-08-28 21:40:38 +00002786 if (N1C && N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
Chris Lattner731d3482005-10-27 05:06:38 +00002787 isa<ConstantSDNode>(N0.getOperand(1))) {
Chris Lattner731d3482005-10-27 05:06:38 +00002788 ConstantSDNode *C1 = cast<ConstantSDNode>(N0.getOperand(1));
Bill Wendling32f9eb22010-03-03 01:58:01 +00002789 if ((C1->getAPIntValue() & N1C->getAPIntValue()) != 0)
Bill Wendling7d9f2b92010-03-03 00:35:56 +00002790 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
2791 DAG.getNode(ISD::OR, N0.getDebugLoc(), VT,
2792 N0.getOperand(0), N1),
2793 DAG.FoldConstantArithmetic(ISD::OR, VT, N1C, C1));
Nate Begeman223df222005-09-08 20:18:10 +00002794 }
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002795 // fold (or (setcc x), (setcc y)) -> (setcc (or x, y))
2796 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
2797 ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
2798 ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
Scott Michelfdc40a02009-02-17 22:15:04 +00002799
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002800 if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00002801 LL.getValueType().isInteger()) {
Bill Wendling09025642009-01-30 20:59:34 +00002802 // fold (or (setne X, 0), (setne Y, 0)) -> (setne (or X, Y), 0)
2803 // fold (or (setlt X, 0), (setlt Y, 0)) -> (setne (or X, Y), 0)
Scott Michelfdc40a02009-02-17 22:15:04 +00002804 if (cast<ConstantSDNode>(LR)->isNullValue() &&
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002805 (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) {
Bill Wendling09025642009-01-30 20:59:34 +00002806 SDValue ORNode = DAG.getNode(ISD::OR, LR.getDebugLoc(),
2807 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00002808 AddToWorkList(ORNode.getNode());
Bill Wendling09025642009-01-30 20:59:34 +00002809 return DAG.getSetCC(N->getDebugLoc(), VT, ORNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002810 }
Bill Wendling09025642009-01-30 20:59:34 +00002811 // fold (or (setne X, -1), (setne Y, -1)) -> (setne (and X, Y), -1)
2812 // fold (or (setgt X, -1), (setgt Y -1)) -> (setgt (and X, Y), -1)
Scott Michelfdc40a02009-02-17 22:15:04 +00002813 if (cast<ConstantSDNode>(LR)->isAllOnesValue() &&
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002814 (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) {
Bill Wendling09025642009-01-30 20:59:34 +00002815 SDValue ANDNode = DAG.getNode(ISD::AND, LR.getDebugLoc(),
2816 LR.getValueType(), LL, RL);
Gabor Greifba36cb52008-08-28 21:40:38 +00002817 AddToWorkList(ANDNode.getNode());
Bill Wendling09025642009-01-30 20:59:34 +00002818 return DAG.getSetCC(N->getDebugLoc(), VT, ANDNode, LR, Op1);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002819 }
2820 }
2821 // canonicalize equivalent to ll == rl
2822 if (LL == RR && LR == RL) {
2823 Op1 = ISD::getSetCCSwappedOperands(Op1);
2824 std::swap(RL, RR);
2825 }
2826 if (LL == RL && LR == RR) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00002827 bool isInteger = LL.getValueType().isInteger();
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002828 ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger);
Chris Lattner6e1c6232008-10-28 07:11:07 +00002829 if (Result != ISD::SETCC_INVALID &&
Duncan Sands25cf2272008-11-24 14:53:14 +00002830 (!LegalOperations || TLI.isCondCodeLegal(Result, LL.getValueType())))
Bill Wendling09025642009-01-30 20:59:34 +00002831 return DAG.getSetCC(N->getDebugLoc(), N0.getValueType(),
2832 LL, LR, Result);
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002833 }
2834 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002835
Bill Wendling09025642009-01-30 20:59:34 +00002836 // Simplify: (or (op x...), (op y...)) -> (op (or x, y))
Chris Lattner35e5c142006-05-05 05:51:50 +00002837 if (N0.getOpcode() == N1.getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00002838 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00002839 if (Tmp.getNode()) return Tmp;
Nate Begeman39ee1ac2005-09-09 19:49:52 +00002840 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002841
Bill Wendling09025642009-01-30 20:59:34 +00002842 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible.
Chris Lattner1ec72732006-09-14 21:11:37 +00002843 if (N0.getOpcode() == ISD::AND &&
2844 N1.getOpcode() == ISD::AND &&
2845 N0.getOperand(1).getOpcode() == ISD::Constant &&
2846 N1.getOperand(1).getOpcode() == ISD::Constant &&
2847 // Don't increase # computations.
Gabor Greifba36cb52008-08-28 21:40:38 +00002848 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
Chris Lattner1ec72732006-09-14 21:11:37 +00002849 // We can only do this xform if we know that bits from X that are set in C2
2850 // but not in C1 are already zero. Likewise for Y.
Dan Gohman2e68b6f2008-02-25 21:11:39 +00002851 const APInt &LHSMask =
2852 cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
2853 const APInt &RHSMask =
2854 cast<ConstantSDNode>(N1.getOperand(1))->getAPIntValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00002855
Dan Gohmanea859be2007-06-22 14:59:07 +00002856 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
2857 DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
Bill Wendling09025642009-01-30 20:59:34 +00002858 SDValue X = DAG.getNode(ISD::OR, N0.getDebugLoc(), VT,
2859 N0.getOperand(0), N1.getOperand(0));
2860 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, X,
2861 DAG.getConstant(LHSMask | RHSMask, VT));
Chris Lattner1ec72732006-09-14 21:11:37 +00002862 }
2863 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002864
Chris Lattner516b9622006-09-14 20:50:57 +00002865 // See if this is some rotate idiom.
Bill Wendling317bd702009-01-30 21:14:50 +00002866 if (SDNode *Rot = MatchRotate(N0, N1, N->getDebugLoc()))
Dan Gohman475871a2008-07-27 21:46:04 +00002867 return SDValue(Rot, 0);
Chris Lattner35e5c142006-05-05 05:51:50 +00002868
Dan Gohman4e39e9d2010-06-24 14:30:44 +00002869 // Simplify the operands using demanded-bits information.
2870 if (!VT.isVector() &&
2871 SimplifyDemandedBits(SDValue(N, 0)))
2872 return SDValue(N, 0);
2873
Evan Chengb3a3d5e2010-04-28 07:10:39 +00002874 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00002875}
2876
Chris Lattner516b9622006-09-14 20:50:57 +00002877/// MatchRotateHalf - Match "(X shl/srl V1) & V2" where V2 may not be present.
Dan Gohman475871a2008-07-27 21:46:04 +00002878static bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) {
Chris Lattner516b9622006-09-14 20:50:57 +00002879 if (Op.getOpcode() == ISD::AND) {
Reid Spencer3ed469c2006-11-02 20:25:50 +00002880 if (isa<ConstantSDNode>(Op.getOperand(1))) {
Chris Lattner516b9622006-09-14 20:50:57 +00002881 Mask = Op.getOperand(1);
2882 Op = Op.getOperand(0);
2883 } else {
2884 return false;
2885 }
2886 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002887
Chris Lattner516b9622006-09-14 20:50:57 +00002888 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
2889 Shift = Op;
2890 return true;
2891 }
Bill Wendling09025642009-01-30 20:59:34 +00002892
Scott Michelfdc40a02009-02-17 22:15:04 +00002893 return false;
Chris Lattner516b9622006-09-14 20:50:57 +00002894}
2895
Chris Lattner516b9622006-09-14 20:50:57 +00002896// MatchRotate - Handle an 'or' of two operands. If this is one of the many
2897// idioms for rotate, and if the target supports rotation instructions, generate
2898// a rot[lr].
Bill Wendling317bd702009-01-30 21:14:50 +00002899SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, DebugLoc DL) {
Duncan Sandsd4b9c172008-06-13 19:07:40 +00002900 // Must be a legal type. Expanded 'n promoted things won't work with rotates.
Owen Andersone50ed302009-08-10 22:56:29 +00002901 EVT VT = LHS.getValueType();
Chris Lattner516b9622006-09-14 20:50:57 +00002902 if (!TLI.isTypeLegal(VT)) return 0;
2903
2904 // The target must have at least one rotate flavor.
Dan Gohmanf560ffa2009-01-28 17:46:25 +00002905 bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT);
2906 bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT);
Chris Lattner516b9622006-09-14 20:50:57 +00002907 if (!HasROTL && !HasROTR) return 0;
Duncan Sandsd4b9c172008-06-13 19:07:40 +00002908
Chris Lattner516b9622006-09-14 20:50:57 +00002909 // Match "(X shl/srl V1) & V2" where V2 may not be present.
Dan Gohman475871a2008-07-27 21:46:04 +00002910 SDValue LHSShift; // The shift.
2911 SDValue LHSMask; // AND value if any.
Chris Lattner516b9622006-09-14 20:50:57 +00002912 if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
2913 return 0; // Not part of a rotate.
2914
Dan Gohman475871a2008-07-27 21:46:04 +00002915 SDValue RHSShift; // The shift.
2916 SDValue RHSMask; // AND value if any.
Chris Lattner516b9622006-09-14 20:50:57 +00002917 if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
2918 return 0; // Not part of a rotate.
Scott Michelfdc40a02009-02-17 22:15:04 +00002919
Chris Lattner516b9622006-09-14 20:50:57 +00002920 if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
2921 return 0; // Not shifting the same value.
2922
2923 if (LHSShift.getOpcode() == RHSShift.getOpcode())
2924 return 0; // Shifts must disagree.
Scott Michelfdc40a02009-02-17 22:15:04 +00002925
Chris Lattner516b9622006-09-14 20:50:57 +00002926 // Canonicalize shl to left side in a shl/srl pair.
2927 if (RHSShift.getOpcode() == ISD::SHL) {
2928 std::swap(LHS, RHS);
2929 std::swap(LHSShift, RHSShift);
2930 std::swap(LHSMask , RHSMask );
2931 }
2932
Duncan Sands83ec4b62008-06-06 12:08:01 +00002933 unsigned OpSizeInBits = VT.getSizeInBits();
Dan Gohman475871a2008-07-27 21:46:04 +00002934 SDValue LHSShiftArg = LHSShift.getOperand(0);
2935 SDValue LHSShiftAmt = LHSShift.getOperand(1);
2936 SDValue RHSShiftAmt = RHSShift.getOperand(1);
Chris Lattner516b9622006-09-14 20:50:57 +00002937
2938 // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
2939 // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
Scott Michelc9dc1142007-04-02 21:36:32 +00002940 if (LHSShiftAmt.getOpcode() == ISD::Constant &&
2941 RHSShiftAmt.getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00002942 uint64_t LShVal = cast<ConstantSDNode>(LHSShiftAmt)->getZExtValue();
2943 uint64_t RShVal = cast<ConstantSDNode>(RHSShiftAmt)->getZExtValue();
Chris Lattner516b9622006-09-14 20:50:57 +00002944 if ((LShVal + RShVal) != OpSizeInBits)
2945 return 0;
2946
Dan Gohman475871a2008-07-27 21:46:04 +00002947 SDValue Rot;
Chris Lattner516b9622006-09-14 20:50:57 +00002948 if (HasROTL)
Bill Wendling317bd702009-01-30 21:14:50 +00002949 Rot = DAG.getNode(ISD::ROTL, DL, VT, LHSShiftArg, LHSShiftAmt);
Chris Lattner516b9622006-09-14 20:50:57 +00002950 else
Bill Wendling317bd702009-01-30 21:14:50 +00002951 Rot = DAG.getNode(ISD::ROTR, DL, VT, LHSShiftArg, RHSShiftAmt);
Scott Michelfdc40a02009-02-17 22:15:04 +00002952
Chris Lattner516b9622006-09-14 20:50:57 +00002953 // If there is an AND of either shifted operand, apply it to the result.
Gabor Greifba36cb52008-08-28 21:40:38 +00002954 if (LHSMask.getNode() || RHSMask.getNode()) {
Dan Gohman220a8232008-03-03 23:51:38 +00002955 APInt Mask = APInt::getAllOnesValue(OpSizeInBits);
Scott Michelfdc40a02009-02-17 22:15:04 +00002956
Gabor Greifba36cb52008-08-28 21:40:38 +00002957 if (LHSMask.getNode()) {
Dan Gohman220a8232008-03-03 23:51:38 +00002958 APInt RHSBits = APInt::getLowBitsSet(OpSizeInBits, LShVal);
2959 Mask &= cast<ConstantSDNode>(LHSMask)->getAPIntValue() | RHSBits;
Chris Lattner516b9622006-09-14 20:50:57 +00002960 }
Gabor Greifba36cb52008-08-28 21:40:38 +00002961 if (RHSMask.getNode()) {
Dan Gohman220a8232008-03-03 23:51:38 +00002962 APInt LHSBits = APInt::getHighBitsSet(OpSizeInBits, RShVal);
2963 Mask &= cast<ConstantSDNode>(RHSMask)->getAPIntValue() | LHSBits;
Chris Lattner516b9622006-09-14 20:50:57 +00002964 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002965
Bill Wendling317bd702009-01-30 21:14:50 +00002966 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, DAG.getConstant(Mask, VT));
Chris Lattner516b9622006-09-14 20:50:57 +00002967 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002968
Gabor Greifba36cb52008-08-28 21:40:38 +00002969 return Rot.getNode();
Chris Lattner516b9622006-09-14 20:50:57 +00002970 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002971
Chris Lattner516b9622006-09-14 20:50:57 +00002972 // If there is a mask here, and we have a variable shift, we can't be sure
2973 // that we're masking out the right stuff.
Gabor Greifba36cb52008-08-28 21:40:38 +00002974 if (LHSMask.getNode() || RHSMask.getNode())
Chris Lattner516b9622006-09-14 20:50:57 +00002975 return 0;
Scott Michelfdc40a02009-02-17 22:15:04 +00002976
Chris Lattner516b9622006-09-14 20:50:57 +00002977 // fold (or (shl x, y), (srl x, (sub 32, y))) -> (rotl x, y)
2978 // fold (or (shl x, y), (srl x, (sub 32, y))) -> (rotr x, (sub 32, y))
Scott Michelc9dc1142007-04-02 21:36:32 +00002979 if (RHSShiftAmt.getOpcode() == ISD::SUB &&
2980 LHSShiftAmt == RHSShiftAmt.getOperand(1)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00002981 if (ConstantSDNode *SUBC =
Scott Michelc9dc1142007-04-02 21:36:32 +00002982 dyn_cast<ConstantSDNode>(RHSShiftAmt.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00002983 if (SUBC->getAPIntValue() == OpSizeInBits) {
Chris Lattner516b9622006-09-14 20:50:57 +00002984 if (HasROTL)
Bill Wendling317bd702009-01-30 21:14:50 +00002985 return DAG.getNode(ISD::ROTL, DL, VT,
2986 LHSShiftArg, LHSShiftAmt).getNode();
Chris Lattner516b9622006-09-14 20:50:57 +00002987 else
Bill Wendling317bd702009-01-30 21:14:50 +00002988 return DAG.getNode(ISD::ROTR, DL, VT,
2989 LHSShiftArg, RHSShiftAmt).getNode();
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00002990 }
Chris Lattner516b9622006-09-14 20:50:57 +00002991 }
2992 }
Scott Michelfdc40a02009-02-17 22:15:04 +00002993
Chris Lattner516b9622006-09-14 20:50:57 +00002994 // fold (or (shl x, (sub 32, y)), (srl x, r)) -> (rotr x, y)
2995 // fold (or (shl x, (sub 32, y)), (srl x, r)) -> (rotl x, (sub 32, y))
Scott Michelc9dc1142007-04-02 21:36:32 +00002996 if (LHSShiftAmt.getOpcode() == ISD::SUB &&
2997 RHSShiftAmt == LHSShiftAmt.getOperand(1)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00002998 if (ConstantSDNode *SUBC =
Scott Michelc9dc1142007-04-02 21:36:32 +00002999 dyn_cast<ConstantSDNode>(LHSShiftAmt.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003000 if (SUBC->getAPIntValue() == OpSizeInBits) {
Bill Wendling2692d592008-08-31 01:13:31 +00003001 if (HasROTR)
Bill Wendling317bd702009-01-30 21:14:50 +00003002 return DAG.getNode(ISD::ROTR, DL, VT,
3003 LHSShiftArg, RHSShiftAmt).getNode();
Bill Wendling2692d592008-08-31 01:13:31 +00003004 else
Bill Wendling317bd702009-01-30 21:14:50 +00003005 return DAG.getNode(ISD::ROTL, DL, VT,
3006 LHSShiftArg, LHSShiftAmt).getNode();
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00003007 }
Scott Michelc9dc1142007-04-02 21:36:32 +00003008 }
3009 }
3010
Dan Gohman74feef22008-10-17 01:23:35 +00003011 // Look for sign/zext/any-extended or truncate cases:
Scott Michelc9dc1142007-04-02 21:36:32 +00003012 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND
3013 || LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND
Dan Gohman74feef22008-10-17 01:23:35 +00003014 || LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND
3015 || LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
Scott Michelc9dc1142007-04-02 21:36:32 +00003016 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND
3017 || RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND
Dan Gohman74feef22008-10-17 01:23:35 +00003018 || RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND
3019 || RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
Dan Gohman475871a2008-07-27 21:46:04 +00003020 SDValue LExtOp0 = LHSShiftAmt.getOperand(0);
3021 SDValue RExtOp0 = RHSShiftAmt.getOperand(0);
Scott Michelc9dc1142007-04-02 21:36:32 +00003022 if (RExtOp0.getOpcode() == ISD::SUB &&
3023 RExtOp0.getOperand(1) == LExtOp0) {
3024 // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003025 // (rotl x, y)
Scott Michelc9dc1142007-04-02 21:36:32 +00003026 // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003027 // (rotr x, (sub 32, y))
Dan Gohman74feef22008-10-17 01:23:35 +00003028 if (ConstantSDNode *SUBC =
3029 dyn_cast<ConstantSDNode>(RExtOp0.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003030 if (SUBC->getAPIntValue() == OpSizeInBits) {
Bill Wendling317bd702009-01-30 21:14:50 +00003031 return DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
3032 LHSShiftArg,
Gabor Greif12632d22008-08-30 19:29:20 +00003033 HasROTL ? LHSShiftAmt : RHSShiftAmt).getNode();
Scott Michelc9dc1142007-04-02 21:36:32 +00003034 }
3035 }
3036 } else if (LExtOp0.getOpcode() == ISD::SUB &&
3037 RExtOp0 == LExtOp0.getOperand(1)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00003038 // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext y))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003039 // (rotr x, y)
Bill Wendling353dea22008-08-31 01:04:56 +00003040 // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext y))) ->
Bill Wendlingc5cbda12008-08-31 00:37:27 +00003041 // (rotl x, (sub 32, y))
Dan Gohman74feef22008-10-17 01:23:35 +00003042 if (ConstantSDNode *SUBC =
3043 dyn_cast<ConstantSDNode>(LExtOp0.getOperand(0))) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003044 if (SUBC->getAPIntValue() == OpSizeInBits) {
Bill Wendling317bd702009-01-30 21:14:50 +00003045 return DAG.getNode(HasROTR ? ISD::ROTR : ISD::ROTL, DL, VT,
3046 LHSShiftArg,
Bill Wendling353dea22008-08-31 01:04:56 +00003047 HasROTR ? RHSShiftAmt : LHSShiftAmt).getNode();
Scott Michelc9dc1142007-04-02 21:36:32 +00003048 }
3049 }
Chris Lattner516b9622006-09-14 20:50:57 +00003050 }
3051 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003052
Chris Lattner516b9622006-09-14 20:50:57 +00003053 return 0;
3054}
3055
Dan Gohman475871a2008-07-27 21:46:04 +00003056SDValue DAGCombiner::visitXOR(SDNode *N) {
3057 SDValue N0 = N->getOperand(0);
3058 SDValue N1 = N->getOperand(1);
3059 SDValue LHS, RHS, CC;
Nate Begeman646d7e22005-09-02 21:18:40 +00003060 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3061 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003062 EVT VT = N0.getValueType();
Scott Michelfdc40a02009-02-17 22:15:04 +00003063
Dan Gohman7f321562007-06-25 16:23:39 +00003064 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00003065 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003066 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00003067 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00003068 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003069
Evan Cheng26471c42008-03-25 20:08:07 +00003070 // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
3071 if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF)
3072 return DAG.getConstant(0, VT);
Dan Gohman613e0d82007-07-03 14:03:57 +00003073 // fold (xor x, undef) -> undef
Dan Gohman70fb1ae2007-07-10 15:19:29 +00003074 if (N0.getOpcode() == ISD::UNDEF)
3075 return N0;
3076 if (N1.getOpcode() == ISD::UNDEF)
Dan Gohman613e0d82007-07-03 14:03:57 +00003077 return N1;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003078 // fold (xor c1, c2) -> c1^c2
Nate Begeman646d7e22005-09-02 21:18:40 +00003079 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003080 return DAG.FoldConstantArithmetic(ISD::XOR, VT, N0C, N1C);
Nate Begeman99801192005-09-07 23:25:52 +00003081 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00003082 if (N0C && !N1C)
Bill Wendling317bd702009-01-30 21:14:50 +00003083 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N1, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003084 // fold (xor x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003085 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003086 return N0;
Nate Begemancd4d58c2006-02-03 06:46:56 +00003087 // reassociate xor
Bill Wendling35247c32009-01-30 00:45:56 +00003088 SDValue RXOR = ReassociateOps(ISD::XOR, N->getDebugLoc(), N0, N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00003089 if (RXOR.getNode() != 0)
Nate Begemancd4d58c2006-02-03 06:46:56 +00003090 return RXOR;
Bill Wendlingae89bb12008-11-11 08:25:46 +00003091
Nate Begeman1d4d4142005-09-01 00:19:25 +00003092 // fold !(x cc y) -> (x !cc y)
Dan Gohman002e5d02008-03-13 22:13:53 +00003093 if (N1C && N1C->getAPIntValue() == 1 && isSetCCEquivalent(N0, LHS, RHS, CC)) {
Duncan Sands83ec4b62008-06-06 12:08:01 +00003094 bool isInt = LHS.getValueType().isInteger();
Nate Begeman646d7e22005-09-02 21:18:40 +00003095 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
3096 isInt);
Bill Wendlingae89bb12008-11-11 08:25:46 +00003097
Duncan Sands25cf2272008-11-24 14:53:14 +00003098 if (!LegalOperations || TLI.isCondCodeLegal(NotCC, LHS.getValueType())) {
Bill Wendlingae89bb12008-11-11 08:25:46 +00003099 switch (N0.getOpcode()) {
3100 default:
Torok Edwinc23197a2009-07-14 16:55:14 +00003101 llvm_unreachable("Unhandled SetCC Equivalent!");
Bill Wendlingae89bb12008-11-11 08:25:46 +00003102 case ISD::SETCC:
Bill Wendling317bd702009-01-30 21:14:50 +00003103 return DAG.getSetCC(N->getDebugLoc(), VT, LHS, RHS, NotCC);
Bill Wendlingae89bb12008-11-11 08:25:46 +00003104 case ISD::SELECT_CC:
Bill Wendling317bd702009-01-30 21:14:50 +00003105 return DAG.getSelectCC(N->getDebugLoc(), LHS, RHS, N0.getOperand(2),
Bill Wendlingae89bb12008-11-11 08:25:46 +00003106 N0.getOperand(3), NotCC);
3107 }
3108 }
Nate Begeman1d4d4142005-09-01 00:19:25 +00003109 }
Bill Wendlingae89bb12008-11-11 08:25:46 +00003110
Chris Lattner61c5ff42007-09-10 21:39:07 +00003111 // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
Dan Gohman002e5d02008-03-13 22:13:53 +00003112 if (N1C && N1C->getAPIntValue() == 1 && N0.getOpcode() == ISD::ZERO_EXTEND &&
Gabor Greif12632d22008-08-30 19:29:20 +00003113 N0.getNode()->hasOneUse() &&
3114 isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
Dan Gohman475871a2008-07-27 21:46:04 +00003115 SDValue V = N0.getOperand(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003116 V = DAG.getNode(ISD::XOR, N0.getDebugLoc(), V.getValueType(), V,
Duncan Sands272dce02007-10-10 09:54:50 +00003117 DAG.getConstant(1, V.getValueType()));
Gabor Greifba36cb52008-08-28 21:40:38 +00003118 AddToWorkList(V.getNode());
Bill Wendling317bd702009-01-30 21:14:50 +00003119 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, V);
Chris Lattner61c5ff42007-09-10 21:39:07 +00003120 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003121
Bill Wendling317bd702009-01-30 21:14:50 +00003122 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
Owen Anderson825b72b2009-08-11 20:47:22 +00003123 if (N1C && N1C->getAPIntValue() == 1 && VT == MVT::i1 &&
Nate Begeman99801192005-09-07 23:25:52 +00003124 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
Dan Gohman475871a2008-07-27 21:46:04 +00003125 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
Nate Begeman99801192005-09-07 23:25:52 +00003126 if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
3127 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
Bill Wendling317bd702009-01-30 21:14:50 +00003128 LHS = DAG.getNode(ISD::XOR, LHS.getDebugLoc(), VT, LHS, N1); // LHS = ~LHS
3129 RHS = DAG.getNode(ISD::XOR, RHS.getDebugLoc(), VT, RHS, N1); // RHS = ~RHS
Gabor Greifba36cb52008-08-28 21:40:38 +00003130 AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode());
Bill Wendling317bd702009-01-30 21:14:50 +00003131 return DAG.getNode(NewOpcode, N->getDebugLoc(), VT, LHS, RHS);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003132 }
3133 }
Bill Wendling317bd702009-01-30 21:14:50 +00003134 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
Scott Michelfdc40a02009-02-17 22:15:04 +00003135 if (N1C && N1C->isAllOnesValue() &&
Nate Begeman99801192005-09-07 23:25:52 +00003136 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
Dan Gohman475871a2008-07-27 21:46:04 +00003137 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
Nate Begeman99801192005-09-07 23:25:52 +00003138 if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
3139 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
Bill Wendling317bd702009-01-30 21:14:50 +00003140 LHS = DAG.getNode(ISD::XOR, LHS.getDebugLoc(), VT, LHS, N1); // LHS = ~LHS
3141 RHS = DAG.getNode(ISD::XOR, RHS.getDebugLoc(), VT, RHS, N1); // RHS = ~RHS
Gabor Greifba36cb52008-08-28 21:40:38 +00003142 AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode());
Bill Wendling317bd702009-01-30 21:14:50 +00003143 return DAG.getNode(NewOpcode, N->getDebugLoc(), VT, LHS, RHS);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003144 }
3145 }
Bill Wendling317bd702009-01-30 21:14:50 +00003146 // fold (xor (xor x, c1), c2) -> (xor x, (xor c1, c2))
Nate Begeman223df222005-09-08 20:18:10 +00003147 if (N1C && N0.getOpcode() == ISD::XOR) {
3148 ConstantSDNode *N00C = dyn_cast<ConstantSDNode>(N0.getOperand(0));
3149 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3150 if (N00C)
Bill Wendling317bd702009-01-30 21:14:50 +00003151 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N0.getOperand(1),
3152 DAG.getConstant(N1C->getAPIntValue() ^
Dan Gohman002e5d02008-03-13 22:13:53 +00003153 N00C->getAPIntValue(), VT));
Nate Begeman223df222005-09-08 20:18:10 +00003154 if (N01C)
Bill Wendling317bd702009-01-30 21:14:50 +00003155 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT, N0.getOperand(0),
3156 DAG.getConstant(N1C->getAPIntValue() ^
Dan Gohman002e5d02008-03-13 22:13:53 +00003157 N01C->getAPIntValue(), VT));
Nate Begeman223df222005-09-08 20:18:10 +00003158 }
3159 // fold (xor x, x) -> 0
Eric Christopher7bccf6a2011-02-16 04:50:12 +00003160 if (N0 == N1)
3161 return tryFoldToZero(N->getDebugLoc(), TLI, VT, DAG, LegalOperations);
Scott Michelfdc40a02009-02-17 22:15:04 +00003162
Chris Lattner35e5c142006-05-05 05:51:50 +00003163 // Simplify: xor (op x...), (op y...) -> (op (xor x, y))
3164 if (N0.getOpcode() == N1.getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003165 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00003166 if (Tmp.getNode()) return Tmp;
Nate Begeman39ee1ac2005-09-09 19:49:52 +00003167 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003168
Chris Lattner3e104b12006-04-08 04:15:24 +00003169 // Simplify the expression using non-local knowledge.
Duncan Sands83ec4b62008-06-06 12:08:01 +00003170 if (!VT.isVector() &&
Dan Gohman475871a2008-07-27 21:46:04 +00003171 SimplifyDemandedBits(SDValue(N, 0)))
3172 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003173
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003174 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003175}
3176
Chris Lattnere70da202007-12-06 07:33:36 +00003177/// visitShiftByConstant - Handle transforms common to the three shifts, when
3178/// the shift amount is a constant.
Dan Gohman475871a2008-07-27 21:46:04 +00003179SDValue DAGCombiner::visitShiftByConstant(SDNode *N, unsigned Amt) {
Gabor Greifba36cb52008-08-28 21:40:38 +00003180 SDNode *LHS = N->getOperand(0).getNode();
Dan Gohman475871a2008-07-27 21:46:04 +00003181 if (!LHS->hasOneUse()) return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00003182
Chris Lattnere70da202007-12-06 07:33:36 +00003183 // We want to pull some binops through shifts, so that we have (and (shift))
3184 // instead of (shift (and)), likewise for add, or, xor, etc. This sort of
3185 // thing happens with address calculations, so it's important to canonicalize
3186 // it.
3187 bool HighBitSet = false; // Can we transform this if the high bit is set?
Scott Michelfdc40a02009-02-17 22:15:04 +00003188
Chris Lattnere70da202007-12-06 07:33:36 +00003189 switch (LHS->getOpcode()) {
Dan Gohman475871a2008-07-27 21:46:04 +00003190 default: return SDValue();
Chris Lattnere70da202007-12-06 07:33:36 +00003191 case ISD::OR:
3192 case ISD::XOR:
3193 HighBitSet = false; // We can only transform sra if the high bit is clear.
3194 break;
3195 case ISD::AND:
3196 HighBitSet = true; // We can only transform sra if the high bit is set.
3197 break;
3198 case ISD::ADD:
Scott Michelfdc40a02009-02-17 22:15:04 +00003199 if (N->getOpcode() != ISD::SHL)
Dan Gohman475871a2008-07-27 21:46:04 +00003200 return SDValue(); // only shl(add) not sr[al](add).
Chris Lattnere70da202007-12-06 07:33:36 +00003201 HighBitSet = false; // We can only transform sra if the high bit is clear.
3202 break;
3203 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003204
Chris Lattnere70da202007-12-06 07:33:36 +00003205 // We require the RHS of the binop to be a constant as well.
3206 ConstantSDNode *BinOpCst = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
Dan Gohman475871a2008-07-27 21:46:04 +00003207 if (!BinOpCst) return SDValue();
Bill Wendling88103372009-01-30 21:37:17 +00003208
3209 // FIXME: disable this unless the input to the binop is a shift by a constant.
3210 // If it is not a shift, it pessimizes some common cases like:
Chris Lattnerd3fd6d22007-12-06 07:47:55 +00003211 //
Bill Wendling88103372009-01-30 21:37:17 +00003212 // void foo(int *X, int i) { X[i & 1235] = 1; }
3213 // int bar(int *X, int i) { return X[i & 255]; }
Gabor Greifba36cb52008-08-28 21:40:38 +00003214 SDNode *BinOpLHSVal = LHS->getOperand(0).getNode();
Scott Michelfdc40a02009-02-17 22:15:04 +00003215 if ((BinOpLHSVal->getOpcode() != ISD::SHL &&
Chris Lattnerd3fd6d22007-12-06 07:47:55 +00003216 BinOpLHSVal->getOpcode() != ISD::SRA &&
3217 BinOpLHSVal->getOpcode() != ISD::SRL) ||
3218 !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1)))
Dan Gohman475871a2008-07-27 21:46:04 +00003219 return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00003220
Owen Andersone50ed302009-08-10 22:56:29 +00003221 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003222
Bill Wendling88103372009-01-30 21:37:17 +00003223 // If this is a signed shift right, and the high bit is modified by the
3224 // logical operation, do not perform the transformation. The highBitSet
3225 // boolean indicates the value of the high bit of the constant which would
3226 // cause it to be modified for this operation.
Chris Lattnere70da202007-12-06 07:33:36 +00003227 if (N->getOpcode() == ISD::SRA) {
Dan Gohman220a8232008-03-03 23:51:38 +00003228 bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative();
3229 if (BinOpRHSSignSet != HighBitSet)
Dan Gohman475871a2008-07-27 21:46:04 +00003230 return SDValue();
Chris Lattnere70da202007-12-06 07:33:36 +00003231 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003232
Chris Lattnere70da202007-12-06 07:33:36 +00003233 // Fold the constants, shifting the binop RHS by the shift amount.
Bill Wendling88103372009-01-30 21:37:17 +00003234 SDValue NewRHS = DAG.getNode(N->getOpcode(), LHS->getOperand(1).getDebugLoc(),
3235 N->getValueType(0),
3236 LHS->getOperand(1), N->getOperand(1));
Chris Lattnere70da202007-12-06 07:33:36 +00003237
3238 // Create the new shift.
Eric Christopher503a64d2010-12-09 04:48:06 +00003239 SDValue NewShift = DAG.getNode(N->getOpcode(),
3240 LHS->getOperand(0).getDebugLoc(),
Bill Wendling88103372009-01-30 21:37:17 +00003241 VT, LHS->getOperand(0), N->getOperand(1));
Chris Lattnere70da202007-12-06 07:33:36 +00003242
3243 // Create the new binop.
Bill Wendling88103372009-01-30 21:37:17 +00003244 return DAG.getNode(LHS->getOpcode(), N->getDebugLoc(), VT, NewShift, NewRHS);
Chris Lattnere70da202007-12-06 07:33:36 +00003245}
3246
Dan Gohman475871a2008-07-27 21:46:04 +00003247SDValue DAGCombiner::visitSHL(SDNode *N) {
3248 SDValue N0 = N->getOperand(0);
3249 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00003250 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3251 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003252 EVT VT = N0.getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00003253 unsigned OpSizeInBits = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00003254
Nate Begeman1d4d4142005-09-01 00:19:25 +00003255 // fold (shl c1, c2) -> c1<<c2
Nate Begeman646d7e22005-09-02 21:18:40 +00003256 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003257 return DAG.FoldConstantArithmetic(ISD::SHL, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003258 // fold (shl 0, x) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00003259 if (N0C && N0C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003260 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003261 // fold (shl x, c >= size(x)) -> undef
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003262 if (N1C && N1C->getZExtValue() >= OpSizeInBits)
Dale Johannesene8d72302009-02-06 23:05:02 +00003263 return DAG.getUNDEF(VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003264 // fold (shl x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003265 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003266 return N0;
Chad Rosier92bcd962011-06-14 22:29:10 +00003267 // fold (shl undef, x) -> 0
3268 if (N0.getOpcode() == ISD::UNDEF)
3269 return DAG.getConstant(0, VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003270 // if (shl x, c) is known to be zero, return 0
Dan Gohman475871a2008-07-27 21:46:04 +00003271 if (DAG.MaskedValueIsZero(SDValue(N, 0),
Dan Gohman87862e72009-12-11 21:31:27 +00003272 APInt::getAllOnesValue(OpSizeInBits)))
Nate Begeman83e75ec2005-09-06 04:43:02 +00003273 return DAG.getConstant(0, VT);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003274 // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
Evan Chengeb9f8922008-08-30 02:03:58 +00003275 if (N1.getOpcode() == ISD::TRUNCATE &&
Evan Cheng242ebd12008-09-22 18:19:24 +00003276 N1.getOperand(0).getOpcode() == ISD::AND &&
3277 N1.hasOneUse() && N1.getOperand(0).hasOneUse()) {
Evan Chengeb9f8922008-08-30 02:03:58 +00003278 SDValue N101 = N1.getOperand(0).getOperand(1);
Evan Cheng242ebd12008-09-22 18:19:24 +00003279 if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) {
Owen Andersone50ed302009-08-10 22:56:29 +00003280 EVT TruncVT = N1.getValueType();
Evan Cheng242ebd12008-09-22 18:19:24 +00003281 SDValue N100 = N1.getOperand(0).getOperand(0);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003282 APInt TruncC = N101C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00003283 TruncC = TruncC.trunc(TruncVT.getSizeInBits());
Bill Wendling88103372009-01-30 21:37:17 +00003284 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0,
Bill Wendlingfc4b6772009-02-01 11:19:36 +00003285 DAG.getNode(ISD::AND, N->getDebugLoc(), TruncVT,
3286 DAG.getNode(ISD::TRUNCATE,
3287 N->getDebugLoc(),
3288 TruncVT, N100),
Dan Gohmance9bc122009-01-27 20:39:34 +00003289 DAG.getConstant(TruncC, TruncVT)));
Evan Chengeb9f8922008-08-30 02:03:58 +00003290 }
3291 }
3292
Dan Gohman475871a2008-07-27 21:46:04 +00003293 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
3294 return SDValue(N, 0);
Bill Wendling88103372009-01-30 21:37:17 +00003295
3296 // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
Scott Michelfdc40a02009-02-17 22:15:04 +00003297 if (N1C && N0.getOpcode() == ISD::SHL &&
Nate Begeman1d4d4142005-09-01 00:19:25 +00003298 N0.getOperand(1).getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003299 uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
3300 uint64_t c2 = N1C->getZExtValue();
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003301 if (c1 + c2 >= OpSizeInBits)
Nate Begeman83e75ec2005-09-06 04:43:02 +00003302 return DAG.getConstant(0, VT);
Bill Wendling88103372009-01-30 21:37:17 +00003303 return DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0.getOperand(0),
Nate Begeman83e75ec2005-09-06 04:43:02 +00003304 DAG.getConstant(c1 + c2, N1.getValueType()));
Nate Begeman1d4d4142005-09-01 00:19:25 +00003305 }
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003306
3307 // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2)))
3308 // For this to be valid, the second form must not preserve any of the bits
3309 // that are shifted out by the inner shift in the first form. This means
3310 // the outer shift size must be >= the number of bits added by the ext.
3311 // As a corollary, we don't care what kind of ext it is.
3312 if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
3313 N0.getOpcode() == ISD::ANY_EXTEND ||
3314 N0.getOpcode() == ISD::SIGN_EXTEND) &&
3315 N0.getOperand(0).getOpcode() == ISD::SHL &&
3316 isa<ConstantSDNode>(N0.getOperand(0)->getOperand(1))) {
Owen Anderson95771af2011-02-25 21:41:48 +00003317 uint64_t c1 =
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003318 cast<ConstantSDNode>(N0.getOperand(0)->getOperand(1))->getZExtValue();
3319 uint64_t c2 = N1C->getZExtValue();
3320 EVT InnerShiftVT = N0.getOperand(0).getValueType();
3321 uint64_t InnerShiftSize = InnerShiftVT.getScalarType().getSizeInBits();
3322 if (c2 >= OpSizeInBits - InnerShiftSize) {
3323 if (c1 + c2 >= OpSizeInBits)
3324 return DAG.getConstant(0, VT);
3325 return DAG.getNode(ISD::SHL, N0->getDebugLoc(), VT,
3326 DAG.getNode(N0.getOpcode(), N0->getDebugLoc(), VT,
3327 N0.getOperand(0)->getOperand(0)),
3328 DAG.getConstant(c1 + c2, N1.getValueType()));
3329 }
3330 }
3331
Eli Friedman2a6d9eb2011-06-09 22:14:44 +00003332 // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
3333 // (and (srl x, (sub c1, c2), MASK)
Chandler Carruth62dfc512012-01-05 11:05:55 +00003334 // Only fold this if the inner shift has no other uses -- if it does, folding
3335 // this will increase the total number of instructions.
3336 if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse() &&
Nate Begeman1d4d4142005-09-01 00:19:25 +00003337 N0.getOperand(1).getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003338 uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
Evan Chengd101a722009-07-21 05:40:15 +00003339 if (c1 < VT.getSizeInBits()) {
3340 uint64_t c2 = N1C->getZExtValue();
Eli Friedman2a6d9eb2011-06-09 22:14:44 +00003341 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
3342 VT.getSizeInBits() - c1);
3343 SDValue Shift;
3344 if (c2 > c1) {
3345 Mask = Mask.shl(c2-c1);
3346 Shift = DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, N0.getOperand(0),
3347 DAG.getConstant(c2-c1, N1.getValueType()));
3348 } else {
3349 Mask = Mask.lshr(c1-c2);
3350 Shift = DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0.getOperand(0),
3351 DAG.getConstant(c1-c2, N1.getValueType()));
3352 }
3353 return DAG.getNode(ISD::AND, N0.getDebugLoc(), VT, Shift,
3354 DAG.getConstant(Mask, VT));
Evan Chengd101a722009-07-21 05:40:15 +00003355 }
Nate Begeman1d4d4142005-09-01 00:19:25 +00003356 }
Bill Wendling88103372009-01-30 21:37:17 +00003357 // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
Dan Gohman5cbd37e2009-08-06 09:18:59 +00003358 if (N1C && N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1)) {
3359 SDValue HiBitsMask =
3360 DAG.getConstant(APInt::getHighBitsSet(VT.getSizeInBits(),
3361 VT.getSizeInBits() -
3362 N1C->getZExtValue()),
3363 VT);
Bill Wendling88103372009-01-30 21:37:17 +00003364 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0.getOperand(0),
Dan Gohman5cbd37e2009-08-06 09:18:59 +00003365 HiBitsMask);
3366 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003367
Evan Chenge5b51ac2010-04-17 06:13:15 +00003368 if (N1C) {
3369 SDValue NewSHL = visitShiftByConstant(N, N1C->getZExtValue());
3370 if (NewSHL.getNode())
3371 return NewSHL;
3372 }
3373
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003374 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003375}
3376
Dan Gohman475871a2008-07-27 21:46:04 +00003377SDValue DAGCombiner::visitSRA(SDNode *N) {
3378 SDValue N0 = N->getOperand(0);
3379 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00003380 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3381 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003382 EVT VT = N0.getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00003383 unsigned OpSizeInBits = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00003384
Bill Wendling88103372009-01-30 21:37:17 +00003385 // fold (sra c1, c2) -> (sra c1, c2)
Nate Begeman646d7e22005-09-02 21:18:40 +00003386 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003387 return DAG.FoldConstantArithmetic(ISD::SRA, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003388 // fold (sra 0, x) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00003389 if (N0C && N0C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003390 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003391 // fold (sra -1, x) -> -1
Nate Begeman646d7e22005-09-02 21:18:40 +00003392 if (N0C && N0C->isAllOnesValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003393 return N0;
Bill Wendling88103372009-01-30 21:37:17 +00003394 // fold (sra x, (setge c, size(x))) -> undef
Dan Gohman87862e72009-12-11 21:31:27 +00003395 if (N1C && N1C->getZExtValue() >= OpSizeInBits)
Dale Johannesene8d72302009-02-06 23:05:02 +00003396 return DAG.getUNDEF(VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003397 // fold (sra x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003398 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003399 return N0;
Nate Begemanfb7217b2006-02-17 19:54:08 +00003400 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
3401 // sext_inreg.
3402 if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
Dan Gohman87862e72009-12-11 21:31:27 +00003403 unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
Dan Gohmand1996362010-01-09 02:13:55 +00003404 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
3405 if (VT.isVector())
3406 ExtVT = EVT::getVectorVT(*DAG.getContext(),
3407 ExtVT, VT.getVectorNumElements());
3408 if ((!LegalOperations ||
3409 TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT)))
Bill Wendling88103372009-01-30 21:37:17 +00003410 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT,
Dan Gohmand1996362010-01-09 02:13:55 +00003411 N0.getOperand(0), DAG.getValueType(ExtVT));
Nate Begemanfb7217b2006-02-17 19:54:08 +00003412 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00003413
Bill Wendling88103372009-01-30 21:37:17 +00003414 // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
Chris Lattner71d9ebc2006-02-28 06:23:04 +00003415 if (N1C && N0.getOpcode() == ISD::SRA) {
3416 if (ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003417 unsigned Sum = N1C->getZExtValue() + C1->getZExtValue();
Dan Gohman87862e72009-12-11 21:31:27 +00003418 if (Sum >= OpSizeInBits) Sum = OpSizeInBits-1;
Bill Wendling88103372009-01-30 21:37:17 +00003419 return DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, N0.getOperand(0),
Chris Lattner71d9ebc2006-02-28 06:23:04 +00003420 DAG.getConstant(Sum, N1C->getValueType(0)));
3421 }
3422 }
Christopher Lamb15cbde32008-03-19 08:30:06 +00003423
Bill Wendling88103372009-01-30 21:37:17 +00003424 // fold (sra (shl X, m), (sub result_size, n))
3425 // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for
Scott Michelfdc40a02009-02-17 22:15:04 +00003426 // result_size - n != m.
3427 // If truncate is free for the target sext(shl) is likely to result in better
Christopher Lambb9b04282008-03-20 04:31:39 +00003428 // code.
Christopher Lamb15cbde32008-03-19 08:30:06 +00003429 if (N0.getOpcode() == ISD::SHL) {
3430 // Get the two constanst of the shifts, CN0 = m, CN = n.
3431 const ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3432 if (N01C && N1C) {
Christopher Lambb9b04282008-03-20 04:31:39 +00003433 // Determine what the truncate's result bitsize and type would be.
Owen Andersone50ed302009-08-10 22:56:29 +00003434 EVT TruncVT =
Eric Christopher503a64d2010-12-09 04:48:06 +00003435 EVT::getIntegerVT(*DAG.getContext(),
3436 OpSizeInBits - N1C->getZExtValue());
Christopher Lambb9b04282008-03-20 04:31:39 +00003437 // Determine the residual right-shift amount.
Torok Edwin6bb49582009-05-23 17:29:48 +00003438 signed ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue();
Duncan Sandsd4b9c172008-06-13 19:07:40 +00003439
Scott Michelfdc40a02009-02-17 22:15:04 +00003440 // If the shift is not a no-op (in which case this should be just a sign
3441 // extend already), the truncated to type is legal, sign_extend is legal
Dan Gohmanf451cb82010-02-10 16:03:48 +00003442 // on that type, and the truncate to that type is both legal and free,
Christopher Lambb9b04282008-03-20 04:31:39 +00003443 // perform the transform.
Torok Edwin6bb49582009-05-23 17:29:48 +00003444 if ((ShiftAmt > 0) &&
Dan Gohmanf560ffa2009-01-28 17:46:25 +00003445 TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) &&
3446 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) &&
Evan Cheng260e07e2008-03-20 02:18:41 +00003447 TLI.isTruncateFree(VT, TruncVT)) {
Christopher Lambb9b04282008-03-20 04:31:39 +00003448
Owen Anderson95771af2011-02-25 21:41:48 +00003449 SDValue Amt = DAG.getConstant(ShiftAmt,
3450 getShiftAmountTy(N0.getOperand(0).getValueType()));
Bill Wendling88103372009-01-30 21:37:17 +00003451 SDValue Shift = DAG.getNode(ISD::SRL, N0.getDebugLoc(), VT,
3452 N0.getOperand(0), Amt);
3453 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(), TruncVT,
3454 Shift);
3455 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(),
3456 N->getValueType(0), Trunc);
Christopher Lamb15cbde32008-03-19 08:30:06 +00003457 }
3458 }
3459 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003460
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003461 // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))).
Evan Chengeb9f8922008-08-30 02:03:58 +00003462 if (N1.getOpcode() == ISD::TRUNCATE &&
Evan Cheng242ebd12008-09-22 18:19:24 +00003463 N1.getOperand(0).getOpcode() == ISD::AND &&
3464 N1.hasOneUse() && N1.getOperand(0).hasOneUse()) {
Evan Chengeb9f8922008-08-30 02:03:58 +00003465 SDValue N101 = N1.getOperand(0).getOperand(1);
Evan Cheng242ebd12008-09-22 18:19:24 +00003466 if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) {
Owen Andersone50ed302009-08-10 22:56:29 +00003467 EVT TruncVT = N1.getValueType();
Evan Cheng242ebd12008-09-22 18:19:24 +00003468 SDValue N100 = N1.getOperand(0).getOperand(0);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003469 APInt TruncC = N101C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00003470 TruncC = TruncC.trunc(TruncVT.getScalarType().getSizeInBits());
Bill Wendling88103372009-01-30 21:37:17 +00003471 return DAG.getNode(ISD::SRA, N->getDebugLoc(), VT, N0,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003472 DAG.getNode(ISD::AND, N->getDebugLoc(),
Bill Wendling88103372009-01-30 21:37:17 +00003473 TruncVT,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003474 DAG.getNode(ISD::TRUNCATE,
3475 N->getDebugLoc(),
3476 TruncVT, N100),
Dan Gohmance9bc122009-01-27 20:39:34 +00003477 DAG.getConstant(TruncC, TruncVT)));
Evan Chengeb9f8922008-08-30 02:03:58 +00003478 }
3479 }
3480
Benjamin Kramer9b108a32011-01-30 16:38:43 +00003481 // fold (sra (trunc (sr x, c1)), c2) -> (trunc (sra x, c1+c2))
3482 // if c1 is equal to the number of bits the trunc removes
3483 if (N0.getOpcode() == ISD::TRUNCATE &&
3484 (N0.getOperand(0).getOpcode() == ISD::SRL ||
3485 N0.getOperand(0).getOpcode() == ISD::SRA) &&
3486 N0.getOperand(0).hasOneUse() &&
3487 N0.getOperand(0).getOperand(1).hasOneUse() &&
3488 N1C && isa<ConstantSDNode>(N0.getOperand(0).getOperand(1))) {
3489 EVT LargeVT = N0.getOperand(0).getValueType();
3490 ConstantSDNode *LargeShiftAmt =
3491 cast<ConstantSDNode>(N0.getOperand(0).getOperand(1));
3492
3493 if (LargeVT.getScalarType().getSizeInBits() - OpSizeInBits ==
3494 LargeShiftAmt->getZExtValue()) {
3495 SDValue Amt =
3496 DAG.getConstant(LargeShiftAmt->getZExtValue() + N1C->getZExtValue(),
Owen Anderson95771af2011-02-25 21:41:48 +00003497 getShiftAmountTy(N0.getOperand(0).getOperand(0).getValueType()));
Benjamin Kramer9b108a32011-01-30 16:38:43 +00003498 SDValue SRA = DAG.getNode(ISD::SRA, N->getDebugLoc(), LargeVT,
3499 N0.getOperand(0).getOperand(0), Amt);
3500 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, SRA);
3501 }
3502 }
3503
Scott Michelfdc40a02009-02-17 22:15:04 +00003504 // Simplify, based on bits shifted out of the LHS.
Dan Gohman475871a2008-07-27 21:46:04 +00003505 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
3506 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003507
3508
Nate Begeman1d4d4142005-09-01 00:19:25 +00003509 // If the sign bit is known to be zero, switch this to a SRL.
Dan Gohman2e68b6f2008-02-25 21:11:39 +00003510 if (DAG.SignBitIsZero(N0))
Bill Wendling88103372009-01-30 21:37:17 +00003511 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0, N1);
Chris Lattnere70da202007-12-06 07:33:36 +00003512
Evan Chenge5b51ac2010-04-17 06:13:15 +00003513 if (N1C) {
3514 SDValue NewSRA = visitShiftByConstant(N, N1C->getZExtValue());
3515 if (NewSRA.getNode())
3516 return NewSRA;
3517 }
3518
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003519 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003520}
3521
Dan Gohman475871a2008-07-27 21:46:04 +00003522SDValue DAGCombiner::visitSRL(SDNode *N) {
3523 SDValue N0 = N->getOperand(0);
3524 SDValue N1 = N->getOperand(1);
Nate Begeman646d7e22005-09-02 21:18:40 +00003525 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3526 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00003527 EVT VT = N0.getValueType();
Dan Gohman87862e72009-12-11 21:31:27 +00003528 unsigned OpSizeInBits = VT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00003529
Nate Begeman1d4d4142005-09-01 00:19:25 +00003530 // fold (srl c1, c2) -> c1 >>u c2
Nate Begeman646d7e22005-09-02 21:18:40 +00003531 if (N0C && N1C)
Bill Wendlingf3cbca22008-09-24 10:25:02 +00003532 return DAG.FoldConstantArithmetic(ISD::SRL, VT, N0C, N1C);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003533 // fold (srl 0, x) -> 0
Nate Begeman646d7e22005-09-02 21:18:40 +00003534 if (N0C && N0C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003535 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003536 // fold (srl x, c >= size(x)) -> undef
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003537 if (N1C && N1C->getZExtValue() >= OpSizeInBits)
Dale Johannesene8d72302009-02-06 23:05:02 +00003538 return DAG.getUNDEF(VT);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003539 // fold (srl x, 0) -> x
Nate Begeman646d7e22005-09-02 21:18:40 +00003540 if (N1C && N1C->isNullValue())
Nate Begeman83e75ec2005-09-06 04:43:02 +00003541 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00003542 // if (srl x, c) is known to be zero, return 0
Dan Gohman475871a2008-07-27 21:46:04 +00003543 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
Dan Gohman2e68b6f2008-02-25 21:11:39 +00003544 APInt::getAllOnesValue(OpSizeInBits)))
Nate Begeman83e75ec2005-09-06 04:43:02 +00003545 return DAG.getConstant(0, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00003546
Bill Wendling88103372009-01-30 21:37:17 +00003547 // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
Scott Michelfdc40a02009-02-17 22:15:04 +00003548 if (N1C && N0.getOpcode() == ISD::SRL &&
Nate Begeman1d4d4142005-09-01 00:19:25 +00003549 N0.getOperand(1).getOpcode() == ISD::Constant) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003550 uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
3551 uint64_t c2 = N1C->getZExtValue();
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003552 if (c1 + c2 >= OpSizeInBits)
Nate Begeman83e75ec2005-09-06 04:43:02 +00003553 return DAG.getConstant(0, VT);
Bill Wendling88103372009-01-30 21:37:17 +00003554 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0.getOperand(0),
Nate Begeman83e75ec2005-09-06 04:43:02 +00003555 DAG.getConstant(c1 + c2, N1.getValueType()));
Nate Begeman1d4d4142005-09-01 00:19:25 +00003556 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003557
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003558 // fold (srl (trunc (srl x, c1)), c2) -> 0 or (trunc (srl x, (add c1, c2)))
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003559 if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
3560 N0.getOperand(0).getOpcode() == ISD::SRL &&
Dale Johannesen025cc6e2010-12-20 20:10:50 +00003561 isa<ConstantSDNode>(N0.getOperand(0)->getOperand(1))) {
Owen Anderson95771af2011-02-25 21:41:48 +00003562 uint64_t c1 =
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003563 cast<ConstantSDNode>(N0.getOperand(0)->getOperand(1))->getZExtValue();
3564 uint64_t c2 = N1C->getZExtValue();
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003565 EVT InnerShiftVT = N0.getOperand(0).getValueType();
3566 EVT ShiftCountVT = N0.getOperand(0)->getOperand(1).getValueType();
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003567 uint64_t InnerShiftSize = InnerShiftVT.getScalarType().getSizeInBits();
Dale Johannesen025cc6e2010-12-20 20:10:50 +00003568 // This is only valid if the OpSizeInBits + c1 = size of inner shift.
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003569 if (c1 + OpSizeInBits == InnerShiftSize) {
3570 if (c1 + c2 >= InnerShiftSize)
3571 return DAG.getConstant(0, VT);
3572 return DAG.getNode(ISD::TRUNCATE, N0->getDebugLoc(), VT,
Owen Anderson95771af2011-02-25 21:41:48 +00003573 DAG.getNode(ISD::SRL, N0->getDebugLoc(), InnerShiftVT,
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003574 N0.getOperand(0)->getOperand(0),
Dale Johannesenc72b18c2010-12-21 21:55:50 +00003575 DAG.getConstant(c1 + c2, ShiftCountVT)));
Dale Johannesenf5daf8b2010-12-17 21:45:49 +00003576 }
3577 }
3578
Chris Lattnerefcddc32010-04-15 05:28:43 +00003579 // fold (srl (shl x, c), c) -> (and x, cst2)
3580 if (N1C && N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1 &&
3581 N0.getValueSizeInBits() <= 64) {
3582 uint64_t ShAmt = N1C->getZExtValue()+64-N0.getValueSizeInBits();
3583 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0.getOperand(0),
3584 DAG.getConstant(~0ULL >> ShAmt, VT));
3585 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003586
Scott Michelfdc40a02009-02-17 22:15:04 +00003587
Chris Lattner06afe072006-05-05 22:53:17 +00003588 // fold (srl (anyextend x), c) -> (anyextend (srl x, c))
3589 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
3590 // Shifting in all undef bits?
Owen Andersone50ed302009-08-10 22:56:29 +00003591 EVT SmallVT = N0.getOperand(0).getValueType();
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00003592 if (N1C->getZExtValue() >= SmallVT.getSizeInBits())
Dale Johannesene8d72302009-02-06 23:05:02 +00003593 return DAG.getUNDEF(VT);
Chris Lattner06afe072006-05-05 22:53:17 +00003594
Evan Chenge5b51ac2010-04-17 06:13:15 +00003595 if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
Owen Andersona34d9362011-04-14 17:30:49 +00003596 uint64_t ShiftAmt = N1C->getZExtValue();
Evan Chenge5b51ac2010-04-17 06:13:15 +00003597 SDValue SmallShift = DAG.getNode(ISD::SRL, N0.getDebugLoc(), SmallVT,
Owen Andersona34d9362011-04-14 17:30:49 +00003598 N0.getOperand(0),
3599 DAG.getConstant(ShiftAmt, getShiftAmountTy(SmallVT)));
Evan Chenge5b51ac2010-04-17 06:13:15 +00003600 AddToWorkList(SmallShift.getNode());
3601 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, SmallShift);
3602 }
Chris Lattner06afe072006-05-05 22:53:17 +00003603 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003604
Chris Lattner3657ffe2006-10-12 20:23:19 +00003605 // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign
3606 // bit, which is unmodified by sra.
Bill Wendling88103372009-01-30 21:37:17 +00003607 if (N1C && N1C->getZExtValue() + 1 == VT.getSizeInBits()) {
Chris Lattner3657ffe2006-10-12 20:23:19 +00003608 if (N0.getOpcode() == ISD::SRA)
Bill Wendling88103372009-01-30 21:37:17 +00003609 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0.getOperand(0), N1);
Chris Lattner3657ffe2006-10-12 20:23:19 +00003610 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003611
Chris Lattner350bec02006-04-02 06:11:11 +00003612 // fold (srl (ctlz x), "5") -> x iff x has one bit set (the low bit).
Scott Michelfdc40a02009-02-17 22:15:04 +00003613 if (N1C && N0.getOpcode() == ISD::CTLZ &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00003614 N1C->getAPIntValue() == Log2_32(VT.getSizeInBits())) {
Dan Gohman948d8ea2008-02-20 16:33:30 +00003615 APInt KnownZero, KnownOne;
Dan Gohman5b870af2010-03-02 02:14:38 +00003616 APInt Mask = APInt::getAllOnesValue(VT.getScalarType().getSizeInBits());
Dan Gohmanea859be2007-06-22 14:59:07 +00003617 DAG.ComputeMaskedBits(N0.getOperand(0), Mask, KnownZero, KnownOne);
Scott Michelfdc40a02009-02-17 22:15:04 +00003618
Chris Lattner350bec02006-04-02 06:11:11 +00003619 // If any of the input bits are KnownOne, then the input couldn't be all
3620 // zeros, thus the result of the srl will always be zero.
Dan Gohman948d8ea2008-02-20 16:33:30 +00003621 if (KnownOne.getBoolValue()) return DAG.getConstant(0, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00003622
Chris Lattner350bec02006-04-02 06:11:11 +00003623 // If all of the bits input the to ctlz node are known to be zero, then
3624 // the result of the ctlz is "32" and the result of the shift is one.
Dan Gohman948d8ea2008-02-20 16:33:30 +00003625 APInt UnknownBits = ~KnownZero & Mask;
Chris Lattner350bec02006-04-02 06:11:11 +00003626 if (UnknownBits == 0) return DAG.getConstant(1, VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00003627
Chris Lattner350bec02006-04-02 06:11:11 +00003628 // Otherwise, check to see if there is exactly one bit input to the ctlz.
Bill Wendling88103372009-01-30 21:37:17 +00003629 if ((UnknownBits & (UnknownBits - 1)) == 0) {
Chris Lattner350bec02006-04-02 06:11:11 +00003630 // Okay, we know that only that the single bit specified by UnknownBits
Bill Wendling88103372009-01-30 21:37:17 +00003631 // could be set on input to the CTLZ node. If this bit is set, the SRL
3632 // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
3633 // to an SRL/XOR pair, which is likely to simplify more.
Dan Gohman948d8ea2008-02-20 16:33:30 +00003634 unsigned ShAmt = UnknownBits.countTrailingZeros();
Dan Gohman475871a2008-07-27 21:46:04 +00003635 SDValue Op = N0.getOperand(0);
Bill Wendling88103372009-01-30 21:37:17 +00003636
Chris Lattner350bec02006-04-02 06:11:11 +00003637 if (ShAmt) {
Bill Wendling88103372009-01-30 21:37:17 +00003638 Op = DAG.getNode(ISD::SRL, N0.getDebugLoc(), VT, Op,
Owen Anderson95771af2011-02-25 21:41:48 +00003639 DAG.getConstant(ShAmt, getShiftAmountTy(Op.getValueType())));
Gabor Greifba36cb52008-08-28 21:40:38 +00003640 AddToWorkList(Op.getNode());
Chris Lattner350bec02006-04-02 06:11:11 +00003641 }
Bill Wendling88103372009-01-30 21:37:17 +00003642
3643 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT,
3644 Op, DAG.getConstant(1, VT));
Chris Lattner350bec02006-04-02 06:11:11 +00003645 }
3646 }
Evan Chengeb9f8922008-08-30 02:03:58 +00003647
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003648 // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))).
Evan Chengeb9f8922008-08-30 02:03:58 +00003649 if (N1.getOpcode() == ISD::TRUNCATE &&
Evan Cheng242ebd12008-09-22 18:19:24 +00003650 N1.getOperand(0).getOpcode() == ISD::AND &&
3651 N1.hasOneUse() && N1.getOperand(0).hasOneUse()) {
Evan Chengeb9f8922008-08-30 02:03:58 +00003652 SDValue N101 = N1.getOperand(0).getOperand(1);
Evan Cheng242ebd12008-09-22 18:19:24 +00003653 if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) {
Owen Andersone50ed302009-08-10 22:56:29 +00003654 EVT TruncVT = N1.getValueType();
Evan Cheng242ebd12008-09-22 18:19:24 +00003655 SDValue N100 = N1.getOperand(0).getOperand(0);
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00003656 APInt TruncC = N101C->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00003657 TruncC = TruncC.trunc(TruncVT.getSizeInBits());
Bill Wendling88103372009-01-30 21:37:17 +00003658 return DAG.getNode(ISD::SRL, N->getDebugLoc(), VT, N0,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003659 DAG.getNode(ISD::AND, N->getDebugLoc(),
Bill Wendling88103372009-01-30 21:37:17 +00003660 TruncVT,
Bill Wendling9729c5a2009-01-31 03:12:48 +00003661 DAG.getNode(ISD::TRUNCATE,
3662 N->getDebugLoc(),
3663 TruncVT, N100),
Dan Gohmance9bc122009-01-27 20:39:34 +00003664 DAG.getConstant(TruncC, TruncVT)));
Evan Chengeb9f8922008-08-30 02:03:58 +00003665 }
3666 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003667
Chris Lattner61a4c072007-04-18 03:06:49 +00003668 // fold operands of srl based on knowledge that the low bits are not
3669 // demanded.
Dan Gohman475871a2008-07-27 21:46:04 +00003670 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
3671 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003672
Evan Cheng9ab2b982009-12-18 21:31:31 +00003673 if (N1C) {
3674 SDValue NewSRL = visitShiftByConstant(N, N1C->getZExtValue());
3675 if (NewSRL.getNode())
3676 return NewSRL;
3677 }
3678
Dan Gohman4e39e9d2010-06-24 14:30:44 +00003679 // Attempt to convert a srl of a load into a narrower zero-extending load.
3680 SDValue NarrowLoad = ReduceLoadWidth(N);
3681 if (NarrowLoad.getNode())
3682 return NarrowLoad;
3683
Evan Cheng9ab2b982009-12-18 21:31:31 +00003684 // Here is a common situation. We want to optimize:
3685 //
3686 // %a = ...
3687 // %b = and i32 %a, 2
3688 // %c = srl i32 %b, 1
3689 // brcond i32 %c ...
3690 //
3691 // into
Wesley Peckbf17cfa2010-11-23 03:31:01 +00003692 //
Evan Cheng9ab2b982009-12-18 21:31:31 +00003693 // %a = ...
3694 // %b = and %a, 2
3695 // %c = setcc eq %b, 0
3696 // brcond %c ...
3697 //
3698 // However when after the source operand of SRL is optimized into AND, the SRL
3699 // itself may not be optimized further. Look for it and add the BRCOND into
3700 // the worklist.
Evan Chengd40d03e2010-01-06 19:38:29 +00003701 if (N->hasOneUse()) {
3702 SDNode *Use = *N->use_begin();
3703 if (Use->getOpcode() == ISD::BRCOND)
3704 AddToWorkList(Use);
3705 else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) {
3706 // Also look pass the truncate.
3707 Use = *Use->use_begin();
3708 if (Use->getOpcode() == ISD::BRCOND)
3709 AddToWorkList(Use);
3710 }
3711 }
Evan Cheng9ab2b982009-12-18 21:31:31 +00003712
Evan Chengb3a3d5e2010-04-28 07:10:39 +00003713 return SDValue();
Evan Cheng4c26e932010-04-19 19:29:22 +00003714}
3715
Dan Gohman475871a2008-07-27 21:46:04 +00003716SDValue DAGCombiner::visitCTLZ(SDNode *N) {
3717 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00003718 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003719
3720 // fold (ctlz c1) -> c2
Chris Lattner310b5782006-05-06 23:06:26 +00003721 if (isa<ConstantSDNode>(N0))
Bill Wendling34584e62009-01-30 22:02:18 +00003722 return DAG.getNode(ISD::CTLZ, N->getDebugLoc(), VT, N0);
Dan Gohman475871a2008-07-27 21:46:04 +00003723 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003724}
3725
Chandler Carruth63974b22011-12-13 01:56:10 +00003726SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
3727 SDValue N0 = N->getOperand(0);
3728 EVT VT = N->getValueType(0);
3729
3730 // fold (ctlz_zero_undef c1) -> c2
3731 if (isa<ConstantSDNode>(N0))
3732 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, N->getDebugLoc(), VT, N0);
3733 return SDValue();
3734}
3735
Dan Gohman475871a2008-07-27 21:46:04 +00003736SDValue DAGCombiner::visitCTTZ(SDNode *N) {
3737 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00003738 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003739
Nate Begeman1d4d4142005-09-01 00:19:25 +00003740 // fold (cttz c1) -> c2
Chris Lattner310b5782006-05-06 23:06:26 +00003741 if (isa<ConstantSDNode>(N0))
Bill Wendling34584e62009-01-30 22:02:18 +00003742 return DAG.getNode(ISD::CTTZ, N->getDebugLoc(), VT, N0);
Dan Gohman475871a2008-07-27 21:46:04 +00003743 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003744}
3745
Chandler Carruth63974b22011-12-13 01:56:10 +00003746SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
3747 SDValue N0 = N->getOperand(0);
3748 EVT VT = N->getValueType(0);
3749
3750 // fold (cttz_zero_undef c1) -> c2
3751 if (isa<ConstantSDNode>(N0))
3752 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, N->getDebugLoc(), VT, N0);
3753 return SDValue();
3754}
3755
Dan Gohman475871a2008-07-27 21:46:04 +00003756SDValue DAGCombiner::visitCTPOP(SDNode *N) {
3757 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00003758 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003759
Nate Begeman1d4d4142005-09-01 00:19:25 +00003760 // fold (ctpop c1) -> c2
Chris Lattner310b5782006-05-06 23:06:26 +00003761 if (isa<ConstantSDNode>(N0))
Bill Wendling34584e62009-01-30 22:02:18 +00003762 return DAG.getNode(ISD::CTPOP, N->getDebugLoc(), VT, N0);
Dan Gohman475871a2008-07-27 21:46:04 +00003763 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00003764}
3765
Dan Gohman475871a2008-07-27 21:46:04 +00003766SDValue DAGCombiner::visitSELECT(SDNode *N) {
3767 SDValue N0 = N->getOperand(0);
3768 SDValue N1 = N->getOperand(1);
3769 SDValue N2 = N->getOperand(2);
Nate Begeman452d7beb2005-09-16 00:54:12 +00003770 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3771 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3772 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
Owen Andersone50ed302009-08-10 22:56:29 +00003773 EVT VT = N->getValueType(0);
3774 EVT VT0 = N0.getValueType();
Nate Begeman44728a72005-09-19 22:34:01 +00003775
Bill Wendling34584e62009-01-30 22:02:18 +00003776 // fold (select C, X, X) -> X
Nate Begeman452d7beb2005-09-16 00:54:12 +00003777 if (N1 == N2)
3778 return N1;
Bill Wendling34584e62009-01-30 22:02:18 +00003779 // fold (select true, X, Y) -> X
Nate Begeman452d7beb2005-09-16 00:54:12 +00003780 if (N0C && !N0C->isNullValue())
3781 return N1;
Bill Wendling34584e62009-01-30 22:02:18 +00003782 // fold (select false, X, Y) -> Y
Nate Begeman452d7beb2005-09-16 00:54:12 +00003783 if (N0C && N0C->isNullValue())
3784 return N2;
Bill Wendling34584e62009-01-30 22:02:18 +00003785 // fold (select C, 1, X) -> (or C, X)
Owen Anderson825b72b2009-08-11 20:47:22 +00003786 if (VT == MVT::i1 && N1C && N1C->getAPIntValue() == 1)
Bill Wendling34584e62009-01-30 22:02:18 +00003787 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N2);
3788 // fold (select C, 0, 1) -> (xor C, 1)
Bob Wilson67ba2232009-01-22 22:05:48 +00003789 if (VT.isInteger() &&
Owen Anderson825b72b2009-08-11 20:47:22 +00003790 (VT0 == MVT::i1 ||
Bob Wilson67ba2232009-01-22 22:05:48 +00003791 (VT0.isInteger() &&
Duncan Sands28b77e92011-09-06 19:07:46 +00003792 TLI.getBooleanContents(false) == TargetLowering::ZeroOrOneBooleanContent)) &&
Dan Gohman002e5d02008-03-13 22:13:53 +00003793 N1C && N2C && N1C->isNullValue() && N2C->getAPIntValue() == 1) {
Bill Wendling34584e62009-01-30 22:02:18 +00003794 SDValue XORNode;
Evan Cheng571c4782007-08-18 05:57:05 +00003795 if (VT == VT0)
Bill Wendling34584e62009-01-30 22:02:18 +00003796 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT0,
3797 N0, DAG.getConstant(1, VT0));
3798 XORNode = DAG.getNode(ISD::XOR, N0.getDebugLoc(), VT0,
3799 N0, DAG.getConstant(1, VT0));
Gabor Greifba36cb52008-08-28 21:40:38 +00003800 AddToWorkList(XORNode.getNode());
Duncan Sands8e4eb092008-06-08 20:54:56 +00003801 if (VT.bitsGT(VT0))
Bill Wendling34584e62009-01-30 22:02:18 +00003802 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, XORNode);
3803 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, XORNode);
Evan Cheng571c4782007-08-18 05:57:05 +00003804 }
Bill Wendling34584e62009-01-30 22:02:18 +00003805 // fold (select C, 0, X) -> (and (not C), X)
Owen Anderson825b72b2009-08-11 20:47:22 +00003806 if (VT == VT0 && VT == MVT::i1 && N1C && N1C->isNullValue()) {
Bill Wendling7581bfa2009-01-30 23:03:19 +00003807 SDValue NOTNode = DAG.getNOT(N0.getDebugLoc(), N0, VT);
Bob Wilson4c245462009-01-22 17:39:32 +00003808 AddToWorkList(NOTNode.getNode());
Bill Wendling7581bfa2009-01-30 23:03:19 +00003809 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, NOTNode, N2);
Nate Begeman452d7beb2005-09-16 00:54:12 +00003810 }
Bill Wendling34584e62009-01-30 22:02:18 +00003811 // fold (select C, X, 1) -> (or (not C), X)
Owen Anderson825b72b2009-08-11 20:47:22 +00003812 if (VT == VT0 && VT == MVT::i1 && N2C && N2C->getAPIntValue() == 1) {
Bill Wendling34584e62009-01-30 22:02:18 +00003813 SDValue NOTNode = DAG.getNOT(N0.getDebugLoc(), N0, VT);
Bob Wilson4c245462009-01-22 17:39:32 +00003814 AddToWorkList(NOTNode.getNode());
Bill Wendling7581bfa2009-01-30 23:03:19 +00003815 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, NOTNode, N1);
Nate Begeman452d7beb2005-09-16 00:54:12 +00003816 }
Bill Wendling34584e62009-01-30 22:02:18 +00003817 // fold (select C, X, 0) -> (and C, X)
Owen Anderson825b72b2009-08-11 20:47:22 +00003818 if (VT == MVT::i1 && N2C && N2C->isNullValue())
Bill Wendling34584e62009-01-30 22:02:18 +00003819 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0, N1);
3820 // fold (select X, X, Y) -> (or X, Y)
3821 // fold (select X, 1, Y) -> (or X, Y)
Owen Anderson825b72b2009-08-11 20:47:22 +00003822 if (VT == MVT::i1 && (N0 == N1 || (N1C && N1C->getAPIntValue() == 1)))
Bill Wendling34584e62009-01-30 22:02:18 +00003823 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N2);
3824 // fold (select X, Y, X) -> (and X, Y)
3825 // fold (select X, Y, 0) -> (and X, Y)
Owen Anderson825b72b2009-08-11 20:47:22 +00003826 if (VT == MVT::i1 && (N0 == N2 || (N2C && N2C->getAPIntValue() == 0)))
Bill Wendling34584e62009-01-30 22:02:18 +00003827 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00003828
Chris Lattner40c62d52005-10-18 06:04:22 +00003829 // If we can fold this based on the true/false value, do so.
3830 if (SimplifySelectOps(N, N1, N2))
Dan Gohman475871a2008-07-27 21:46:04 +00003831 return SDValue(N, 0); // Don't revisit N.
Duncan Sandsd4b9c172008-06-13 19:07:40 +00003832
Nate Begeman44728a72005-09-19 22:34:01 +00003833 // fold selects based on a setcc into other things, such as min/max/abs
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00003834 if (N0.getOpcode() == ISD::SETCC) {
Nate Begeman750ac1b2006-02-01 07:19:44 +00003835 // FIXME:
Owen Anderson825b72b2009-08-11 20:47:22 +00003836 // Check against MVT::Other for SELECT_CC, which is a workaround for targets
Nate Begeman750ac1b2006-02-01 07:19:44 +00003837 // having to say they don't support SELECT_CC on every type the DAG knows
3838 // about, since there is no way to mark an opcode illegal at all value types
Owen Anderson825b72b2009-08-11 20:47:22 +00003839 if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, MVT::Other) &&
Dan Gohman4ea48042009-08-02 16:19:38 +00003840 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))
Bill Wendling34584e62009-01-30 22:02:18 +00003841 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), VT,
3842 N0.getOperand(0), N0.getOperand(1),
Nate Begeman750ac1b2006-02-01 07:19:44 +00003843 N1, N2, N0.getOperand(2));
Chris Lattner600fec32009-03-11 05:08:08 +00003844 return SimplifySelect(N->getDebugLoc(), N0, N1, N2);
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00003845 }
Bill Wendling34584e62009-01-30 22:02:18 +00003846
Dan Gohman475871a2008-07-27 21:46:04 +00003847 return SDValue();
Nate Begeman452d7beb2005-09-16 00:54:12 +00003848}
3849
Dan Gohman475871a2008-07-27 21:46:04 +00003850SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
3851 SDValue N0 = N->getOperand(0);
3852 SDValue N1 = N->getOperand(1);
3853 SDValue N2 = N->getOperand(2);
3854 SDValue N3 = N->getOperand(3);
3855 SDValue N4 = N->getOperand(4);
Nate Begeman44728a72005-09-19 22:34:01 +00003856 ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get();
Scott Michelfdc40a02009-02-17 22:15:04 +00003857
Nate Begeman44728a72005-09-19 22:34:01 +00003858 // fold select_cc lhs, rhs, x, x, cc -> x
3859 if (N2 == N3)
3860 return N2;
Scott Michelfdc40a02009-02-17 22:15:04 +00003861
Chris Lattner5f42a242006-09-20 06:19:26 +00003862 // Determine if the condition we're dealing with is constant
Duncan Sands5480c042009-01-01 15:52:00 +00003863 SDValue SCC = SimplifySetCC(TLI.getSetCCResultType(N0.getValueType()),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00003864 N0, N1, CC, N->getDebugLoc(), false);
Gabor Greifba36cb52008-08-28 21:40:38 +00003865 if (SCC.getNode()) AddToWorkList(SCC.getNode());
Chris Lattner5f42a242006-09-20 06:19:26 +00003866
Gabor Greifba36cb52008-08-28 21:40:38 +00003867 if (ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.getNode())) {
Dan Gohman002e5d02008-03-13 22:13:53 +00003868 if (!SCCC->isNullValue())
Chris Lattner5f42a242006-09-20 06:19:26 +00003869 return N2; // cond always true -> true val
3870 else
3871 return N3; // cond always false -> false val
3872 }
Scott Michelfdc40a02009-02-17 22:15:04 +00003873
Chris Lattner5f42a242006-09-20 06:19:26 +00003874 // Fold to a simpler select_cc
Gabor Greifba36cb52008-08-28 21:40:38 +00003875 if (SCC.getNode() && SCC.getOpcode() == ISD::SETCC)
Scott Michelfdc40a02009-02-17 22:15:04 +00003876 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), N2.getValueType(),
3877 SCC.getOperand(0), SCC.getOperand(1), N2, N3,
Chris Lattner5f42a242006-09-20 06:19:26 +00003878 SCC.getOperand(2));
Scott Michelfdc40a02009-02-17 22:15:04 +00003879
Chris Lattner40c62d52005-10-18 06:04:22 +00003880 // If we can fold this based on the true/false value, do so.
3881 if (SimplifySelectOps(N, N2, N3))
Dan Gohman475871a2008-07-27 21:46:04 +00003882 return SDValue(N, 0); // Don't revisit N.
Scott Michelfdc40a02009-02-17 22:15:04 +00003883
Nate Begeman44728a72005-09-19 22:34:01 +00003884 // fold select_cc into other things, such as min/max/abs
Bill Wendling836ca7d2009-01-30 23:59:18 +00003885 return SimplifySelectCC(N->getDebugLoc(), N0, N1, N2, N3, CC);
Nate Begeman452d7beb2005-09-16 00:54:12 +00003886}
3887
Dan Gohman475871a2008-07-27 21:46:04 +00003888SDValue DAGCombiner::visitSETCC(SDNode *N) {
Nate Begeman452d7beb2005-09-16 00:54:12 +00003889 return SimplifySetCC(N->getValueType(0), N->getOperand(0), N->getOperand(1),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00003890 cast<CondCodeSDNode>(N->getOperand(2))->get(),
3891 N->getDebugLoc());
Nate Begeman452d7beb2005-09-16 00:54:12 +00003892}
3893
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003894// ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
Dan Gohman57fc82d2009-04-09 03:51:29 +00003895// "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003896// transformation. Returns true if extension are possible and the above
Scott Michelfdc40a02009-02-17 22:15:04 +00003897// mentioned transformation is profitable.
Dan Gohman475871a2008-07-27 21:46:04 +00003898static bool ExtendUsesToFormExtLoad(SDNode *N, SDValue N0,
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003899 unsigned ExtOpc,
3900 SmallVector<SDNode*, 4> &ExtendNodes,
Dan Gohman79ce2762009-01-15 19:20:50 +00003901 const TargetLowering &TLI) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003902 bool HasCopyToRegUses = false;
3903 bool isTruncFree = TLI.isTruncateFree(N->getValueType(0), N0.getValueType());
Gabor Greif12632d22008-08-30 19:29:20 +00003904 for (SDNode::use_iterator UI = N0.getNode()->use_begin(),
3905 UE = N0.getNode()->use_end();
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003906 UI != UE; ++UI) {
Dan Gohman89684502008-07-27 20:43:25 +00003907 SDNode *User = *UI;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003908 if (User == N)
3909 continue;
Dan Gohman57fc82d2009-04-09 03:51:29 +00003910 if (UI.getUse().getResNo() != N0.getResNo())
3911 continue;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003912 // FIXME: Only extend SETCC N, N and SETCC N, c for now.
Dan Gohman57fc82d2009-04-09 03:51:29 +00003913 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003914 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get();
3915 if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC))
3916 // Sign bits will be lost after a zext.
3917 return false;
3918 bool Add = false;
3919 for (unsigned i = 0; i != 2; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00003920 SDValue UseOp = User->getOperand(i);
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003921 if (UseOp == N0)
3922 continue;
3923 if (!isa<ConstantSDNode>(UseOp))
3924 return false;
3925 Add = true;
3926 }
3927 if (Add)
3928 ExtendNodes.push_back(User);
Dan Gohman57fc82d2009-04-09 03:51:29 +00003929 continue;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003930 }
Dan Gohman57fc82d2009-04-09 03:51:29 +00003931 // If truncates aren't free and there are users we can't
3932 // extend, it isn't worthwhile.
3933 if (!isTruncFree)
3934 return false;
3935 // Remember if this value is live-out.
3936 if (User->getOpcode() == ISD::CopyToReg)
3937 HasCopyToRegUses = true;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003938 }
3939
3940 if (HasCopyToRegUses) {
3941 bool BothLiveOut = false;
3942 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
3943 UI != UE; ++UI) {
Dan Gohman57fc82d2009-04-09 03:51:29 +00003944 SDUse &Use = UI.getUse();
3945 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) {
3946 BothLiveOut = true;
3947 break;
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003948 }
3949 }
3950 if (BothLiveOut)
3951 // Both unextended and extended values are live out. There had better be
Bob Wilsonbebfbc52010-11-28 06:51:19 +00003952 // a good reason for the transformation.
Evan Cheng3c3ddb32007-10-29 19:58:20 +00003953 return ExtendNodes.size();
3954 }
3955 return true;
3956}
3957
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00003958void DAGCombiner::ExtendSetCCUses(SmallVector<SDNode*, 4> SetCCs,
3959 SDValue Trunc, SDValue ExtLoad, DebugLoc DL,
3960 ISD::NodeType ExtType) {
3961 // Extend SetCC uses if necessary.
3962 for (unsigned i = 0, e = SetCCs.size(); i != e; ++i) {
3963 SDNode *SetCC = SetCCs[i];
3964 SmallVector<SDValue, 4> Ops;
3965
3966 for (unsigned j = 0; j != 2; ++j) {
3967 SDValue SOp = SetCC->getOperand(j);
3968 if (SOp == Trunc)
3969 Ops.push_back(ExtLoad);
3970 else
3971 Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp));
3972 }
3973
3974 Ops.push_back(SetCC->getOperand(2));
3975 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0),
3976 &Ops[0], Ops.size()));
3977 }
3978}
3979
Dan Gohman475871a2008-07-27 21:46:04 +00003980SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
3981 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00003982 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00003983
Nate Begeman1d4d4142005-09-01 00:19:25 +00003984 // fold (sext c1) -> c1
Reid Spencer3ed469c2006-11-02 20:25:50 +00003985 if (isa<ConstantSDNode>(N0))
Bill Wendling6ce610f2009-01-30 22:23:15 +00003986 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00003987
Nate Begeman1d4d4142005-09-01 00:19:25 +00003988 // fold (sext (sext x)) -> (sext x)
Chris Lattner310b5782006-05-06 23:06:26 +00003989 // fold (sext (aext x)) -> (sext x)
3990 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
Bill Wendling6ce610f2009-01-30 22:23:15 +00003991 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT,
3992 N0.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00003993
Chris Lattner22558872007-02-26 03:13:59 +00003994 if (N0.getOpcode() == ISD::TRUNCATE) {
Dan Gohman1fdfa6a2008-05-20 20:56:33 +00003995 // fold (sext (truncate (load x))) -> (sext (smaller load x))
3996 // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
Gabor Greifba36cb52008-08-28 21:40:38 +00003997 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
3998 if (NarrowLoad.getNode()) {
Dale Johannesen61734eb2010-05-25 17:50:03 +00003999 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4000 if (NarrowLoad.getNode() != N0.getNode()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004001 CombineTo(N0.getNode(), NarrowLoad);
Dale Johannesen61734eb2010-05-25 17:50:03 +00004002 // CombineTo deleted the truncate, if needed, but not what's under it.
4003 AddToWorkList(oye);
4004 }
Dan Gohmanc7b34442009-04-27 02:00:55 +00004005 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng0b063de2007-03-23 02:16:52 +00004006 }
Evan Chengc88138f2007-03-22 01:54:19 +00004007
Dan Gohman1fdfa6a2008-05-20 20:56:33 +00004008 // See if the value being truncated is already sign extended. If so, just
4009 // eliminate the trunc/sext pair.
Dan Gohman475871a2008-07-27 21:46:04 +00004010 SDValue Op = N0.getOperand(0);
Dan Gohmand1996362010-01-09 02:13:55 +00004011 unsigned OpBits = Op.getValueType().getScalarType().getSizeInBits();
4012 unsigned MidBits = N0.getValueType().getScalarType().getSizeInBits();
4013 unsigned DestBits = VT.getScalarType().getSizeInBits();
Dan Gohmanea859be2007-06-22 14:59:07 +00004014 unsigned NumSignBits = DAG.ComputeNumSignBits(Op);
Scott Michelfdc40a02009-02-17 22:15:04 +00004015
Chris Lattner22558872007-02-26 03:13:59 +00004016 if (OpBits == DestBits) {
4017 // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign
4018 // bits, it is already ready.
4019 if (NumSignBits > DestBits-MidBits)
4020 return Op;
4021 } else if (OpBits < DestBits) {
4022 // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign
4023 // bits, just sext from i32.
4024 if (NumSignBits > OpBits-MidBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004025 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, Op);
Chris Lattner22558872007-02-26 03:13:59 +00004026 } else {
4027 // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign
4028 // bits, just truncate to i32.
4029 if (NumSignBits > OpBits-MidBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004030 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, Op);
Chris Lattner6007b842006-09-21 06:00:20 +00004031 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004032
Chris Lattner22558872007-02-26 03:13:59 +00004033 // fold (sext (truncate x)) -> (sextinreg x).
Duncan Sands25cf2272008-11-24 14:53:14 +00004034 if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG,
4035 N0.getValueType())) {
Dan Gohmand1996362010-01-09 02:13:55 +00004036 if (OpBits < DestBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004037 Op = DAG.getNode(ISD::ANY_EXTEND, N0.getDebugLoc(), VT, Op);
Dan Gohmand1996362010-01-09 02:13:55 +00004038 else if (OpBits > DestBits)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004039 Op = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(), VT, Op);
4040 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT, Op,
Dan Gohmand1996362010-01-09 02:13:55 +00004041 DAG.getValueType(N0.getValueType()));
Chris Lattner22558872007-02-26 03:13:59 +00004042 }
Chris Lattner6007b842006-09-21 06:00:20 +00004043 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004044
Evan Cheng110dec22005-12-14 02:19:23 +00004045 // fold (sext (load x)) -> (sext (truncate (sextload x)))
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004046 // None of the supported targets knows how to perform load and sign extend
Nadav Rotemfcd96192011-02-27 07:40:43 +00004047 // on vectors in one instruction. We only perform this transformation on
4048 // scalars.
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004049 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004050 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00004051 TLI.isLoadExtLegal(ISD::SEXTLOAD, N0.getValueType()))) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004052 bool DoXform = true;
4053 SmallVector<SDNode*, 4> SetCCs;
4054 if (!N0.hasOneUse())
4055 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::SIGN_EXTEND, SetCCs, TLI);
4056 if (DoXform) {
4057 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00004058 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Dan Gohman57fc82d2009-04-09 03:51:29 +00004059 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004060 LN0->getBasePtr(), LN0->getPointerInfo(),
Duncan Sands25cf2272008-11-24 14:53:14 +00004061 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00004062 LN0->isVolatile(), LN0->isNonTemporal(),
4063 LN0->getAlignment());
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004064 CombineTo(N, ExtLoad);
Bill Wendling6ce610f2009-01-30 22:23:15 +00004065 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4066 N0.getValueType(), ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00004067 CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1));
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004068 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4069 ISD::SIGN_EXTEND);
Dan Gohman475871a2008-07-27 21:46:04 +00004070 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004071 }
Nate Begeman3df4d522005-10-12 20:40:40 +00004072 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004073
4074 // fold (sext (sextload x)) -> (sext (truncate (sextload x)))
4075 // fold (sext ( extload x)) -> (sext (truncate (sextload x)))
Gabor Greifba36cb52008-08-28 21:40:38 +00004076 if ((ISD::isSEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) &&
4077 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
Evan Cheng466685d2006-10-09 20:57:25 +00004078 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00004079 EVT MemVT = LN0->getMemoryVT();
Duncan Sands25cf2272008-11-24 14:53:14 +00004080 if ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00004081 TLI.isLoadExtLegal(ISD::SEXTLOAD, MemVT)) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00004082 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling6ce610f2009-01-30 22:23:15 +00004083 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004084 LN0->getBasePtr(), LN0->getPointerInfo(),
4085 MemVT,
David Greene1e559442010-02-15 17:00:31 +00004086 LN0->isVolatile(), LN0->isNonTemporal(),
4087 LN0->getAlignment());
Jim Laskeyf6c4ccf2006-12-15 21:38:30 +00004088 CombineTo(N, ExtLoad);
Gabor Greif12632d22008-08-30 19:29:20 +00004089 CombineTo(N0.getNode(),
Bill Wendling6ce610f2009-01-30 22:23:15 +00004090 DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4091 N0.getValueType(), ExtLoad),
Jim Laskeyf6c4ccf2006-12-15 21:38:30 +00004092 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00004093 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Jim Laskeyf6c4ccf2006-12-15 21:38:30 +00004094 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004095 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004096
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004097 // fold (sext (and/or/xor (load x), cst)) ->
4098 // (and/or/xor (sextload x), (sext cst))
4099 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
4100 N0.getOpcode() == ISD::XOR) &&
4101 isa<LoadSDNode>(N0.getOperand(0)) &&
4102 N0.getOperand(1).getOpcode() == ISD::Constant &&
4103 TLI.isLoadExtLegal(ISD::SEXTLOAD, N0.getValueType()) &&
4104 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
4105 LoadSDNode *LN0 = cast<LoadSDNode>(N0.getOperand(0));
4106 if (LN0->getExtensionType() != ISD::ZEXTLOAD) {
4107 bool DoXform = true;
4108 SmallVector<SDNode*, 4> SetCCs;
4109 if (!N0.hasOneUse())
4110 DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::SIGN_EXTEND,
4111 SetCCs, TLI);
4112 if (DoXform) {
4113 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, LN0->getDebugLoc(), VT,
4114 LN0->getChain(), LN0->getBasePtr(),
4115 LN0->getPointerInfo(),
4116 LN0->getMemoryVT(),
4117 LN0->isVolatile(),
4118 LN0->isNonTemporal(),
4119 LN0->getAlignment());
4120 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
4121 Mask = Mask.sext(VT.getSizeInBits());
4122 SDValue And = DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
4123 ExtLoad, DAG.getConstant(Mask, VT));
4124 SDValue Trunc = DAG.getNode(ISD::TRUNCATE,
4125 N0.getOperand(0).getDebugLoc(),
4126 N0.getOperand(0).getValueType(), ExtLoad);
4127 CombineTo(N, And);
4128 CombineTo(N0.getOperand(0).getNode(), Trunc, ExtLoad.getValue(1));
4129 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4130 ISD::SIGN_EXTEND);
4131 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4132 }
4133 }
4134 }
4135
Chris Lattner20a35c32007-04-11 05:32:27 +00004136 if (N0.getOpcode() == ISD::SETCC) {
Chris Lattner2b7a2712009-07-08 00:31:33 +00004137 // sext(setcc) -> sext_in_reg(vsetcc) for vectors.
Dan Gohman3ce89f42010-04-30 17:19:19 +00004138 // Only do this before legalize for now.
4139 if (VT.isVector() && !LegalOperations) {
4140 EVT N0VT = N0.getOperand(0).getValueType();
Chris Lattner2b7a2712009-07-08 00:31:33 +00004141 // We know that the # elements of the results is the same as the
4142 // # elements of the compare (and the # elements of the compare result
4143 // for that matter). Check to see that they are the same size. If so,
4144 // we know that the element size of the sext'd result matches the
4145 // element size of the compare operands.
Dan Gohman3ce89f42010-04-30 17:19:19 +00004146 if (VT.getSizeInBits() == N0VT.getSizeInBits())
Duncan Sands28b77e92011-09-06 19:07:46 +00004147 return DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
Duncan Sands34727662010-07-12 08:16:59 +00004148 N0.getOperand(1),
4149 cast<CondCodeSDNode>(N0.getOperand(2))->get());
Dan Gohman3ce89f42010-04-30 17:19:19 +00004150 // If the desired elements are smaller or larger than the source
4151 // elements we can use a matching integer vector type and then
4152 // truncate/sign extend
4153 else {
Duncan Sands34727662010-07-12 08:16:59 +00004154 EVT MatchingElementType =
4155 EVT::getIntegerVT(*DAG.getContext(),
4156 N0VT.getScalarType().getSizeInBits());
4157 EVT MatchingVectorType =
4158 EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
4159 N0VT.getVectorNumElements());
4160 SDValue VsetCC =
Duncan Sands28b77e92011-09-06 19:07:46 +00004161 DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
Duncan Sands34727662010-07-12 08:16:59 +00004162 N0.getOperand(1),
4163 cast<CondCodeSDNode>(N0.getOperand(2))->get());
4164 return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT);
Dan Gohman3ce89f42010-04-30 17:19:19 +00004165 }
Chris Lattner2b7a2712009-07-08 00:31:33 +00004166 }
Dan Gohman3ce89f42010-04-30 17:19:19 +00004167
Chris Lattner2b7a2712009-07-08 00:31:33 +00004168 // sext(setcc x, y, cc) -> (select_cc x, y, -1, 0, cc)
Dan Gohmana7bcef12010-04-24 01:17:30 +00004169 unsigned ElementWidth = VT.getScalarType().getSizeInBits();
Dan Gohman5cbd37e2009-08-06 09:18:59 +00004170 SDValue NegOne =
Dan Gohmana7bcef12010-04-24 01:17:30 +00004171 DAG.getConstant(APInt::getAllOnesValue(ElementWidth), VT);
Scott Michelfdc40a02009-02-17 22:15:04 +00004172 SDValue SCC =
Bill Wendling836ca7d2009-01-30 23:59:18 +00004173 SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1),
Dan Gohman5cbd37e2009-08-06 09:18:59 +00004174 NegOne, DAG.getConstant(0, VT),
Chris Lattner1eba01e2007-04-11 06:50:51 +00004175 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
Gabor Greifba36cb52008-08-28 21:40:38 +00004176 if (SCC.getNode()) return SCC;
Evan Cheng8c7ecaf2010-01-26 02:00:44 +00004177 if (!LegalOperations ||
4178 TLI.isOperationLegal(ISD::SETCC, TLI.getSetCCResultType(VT)))
4179 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), VT,
4180 DAG.getSetCC(N->getDebugLoc(),
4181 TLI.getSetCCResultType(VT),
4182 N0.getOperand(0), N0.getOperand(1),
4183 cast<CondCodeSDNode>(N0.getOperand(2))->get()),
4184 NegOne, DAG.getConstant(0, VT));
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004185 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004186
Dan Gohman8f0ad582008-04-28 16:58:24 +00004187 // fold (sext x) -> (zext x) if the sign bit is known zero.
Duncan Sands25cf2272008-11-24 14:53:14 +00004188 if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) &&
Dan Gohman187db7b2008-04-28 18:47:17 +00004189 DAG.SignBitIsZero(N0))
Bill Wendling6ce610f2009-01-30 22:23:15 +00004190 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00004191
Evan Chengb3a3d5e2010-04-28 07:10:39 +00004192 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00004193}
4194
Dan Gohman475871a2008-07-27 21:46:04 +00004195SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
4196 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004197 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00004198
Nate Begeman1d4d4142005-09-01 00:19:25 +00004199 // fold (zext c1) -> c1
Reid Spencer3ed469c2006-11-02 20:25:50 +00004200 if (isa<ConstantSDNode>(N0))
Bill Wendling6ce610f2009-01-30 22:23:15 +00004201 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00004202 // fold (zext (zext x)) -> (zext x)
Chris Lattner310b5782006-05-06 23:06:26 +00004203 // fold (zext (aext x)) -> (zext x)
4204 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
Bill Wendling6ce610f2009-01-30 22:23:15 +00004205 return DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT,
4206 N0.getOperand(0));
Chris Lattner6007b842006-09-21 06:00:20 +00004207
Evan Chengc88138f2007-03-22 01:54:19 +00004208 // fold (zext (truncate (load x))) -> (zext (smaller load x))
4209 // fold (zext (truncate (srl (load x), c))) -> (zext (small load (x+c/n)))
Dale Johannesen2041a0e2007-03-30 21:38:07 +00004210 if (N0.getOpcode() == ISD::TRUNCATE) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004211 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
4212 if (NarrowLoad.getNode()) {
Dale Johannesen61734eb2010-05-25 17:50:03 +00004213 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4214 if (NarrowLoad.getNode() != N0.getNode()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004215 CombineTo(N0.getNode(), NarrowLoad);
Dale Johannesen61734eb2010-05-25 17:50:03 +00004216 // CombineTo deleted the truncate, if needed, but not what's under it.
4217 AddToWorkList(oye);
4218 }
Eli Friedmane545d382011-04-16 23:25:34 +00004219 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng0b063de2007-03-23 02:16:52 +00004220 }
Evan Chengc88138f2007-03-22 01:54:19 +00004221 }
4222
Chris Lattner6007b842006-09-21 06:00:20 +00004223 // fold (zext (truncate x)) -> (and x, mask)
4224 if (N0.getOpcode() == ISD::TRUNCATE &&
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004225 (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT))) {
Dan Gohman394d6292010-11-03 01:47:46 +00004226
4227 // fold (zext (truncate (load x))) -> (zext (smaller load x))
4228 // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
4229 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
4230 if (NarrowLoad.getNode()) {
4231 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4232 if (NarrowLoad.getNode() != N0.getNode()) {
4233 CombineTo(N0.getNode(), NarrowLoad);
4234 // CombineTo deleted the truncate, if needed, but not what's under it.
4235 AddToWorkList(oye);
4236 }
4237 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4238 }
4239
Dan Gohman475871a2008-07-27 21:46:04 +00004240 SDValue Op = N0.getOperand(0);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004241 if (Op.getValueType().bitsLT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004242 Op = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, Op);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004243 } else if (Op.getValueType().bitsGT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004244 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, Op);
Chris Lattner6007b842006-09-21 06:00:20 +00004245 }
Dan Gohman87862e72009-12-11 21:31:27 +00004246 return DAG.getZeroExtendInReg(Op, N->getDebugLoc(),
4247 N0.getValueType().getScalarType());
Chris Lattner6007b842006-09-21 06:00:20 +00004248 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004249
Dan Gohman97121ba2009-04-08 00:15:30 +00004250 // Fold (zext (and (trunc x), cst)) -> (and x, cst),
4251 // if either of the casts is not free.
Chris Lattner111c2282006-09-21 06:14:31 +00004252 if (N0.getOpcode() == ISD::AND &&
4253 N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
Dan Gohman97121ba2009-04-08 00:15:30 +00004254 N0.getOperand(1).getOpcode() == ISD::Constant &&
4255 (!TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
4256 N0.getValueType()) ||
4257 !TLI.isZExtFree(N0.getValueType(), VT))) {
Dan Gohman475871a2008-07-27 21:46:04 +00004258 SDValue X = N0.getOperand(0).getOperand(0);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004259 if (X.getValueType().bitsLT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004260 X = DAG.getNode(ISD::ANY_EXTEND, X.getDebugLoc(), VT, X);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004261 } else if (X.getValueType().bitsGT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004262 X = DAG.getNode(ISD::TRUNCATE, X.getDebugLoc(), VT, X);
Chris Lattner111c2282006-09-21 06:14:31 +00004263 }
Dan Gohman220a8232008-03-03 23:51:38 +00004264 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00004265 Mask = Mask.zext(VT.getSizeInBits());
Bill Wendling6ce610f2009-01-30 22:23:15 +00004266 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
4267 X, DAG.getConstant(Mask, VT));
Chris Lattner111c2282006-09-21 06:14:31 +00004268 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004269
Evan Cheng110dec22005-12-14 02:19:23 +00004270 // fold (zext (load x)) -> (zext (truncate (zextload x)))
Nadav Rotemed9b9342011-02-20 12:37:50 +00004271 // None of the supported targets knows how to perform load and vector_zext
Nadav Rotemfcd96192011-02-27 07:40:43 +00004272 // on vectors in one instruction. We only perform this transformation on
4273 // scalars.
Nadav Rotemed9b9342011-02-20 12:37:50 +00004274 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004275 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00004276 TLI.isLoadExtLegal(ISD::ZEXTLOAD, N0.getValueType()))) {
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004277 bool DoXform = true;
4278 SmallVector<SDNode*, 4> SetCCs;
4279 if (!N0.hasOneUse())
4280 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ZERO_EXTEND, SetCCs, TLI);
4281 if (DoXform) {
4282 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00004283 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling6ce610f2009-01-30 22:23:15 +00004284 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004285 LN0->getBasePtr(), LN0->getPointerInfo(),
Duncan Sands25cf2272008-11-24 14:53:14 +00004286 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00004287 LN0->isVolatile(), LN0->isNonTemporal(),
4288 LN0->getAlignment());
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004289 CombineTo(N, ExtLoad);
Bill Wendling6ce610f2009-01-30 22:23:15 +00004290 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4291 N0.getValueType(), ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00004292 CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1));
Bill Wendling6ce610f2009-01-30 22:23:15 +00004293
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004294 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4295 ISD::ZERO_EXTEND);
Dan Gohman475871a2008-07-27 21:46:04 +00004296 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng3c3ddb32007-10-29 19:58:20 +00004297 }
Evan Cheng110dec22005-12-14 02:19:23 +00004298 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004299
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004300 // fold (zext (and/or/xor (load x), cst)) ->
4301 // (and/or/xor (zextload x), (zext cst))
4302 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
4303 N0.getOpcode() == ISD::XOR) &&
4304 isa<LoadSDNode>(N0.getOperand(0)) &&
4305 N0.getOperand(1).getOpcode() == ISD::Constant &&
4306 TLI.isLoadExtLegal(ISD::ZEXTLOAD, N0.getValueType()) &&
4307 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
4308 LoadSDNode *LN0 = cast<LoadSDNode>(N0.getOperand(0));
4309 if (LN0->getExtensionType() != ISD::SEXTLOAD) {
4310 bool DoXform = true;
4311 SmallVector<SDNode*, 4> SetCCs;
4312 if (!N0.hasOneUse())
4313 DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::ZERO_EXTEND,
4314 SetCCs, TLI);
4315 if (DoXform) {
4316 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, LN0->getDebugLoc(), VT,
4317 LN0->getChain(), LN0->getBasePtr(),
4318 LN0->getPointerInfo(),
4319 LN0->getMemoryVT(),
4320 LN0->isVolatile(),
4321 LN0->isNonTemporal(),
4322 LN0->getAlignment());
4323 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
4324 Mask = Mask.zext(VT.getSizeInBits());
4325 SDValue And = DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
4326 ExtLoad, DAG.getConstant(Mask, VT));
4327 SDValue Trunc = DAG.getNode(ISD::TRUNCATE,
4328 N0.getOperand(0).getDebugLoc(),
4329 N0.getOperand(0).getValueType(), ExtLoad);
4330 CombineTo(N, And);
4331 CombineTo(N0.getOperand(0).getNode(), Trunc, ExtLoad.getValue(1));
4332 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4333 ISD::ZERO_EXTEND);
4334 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4335 }
4336 }
4337 }
4338
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004339 // fold (zext (zextload x)) -> (zext (truncate (zextload x)))
4340 // fold (zext ( extload x)) -> (zext (truncate (zextload x)))
Gabor Greifba36cb52008-08-28 21:40:38 +00004341 if ((ISD::isZEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) &&
4342 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
Evan Cheng466685d2006-10-09 20:57:25 +00004343 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00004344 EVT MemVT = LN0->getMemoryVT();
Duncan Sands25cf2272008-11-24 14:53:14 +00004345 if ((!LegalOperations && !LN0->isVolatile()) ||
Dan Gohman8a55ce42009-09-23 21:02:20 +00004346 TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT)) {
Stuart Hastingsa9011292011-02-16 16:23:55 +00004347 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling6ce610f2009-01-30 22:23:15 +00004348 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004349 LN0->getBasePtr(), LN0->getPointerInfo(),
4350 MemVT,
David Greene1e559442010-02-15 17:00:31 +00004351 LN0->isVolatile(), LN0->isNonTemporal(),
4352 LN0->getAlignment());
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004353 CombineTo(N, ExtLoad);
Gabor Greif12632d22008-08-30 19:29:20 +00004354 CombineTo(N0.getNode(),
Bill Wendling6ce610f2009-01-30 22:23:15 +00004355 DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(), N0.getValueType(),
4356 ExtLoad),
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004357 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00004358 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004359 }
Chris Lattnerad25d4e2005-12-14 19:05:06 +00004360 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004361
Chris Lattner20a35c32007-04-11 05:32:27 +00004362 if (N0.getOpcode() == ISD::SETCC) {
Evan Cheng0a942db2010-05-19 01:08:17 +00004363 if (!LegalOperations && VT.isVector()) {
4364 // zext(setcc) -> (and (vsetcc), (1, 1, ...) for vectors.
4365 // Only do this before legalize for now.
4366 EVT N0VT = N0.getOperand(0).getValueType();
4367 EVT EltVT = VT.getVectorElementType();
4368 SmallVector<SDValue,8> OneOps(VT.getVectorNumElements(),
4369 DAG.getConstant(1, EltVT));
Dan Gohman71dc7c92011-05-17 22:20:36 +00004370 if (VT.getSizeInBits() == N0VT.getSizeInBits())
Evan Cheng0a942db2010-05-19 01:08:17 +00004371 // We know that the # elements of the results is the same as the
4372 // # elements of the compare (and the # elements of the compare result
4373 // for that matter). Check to see that they are the same size. If so,
4374 // we know that the element size of the sext'd result matches the
4375 // element size of the compare operands.
4376 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
Duncan Sands28b77e92011-09-06 19:07:46 +00004377 DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
Evan Cheng0a942db2010-05-19 01:08:17 +00004378 N0.getOperand(1),
4379 cast<CondCodeSDNode>(N0.getOperand(2))->get()),
4380 DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT,
4381 &OneOps[0], OneOps.size()));
Dan Gohman71dc7c92011-05-17 22:20:36 +00004382
4383 // If the desired elements are smaller or larger than the source
4384 // elements we can use a matching integer vector type and then
4385 // truncate/sign extend
4386 EVT MatchingElementType =
4387 EVT::getIntegerVT(*DAG.getContext(),
4388 N0VT.getScalarType().getSizeInBits());
4389 EVT MatchingVectorType =
4390 EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
4391 N0VT.getVectorNumElements());
4392 SDValue VsetCC =
Duncan Sands28b77e92011-09-06 19:07:46 +00004393 DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
Dan Gohman71dc7c92011-05-17 22:20:36 +00004394 N0.getOperand(1),
4395 cast<CondCodeSDNode>(N0.getOperand(2))->get());
4396 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
4397 DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT),
4398 DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT,
4399 &OneOps[0], OneOps.size()));
Evan Cheng0a942db2010-05-19 01:08:17 +00004400 }
4401
4402 // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
Scott Michelfdc40a02009-02-17 22:15:04 +00004403 SDValue SCC =
Bill Wendling836ca7d2009-01-30 23:59:18 +00004404 SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1),
Chris Lattner20a35c32007-04-11 05:32:27 +00004405 DAG.getConstant(1, VT), DAG.getConstant(0, VT),
Chris Lattner1eba01e2007-04-11 06:50:51 +00004406 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
Gabor Greifba36cb52008-08-28 21:40:38 +00004407 if (SCC.getNode()) return SCC;
Chris Lattner20a35c32007-04-11 05:32:27 +00004408 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004409
Evan Cheng9818c042009-12-15 03:00:32 +00004410 // (zext (shl (zext x), cst)) -> (shl (zext x), cst)
Evan Cheng99b653c2009-12-15 00:41:36 +00004411 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
Evan Cheng9818c042009-12-15 03:00:32 +00004412 isa<ConstantSDNode>(N0.getOperand(1)) &&
Evan Cheng99b653c2009-12-15 00:41:36 +00004413 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND &&
4414 N0.hasOneUse()) {
Chris Lattnere0751182011-02-13 19:09:16 +00004415 SDValue ShAmt = N0.getOperand(1);
4416 unsigned ShAmtVal = cast<ConstantSDNode>(ShAmt)->getZExtValue();
Evan Cheng9818c042009-12-15 03:00:32 +00004417 if (N0.getOpcode() == ISD::SHL) {
Chris Lattnere0751182011-02-13 19:09:16 +00004418 SDValue InnerZExt = N0.getOperand(0);
Evan Cheng9818c042009-12-15 03:00:32 +00004419 // If the original shl may be shifting out bits, do not perform this
4420 // transformation.
Chris Lattnere0751182011-02-13 19:09:16 +00004421 unsigned KnownZeroBits = InnerZExt.getValueType().getSizeInBits() -
4422 InnerZExt.getOperand(0).getValueType().getSizeInBits();
4423 if (ShAmtVal > KnownZeroBits)
Evan Cheng9818c042009-12-15 03:00:32 +00004424 return SDValue();
4425 }
Chris Lattnere0751182011-02-13 19:09:16 +00004426
4427 DebugLoc DL = N->getDebugLoc();
Owen Anderson95771af2011-02-25 21:41:48 +00004428
4429 // Ensure that the shift amount is wide enough for the shifted value.
Chris Lattnere0751182011-02-13 19:09:16 +00004430 if (VT.getSizeInBits() >= 256)
4431 ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt);
Owen Anderson95771af2011-02-25 21:41:48 +00004432
Chris Lattnere0751182011-02-13 19:09:16 +00004433 return DAG.getNode(N0.getOpcode(), DL, VT,
4434 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)),
4435 ShAmt);
Evan Cheng99b653c2009-12-15 00:41:36 +00004436 }
4437
Evan Chengb3a3d5e2010-04-28 07:10:39 +00004438 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00004439}
4440
Dan Gohman475871a2008-07-27 21:46:04 +00004441SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
4442 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004443 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00004444
Chris Lattner5ffc0662006-05-05 05:58:59 +00004445 // fold (aext c1) -> c1
Chris Lattner310b5782006-05-06 23:06:26 +00004446 if (isa<ConstantSDNode>(N0))
Bill Wendlingfc4b6772009-02-01 11:19:36 +00004447 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, N0);
Chris Lattner5ffc0662006-05-05 05:58:59 +00004448 // fold (aext (aext x)) -> (aext x)
4449 // fold (aext (zext x)) -> (zext x)
4450 // fold (aext (sext x)) -> (sext x)
4451 if (N0.getOpcode() == ISD::ANY_EXTEND ||
4452 N0.getOpcode() == ISD::ZERO_EXTEND ||
4453 N0.getOpcode() == ISD::SIGN_EXTEND)
Bill Wendling683c9572009-01-30 22:27:33 +00004454 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT, N0.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00004455
Evan Chengc88138f2007-03-22 01:54:19 +00004456 // fold (aext (truncate (load x))) -> (aext (smaller load x))
4457 // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n)))
4458 if (N0.getOpcode() == ISD::TRUNCATE) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004459 SDValue NarrowLoad = ReduceLoadWidth(N0.getNode());
4460 if (NarrowLoad.getNode()) {
Dale Johannesen86234c32010-05-25 18:47:23 +00004461 SDNode* oye = N0.getNode()->getOperand(0).getNode();
4462 if (NarrowLoad.getNode() != N0.getNode()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00004463 CombineTo(N0.getNode(), NarrowLoad);
Dale Johannesen86234c32010-05-25 18:47:23 +00004464 // CombineTo deleted the truncate, if needed, but not what's under it.
4465 AddToWorkList(oye);
4466 }
Eli Friedmane545d382011-04-16 23:25:34 +00004467 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng0b063de2007-03-23 02:16:52 +00004468 }
Evan Chengc88138f2007-03-22 01:54:19 +00004469 }
4470
Chris Lattner84750582006-09-20 06:29:17 +00004471 // fold (aext (truncate x))
4472 if (N0.getOpcode() == ISD::TRUNCATE) {
Dan Gohman475871a2008-07-27 21:46:04 +00004473 SDValue TruncOp = N0.getOperand(0);
Chris Lattner84750582006-09-20 06:29:17 +00004474 if (TruncOp.getValueType() == VT)
4475 return TruncOp; // x iff x size == zext size.
Duncan Sands8e4eb092008-06-08 20:54:56 +00004476 if (TruncOp.getValueType().bitsGT(VT))
Bill Wendling683c9572009-01-30 22:27:33 +00004477 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, TruncOp);
4478 return DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, TruncOp);
Chris Lattner84750582006-09-20 06:29:17 +00004479 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004480
Dan Gohman97121ba2009-04-08 00:15:30 +00004481 // Fold (aext (and (trunc x), cst)) -> (and x, cst)
4482 // if the trunc is not free.
Chris Lattner0e4b9222006-09-21 06:40:43 +00004483 if (N0.getOpcode() == ISD::AND &&
4484 N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
Dan Gohman97121ba2009-04-08 00:15:30 +00004485 N0.getOperand(1).getOpcode() == ISD::Constant &&
4486 !TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
4487 N0.getValueType())) {
Dan Gohman475871a2008-07-27 21:46:04 +00004488 SDValue X = N0.getOperand(0).getOperand(0);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004489 if (X.getValueType().bitsLT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004490 X = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), VT, X);
Duncan Sands8e4eb092008-06-08 20:54:56 +00004491 } else if (X.getValueType().bitsGT(VT)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00004492 X = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, X);
Chris Lattner0e4b9222006-09-21 06:40:43 +00004493 }
Dan Gohman220a8232008-03-03 23:51:38 +00004494 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
Jay Foad40f8f622010-12-07 08:25:19 +00004495 Mask = Mask.zext(VT.getSizeInBits());
Bill Wendling683c9572009-01-30 22:27:33 +00004496 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
4497 X, DAG.getConstant(Mask, VT));
Chris Lattner0e4b9222006-09-21 06:40:43 +00004498 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004499
Chris Lattner5ffc0662006-05-05 05:58:59 +00004500 // fold (aext (load x)) -> (aext (truncate (extload x)))
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004501 // None of the supported targets knows how to perform load and any_ext
Nadav Rotemfcd96192011-02-27 07:40:43 +00004502 // on vectors in one instruction. We only perform this transformation on
4503 // scalars.
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004504 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004505 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00004506 TLI.isLoadExtLegal(ISD::EXTLOAD, N0.getValueType()))) {
Dan Gohman57fc82d2009-04-09 03:51:29 +00004507 bool DoXform = true;
4508 SmallVector<SDNode*, 4> SetCCs;
4509 if (!N0.hasOneUse())
4510 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ANY_EXTEND, SetCCs, TLI);
4511 if (DoXform) {
4512 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00004513 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, N->getDebugLoc(), VT,
Dan Gohman57fc82d2009-04-09 03:51:29 +00004514 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004515 LN0->getBasePtr(), LN0->getPointerInfo(),
Dan Gohman57fc82d2009-04-09 03:51:29 +00004516 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00004517 LN0->isVolatile(), LN0->isNonTemporal(),
4518 LN0->getAlignment());
Dan Gohman57fc82d2009-04-09 03:51:29 +00004519 CombineTo(N, ExtLoad);
4520 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4521 N0.getValueType(), ExtLoad);
4522 CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1));
Nick Lewyckyc06b5bf2011-06-16 01:15:49 +00004523 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, N->getDebugLoc(),
4524 ISD::ANY_EXTEND);
Dan Gohman57fc82d2009-04-09 03:51:29 +00004525 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4526 }
Chris Lattner5ffc0662006-05-05 05:58:59 +00004527 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004528
Chris Lattner5ffc0662006-05-05 05:58:59 +00004529 // fold (aext (zextload x)) -> (aext (truncate (zextload x)))
4530 // fold (aext (sextload x)) -> (aext (truncate (sextload x)))
4531 // fold (aext ( extload x)) -> (aext (truncate (extload x)))
Evan Cheng83060c52007-03-07 08:07:03 +00004532 if (N0.getOpcode() == ISD::LOAD &&
Gabor Greifba36cb52008-08-28 21:40:38 +00004533 !ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
Evan Cheng466685d2006-10-09 20:57:25 +00004534 N0.hasOneUse()) {
4535 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman8a55ce42009-09-23 21:02:20 +00004536 EVT MemVT = LN0->getMemoryVT();
Stuart Hastingsa9011292011-02-16 16:23:55 +00004537 SDValue ExtLoad = DAG.getExtLoad(LN0->getExtensionType(), N->getDebugLoc(),
4538 VT, LN0->getChain(), LN0->getBasePtr(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004539 LN0->getPointerInfo(), MemVT,
David Greene1e559442010-02-15 17:00:31 +00004540 LN0->isVolatile(), LN0->isNonTemporal(),
4541 LN0->getAlignment());
Chris Lattner5ffc0662006-05-05 05:58:59 +00004542 CombineTo(N, ExtLoad);
Evan Cheng45299662008-08-29 23:20:46 +00004543 CombineTo(N0.getNode(),
Bill Wendling683c9572009-01-30 22:27:33 +00004544 DAG.getNode(ISD::TRUNCATE, N0.getDebugLoc(),
4545 N0.getValueType(), ExtLoad),
Chris Lattner5ffc0662006-05-05 05:58:59 +00004546 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00004547 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattner5ffc0662006-05-05 05:58:59 +00004548 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004549
Chris Lattner20a35c32007-04-11 05:32:27 +00004550 if (N0.getOpcode() == ISD::SETCC) {
Evan Cheng0a942db2010-05-19 01:08:17 +00004551 // aext(setcc) -> sext_in_reg(vsetcc) for vectors.
4552 // Only do this before legalize for now.
4553 if (VT.isVector() && !LegalOperations) {
4554 EVT N0VT = N0.getOperand(0).getValueType();
4555 // We know that the # elements of the results is the same as the
4556 // # elements of the compare (and the # elements of the compare result
4557 // for that matter). Check to see that they are the same size. If so,
4558 // we know that the element size of the sext'd result matches the
4559 // element size of the compare operands.
4560 if (VT.getSizeInBits() == N0VT.getSizeInBits())
Duncan Sands28b77e92011-09-06 19:07:46 +00004561 return DAG.getSetCC(N->getDebugLoc(), VT, N0.getOperand(0),
Duncan Sands34727662010-07-12 08:16:59 +00004562 N0.getOperand(1),
4563 cast<CondCodeSDNode>(N0.getOperand(2))->get());
Evan Cheng0a942db2010-05-19 01:08:17 +00004564 // If the desired elements are smaller or larger than the source
4565 // elements we can use a matching integer vector type and then
4566 // truncate/sign extend
4567 else {
Duncan Sands34727662010-07-12 08:16:59 +00004568 EVT MatchingElementType =
4569 EVT::getIntegerVT(*DAG.getContext(),
4570 N0VT.getScalarType().getSizeInBits());
4571 EVT MatchingVectorType =
4572 EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
4573 N0VT.getVectorNumElements());
4574 SDValue VsetCC =
Duncan Sands28b77e92011-09-06 19:07:46 +00004575 DAG.getSetCC(N->getDebugLoc(), MatchingVectorType, N0.getOperand(0),
Duncan Sands34727662010-07-12 08:16:59 +00004576 N0.getOperand(1),
4577 cast<CondCodeSDNode>(N0.getOperand(2))->get());
4578 return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT);
Evan Cheng0a942db2010-05-19 01:08:17 +00004579 }
4580 }
4581
4582 // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
Scott Michelfdc40a02009-02-17 22:15:04 +00004583 SDValue SCC =
Bill Wendling836ca7d2009-01-30 23:59:18 +00004584 SimplifySelectCC(N->getDebugLoc(), N0.getOperand(0), N0.getOperand(1),
Chris Lattner1eba01e2007-04-11 06:50:51 +00004585 DAG.getConstant(1, VT), DAG.getConstant(0, VT),
Chris Lattnerc24bbad2007-04-11 16:51:53 +00004586 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true);
Gabor Greifba36cb52008-08-28 21:40:38 +00004587 if (SCC.getNode())
Chris Lattnerc56a81d2007-04-11 06:43:25 +00004588 return SCC;
Chris Lattner20a35c32007-04-11 05:32:27 +00004589 }
Scott Michelfdc40a02009-02-17 22:15:04 +00004590
Evan Chengb3a3d5e2010-04-28 07:10:39 +00004591 return SDValue();
Chris Lattner5ffc0662006-05-05 05:58:59 +00004592}
4593
Chris Lattner2b4c2792007-10-13 06:35:54 +00004594/// GetDemandedBits - See if the specified operand can be simplified with the
4595/// knowledge that only the bits specified by Mask are used. If so, return the
Dan Gohman475871a2008-07-27 21:46:04 +00004596/// simpler operand, otherwise return a null SDValue.
4597SDValue DAGCombiner::GetDemandedBits(SDValue V, const APInt &Mask) {
Chris Lattner2b4c2792007-10-13 06:35:54 +00004598 switch (V.getOpcode()) {
4599 default: break;
Lang Hames5207bf22011-11-08 18:56:23 +00004600 case ISD::Constant: {
4601 const ConstantSDNode *CV = cast<ConstantSDNode>(V.getNode());
4602 assert(CV != 0 && "Const value should be ConstSDNode.");
4603 const APInt &CVal = CV->getAPIntValue();
4604 APInt NewVal = CVal & Mask;
4605 if (NewVal != CVal) {
4606 return DAG.getConstant(NewVal, V.getValueType());
4607 }
4608 break;
4609 }
Chris Lattner2b4c2792007-10-13 06:35:54 +00004610 case ISD::OR:
4611 case ISD::XOR:
4612 // If the LHS or RHS don't contribute bits to the or, drop them.
4613 if (DAG.MaskedValueIsZero(V.getOperand(0), Mask))
4614 return V.getOperand(1);
4615 if (DAG.MaskedValueIsZero(V.getOperand(1), Mask))
4616 return V.getOperand(0);
4617 break;
Chris Lattnere33544c2007-10-13 06:58:48 +00004618 case ISD::SRL:
4619 // Only look at single-use SRLs.
Gabor Greifba36cb52008-08-28 21:40:38 +00004620 if (!V.getNode()->hasOneUse())
Chris Lattnere33544c2007-10-13 06:58:48 +00004621 break;
4622 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
4623 // See if we can recursively simplify the LHS.
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00004624 unsigned Amt = RHSC->getZExtValue();
Bill Wendling8509c902009-01-30 22:33:24 +00004625
Dan Gohmancc91d632009-01-03 19:22:06 +00004626 // Watch out for shift count overflow though.
4627 if (Amt >= Mask.getBitWidth()) break;
Dan Gohman2e68b6f2008-02-25 21:11:39 +00004628 APInt NewMask = Mask << Amt;
Dan Gohman475871a2008-07-27 21:46:04 +00004629 SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask);
Bill Wendling8509c902009-01-30 22:33:24 +00004630 if (SimplifyLHS.getNode())
Scott Michelfdc40a02009-02-17 22:15:04 +00004631 return DAG.getNode(ISD::SRL, V.getDebugLoc(), V.getValueType(),
Chris Lattnere33544c2007-10-13 06:58:48 +00004632 SimplifyLHS, V.getOperand(1));
Chris Lattnere33544c2007-10-13 06:58:48 +00004633 }
Chris Lattner2b4c2792007-10-13 06:35:54 +00004634 }
Dan Gohman475871a2008-07-27 21:46:04 +00004635 return SDValue();
Chris Lattner2b4c2792007-10-13 06:35:54 +00004636}
4637
Evan Chengc88138f2007-03-22 01:54:19 +00004638/// ReduceLoadWidth - If the result of a wider load is shifted to right of N
4639/// bits and then truncated to a narrower type and where N is a multiple
4640/// of number of bits of the narrower type, transform it to a narrower load
4641/// from address + N / num of bits of new type. If the result is to be
4642/// extended, also fold the extension to form a extending load.
Dan Gohman475871a2008-07-27 21:46:04 +00004643SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
Evan Chengc88138f2007-03-22 01:54:19 +00004644 unsigned Opc = N->getOpcode();
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004645
Evan Chengc88138f2007-03-22 01:54:19 +00004646 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
Dan Gohman475871a2008-07-27 21:46:04 +00004647 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004648 EVT VT = N->getValueType(0);
4649 EVT ExtVT = VT;
Evan Chengc88138f2007-03-22 01:54:19 +00004650
Dan Gohman7f8613e2008-08-14 20:04:46 +00004651 // This transformation isn't valid for vector loads.
4652 if (VT.isVector())
4653 return SDValue();
4654
Dan Gohmand1996362010-01-09 02:13:55 +00004655 // Special case: SIGN_EXTEND_INREG is basically truncating to ExtVT then
Evan Chenge177e302007-03-23 22:13:36 +00004656 // extended to VT.
Evan Chengc88138f2007-03-22 01:54:19 +00004657 if (Opc == ISD::SIGN_EXTEND_INREG) {
4658 ExtType = ISD::SEXTLOAD;
Owen Andersone50ed302009-08-10 22:56:29 +00004659 ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT();
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004660 } else if (Opc == ISD::SRL) {
Chris Lattner90b03642010-12-21 18:05:22 +00004661 // Another special-case: SRL is basically zero-extending a narrower value.
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004662 ExtType = ISD::ZEXTLOAD;
4663 N0 = SDValue(N, 0);
4664 ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4665 if (!N01) return SDValue();
4666 ExtVT = EVT::getIntegerVT(*DAG.getContext(),
4667 VT.getSizeInBits() - N01->getZExtValue());
Evan Chengc88138f2007-03-22 01:54:19 +00004668 }
Richard Osborne4e3740e2011-01-31 17:41:44 +00004669 if (LegalOperations && !TLI.isLoadExtLegal(ExtType, ExtVT))
4670 return SDValue();
Evan Chengc88138f2007-03-22 01:54:19 +00004671
Owen Andersone50ed302009-08-10 22:56:29 +00004672 unsigned EVTBits = ExtVT.getSizeInBits();
Owen Anderson95771af2011-02-25 21:41:48 +00004673
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00004674 // Do not generate loads of non-round integer types since these can
4675 // be expensive (and would be wrong if the type is not byte sized).
4676 if (!ExtVT.isRound())
4677 return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00004678
Evan Chengc88138f2007-03-22 01:54:19 +00004679 unsigned ShAmt = 0;
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00004680 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
Evan Chengc88138f2007-03-22 01:54:19 +00004681 if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00004682 ShAmt = N01->getZExtValue();
Evan Chengc88138f2007-03-22 01:54:19 +00004683 // Is the shift amount a multiple of size of VT?
4684 if ((ShAmt & (EVTBits-1)) == 0) {
4685 N0 = N0.getOperand(0);
Eli Friedmand68eea22009-08-19 08:46:10 +00004686 // Is the load width a multiple of size of VT?
4687 if ((N0.getValueType().getSizeInBits() & (EVTBits-1)) != 0)
Dan Gohman475871a2008-07-27 21:46:04 +00004688 return SDValue();
Evan Chengc88138f2007-03-22 01:54:19 +00004689 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004690
Chris Lattnercbf68df2010-12-22 08:02:57 +00004691 // At this point, we must have a load or else we can't do the transform.
4692 if (!isa<LoadSDNode>(N0)) return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00004693
Chris Lattner2831a192010-10-01 05:36:09 +00004694 // If the shift amount is larger than the input type then we're not
4695 // accessing any of the loaded bytes. If the load was a zextload/extload
4696 // then the result of the shift+trunc is zero/undef (handled elsewhere).
4697 // If the load was a sextload then the result is a splat of the sign bit
4698 // of the extended byte. This is not worth optimizing for.
Chris Lattnercbf68df2010-12-22 08:02:57 +00004699 if (ShAmt >= cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits())
Chris Lattner2831a192010-10-01 05:36:09 +00004700 return SDValue();
Evan Chengc88138f2007-03-22 01:54:19 +00004701 }
4702 }
4703
Dan Gohman394d6292010-11-03 01:47:46 +00004704 // If the load is shifted left (and the result isn't shifted back right),
4705 // we can fold the truncate through the shift.
4706 unsigned ShLeftAmt = 0;
4707 if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
Chris Lattner4c32bc22010-12-22 07:36:50 +00004708 ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) {
Dan Gohman394d6292010-11-03 01:47:46 +00004709 if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4710 ShLeftAmt = N01->getZExtValue();
4711 N0 = N0.getOperand(0);
4712 }
4713 }
Owen Anderson95771af2011-02-25 21:41:48 +00004714
Chris Lattner4c32bc22010-12-22 07:36:50 +00004715 // If we haven't found a load, we can't narrow it. Don't transform one with
4716 // multiple uses, this would require adding a new load.
4717 if (!isa<LoadSDNode>(N0) || !N0.hasOneUse() ||
4718 // Don't change the width of a volatile load.
4719 cast<LoadSDNode>(N0)->isVolatile())
4720 return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00004721
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00004722 // Verify that we are actually reducing a load width here.
4723 if (cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits() < EVTBits)
Chris Lattner4c32bc22010-12-22 07:36:50 +00004724 return SDValue();
Owen Anderson95771af2011-02-25 21:41:48 +00004725
Chris Lattner4c32bc22010-12-22 07:36:50 +00004726 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
4727 EVT PtrType = N0.getOperand(1).getValueType();
Bill Wendling8509c902009-01-30 22:33:24 +00004728
Chris Lattner4c32bc22010-12-22 07:36:50 +00004729 // For big endian targets, we need to adjust the offset to the pointer to
4730 // load the correct bytes.
4731 if (TLI.isBigEndian()) {
4732 unsigned LVTStoreBits = LN0->getMemoryVT().getStoreSizeInBits();
4733 unsigned EVTStoreBits = ExtVT.getStoreSizeInBits();
4734 ShAmt = LVTStoreBits - EVTStoreBits - ShAmt;
Evan Chengc88138f2007-03-22 01:54:19 +00004735 }
4736
Chris Lattner4c32bc22010-12-22 07:36:50 +00004737 uint64_t PtrOff = ShAmt / 8;
4738 unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff);
4739 SDValue NewPtr = DAG.getNode(ISD::ADD, LN0->getDebugLoc(),
4740 PtrType, LN0->getBasePtr(),
4741 DAG.getConstant(PtrOff, PtrType));
4742 AddToWorkList(NewPtr.getNode());
4743
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00004744 SDValue Load;
4745 if (ExtType == ISD::NON_EXTLOAD)
4746 Load = DAG.getLoad(VT, N0.getDebugLoc(), LN0->getChain(), NewPtr,
4747 LN0->getPointerInfo().getWithOffset(PtrOff),
Pete Cooperd752e0f2011-11-08 18:42:53 +00004748 LN0->isVolatile(), LN0->isNonTemporal(),
4749 LN0->isInvariant(), NewAlign);
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00004750 else
Stuart Hastingsa9011292011-02-16 16:23:55 +00004751 Load = DAG.getExtLoad(ExtType, N0.getDebugLoc(), VT, LN0->getChain(),NewPtr,
Chris Lattner7a2a7fa2010-12-22 08:01:44 +00004752 LN0->getPointerInfo().getWithOffset(PtrOff),
4753 ExtVT, LN0->isVolatile(), LN0->isNonTemporal(),
4754 NewAlign);
Chris Lattner4c32bc22010-12-22 07:36:50 +00004755
4756 // Replace the old load's chain with the new load's chain.
4757 WorkListRemover DeadNodes(*this);
4758 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1),
4759 &DeadNodes);
4760
4761 // Shift the result left, if we've swallowed a left shift.
4762 SDValue Result = Load;
4763 if (ShLeftAmt != 0) {
Owen Anderson95771af2011-02-25 21:41:48 +00004764 EVT ShImmTy = getShiftAmountTy(Result.getValueType());
Chris Lattner4c32bc22010-12-22 07:36:50 +00004765 if (!isUIntN(ShImmTy.getSizeInBits(), ShLeftAmt))
4766 ShImmTy = VT;
4767 Result = DAG.getNode(ISD::SHL, N0.getDebugLoc(), VT,
4768 Result, DAG.getConstant(ShLeftAmt, ShImmTy));
4769 }
4770
4771 // Return the new loaded value.
4772 return Result;
Evan Chengc88138f2007-03-22 01:54:19 +00004773}
4774
Dan Gohman475871a2008-07-27 21:46:04 +00004775SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
4776 SDValue N0 = N->getOperand(0);
4777 SDValue N1 = N->getOperand(1);
Owen Andersone50ed302009-08-10 22:56:29 +00004778 EVT VT = N->getValueType(0);
4779 EVT EVT = cast<VTSDNode>(N1)->getVT();
Dan Gohman87862e72009-12-11 21:31:27 +00004780 unsigned VTBits = VT.getScalarType().getSizeInBits();
Dan Gohmand1996362010-01-09 02:13:55 +00004781 unsigned EVTBits = EVT.getScalarType().getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00004782
Nate Begeman1d4d4142005-09-01 00:19:25 +00004783 // fold (sext_in_reg c1) -> c1
Chris Lattnereaeda562006-05-08 20:59:41 +00004784 if (isa<ConstantSDNode>(N0) || N0.getOpcode() == ISD::UNDEF)
Bill Wendling8509c902009-01-30 22:33:24 +00004785 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00004786
Chris Lattner541a24f2006-05-06 22:43:44 +00004787 // If the input is already sign extended, just drop the extension.
Dan Gohman87862e72009-12-11 21:31:27 +00004788 if (DAG.ComputeNumSignBits(N0) >= VTBits-EVTBits+1)
Chris Lattneree4ea922006-05-06 09:30:03 +00004789 return N0;
Scott Michelfdc40a02009-02-17 22:15:04 +00004790
Nate Begeman646d7e22005-09-02 21:18:40 +00004791 // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
4792 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
Duncan Sands8e4eb092008-06-08 20:54:56 +00004793 EVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT())) {
Bill Wendling8509c902009-01-30 22:33:24 +00004794 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT,
4795 N0.getOperand(0), N1);
Nate Begeman646d7e22005-09-02 21:18:40 +00004796 }
Chris Lattner4b37e872006-05-08 21:18:59 +00004797
Dan Gohman75dcf082008-07-31 00:50:31 +00004798 // fold (sext_in_reg (sext x)) -> (sext x)
4799 // fold (sext_in_reg (aext x)) -> (sext x)
4800 // if x is small enough.
4801 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
4802 SDValue N00 = N0.getOperand(0);
Evan Cheng003d7c42010-04-16 22:26:19 +00004803 if (N00.getValueType().getScalarType().getSizeInBits() <= EVTBits &&
4804 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
Bill Wendling8509c902009-01-30 22:33:24 +00004805 return DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, N00, N1);
Dan Gohman75dcf082008-07-31 00:50:31 +00004806 }
4807
Chris Lattner95a5e052007-04-17 19:03:21 +00004808 // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero.
Dan Gohman2e68b6f2008-02-25 21:11:39 +00004809 if (DAG.MaskedValueIsZero(N0, APInt::getBitsSet(VTBits, EVTBits-1, EVTBits)))
Bill Wendlingfc4b6772009-02-01 11:19:36 +00004810 return DAG.getZeroExtendInReg(N0, N->getDebugLoc(), EVT);
Scott Michelfdc40a02009-02-17 22:15:04 +00004811
Chris Lattner95a5e052007-04-17 19:03:21 +00004812 // fold operands of sext_in_reg based on knowledge that the top bits are not
4813 // demanded.
Dan Gohman475871a2008-07-27 21:46:04 +00004814 if (SimplifyDemandedBits(SDValue(N, 0)))
4815 return SDValue(N, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00004816
Evan Chengc88138f2007-03-22 01:54:19 +00004817 // fold (sext_in_reg (load x)) -> (smaller sextload x)
4818 // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
Dan Gohman475871a2008-07-27 21:46:04 +00004819 SDValue NarrowLoad = ReduceLoadWidth(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00004820 if (NarrowLoad.getNode())
Evan Chengc88138f2007-03-22 01:54:19 +00004821 return NarrowLoad;
4822
Bill Wendling8509c902009-01-30 22:33:24 +00004823 // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24)
4824 // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible.
Chris Lattner4b37e872006-05-08 21:18:59 +00004825 // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above.
4826 if (N0.getOpcode() == ISD::SRL) {
4827 if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
Dan Gohman87862e72009-12-11 21:31:27 +00004828 if (ShAmt->getZExtValue()+EVTBits <= VTBits) {
Chris Lattner4b37e872006-05-08 21:18:59 +00004829 // We can turn this into an SRA iff the input to the SRL is already sign
4830 // extended enough.
Dan Gohmanea859be2007-06-22 14:59:07 +00004831 unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
Dan Gohman87862e72009-12-11 21:31:27 +00004832 if (VTBits-(ShAmt->getZExtValue()+EVTBits) < InSignBits)
Bill Wendling8509c902009-01-30 22:33:24 +00004833 return DAG.getNode(ISD::SRA, N->getDebugLoc(), VT,
4834 N0.getOperand(0), N0.getOperand(1));
Chris Lattner4b37e872006-05-08 21:18:59 +00004835 }
4836 }
Evan Chengc88138f2007-03-22 01:54:19 +00004837
Nate Begemanded49632005-10-13 03:11:28 +00004838 // fold (sext_inreg (extload x)) -> (sextload x)
Scott Michelfdc40a02009-02-17 22:15:04 +00004839 if (ISD::isEXTLoad(N0.getNode()) &&
Gabor Greifba36cb52008-08-28 21:40:38 +00004840 ISD::isUNINDEXEDLoad(N0.getNode()) &&
Dan Gohmanb625f2f2008-01-30 00:15:11 +00004841 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004842 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00004843 TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT))) {
Evan Cheng466685d2006-10-09 20:57:25 +00004844 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00004845 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling8509c902009-01-30 22:33:24 +00004846 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004847 LN0->getBasePtr(), LN0->getPointerInfo(),
4848 EVT,
David Greene1e559442010-02-15 17:00:31 +00004849 LN0->isVolatile(), LN0->isNonTemporal(),
4850 LN0->getAlignment());
Chris Lattnerd4771842005-12-14 19:25:30 +00004851 CombineTo(N, ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00004852 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00004853 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00004854 }
4855 // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
Gabor Greifba36cb52008-08-28 21:40:38 +00004856 if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
Evan Cheng83060c52007-03-07 08:07:03 +00004857 N0.hasOneUse() &&
Dan Gohmanb625f2f2008-01-30 00:15:11 +00004858 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004859 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00004860 TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT))) {
Evan Cheng466685d2006-10-09 20:57:25 +00004861 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00004862 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, N->getDebugLoc(), VT,
Bill Wendling8509c902009-01-30 22:33:24 +00004863 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00004864 LN0->getBasePtr(), LN0->getPointerInfo(),
4865 EVT,
David Greene1e559442010-02-15 17:00:31 +00004866 LN0->isVolatile(), LN0->isNonTemporal(),
4867 LN0->getAlignment());
Chris Lattnerd4771842005-12-14 19:25:30 +00004868 CombineTo(N, ExtLoad);
Gabor Greifba36cb52008-08-28 21:40:38 +00004869 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00004870 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Nate Begemanded49632005-10-13 03:11:28 +00004871 }
Evan Cheng9568e5c2011-06-21 06:01:08 +00004872
4873 // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
4874 if (EVTBits <= 16 && N0.getOpcode() == ISD::OR) {
4875 SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
4876 N0.getOperand(1), false);
4877 if (BSwap.getNode() != 0)
4878 return DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), VT,
4879 BSwap, N1);
4880 }
4881
Dan Gohman475871a2008-07-27 21:46:04 +00004882 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00004883}
4884
Dan Gohman475871a2008-07-27 21:46:04 +00004885SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
4886 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004887 EVT VT = N->getValueType(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00004888
4889 // noop truncate
4890 if (N0.getValueType() == N->getValueType(0))
Nate Begeman83e75ec2005-09-06 04:43:02 +00004891 return N0;
Nate Begeman1d4d4142005-09-01 00:19:25 +00004892 // fold (truncate c1) -> c1
Chris Lattner310b5782006-05-06 23:06:26 +00004893 if (isa<ConstantSDNode>(N0))
Bill Wendling67a67682009-01-30 22:44:24 +00004894 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00004895 // fold (truncate (truncate x)) -> (truncate x)
4896 if (N0.getOpcode() == ISD::TRUNCATE)
Bill Wendling67a67682009-01-30 22:44:24 +00004897 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, N0.getOperand(0));
Nate Begeman1d4d4142005-09-01 00:19:25 +00004898 // fold (truncate (ext x)) -> (ext x) or (truncate x) or x
Chris Lattner7f893c02010-04-07 18:13:33 +00004899 if (N0.getOpcode() == ISD::ZERO_EXTEND ||
4900 N0.getOpcode() == ISD::SIGN_EXTEND ||
Chris Lattnerb72773b2006-05-05 22:56:26 +00004901 N0.getOpcode() == ISD::ANY_EXTEND) {
Duncan Sands8e4eb092008-06-08 20:54:56 +00004902 if (N0.getOperand(0).getValueType().bitsLT(VT))
Nate Begeman1d4d4142005-09-01 00:19:25 +00004903 // if the source is smaller than the dest, we still need an extend
Bill Wendling67a67682009-01-30 22:44:24 +00004904 return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
4905 N0.getOperand(0));
Duncan Sands8e4eb092008-06-08 20:54:56 +00004906 else if (N0.getOperand(0).getValueType().bitsGT(VT))
Nate Begeman1d4d4142005-09-01 00:19:25 +00004907 // if the source is larger than the dest, than we just need the truncate
Bill Wendling67a67682009-01-30 22:44:24 +00004908 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, N0.getOperand(0));
Nate Begeman1d4d4142005-09-01 00:19:25 +00004909 else
4910 // if the source and dest are the same type, we can drop both the extend
Evan Chengd40d03e2010-01-06 19:38:29 +00004911 // and the truncate.
Nate Begeman83e75ec2005-09-06 04:43:02 +00004912 return N0.getOperand(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00004913 }
Evan Cheng007b69e2007-03-21 20:14:05 +00004914
Chris Lattner2b4c2792007-10-13 06:35:54 +00004915 // See if we can simplify the input to this truncate through knowledge that
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004916 // only the low bits are being used.
4917 // For example "trunc (or (shl x, 8), y)" // -> trunc y
Nadav Rotemfcd96192011-02-27 07:40:43 +00004918 // Currently we only perform this optimization on scalars because vectors
Nadav Rotem8c20ec52011-02-24 21:01:34 +00004919 // may have different active low bits.
4920 if (!VT.isVector()) {
4921 SDValue Shorter =
4922 GetDemandedBits(N0, APInt::getLowBitsSet(N0.getValueSizeInBits(),
4923 VT.getSizeInBits()));
4924 if (Shorter.getNode())
4925 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, Shorter);
4926 }
Nate Begeman3df4d522005-10-12 20:40:40 +00004927 // fold (truncate (load x)) -> (smaller load x)
Evan Cheng007b69e2007-03-21 20:14:05 +00004928 // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
Dan Gohman4e39e9d2010-06-24 14:30:44 +00004929 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) {
4930 SDValue Reduced = ReduceLoadWidth(N);
4931 if (Reduced.getNode())
4932 return Reduced;
4933 }
4934
4935 // Simplify the operands using demanded-bits information.
4936 if (!VT.isVector() &&
4937 SimplifyDemandedBits(SDValue(N, 0)))
4938 return SDValue(N, 0);
4939
Evan Chenge5b51ac2010-04-17 06:13:15 +00004940 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00004941}
4942
Evan Cheng9bfa03c2008-05-12 23:04:07 +00004943static SDNode *getBuildPairElt(SDNode *N, unsigned i) {
Dan Gohman475871a2008-07-27 21:46:04 +00004944 SDValue Elt = N->getOperand(i);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00004945 if (Elt.getOpcode() != ISD::MERGE_VALUES)
Gabor Greifba36cb52008-08-28 21:40:38 +00004946 return Elt.getNode();
4947 return Elt.getOperand(Elt.getResNo()).getNode();
Evan Cheng9bfa03c2008-05-12 23:04:07 +00004948}
4949
4950/// CombineConsecutiveLoads - build_pair (load, load) -> load
Scott Michelfdc40a02009-02-17 22:15:04 +00004951/// if load locations are consecutive.
Owen Andersone50ed302009-08-10 22:56:29 +00004952SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
Evan Cheng9bfa03c2008-05-12 23:04:07 +00004953 assert(N->getOpcode() == ISD::BUILD_PAIR);
4954
Nate Begemanabc01992009-06-05 21:37:30 +00004955 LoadSDNode *LD1 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 0));
4956 LoadSDNode *LD2 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 1));
Chris Lattnerfa459012010-09-21 16:08:50 +00004957 if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !LD1->hasOneUse() ||
4958 LD1->getPointerInfo().getAddrSpace() !=
4959 LD2->getPointerInfo().getAddrSpace())
Dan Gohman475871a2008-07-27 21:46:04 +00004960 return SDValue();
Owen Andersone50ed302009-08-10 22:56:29 +00004961 EVT LD1VT = LD1->getValueType(0);
Bill Wendling67a67682009-01-30 22:44:24 +00004962
Evan Cheng9bfa03c2008-05-12 23:04:07 +00004963 if (ISD::isNON_EXTLoad(LD2) &&
4964 LD2->hasOneUse() &&
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004965 // If both are volatile this would reduce the number of volatile loads.
4966 // If one is volatile it might be ok, but play conservative and bail out.
Nate Begemanabc01992009-06-05 21:37:30 +00004967 !LD1->isVolatile() &&
4968 !LD2->isVolatile() &&
Evan Cheng64fa4a92009-12-09 01:36:00 +00004969 DAG.isConsecutiveLoad(LD2, LD1, LD1VT.getSizeInBits()/8, 1)) {
Nate Begemanabc01992009-06-05 21:37:30 +00004970 unsigned Align = LD1->getAlignment();
Dan Gohman6448d912008-09-04 15:39:15 +00004971 unsigned NewAlign = TLI.getTargetData()->
Owen Anderson23b9b192009-08-12 00:36:31 +00004972 getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
Bill Wendling67a67682009-01-30 22:44:24 +00004973
Duncan Sandsd4b9c172008-06-13 19:07:40 +00004974 if (NewAlign <= Align &&
Duncan Sands25cf2272008-11-24 14:53:14 +00004975 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)))
Nate Begemanabc01992009-06-05 21:37:30 +00004976 return DAG.getLoad(VT, N->getDebugLoc(), LD1->getChain(),
Chris Lattnerfa459012010-09-21 16:08:50 +00004977 LD1->getBasePtr(), LD1->getPointerInfo(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00004978 false, false, false, Align);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00004979 }
Bill Wendling67a67682009-01-30 22:44:24 +00004980
Dan Gohman475871a2008-07-27 21:46:04 +00004981 return SDValue();
Evan Cheng9bfa03c2008-05-12 23:04:07 +00004982}
4983
Wesley Peckbf17cfa2010-11-23 03:31:01 +00004984SDValue DAGCombiner::visitBITCAST(SDNode *N) {
Dan Gohman475871a2008-07-27 21:46:04 +00004985 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00004986 EVT VT = N->getValueType(0);
Chris Lattner94683772005-12-23 05:30:37 +00004987
Dan Gohman7f321562007-06-25 16:23:39 +00004988 // If the input is a BUILD_VECTOR with all constant elements, fold this now.
4989 // Only do this before legalize, since afterward the target may be depending
4990 // on the bitconvert.
4991 // First check to see if this is all constant.
Duncan Sands25cf2272008-11-24 14:53:14 +00004992 if (!LegalTypes &&
Gabor Greifba36cb52008-08-28 21:40:38 +00004993 N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNode()->hasOneUse() &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00004994 VT.isVector()) {
Dan Gohman7f321562007-06-25 16:23:39 +00004995 bool isSimple = true;
4996 for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i)
4997 if (N0.getOperand(i).getOpcode() != ISD::UNDEF &&
4998 N0.getOperand(i).getOpcode() != ISD::Constant &&
4999 N0.getOperand(i).getOpcode() != ISD::ConstantFP) {
Scott Michelfdc40a02009-02-17 22:15:04 +00005000 isSimple = false;
Dan Gohman7f321562007-06-25 16:23:39 +00005001 break;
5002 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005003
Owen Andersone50ed302009-08-10 22:56:29 +00005004 EVT DestEltVT = N->getValueType(0).getVectorElementType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00005005 assert(!DestEltVT.isVector() &&
Dan Gohman7f321562007-06-25 16:23:39 +00005006 "Element type of vector ValueType must not be vector!");
Bill Wendling67a67682009-01-30 22:44:24 +00005007 if (isSimple)
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005008 return ConstantFoldBITCASTofBUILD_VECTOR(N0.getNode(), DestEltVT);
Dan Gohman7f321562007-06-25 16:23:39 +00005009 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005010
Dan Gohman3dd168d2008-09-05 01:58:21 +00005011 // If the input is a constant, let getNode fold it.
Chris Lattner94683772005-12-23 05:30:37 +00005012 if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005013 SDValue Res = DAG.getNode(ISD::BITCAST, N->getDebugLoc(), VT, N0);
Dan Gohmana407ca12009-08-10 23:15:10 +00005014 if (Res.getNode() != N) {
5015 if (!LegalOperations ||
5016 TLI.isOperationLegal(Res.getNode()->getOpcode(), VT))
5017 return Res;
5018
5019 // Folding it resulted in an illegal node, and it's too late to
5020 // do that. Clean up the old node and forego the transformation.
5021 // Ideally this won't happen very often, because instcombine
5022 // and the earlier dagcombine runs (where illegal nodes are
5023 // permitted) should have folded most of them already.
5024 DAG.DeleteNode(Res.getNode());
5025 }
Chris Lattner94683772005-12-23 05:30:37 +00005026 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005027
Bill Wendling67a67682009-01-30 22:44:24 +00005028 // (conv (conv x, t1), t2) -> (conv x, t2)
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005029 if (N0.getOpcode() == ISD::BITCAST)
5030 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005031 N0.getOperand(0));
Chris Lattner6258fb22006-04-02 02:53:43 +00005032
Chris Lattner57104102005-12-23 05:44:41 +00005033 // fold (conv (load x)) -> (load (conv*)x)
Evan Cheng513da432007-10-06 08:19:55 +00005034 // If the resultant load doesn't need a higher alignment than the original!
Gabor Greifba36cb52008-08-28 21:40:38 +00005035 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
Duncan Sandsd4b9c172008-06-13 19:07:40 +00005036 // Do not change the width of a volatile load.
5037 !cast<LoadSDNode>(N0)->isVolatile() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00005038 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT))) {
Evan Cheng466685d2006-10-09 20:57:25 +00005039 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Dan Gohman6448d912008-09-04 15:39:15 +00005040 unsigned Align = TLI.getTargetData()->
Owen Anderson23b9b192009-08-12 00:36:31 +00005041 getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
Evan Cheng59d5b682007-05-07 21:27:48 +00005042 unsigned OrigAlign = LN0->getAlignment();
Bill Wendling67a67682009-01-30 22:44:24 +00005043
Evan Cheng59d5b682007-05-07 21:27:48 +00005044 if (Align <= OrigAlign) {
Bill Wendling67a67682009-01-30 22:44:24 +00005045 SDValue Load = DAG.getLoad(VT, N->getDebugLoc(), LN0->getChain(),
Chris Lattnerfa459012010-09-21 16:08:50 +00005046 LN0->getBasePtr(), LN0->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00005047 LN0->isVolatile(), LN0->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00005048 LN0->isInvariant(), OrigAlign);
Evan Cheng59d5b682007-05-07 21:27:48 +00005049 AddToWorkList(N);
Gabor Greif12632d22008-08-30 19:29:20 +00005050 CombineTo(N0.getNode(),
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005051 DAG.getNode(ISD::BITCAST, N0.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005052 N0.getValueType(), Load),
Evan Cheng59d5b682007-05-07 21:27:48 +00005053 Load.getValue(1));
5054 return Load;
5055 }
Chris Lattner57104102005-12-23 05:44:41 +00005056 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00005057
Bill Wendling67a67682009-01-30 22:44:24 +00005058 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
5059 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
Chris Lattner3bd39d42008-01-27 17:42:27 +00005060 // This often reduces constant pool loads.
5061 if ((N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FABS) &&
Gabor Greifba36cb52008-08-28 21:40:38 +00005062 N0.getNode()->hasOneUse() && VT.isInteger() && !VT.isVector()) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005063 SDValue NewConv = DAG.getNode(ISD::BITCAST, N0.getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005064 N0.getOperand(0));
Gabor Greifba36cb52008-08-28 21:40:38 +00005065 AddToWorkList(NewConv.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00005066
Duncan Sands83ec4b62008-06-06 12:08:01 +00005067 APInt SignBit = APInt::getSignBit(VT.getSizeInBits());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005068 if (N0.getOpcode() == ISD::FNEG)
Bill Wendling67a67682009-01-30 22:44:24 +00005069 return DAG.getNode(ISD::XOR, N->getDebugLoc(), VT,
5070 NewConv, DAG.getConstant(SignBit, VT));
Chris Lattner3bd39d42008-01-27 17:42:27 +00005071 assert(N0.getOpcode() == ISD::FABS);
Bill Wendling67a67682009-01-30 22:44:24 +00005072 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
5073 NewConv, DAG.getConstant(~SignBit, VT));
Chris Lattner3bd39d42008-01-27 17:42:27 +00005074 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005075
Bill Wendling67a67682009-01-30 22:44:24 +00005076 // fold (bitconvert (fcopysign cst, x)) ->
5077 // (or (and (bitconvert x), sign), (and cst, (not sign)))
5078 // Note that we don't handle (copysign x, cst) because this can always be
5079 // folded to an fneg or fabs.
Gabor Greifba36cb52008-08-28 21:40:38 +00005080 if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse() &&
Chris Lattnerf32aac32008-01-27 23:32:17 +00005081 isa<ConstantFPSDNode>(N0.getOperand(0)) &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00005082 VT.isInteger() && !VT.isVector()) {
5083 unsigned OrigXWidth = N0.getOperand(1).getValueType().getSizeInBits();
Owen Anderson23b9b192009-08-12 00:36:31 +00005084 EVT IntXVT = EVT::getIntegerVT(*DAG.getContext(), OrigXWidth);
Chris Lattner2392ae72010-04-15 04:48:01 +00005085 if (isTypeLegal(IntXVT)) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005086 SDValue X = DAG.getNode(ISD::BITCAST, N0.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005087 IntXVT, N0.getOperand(1));
Duncan Sands25cf2272008-11-24 14:53:14 +00005088 AddToWorkList(X.getNode());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005089
Duncan Sands25cf2272008-11-24 14:53:14 +00005090 // If X has a different width than the result/lhs, sext it or truncate it.
5091 unsigned VTWidth = VT.getSizeInBits();
5092 if (OrigXWidth < VTWidth) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00005093 X = DAG.getNode(ISD::SIGN_EXTEND, N->getDebugLoc(), VT, X);
Duncan Sands25cf2272008-11-24 14:53:14 +00005094 AddToWorkList(X.getNode());
5095 } else if (OrigXWidth > VTWidth) {
5096 // To get the sign bit in the right place, we have to shift it right
5097 // before truncating.
Bill Wendling9729c5a2009-01-31 03:12:48 +00005098 X = DAG.getNode(ISD::SRL, X.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005099 X.getValueType(), X,
Duncan Sands25cf2272008-11-24 14:53:14 +00005100 DAG.getConstant(OrigXWidth-VTWidth, X.getValueType()));
5101 AddToWorkList(X.getNode());
Bill Wendling9729c5a2009-01-31 03:12:48 +00005102 X = DAG.getNode(ISD::TRUNCATE, X.getDebugLoc(), VT, X);
Duncan Sands25cf2272008-11-24 14:53:14 +00005103 AddToWorkList(X.getNode());
5104 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005105
Duncan Sands25cf2272008-11-24 14:53:14 +00005106 APInt SignBit = APInt::getSignBit(VT.getSizeInBits());
Bill Wendling9729c5a2009-01-31 03:12:48 +00005107 X = DAG.getNode(ISD::AND, X.getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005108 X, DAG.getConstant(SignBit, VT));
Duncan Sands25cf2272008-11-24 14:53:14 +00005109 AddToWorkList(X.getNode());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005110
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005111 SDValue Cst = DAG.getNode(ISD::BITCAST, N0.getDebugLoc(),
Bill Wendling67a67682009-01-30 22:44:24 +00005112 VT, N0.getOperand(0));
Bill Wendling9729c5a2009-01-31 03:12:48 +00005113 Cst = DAG.getNode(ISD::AND, Cst.getDebugLoc(), VT,
Bill Wendling67a67682009-01-30 22:44:24 +00005114 Cst, DAG.getConstant(~SignBit, VT));
Duncan Sands25cf2272008-11-24 14:53:14 +00005115 AddToWorkList(Cst.getNode());
Chris Lattner3bd39d42008-01-27 17:42:27 +00005116
Bill Wendling67a67682009-01-30 22:44:24 +00005117 return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, X, Cst);
Duncan Sands25cf2272008-11-24 14:53:14 +00005118 }
Chris Lattner3bd39d42008-01-27 17:42:27 +00005119 }
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005120
Scott Michelfdc40a02009-02-17 22:15:04 +00005121 // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive.
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005122 if (N0.getOpcode() == ISD::BUILD_PAIR) {
Gabor Greifba36cb52008-08-28 21:40:38 +00005123 SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT);
5124 if (CombineLD.getNode())
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005125 return CombineLD;
5126 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005127
Dan Gohman475871a2008-07-27 21:46:04 +00005128 return SDValue();
Chris Lattner94683772005-12-23 05:30:37 +00005129}
5130
Dan Gohman475871a2008-07-27 21:46:04 +00005131SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
Owen Andersone50ed302009-08-10 22:56:29 +00005132 EVT VT = N->getValueType(0);
Evan Cheng9bfa03c2008-05-12 23:04:07 +00005133 return CombineConsecutiveLoads(N, VT);
5134}
5135
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005136/// ConstantFoldBITCASTofBUILD_VECTOR - We know that BV is a build_vector
Scott Michelfdc40a02009-02-17 22:15:04 +00005137/// node with Constant, ConstantFP or Undef operands. DstEltVT indicates the
Chris Lattner6258fb22006-04-02 02:53:43 +00005138/// destination element value type.
Dan Gohman475871a2008-07-27 21:46:04 +00005139SDValue DAGCombiner::
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005140ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
Owen Andersone50ed302009-08-10 22:56:29 +00005141 EVT SrcEltVT = BV->getValueType(0).getVectorElementType();
Scott Michelfdc40a02009-02-17 22:15:04 +00005142
Chris Lattner6258fb22006-04-02 02:53:43 +00005143 // If this is already the right type, we're done.
Dan Gohman475871a2008-07-27 21:46:04 +00005144 if (SrcEltVT == DstEltVT) return SDValue(BV, 0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005145
Duncan Sands83ec4b62008-06-06 12:08:01 +00005146 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
5147 unsigned DstBitSize = DstEltVT.getSizeInBits();
Scott Michelfdc40a02009-02-17 22:15:04 +00005148
Chris Lattner6258fb22006-04-02 02:53:43 +00005149 // If this is a conversion of N elements of one type to N elements of another
5150 // type, convert each element. This handles FP<->INT cases.
5151 if (SrcBitSize == DstBitSize) {
Nate Begemane0efc212010-07-27 18:02:18 +00005152 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
5153 BV->getValueType(0).getVectorNumElements());
5154
5155 // Due to the FP element handling below calling this routine recursively,
5156 // we can end up with a scalar-to-vector node here.
5157 if (BV->getOpcode() == ISD::SCALAR_TO_VECTOR)
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005158 return DAG.getNode(ISD::SCALAR_TO_VECTOR, BV->getDebugLoc(), VT,
5159 DAG.getNode(ISD::BITCAST, BV->getDebugLoc(),
Nate Begemane0efc212010-07-27 18:02:18 +00005160 DstEltVT, BV->getOperand(0)));
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005161
Dan Gohman475871a2008-07-27 21:46:04 +00005162 SmallVector<SDValue, 8> Ops;
Dan Gohman7f321562007-06-25 16:23:39 +00005163 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
Bob Wilsonb1303d02009-04-13 22:05:19 +00005164 SDValue Op = BV->getOperand(i);
5165 // If the vector element type is not legal, the BUILD_VECTOR operands
5166 // are promoted and implicitly truncated. Make that explicit here.
Bob Wilsonc8851652009-04-20 17:27:09 +00005167 if (Op.getValueType() != SrcEltVT)
5168 Op = DAG.getNode(ISD::TRUNCATE, BV->getDebugLoc(), SrcEltVT, Op);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005169 Ops.push_back(DAG.getNode(ISD::BITCAST, BV->getDebugLoc(),
Bob Wilsonb1303d02009-04-13 22:05:19 +00005170 DstEltVT, Op));
Gabor Greifba36cb52008-08-28 21:40:38 +00005171 AddToWorkList(Ops.back().getNode());
Chris Lattner3e104b12006-04-08 04:15:24 +00005172 }
Evan Chenga87008d2009-02-25 22:49:59 +00005173 return DAG.getNode(ISD::BUILD_VECTOR, BV->getDebugLoc(), VT,
5174 &Ops[0], Ops.size());
Chris Lattner6258fb22006-04-02 02:53:43 +00005175 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005176
Chris Lattner6258fb22006-04-02 02:53:43 +00005177 // Otherwise, we're growing or shrinking the elements. To avoid having to
5178 // handle annoying details of growing/shrinking FP values, we convert them to
5179 // int first.
Duncan Sands83ec4b62008-06-06 12:08:01 +00005180 if (SrcEltVT.isFloatingPoint()) {
Chris Lattner6258fb22006-04-02 02:53:43 +00005181 // Convert the input float vector to a int vector where the elements are the
5182 // same sizes.
Owen Anderson825b72b2009-08-11 20:47:22 +00005183 assert((SrcEltVT == MVT::f32 || SrcEltVT == MVT::f64) && "Unknown FP VT!");
Owen Anderson23b9b192009-08-12 00:36:31 +00005184 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltVT.getSizeInBits());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005185 BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
Chris Lattner6258fb22006-04-02 02:53:43 +00005186 SrcEltVT = IntVT;
5187 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005188
Chris Lattner6258fb22006-04-02 02:53:43 +00005189 // Now we know the input is an integer vector. If the output is a FP type,
5190 // convert to integer first, then to FP of the right size.
Duncan Sands83ec4b62008-06-06 12:08:01 +00005191 if (DstEltVT.isFloatingPoint()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00005192 assert((DstEltVT == MVT::f32 || DstEltVT == MVT::f64) && "Unknown FP VT!");
Owen Anderson23b9b192009-08-12 00:36:31 +00005193 EVT TmpVT = EVT::getIntegerVT(*DAG.getContext(), DstEltVT.getSizeInBits());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005194 SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
Scott Michelfdc40a02009-02-17 22:15:04 +00005195
Chris Lattner6258fb22006-04-02 02:53:43 +00005196 // Next, convert to FP elements of the same size.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005197 return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
Chris Lattner6258fb22006-04-02 02:53:43 +00005198 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005199
Chris Lattner6258fb22006-04-02 02:53:43 +00005200 // Okay, we know the src/dst types are both integers of differing types.
5201 // Handling growing first.
Duncan Sands83ec4b62008-06-06 12:08:01 +00005202 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
Chris Lattner6258fb22006-04-02 02:53:43 +00005203 if (SrcBitSize < DstBitSize) {
5204 unsigned NumInputsPerOutput = DstBitSize/SrcBitSize;
Scott Michelfdc40a02009-02-17 22:15:04 +00005205
Dan Gohman475871a2008-07-27 21:46:04 +00005206 SmallVector<SDValue, 8> Ops;
Dan Gohman7f321562007-06-25 16:23:39 +00005207 for (unsigned i = 0, e = BV->getNumOperands(); i != e;
Chris Lattner6258fb22006-04-02 02:53:43 +00005208 i += NumInputsPerOutput) {
5209 bool isLE = TLI.isLittleEndian();
Dan Gohman220a8232008-03-03 23:51:38 +00005210 APInt NewBits = APInt(DstBitSize, 0);
Chris Lattner6258fb22006-04-02 02:53:43 +00005211 bool EltIsUndef = true;
5212 for (unsigned j = 0; j != NumInputsPerOutput; ++j) {
5213 // Shift the previously computed bits over.
5214 NewBits <<= SrcBitSize;
Dan Gohman475871a2008-07-27 21:46:04 +00005215 SDValue Op = BV->getOperand(i+ (isLE ? (NumInputsPerOutput-j-1) : j));
Chris Lattner6258fb22006-04-02 02:53:43 +00005216 if (Op.getOpcode() == ISD::UNDEF) continue;
5217 EltIsUndef = false;
Scott Michelfdc40a02009-02-17 22:15:04 +00005218
Jay Foad40f8f622010-12-07 08:25:19 +00005219 NewBits |= cast<ConstantSDNode>(Op)->getAPIntValue().
Dan Gohman58c25872010-04-12 02:24:01 +00005220 zextOrTrunc(SrcBitSize).zext(DstBitSize);
Chris Lattner6258fb22006-04-02 02:53:43 +00005221 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005222
Chris Lattner6258fb22006-04-02 02:53:43 +00005223 if (EltIsUndef)
Dale Johannesene8d72302009-02-06 23:05:02 +00005224 Ops.push_back(DAG.getUNDEF(DstEltVT));
Chris Lattner6258fb22006-04-02 02:53:43 +00005225 else
5226 Ops.push_back(DAG.getConstant(NewBits, DstEltVT));
5227 }
5228
Owen Anderson23b9b192009-08-12 00:36:31 +00005229 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size());
Evan Chenga87008d2009-02-25 22:49:59 +00005230 return DAG.getNode(ISD::BUILD_VECTOR, BV->getDebugLoc(), VT,
5231 &Ops[0], Ops.size());
Chris Lattner6258fb22006-04-02 02:53:43 +00005232 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005233
Chris Lattner6258fb22006-04-02 02:53:43 +00005234 // Finally, this must be the case where we are shrinking elements: each input
5235 // turns into multiple outputs.
Evan Chengefec7512008-02-18 23:04:32 +00005236 bool isS2V = ISD::isScalarToVector(BV);
Chris Lattner6258fb22006-04-02 02:53:43 +00005237 unsigned NumOutputsPerInput = SrcBitSize/DstBitSize;
Owen Anderson23b9b192009-08-12 00:36:31 +00005238 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
5239 NumOutputsPerInput*BV->getNumOperands());
Dan Gohman475871a2008-07-27 21:46:04 +00005240 SmallVector<SDValue, 8> Ops;
Bill Wendlingb0162f52009-01-30 22:53:48 +00005241
Dan Gohman7f321562007-06-25 16:23:39 +00005242 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
Chris Lattner6258fb22006-04-02 02:53:43 +00005243 if (BV->getOperand(i).getOpcode() == ISD::UNDEF) {
5244 for (unsigned j = 0; j != NumOutputsPerInput; ++j)
Dale Johannesene8d72302009-02-06 23:05:02 +00005245 Ops.push_back(DAG.getUNDEF(DstEltVT));
Chris Lattner6258fb22006-04-02 02:53:43 +00005246 continue;
5247 }
Bill Wendlingb0162f52009-01-30 22:53:48 +00005248
Jay Foad40f8f622010-12-07 08:25:19 +00005249 APInt OpVal = cast<ConstantSDNode>(BV->getOperand(i))->
5250 getAPIntValue().zextOrTrunc(SrcBitSize);
Bill Wendlingb0162f52009-01-30 22:53:48 +00005251
Chris Lattner6258fb22006-04-02 02:53:43 +00005252 for (unsigned j = 0; j != NumOutputsPerInput; ++j) {
Jay Foad40f8f622010-12-07 08:25:19 +00005253 APInt ThisVal = OpVal.trunc(DstBitSize);
Chris Lattner6258fb22006-04-02 02:53:43 +00005254 Ops.push_back(DAG.getConstant(ThisVal, DstEltVT));
Jay Foad40f8f622010-12-07 08:25:19 +00005255 if (isS2V && i == 0 && j == 0 && ThisVal.zext(SrcBitSize) == OpVal)
Evan Chengefec7512008-02-18 23:04:32 +00005256 // Simply turn this into a SCALAR_TO_VECTOR of the new type.
Bill Wendlingb0162f52009-01-30 22:53:48 +00005257 return DAG.getNode(ISD::SCALAR_TO_VECTOR, BV->getDebugLoc(), VT,
5258 Ops[0]);
Dan Gohman220a8232008-03-03 23:51:38 +00005259 OpVal = OpVal.lshr(DstBitSize);
Chris Lattner6258fb22006-04-02 02:53:43 +00005260 }
5261
5262 // For big endian targets, swap the order of the pieces of each element.
Duncan Sands0753fc12008-02-11 10:37:04 +00005263 if (TLI.isBigEndian())
Chris Lattner6258fb22006-04-02 02:53:43 +00005264 std::reverse(Ops.end()-NumOutputsPerInput, Ops.end());
5265 }
Bill Wendlingb0162f52009-01-30 22:53:48 +00005266
Evan Chenga87008d2009-02-25 22:49:59 +00005267 return DAG.getNode(ISD::BUILD_VECTOR, BV->getDebugLoc(), VT,
5268 &Ops[0], Ops.size());
Chris Lattner6258fb22006-04-02 02:53:43 +00005269}
5270
Dan Gohman475871a2008-07-27 21:46:04 +00005271SDValue DAGCombiner::visitFADD(SDNode *N) {
5272 SDValue N0 = N->getOperand(0);
5273 SDValue N1 = N->getOperand(1);
Nate Begemana0e221d2005-10-18 00:28:13 +00005274 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5275 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005276 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005277
Dan Gohman7f321562007-06-25 16:23:39 +00005278 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00005279 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005280 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005281 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00005282 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005283
Bill Wendlingb0162f52009-01-30 22:53:48 +00005284 // fold (fadd c1, c2) -> (fadd c1, c2)
Owen Anderson825b72b2009-08-11 20:47:22 +00005285 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005286 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0, N1);
Nate Begemana0e221d2005-10-18 00:28:13 +00005287 // canonicalize constant to RHS
5288 if (N0CFP && !N1CFP)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005289 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N1, N0);
5290 // fold (fadd A, 0) -> A
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005291 if (DAG.getTarget().Options.UnsafeFPMath && N1CFP &&
5292 N1CFP->getValueAPF().isZero())
Dan Gohman760f86f2009-01-22 21:58:43 +00005293 return N0;
Bill Wendlingb0162f52009-01-30 22:53:48 +00005294 // fold (fadd A, (fneg B)) -> (fsub A, B)
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005295 if (isNegatibleForFree(N1, LegalOperations, &DAG.getTarget().Options) == 2)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005296 return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N0,
Duncan Sands25cf2272008-11-24 14:53:14 +00005297 GetNegatedExpression(N1, DAG, LegalOperations));
Bill Wendlingb0162f52009-01-30 22:53:48 +00005298 // fold (fadd (fneg A), B) -> (fsub B, A)
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005299 if (isNegatibleForFree(N0, LegalOperations, &DAG.getTarget().Options) == 2)
Bill Wendlingb0162f52009-01-30 22:53:48 +00005300 return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N1,
Duncan Sands25cf2272008-11-24 14:53:14 +00005301 GetNegatedExpression(N0, DAG, LegalOperations));
Scott Michelfdc40a02009-02-17 22:15:04 +00005302
Chris Lattnerddae4bd2007-01-08 23:04:05 +00005303 // If allowed, fold (fadd (fadd x, c1), c2) -> (fadd x, (fadd c1, c2))
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005304 if (DAG.getTarget().Options.UnsafeFPMath && N1CFP &&
5305 N0.getOpcode() == ISD::FADD && N0.getNode()->hasOneUse() &&
5306 isa<ConstantFPSDNode>(N0.getOperand(1)))
Bill Wendlingb0162f52009-01-30 22:53:48 +00005307 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0.getOperand(0),
Bill Wendlingfc4b6772009-02-01 11:19:36 +00005308 DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
5309 N0.getOperand(1), N1));
Scott Michelfdc40a02009-02-17 22:15:04 +00005310
Dan Gohman475871a2008-07-27 21:46:04 +00005311 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005312}
5313
Dan Gohman475871a2008-07-27 21:46:04 +00005314SDValue DAGCombiner::visitFSUB(SDNode *N) {
5315 SDValue N0 = N->getOperand(0);
5316 SDValue N1 = N->getOperand(1);
Nate Begemana0e221d2005-10-18 00:28:13 +00005317 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5318 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005319 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005320
Dan Gohman7f321562007-06-25 16:23:39 +00005321 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00005322 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005323 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005324 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00005325 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005326
Nate Begemana0e221d2005-10-18 00:28:13 +00005327 // fold (fsub c1, c2) -> c1-c2
Owen Anderson825b72b2009-08-11 20:47:22 +00005328 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlingfc4b6772009-02-01 11:19:36 +00005329 return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N0, N1);
Bill Wendlingb0162f52009-01-30 22:53:48 +00005330 // fold (fsub A, 0) -> A
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005331 if (DAG.getTarget().Options.UnsafeFPMath &&
5332 N1CFP && N1CFP->getValueAPF().isZero())
Dan Gohmana90c8e62009-01-23 19:10:37 +00005333 return N0;
Bill Wendlingb0162f52009-01-30 22:53:48 +00005334 // fold (fsub 0, B) -> -B
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005335 if (DAG.getTarget().Options.UnsafeFPMath &&
5336 N0CFP && N0CFP->getValueAPF().isZero()) {
5337 if (isNegatibleForFree(N1, LegalOperations, &DAG.getTarget().Options))
Duncan Sands25cf2272008-11-24 14:53:14 +00005338 return GetNegatedExpression(N1, DAG, LegalOperations);
Dan Gohman760f86f2009-01-22 21:58:43 +00005339 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
Bill Wendlingb0162f52009-01-30 22:53:48 +00005340 return DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT, N1);
Dan Gohman23ff1822007-07-02 15:48:56 +00005341 }
Bill Wendlingb0162f52009-01-30 22:53:48 +00005342 // fold (fsub A, (fneg B)) -> (fadd A, B)
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005343 if (isNegatibleForFree(N1, LegalOperations, &DAG.getTarget().Options))
Bill Wendlingb0162f52009-01-30 22:53:48 +00005344 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0,
Duncan Sands25cf2272008-11-24 14:53:14 +00005345 GetNegatedExpression(N1, DAG, LegalOperations));
Scott Michelfdc40a02009-02-17 22:15:04 +00005346
Dan Gohman475871a2008-07-27 21:46:04 +00005347 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005348}
5349
Dan Gohman475871a2008-07-27 21:46:04 +00005350SDValue DAGCombiner::visitFMUL(SDNode *N) {
5351 SDValue N0 = N->getOperand(0);
5352 SDValue N1 = N->getOperand(1);
Nate Begeman11af4ea2005-10-17 20:40:11 +00005353 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5354 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005355 EVT VT = N->getValueType(0);
Chris Lattner01b3d732005-09-28 22:28:18 +00005356
Dan Gohman7f321562007-06-25 16:23:39 +00005357 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00005358 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005359 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005360 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00005361 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005362
Nate Begeman11af4ea2005-10-17 20:40:11 +00005363 // fold (fmul c1, c2) -> c1*c2
Owen Anderson825b72b2009-08-11 20:47:22 +00005364 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005365 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N0, N1);
Nate Begeman11af4ea2005-10-17 20:40:11 +00005366 // canonicalize constant to RHS
Nate Begemana0e221d2005-10-18 00:28:13 +00005367 if (N0CFP && !N1CFP)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005368 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N1, N0);
5369 // fold (fmul A, 0) -> 0
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005370 if (DAG.getTarget().Options.UnsafeFPMath &&
5371 N1CFP && N1CFP->getValueAPF().isZero())
Dan Gohman760f86f2009-01-22 21:58:43 +00005372 return N1;
Dan Gohman77b81fe2009-06-04 17:12:12 +00005373 // fold (fmul A, 0) -> 0, vector edition.
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005374 if (DAG.getTarget().Options.UnsafeFPMath &&
5375 ISD::isBuildVectorAllZeros(N1.getNode()))
Dan Gohman77b81fe2009-06-04 17:12:12 +00005376 return N1;
Nate Begeman11af4ea2005-10-17 20:40:11 +00005377 // fold (fmul X, 2.0) -> (fadd X, X)
5378 if (N1CFP && N1CFP->isExactlyValue(+2.0))
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005379 return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0, N0);
Dan Gohmaneb1fedc2009-08-10 16:50:32 +00005380 // fold (fmul X, -1.0) -> (fneg X)
Chris Lattner29446522007-05-14 22:04:50 +00005381 if (N1CFP && N1CFP->isExactlyValue(-1.0))
Dan Gohman760f86f2009-01-22 21:58:43 +00005382 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005383 return DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005384
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005385 // fold (fmul (fneg X), (fneg Y)) -> (fmul X, Y)
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005386 if (char LHSNeg = isNegatibleForFree(N0, LegalOperations,
5387 &DAG.getTarget().Options)) {
5388 if (char RHSNeg = isNegatibleForFree(N1, LegalOperations,
5389 &DAG.getTarget().Options)) {
Chris Lattner29446522007-05-14 22:04:50 +00005390 // Both can be negated for free, check to see if at least one is cheaper
5391 // negated.
5392 if (LHSNeg == 2 || RHSNeg == 2)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005393 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
Duncan Sands25cf2272008-11-24 14:53:14 +00005394 GetNegatedExpression(N0, DAG, LegalOperations),
5395 GetNegatedExpression(N1, DAG, LegalOperations));
Chris Lattner29446522007-05-14 22:04:50 +00005396 }
5397 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005398
Chris Lattnerddae4bd2007-01-08 23:04:05 +00005399 // If allowed, fold (fmul (fmul x, c1), c2) -> (fmul x, (fmul c1, c2))
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005400 if (DAG.getTarget().Options.UnsafeFPMath &&
5401 N1CFP && N0.getOpcode() == ISD::FMUL &&
Gabor Greifba36cb52008-08-28 21:40:38 +00005402 N0.getNode()->hasOneUse() && isa<ConstantFPSDNode>(N0.getOperand(1)))
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005403 return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N0.getOperand(0),
Scott Michelfdc40a02009-02-17 22:15:04 +00005404 DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
Dale Johannesende064702009-02-06 21:50:26 +00005405 N0.getOperand(1), N1));
Scott Michelfdc40a02009-02-17 22:15:04 +00005406
Dan Gohman475871a2008-07-27 21:46:04 +00005407 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005408}
5409
Dan Gohman475871a2008-07-27 21:46:04 +00005410SDValue DAGCombiner::visitFDIV(SDNode *N) {
5411 SDValue N0 = N->getOperand(0);
5412 SDValue N1 = N->getOperand(1);
Nate Begemana148d982006-01-18 22:35:16 +00005413 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5414 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005415 EVT VT = N->getValueType(0);
Chris Lattner01b3d732005-09-28 22:28:18 +00005416
Dan Gohman7f321562007-06-25 16:23:39 +00005417 // fold vector ops
Duncan Sands83ec4b62008-06-06 12:08:01 +00005418 if (VT.isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005419 SDValue FoldedVOp = SimplifyVBinOp(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00005420 if (FoldedVOp.getNode()) return FoldedVOp;
Dan Gohman05d92fe2007-07-13 20:03:40 +00005421 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005422
Nate Begemana148d982006-01-18 22:35:16 +00005423 // fold (fdiv c1, c2) -> c1/c2
Owen Anderson825b72b2009-08-11 20:47:22 +00005424 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005425 return DAG.getNode(ISD::FDIV, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00005426
5427
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005428 // (fdiv (fneg X), (fneg Y)) -> (fdiv X, Y)
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005429 if (char LHSNeg = isNegatibleForFree(N0, LegalOperations,
5430 &DAG.getTarget().Options)) {
5431 if (char RHSNeg = isNegatibleForFree(N1, LegalOperations,
5432 &DAG.getTarget().Options)) {
Chris Lattner29446522007-05-14 22:04:50 +00005433 // Both can be negated for free, check to see if at least one is cheaper
5434 // negated.
5435 if (LHSNeg == 2 || RHSNeg == 2)
Scott Michelfdc40a02009-02-17 22:15:04 +00005436 return DAG.getNode(ISD::FDIV, N->getDebugLoc(), VT,
Duncan Sands25cf2272008-11-24 14:53:14 +00005437 GetNegatedExpression(N0, DAG, LegalOperations),
5438 GetNegatedExpression(N1, DAG, LegalOperations));
Chris Lattner29446522007-05-14 22:04:50 +00005439 }
5440 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005441
Dan Gohman475871a2008-07-27 21:46:04 +00005442 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005443}
5444
Dan Gohman475871a2008-07-27 21:46:04 +00005445SDValue DAGCombiner::visitFREM(SDNode *N) {
5446 SDValue N0 = N->getOperand(0);
5447 SDValue N1 = N->getOperand(1);
Nate Begemana148d982006-01-18 22:35:16 +00005448 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5449 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005450 EVT VT = N->getValueType(0);
Chris Lattner01b3d732005-09-28 22:28:18 +00005451
Nate Begemana148d982006-01-18 22:35:16 +00005452 // fold (frem c1, c2) -> fmod(c1,c2)
Owen Anderson825b72b2009-08-11 20:47:22 +00005453 if (N0CFP && N1CFP && VT != MVT::ppcf128)
Bill Wendlinga03e74b2009-01-30 22:57:07 +00005454 return DAG.getNode(ISD::FREM, N->getDebugLoc(), VT, N0, N1);
Dan Gohman7f321562007-06-25 16:23:39 +00005455
Dan Gohman475871a2008-07-27 21:46:04 +00005456 return SDValue();
Chris Lattner01b3d732005-09-28 22:28:18 +00005457}
5458
Dan Gohman475871a2008-07-27 21:46:04 +00005459SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
5460 SDValue N0 = N->getOperand(0);
5461 SDValue N1 = N->getOperand(1);
Chris Lattner12d83032006-03-05 05:30:57 +00005462 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
5463 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
Owen Andersone50ed302009-08-10 22:56:29 +00005464 EVT VT = N->getValueType(0);
Chris Lattner12d83032006-03-05 05:30:57 +00005465
Owen Anderson825b72b2009-08-11 20:47:22 +00005466 if (N0CFP && N1CFP && VT != MVT::ppcf128) // Constant fold
Bill Wendlingfc4b6772009-02-01 11:19:36 +00005467 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00005468
Chris Lattner12d83032006-03-05 05:30:57 +00005469 if (N1CFP) {
Dale Johannesene6c17422007-08-26 01:18:27 +00005470 const APFloat& V = N1CFP->getValueAPF();
Chris Lattner12d83032006-03-05 05:30:57 +00005471 // copysign(x, c1) -> fabs(x) iff ispos(c1)
5472 // copysign(x, c1) -> fneg(fabs(x)) iff isneg(c1)
Dan Gohman760f86f2009-01-22 21:58:43 +00005473 if (!V.isNegative()) {
5474 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005475 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0);
Dan Gohman760f86f2009-01-22 21:58:43 +00005476 } else {
5477 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005478 return DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT,
Bill Wendling9729c5a2009-01-31 03:12:48 +00005479 DAG.getNode(ISD::FABS, N0.getDebugLoc(), VT, N0));
Dan Gohman760f86f2009-01-22 21:58:43 +00005480 }
Chris Lattner12d83032006-03-05 05:30:57 +00005481 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005482
Chris Lattner12d83032006-03-05 05:30:57 +00005483 // copysign(fabs(x), y) -> copysign(x, y)
5484 // copysign(fneg(x), y) -> copysign(x, y)
5485 // copysign(copysign(x,z), y) -> copysign(x, y)
5486 if (N0.getOpcode() == ISD::FABS || N0.getOpcode() == ISD::FNEG ||
5487 N0.getOpcode() == ISD::FCOPYSIGN)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005488 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
5489 N0.getOperand(0), N1);
Chris Lattner12d83032006-03-05 05:30:57 +00005490
5491 // copysign(x, abs(y)) -> abs(x)
5492 if (N1.getOpcode() == ISD::FABS)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005493 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005494
Chris Lattner12d83032006-03-05 05:30:57 +00005495 // copysign(x, copysign(y,z)) -> copysign(x, z)
5496 if (N1.getOpcode() == ISD::FCOPYSIGN)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005497 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
5498 N0, N1.getOperand(1));
Scott Michelfdc40a02009-02-17 22:15:04 +00005499
Chris Lattner12d83032006-03-05 05:30:57 +00005500 // copysign(x, fp_extend(y)) -> copysign(x, y)
5501 // copysign(x, fp_round(y)) -> copysign(x, y)
5502 if (N1.getOpcode() == ISD::FP_EXTEND || N1.getOpcode() == ISD::FP_ROUND)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005503 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
5504 N0, N1.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00005505
Dan Gohman475871a2008-07-27 21:46:04 +00005506 return SDValue();
Chris Lattner12d83032006-03-05 05:30:57 +00005507}
5508
Dan Gohman475871a2008-07-27 21:46:04 +00005509SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
5510 SDValue N0 = N->getOperand(0);
Nate Begeman646d7e22005-09-02 21:18:40 +00005511 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00005512 EVT VT = N->getValueType(0);
5513 EVT OpVT = N0.getValueType();
Chris Lattnercda88752008-06-26 00:16:49 +00005514
Nate Begeman1d4d4142005-09-01 00:19:25 +00005515 // fold (sint_to_fp c1) -> c1fp
Stuart Hastings7e334182011-03-02 19:36:30 +00005516 if (N0C && OpVT != MVT::ppcf128 &&
5517 // ...but only if the target supports immediate floating-point values
Eli Friedman50185242011-11-12 00:35:34 +00005518 (!LegalOperations ||
Evan Cheng9568e5c2011-06-21 06:01:08 +00005519 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT)))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005520 return DAG.getNode(ISD::SINT_TO_FP, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005521
Chris Lattnercda88752008-06-26 00:16:49 +00005522 // If the input is a legal type, and SINT_TO_FP is not legal on this target,
5523 // but UINT_TO_FP is legal on this target, try to convert.
Dan Gohmanf560ffa2009-01-28 17:46:25 +00005524 if (!TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT) &&
5525 TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00005526 // If the sign bit is known to be zero, we can change this to UINT_TO_FP.
Chris Lattnercda88752008-06-26 00:16:49 +00005527 if (DAG.SignBitIsZero(N0))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005528 return DAG.getNode(ISD::UINT_TO_FP, N->getDebugLoc(), VT, N0);
Chris Lattnercda88752008-06-26 00:16:49 +00005529 }
Bill Wendling0225a1d2009-01-30 23:15:49 +00005530
Dan Gohman475871a2008-07-27 21:46:04 +00005531 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005532}
5533
Dan Gohman475871a2008-07-27 21:46:04 +00005534SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
5535 SDValue N0 = N->getOperand(0);
Nate Begeman646d7e22005-09-02 21:18:40 +00005536 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00005537 EVT VT = N->getValueType(0);
5538 EVT OpVT = N0.getValueType();
Nate Begemana148d982006-01-18 22:35:16 +00005539
Nate Begeman1d4d4142005-09-01 00:19:25 +00005540 // fold (uint_to_fp c1) -> c1fp
Stuart Hastings7e334182011-03-02 19:36:30 +00005541 if (N0C && OpVT != MVT::ppcf128 &&
5542 // ...but only if the target supports immediate floating-point values
Eli Friedman50185242011-11-12 00:35:34 +00005543 (!LegalOperations ||
Evan Cheng9568e5c2011-06-21 06:01:08 +00005544 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT)))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005545 return DAG.getNode(ISD::UINT_TO_FP, N->getDebugLoc(), VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005546
Chris Lattnercda88752008-06-26 00:16:49 +00005547 // If the input is a legal type, and UINT_TO_FP is not legal on this target,
5548 // but SINT_TO_FP is legal on this target, try to convert.
Dan Gohmanf560ffa2009-01-28 17:46:25 +00005549 if (!TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT) &&
5550 TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT)) {
Scott Michelfdc40a02009-02-17 22:15:04 +00005551 // If the sign bit is known to be zero, we can change this to SINT_TO_FP.
Chris Lattnercda88752008-06-26 00:16:49 +00005552 if (DAG.SignBitIsZero(N0))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005553 return DAG.getNode(ISD::SINT_TO_FP, N->getDebugLoc(), VT, N0);
Chris Lattnercda88752008-06-26 00:16:49 +00005554 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005555
Dan Gohman475871a2008-07-27 21:46:04 +00005556 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005557}
5558
Dan Gohman475871a2008-07-27 21:46:04 +00005559SDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) {
5560 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00005561 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00005562 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005563
Nate Begeman1d4d4142005-09-01 00:19:25 +00005564 // fold (fp_to_sint c1fp) -> c1
Nate Begeman646d7e22005-09-02 21:18:40 +00005565 if (N0CFP)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005566 return DAG.getNode(ISD::FP_TO_SINT, N->getDebugLoc(), VT, N0);
5567
Dan Gohman475871a2008-07-27 21:46:04 +00005568 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005569}
5570
Dan Gohman475871a2008-07-27 21:46:04 +00005571SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) {
5572 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00005573 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00005574 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005575
Nate Begeman1d4d4142005-09-01 00:19:25 +00005576 // fold (fp_to_uint c1fp) -> c1
Owen Anderson825b72b2009-08-11 20:47:22 +00005577 if (N0CFP && VT != MVT::ppcf128)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005578 return DAG.getNode(ISD::FP_TO_UINT, N->getDebugLoc(), VT, N0);
5579
Dan Gohman475871a2008-07-27 21:46:04 +00005580 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005581}
5582
Dan Gohman475871a2008-07-27 21:46:04 +00005583SDValue DAGCombiner::visitFP_ROUND(SDNode *N) {
5584 SDValue N0 = N->getOperand(0);
5585 SDValue N1 = N->getOperand(1);
Nate Begemana148d982006-01-18 22:35:16 +00005586 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00005587 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005588
Nate Begeman1d4d4142005-09-01 00:19:25 +00005589 // fold (fp_round c1fp) -> c1fp
Owen Anderson825b72b2009-08-11 20:47:22 +00005590 if (N0CFP && N0.getValueType() != MVT::ppcf128)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005591 return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), VT, N0, N1);
Scott Michelfdc40a02009-02-17 22:15:04 +00005592
Chris Lattner79dbea52006-03-13 06:26:26 +00005593 // fold (fp_round (fp_extend x)) -> x
5594 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType())
5595 return N0.getOperand(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005596
Chris Lattner0aa5e6f2008-01-24 06:45:35 +00005597 // fold (fp_round (fp_round x)) -> (fp_round x)
5598 if (N0.getOpcode() == ISD::FP_ROUND) {
5599 // This is a value preserving truncation if both round's are.
5600 bool IsTrunc = N->getConstantOperandVal(1) == 1 &&
Gabor Greifba36cb52008-08-28 21:40:38 +00005601 N0.getNode()->getConstantOperandVal(1) == 1;
Bill Wendling0225a1d2009-01-30 23:15:49 +00005602 return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), VT, N0.getOperand(0),
Chris Lattner0aa5e6f2008-01-24 06:45:35 +00005603 DAG.getIntPtrConstant(IsTrunc));
5604 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005605
Chris Lattner79dbea52006-03-13 06:26:26 +00005606 // fold (fp_round (copysign X, Y)) -> (copysign (fp_round X), Y)
Gabor Greifba36cb52008-08-28 21:40:38 +00005607 if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse()) {
Bill Wendling0225a1d2009-01-30 23:15:49 +00005608 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, N0.getDebugLoc(), VT,
5609 N0.getOperand(0), N1);
Gabor Greifba36cb52008-08-28 21:40:38 +00005610 AddToWorkList(Tmp.getNode());
Bill Wendling0225a1d2009-01-30 23:15:49 +00005611 return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT,
5612 Tmp, N0.getOperand(1));
Chris Lattner79dbea52006-03-13 06:26:26 +00005613 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005614
Dan Gohman475871a2008-07-27 21:46:04 +00005615 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005616}
5617
Dan Gohman475871a2008-07-27 21:46:04 +00005618SDValue DAGCombiner::visitFP_ROUND_INREG(SDNode *N) {
5619 SDValue N0 = N->getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00005620 EVT VT = N->getValueType(0);
5621 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
Nate Begeman646d7e22005-09-02 21:18:40 +00005622 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005623
Nate Begeman1d4d4142005-09-01 00:19:25 +00005624 // fold (fp_round_inreg c1fp) -> c1fp
Chris Lattner2392ae72010-04-15 04:48:01 +00005625 if (N0CFP && isTypeLegal(EVT)) {
Dan Gohman4fbd7962008-09-12 18:08:03 +00005626 SDValue Round = DAG.getConstantFP(*N0CFP->getConstantFPValue(), EVT);
Bill Wendling0225a1d2009-01-30 23:15:49 +00005627 return DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), VT, Round);
Nate Begeman1d4d4142005-09-01 00:19:25 +00005628 }
Bill Wendling0225a1d2009-01-30 23:15:49 +00005629
Dan Gohman475871a2008-07-27 21:46:04 +00005630 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005631}
5632
Dan Gohman475871a2008-07-27 21:46:04 +00005633SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
5634 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00005635 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00005636 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005637
Chris Lattner5938bef2007-12-29 06:55:23 +00005638 // If this is fp_round(fpextend), don't fold it, allow ourselves to be folded.
Scott Michelfdc40a02009-02-17 22:15:04 +00005639 if (N->hasOneUse() &&
Dan Gohmane7852d02009-01-26 04:35:06 +00005640 N->use_begin()->getOpcode() == ISD::FP_ROUND)
Dan Gohman475871a2008-07-27 21:46:04 +00005641 return SDValue();
Chris Lattner0bd48932008-01-17 07:00:52 +00005642
Nate Begeman1d4d4142005-09-01 00:19:25 +00005643 // fold (fp_extend c1fp) -> c1fp
Owen Anderson825b72b2009-08-11 20:47:22 +00005644 if (N0CFP && VT != MVT::ppcf128)
Bill Wendling0225a1d2009-01-30 23:15:49 +00005645 return DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), VT, N0);
Chris Lattner0bd48932008-01-17 07:00:52 +00005646
5647 // Turn fp_extend(fp_round(X, 1)) -> x since the fp_round doesn't affect the
5648 // value of X.
Gabor Greif12632d22008-08-30 19:29:20 +00005649 if (N0.getOpcode() == ISD::FP_ROUND
5650 && N0.getNode()->getConstantOperandVal(1) == 1) {
Dan Gohman475871a2008-07-27 21:46:04 +00005651 SDValue In = N0.getOperand(0);
Chris Lattner0bd48932008-01-17 07:00:52 +00005652 if (In.getValueType() == VT) return In;
Duncan Sands8e4eb092008-06-08 20:54:56 +00005653 if (VT.bitsLT(In.getValueType()))
Bill Wendling0225a1d2009-01-30 23:15:49 +00005654 return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), VT,
5655 In, N0.getOperand(1));
5656 return DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), VT, In);
Chris Lattner0bd48932008-01-17 07:00:52 +00005657 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005658
Chris Lattner0bd48932008-01-17 07:00:52 +00005659 // fold (fpext (load x)) -> (fpext (fptrunc (extload x)))
Gabor Greifba36cb52008-08-28 21:40:38 +00005660 if (ISD::isNON_EXTLoad(N0.getNode()) && N0.hasOneUse() &&
Duncan Sands25cf2272008-11-24 14:53:14 +00005661 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
Evan Cheng03294662008-10-14 21:26:46 +00005662 TLI.isLoadExtLegal(ISD::EXTLOAD, N0.getValueType()))) {
Evan Cheng466685d2006-10-09 20:57:25 +00005663 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
Stuart Hastingsa9011292011-02-16 16:23:55 +00005664 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, N->getDebugLoc(), VT,
Bill Wendling0225a1d2009-01-30 23:15:49 +00005665 LN0->getChain(),
Chris Lattner3d6ccfb2010-09-21 17:04:51 +00005666 LN0->getBasePtr(), LN0->getPointerInfo(),
Duncan Sands25cf2272008-11-24 14:53:14 +00005667 N0.getValueType(),
David Greene1e559442010-02-15 17:00:31 +00005668 LN0->isVolatile(), LN0->isNonTemporal(),
5669 LN0->getAlignment());
Chris Lattnere564dbb2006-05-05 21:34:35 +00005670 CombineTo(N, ExtLoad);
Bill Wendling0225a1d2009-01-30 23:15:49 +00005671 CombineTo(N0.getNode(),
5672 DAG.getNode(ISD::FP_ROUND, N0.getDebugLoc(),
5673 N0.getValueType(), ExtLoad, DAG.getIntPtrConstant(1)),
Chris Lattnere564dbb2006-05-05 21:34:35 +00005674 ExtLoad.getValue(1));
Dan Gohman475871a2008-07-27 21:46:04 +00005675 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Chris Lattnere564dbb2006-05-05 21:34:35 +00005676 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00005677
Dan Gohman475871a2008-07-27 21:46:04 +00005678 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005679}
5680
Dan Gohman475871a2008-07-27 21:46:04 +00005681SDValue DAGCombiner::visitFNEG(SDNode *N) {
5682 SDValue N0 = N->getOperand(0);
Anton Korobeynikov2bcf60a2009-10-20 21:37:45 +00005683 EVT VT = N->getValueType(0);
Nate Begemana148d982006-01-18 22:35:16 +00005684
Nick Lewycky8a8d4792011-12-02 22:16:29 +00005685 if (isNegatibleForFree(N0, LegalOperations, &DAG.getTarget().Options))
Duncan Sands25cf2272008-11-24 14:53:14 +00005686 return GetNegatedExpression(N0, DAG, LegalOperations);
Dan Gohman23ff1822007-07-02 15:48:56 +00005687
Chris Lattner3bd39d42008-01-27 17:42:27 +00005688 // Transform fneg(bitconvert(x)) -> bitconvert(x^sign) to avoid loading
5689 // constant pool values.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005690 if (N0.getOpcode() == ISD::BITCAST &&
Anton Korobeynikov2bcf60a2009-10-20 21:37:45 +00005691 !VT.isVector() &&
5692 N0.getNode()->hasOneUse() &&
5693 N0.getOperand(0).getValueType().isInteger()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005694 SDValue Int = N0.getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00005695 EVT IntVT = Int.getValueType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00005696 if (IntVT.isInteger() && !IntVT.isVector()) {
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00005697 Int = DAG.getNode(ISD::XOR, N0.getDebugLoc(), IntVT, Int,
5698 DAG.getConstant(APInt::getSignBit(IntVT.getSizeInBits()), IntVT));
Gabor Greifba36cb52008-08-28 21:40:38 +00005699 AddToWorkList(Int.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005700 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(),
Anton Korobeynikov2bcf60a2009-10-20 21:37:45 +00005701 VT, Int);
Chris Lattner3bd39d42008-01-27 17:42:27 +00005702 }
5703 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005704
Dan Gohman475871a2008-07-27 21:46:04 +00005705 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005706}
5707
Dan Gohman475871a2008-07-27 21:46:04 +00005708SDValue DAGCombiner::visitFABS(SDNode *N) {
5709 SDValue N0 = N->getOperand(0);
Nate Begemana148d982006-01-18 22:35:16 +00005710 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
Owen Andersone50ed302009-08-10 22:56:29 +00005711 EVT VT = N->getValueType(0);
Scott Michelfdc40a02009-02-17 22:15:04 +00005712
Nate Begeman1d4d4142005-09-01 00:19:25 +00005713 // fold (fabs c1) -> fabs(c1)
Owen Anderson825b72b2009-08-11 20:47:22 +00005714 if (N0CFP && VT != MVT::ppcf128)
Bill Wendlingc0debad2009-01-30 23:27:35 +00005715 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00005716 // fold (fabs (fabs x)) -> (fabs x)
Chris Lattner12d83032006-03-05 05:30:57 +00005717 if (N0.getOpcode() == ISD::FABS)
Nate Begeman83e75ec2005-09-06 04:43:02 +00005718 return N->getOperand(0);
Nate Begeman1d4d4142005-09-01 00:19:25 +00005719 // fold (fabs (fneg x)) -> (fabs x)
Chris Lattner12d83032006-03-05 05:30:57 +00005720 // fold (fabs (fcopysign x, y)) -> (fabs x)
5721 if (N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FCOPYSIGN)
Bill Wendlingc0debad2009-01-30 23:27:35 +00005722 return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0.getOperand(0));
Scott Michelfdc40a02009-02-17 22:15:04 +00005723
Chris Lattner3bd39d42008-01-27 17:42:27 +00005724 // Transform fabs(bitconvert(x)) -> bitconvert(x&~sign) to avoid loading
5725 // constant pool values.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005726 if (N0.getOpcode() == ISD::BITCAST && N0.getNode()->hasOneUse() &&
Duncan Sands83ec4b62008-06-06 12:08:01 +00005727 N0.getOperand(0).getValueType().isInteger() &&
5728 !N0.getOperand(0).getValueType().isVector()) {
Dan Gohman475871a2008-07-27 21:46:04 +00005729 SDValue Int = N0.getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00005730 EVT IntVT = Int.getValueType();
Duncan Sands83ec4b62008-06-06 12:08:01 +00005731 if (IntVT.isInteger() && !IntVT.isVector()) {
Scott Michelfdc40a02009-02-17 22:15:04 +00005732 Int = DAG.getNode(ISD::AND, N0.getDebugLoc(), IntVT, Int,
Duncan Sandsb0d5cdd2009-02-01 18:06:53 +00005733 DAG.getConstant(~APInt::getSignBit(IntVT.getSizeInBits()), IntVT));
Gabor Greifba36cb52008-08-28 21:40:38 +00005734 AddToWorkList(Int.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005735 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(),
Bill Wendlingc0debad2009-01-30 23:27:35 +00005736 N->getValueType(0), Int);
Chris Lattner3bd39d42008-01-27 17:42:27 +00005737 }
5738 }
Scott Michelfdc40a02009-02-17 22:15:04 +00005739
Dan Gohman475871a2008-07-27 21:46:04 +00005740 return SDValue();
Nate Begeman1d4d4142005-09-01 00:19:25 +00005741}
5742
Dan Gohman475871a2008-07-27 21:46:04 +00005743SDValue DAGCombiner::visitBRCOND(SDNode *N) {
5744 SDValue Chain = N->getOperand(0);
5745 SDValue N1 = N->getOperand(1);
5746 SDValue N2 = N->getOperand(2);
Scott Michelfdc40a02009-02-17 22:15:04 +00005747
Dan Gohmane0f06c72009-11-17 00:47:23 +00005748 // If N is a constant we could fold this into a fallthrough or unconditional
5749 // branch. However that doesn't happen very often in normal code, because
5750 // Instcombine/SimplifyCFG should have handled the available opportunities.
5751 // If we did this folding here, it would be necessary to update the
5752 // MachineBasicBlock CFG, which is awkward.
5753
Nate Begeman750ac1b2006-02-01 07:19:44 +00005754 // fold a brcond with a setcc condition into a BR_CC node if BR_CC is legal
5755 // on the target.
Scott Michelfdc40a02009-02-17 22:15:04 +00005756 if (N1.getOpcode() == ISD::SETCC &&
Owen Anderson825b72b2009-08-11 20:47:22 +00005757 TLI.isOperationLegalOrCustom(ISD::BR_CC, MVT::Other)) {
5758 return DAG.getNode(ISD::BR_CC, N->getDebugLoc(), MVT::Other,
Bill Wendlingc0debad2009-01-30 23:27:35 +00005759 Chain, N1.getOperand(2),
Nate Begeman750ac1b2006-02-01 07:19:44 +00005760 N1.getOperand(0), N1.getOperand(1), N2);
5761 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00005762
Evan Cheng2a135ae2010-10-04 22:41:01 +00005763 if ((N1.hasOneUse() && N1.getOpcode() == ISD::SRL) ||
5764 ((N1.getOpcode() == ISD::TRUNCATE && N1.hasOneUse()) &&
5765 (N1.getOperand(0).hasOneUse() &&
5766 N1.getOperand(0).getOpcode() == ISD::SRL))) {
5767 SDNode *Trunc = 0;
5768 if (N1.getOpcode() == ISD::TRUNCATE) {
5769 // Look pass the truncate.
5770 Trunc = N1.getNode();
5771 N1 = N1.getOperand(0);
5772 }
Evan Chengd40d03e2010-01-06 19:38:29 +00005773
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00005774 // Match this pattern so that we can generate simpler code:
5775 //
5776 // %a = ...
5777 // %b = and i32 %a, 2
5778 // %c = srl i32 %b, 1
5779 // brcond i32 %c ...
5780 //
5781 // into
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005782 //
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00005783 // %a = ...
Evan Chengd40d03e2010-01-06 19:38:29 +00005784 // %b = and i32 %a, 2
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00005785 // %c = setcc eq %b, 0
5786 // brcond %c ...
5787 //
5788 // This applies only when the AND constant value has one bit set and the
5789 // SRL constant is equal to the log2 of the AND constant. The back-end is
5790 // smart enough to convert the result into a TEST/JMP sequence.
5791 SDValue Op0 = N1.getOperand(0);
5792 SDValue Op1 = N1.getOperand(1);
5793
5794 if (Op0.getOpcode() == ISD::AND &&
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00005795 Op1.getOpcode() == ISD::Constant) {
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00005796 SDValue AndOp1 = Op0.getOperand(1);
5797
5798 if (AndOp1.getOpcode() == ISD::Constant) {
5799 const APInt &AndConst = cast<ConstantSDNode>(AndOp1)->getAPIntValue();
5800
5801 if (AndConst.isPowerOf2() &&
5802 cast<ConstantSDNode>(Op1)->getAPIntValue()==AndConst.logBase2()) {
5803 SDValue SetCC =
5804 DAG.getSetCC(N->getDebugLoc(),
5805 TLI.getSetCCResultType(Op0.getValueType()),
5806 Op0, DAG.getConstant(0, Op0.getValueType()),
5807 ISD::SETNE);
5808
Evan Chengd40d03e2010-01-06 19:38:29 +00005809 SDValue NewBRCond = DAG.getNode(ISD::BRCOND, N->getDebugLoc(),
5810 MVT::Other, Chain, SetCC, N2);
5811 // Don't add the new BRCond into the worklist or else SimplifySelectCC
5812 // will convert it back to (X & C1) >> C2.
5813 CombineTo(N, NewBRCond, false);
5814 // Truncate is dead.
5815 if (Trunc) {
5816 removeFromWorkList(Trunc);
5817 DAG.DeleteNode(Trunc);
5818 }
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00005819 // Replace the uses of SRL with SETCC
Evan Cheng2c755ba2010-02-27 07:36:59 +00005820 WorkListRemover DeadNodes(*this);
5821 DAG.ReplaceAllUsesOfValueWith(N1, SetCC, &DeadNodes);
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00005822 removeFromWorkList(N1.getNode());
5823 DAG.DeleteNode(N1.getNode());
Evan Chengd40d03e2010-01-06 19:38:29 +00005824 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00005825 }
5826 }
5827 }
Evan Cheng2a135ae2010-10-04 22:41:01 +00005828
5829 if (Trunc)
5830 // Restore N1 if the above transformation doesn't match.
5831 N1 = N->getOperand(1);
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00005832 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00005833
Evan Cheng2c755ba2010-02-27 07:36:59 +00005834 // Transform br(xor(x, y)) -> br(x != y)
5835 // Transform br(xor(xor(x,y), 1)) -> br (x == y)
5836 if (N1.hasOneUse() && N1.getOpcode() == ISD::XOR) {
5837 SDNode *TheXor = N1.getNode();
5838 SDValue Op0 = TheXor->getOperand(0);
5839 SDValue Op1 = TheXor->getOperand(1);
5840 if (Op0.getOpcode() == Op1.getOpcode()) {
5841 // Avoid missing important xor optimizations.
5842 SDValue Tmp = visitXOR(TheXor);
Bill Wendling86c5abb2010-04-20 01:25:01 +00005843 if (Tmp.getNode() && Tmp.getNode() != TheXor) {
Evan Cheng2c755ba2010-02-27 07:36:59 +00005844 DEBUG(dbgs() << "\nReplacing.8 ";
5845 TheXor->dump(&DAG);
5846 dbgs() << "\nWith: ";
5847 Tmp.getNode()->dump(&DAG);
5848 dbgs() << '\n');
5849 WorkListRemover DeadNodes(*this);
5850 DAG.ReplaceAllUsesOfValueWith(N1, Tmp, &DeadNodes);
5851 removeFromWorkList(TheXor);
5852 DAG.DeleteNode(TheXor);
5853 return DAG.getNode(ISD::BRCOND, N->getDebugLoc(),
5854 MVT::Other, Chain, Tmp, N2);
5855 }
5856 }
5857
5858 if (Op0.getOpcode() != ISD::SETCC && Op1.getOpcode() != ISD::SETCC) {
5859 bool Equal = false;
5860 if (ConstantSDNode *RHSCI = dyn_cast<ConstantSDNode>(Op0))
5861 if (RHSCI->getAPIntValue() == 1 && Op0.hasOneUse() &&
5862 Op0.getOpcode() == ISD::XOR) {
5863 TheXor = Op0.getNode();
5864 Equal = true;
5865 }
5866
Evan Cheng2a135ae2010-10-04 22:41:01 +00005867 EVT SetCCVT = N1.getValueType();
Evan Cheng2c755ba2010-02-27 07:36:59 +00005868 if (LegalTypes)
5869 SetCCVT = TLI.getSetCCResultType(SetCCVT);
5870 SDValue SetCC = DAG.getSetCC(TheXor->getDebugLoc(),
5871 SetCCVT,
5872 Op0, Op1,
5873 Equal ? ISD::SETEQ : ISD::SETNE);
5874 // Replace the uses of XOR with SETCC
5875 WorkListRemover DeadNodes(*this);
Evan Cheng2a135ae2010-10-04 22:41:01 +00005876 DAG.ReplaceAllUsesOfValueWith(N1, SetCC, &DeadNodes);
5877 removeFromWorkList(N1.getNode());
5878 DAG.DeleteNode(N1.getNode());
Evan Cheng2c755ba2010-02-27 07:36:59 +00005879 return DAG.getNode(ISD::BRCOND, N->getDebugLoc(),
5880 MVT::Other, Chain, SetCC, N2);
5881 }
5882 }
Bill Wendlinga02a3dd2009-03-26 06:14:09 +00005883
Dan Gohman475871a2008-07-27 21:46:04 +00005884 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00005885}
5886
Chris Lattner3ea0b472005-10-05 06:47:48 +00005887// Operand List for BR_CC: Chain, CondCC, CondLHS, CondRHS, DestBB.
5888//
Dan Gohman475871a2008-07-27 21:46:04 +00005889SDValue DAGCombiner::visitBR_CC(SDNode *N) {
Chris Lattner3ea0b472005-10-05 06:47:48 +00005890 CondCodeSDNode *CC = cast<CondCodeSDNode>(N->getOperand(1));
Dan Gohman475871a2008-07-27 21:46:04 +00005891 SDValue CondLHS = N->getOperand(2), CondRHS = N->getOperand(3);
Scott Michelfdc40a02009-02-17 22:15:04 +00005892
Dan Gohmane0f06c72009-11-17 00:47:23 +00005893 // If N is a constant we could fold this into a fallthrough or unconditional
5894 // branch. However that doesn't happen very often in normal code, because
5895 // Instcombine/SimplifyCFG should have handled the available opportunities.
5896 // If we did this folding here, it would be necessary to update the
5897 // MachineBasicBlock CFG, which is awkward.
5898
Duncan Sands8eab8a22008-06-09 11:32:28 +00005899 // Use SimplifySetCC to simplify SETCC's.
Duncan Sands5480c042009-01-01 15:52:00 +00005900 SDValue Simp = SimplifySetCC(TLI.getSetCCResultType(CondLHS.getValueType()),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00005901 CondLHS, CondRHS, CC->get(), N->getDebugLoc(),
5902 false);
Gabor Greifba36cb52008-08-28 21:40:38 +00005903 if (Simp.getNode()) AddToWorkList(Simp.getNode());
Chris Lattner30f73e72006-10-14 03:52:46 +00005904
Nate Begemane17daeb2005-10-05 21:43:42 +00005905 // fold to a simpler setcc
Gabor Greifba36cb52008-08-28 21:40:38 +00005906 if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC)
Owen Anderson825b72b2009-08-11 20:47:22 +00005907 return DAG.getNode(ISD::BR_CC, N->getDebugLoc(), MVT::Other,
Bill Wendlingc0debad2009-01-30 23:27:35 +00005908 N->getOperand(0), Simp.getOperand(2),
5909 Simp.getOperand(0), Simp.getOperand(1),
5910 N->getOperand(4));
5911
Dan Gohman475871a2008-07-27 21:46:04 +00005912 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00005913}
5914
Duncan Sandsec87aa82008-06-15 20:12:31 +00005915/// CombineToPreIndexedLoadStore - Try turning a load / store into a
5916/// pre-indexed load / store when the base pointer is an add or subtract
Chris Lattner448f2192006-11-11 00:39:41 +00005917/// and it has other uses besides the load / store. After the
5918/// transformation, the new indexed load / store has effectively folded
5919/// the add / subtract in and all of its other uses are redirected to the
5920/// new load / store.
5921bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) {
Eli Friedman50185242011-11-12 00:35:34 +00005922 if (Level < AfterLegalizeDAG)
Chris Lattner448f2192006-11-11 00:39:41 +00005923 return false;
5924
5925 bool isLoad = true;
Dan Gohman475871a2008-07-27 21:46:04 +00005926 SDValue Ptr;
Owen Andersone50ed302009-08-10 22:56:29 +00005927 EVT VT;
Chris Lattner448f2192006-11-11 00:39:41 +00005928 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00005929 if (LD->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00005930 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00005931 VT = LD->getMemoryVT();
Evan Cheng83060c52007-03-07 08:07:03 +00005932 if (!TLI.isIndexedLoadLegal(ISD::PRE_INC, VT) &&
Chris Lattner448f2192006-11-11 00:39:41 +00005933 !TLI.isIndexedLoadLegal(ISD::PRE_DEC, VT))
5934 return false;
5935 Ptr = LD->getBasePtr();
5936 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00005937 if (ST->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00005938 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00005939 VT = ST->getMemoryVT();
Chris Lattner448f2192006-11-11 00:39:41 +00005940 if (!TLI.isIndexedStoreLegal(ISD::PRE_INC, VT) &&
5941 !TLI.isIndexedStoreLegal(ISD::PRE_DEC, VT))
5942 return false;
5943 Ptr = ST->getBasePtr();
5944 isLoad = false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00005945 } else {
Chris Lattner448f2192006-11-11 00:39:41 +00005946 return false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00005947 }
Chris Lattner448f2192006-11-11 00:39:41 +00005948
Chris Lattner9f1794e2006-11-11 00:56:29 +00005949 // If the pointer is not an add/sub, or if it doesn't have multiple uses, bail
5950 // out. There is no reason to make this a preinc/predec.
5951 if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) ||
Gabor Greifba36cb52008-08-28 21:40:38 +00005952 Ptr.getNode()->hasOneUse())
Chris Lattner9f1794e2006-11-11 00:56:29 +00005953 return false;
Chris Lattner448f2192006-11-11 00:39:41 +00005954
Chris Lattner9f1794e2006-11-11 00:56:29 +00005955 // Ask the target to do addressing mode selection.
Dan Gohman475871a2008-07-27 21:46:04 +00005956 SDValue BasePtr;
5957 SDValue Offset;
Chris Lattner9f1794e2006-11-11 00:56:29 +00005958 ISD::MemIndexedMode AM = ISD::UNINDEXED;
5959 if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG))
5960 return false;
Evan Chenga7d4a042007-05-03 23:52:19 +00005961 // Don't create a indexed load / store with zero offset.
5962 if (isa<ConstantSDNode>(Offset) &&
Dan Gohman002e5d02008-03-13 22:13:53 +00005963 cast<ConstantSDNode>(Offset)->isNullValue())
Evan Chenga7d4a042007-05-03 23:52:19 +00005964 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +00005965
Chris Lattner41e53fd2006-11-11 01:00:15 +00005966 // Try turning it into a pre-indexed load / store except when:
Evan Chengc843abe2007-05-24 02:35:39 +00005967 // 1) The new base ptr is a frame index.
5968 // 2) If N is a store and the new base ptr is either the same as or is a
Chris Lattner9f1794e2006-11-11 00:56:29 +00005969 // predecessor of the value being stored.
Evan Chengc843abe2007-05-24 02:35:39 +00005970 // 3) Another use of old base ptr is a predecessor of N. If ptr is folded
Chris Lattner9f1794e2006-11-11 00:56:29 +00005971 // that would create a cycle.
Evan Chengc843abe2007-05-24 02:35:39 +00005972 // 4) All uses are load / store ops that use it as old base ptr.
Chris Lattner448f2192006-11-11 00:39:41 +00005973
Chris Lattner41e53fd2006-11-11 01:00:15 +00005974 // Check #1. Preinc'ing a frame index would require copying the stack pointer
5975 // (plus the implicit offset) to a register to preinc anyway.
Evan Chengcaab1292009-05-06 18:25:01 +00005976 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
Chris Lattner41e53fd2006-11-11 01:00:15 +00005977 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +00005978
Chris Lattner41e53fd2006-11-11 01:00:15 +00005979 // Check #2.
Chris Lattner9f1794e2006-11-11 00:56:29 +00005980 if (!isLoad) {
Dan Gohman475871a2008-07-27 21:46:04 +00005981 SDValue Val = cast<StoreSDNode>(N)->getValue();
Gabor Greifba36cb52008-08-28 21:40:38 +00005982 if (Val == BasePtr || BasePtr.getNode()->isPredecessorOf(Val.getNode()))
Chris Lattner9f1794e2006-11-11 00:56:29 +00005983 return false;
Chris Lattner448f2192006-11-11 00:39:41 +00005984 }
Chris Lattner9f1794e2006-11-11 00:56:29 +00005985
Evan Chengc843abe2007-05-24 02:35:39 +00005986 // Now check for #3 and #4.
Chris Lattner9f1794e2006-11-11 00:56:29 +00005987 bool RealUse = false;
Lang Hames944520f2011-07-07 04:31:51 +00005988
5989 // Caches for hasPredecessorHelper
5990 SmallPtrSet<const SDNode *, 32> Visited;
5991 SmallVector<const SDNode *, 16> Worklist;
5992
Gabor Greifba36cb52008-08-28 21:40:38 +00005993 for (SDNode::use_iterator I = Ptr.getNode()->use_begin(),
5994 E = Ptr.getNode()->use_end(); I != E; ++I) {
Dan Gohman89684502008-07-27 20:43:25 +00005995 SDNode *Use = *I;
Chris Lattner9f1794e2006-11-11 00:56:29 +00005996 if (Use == N)
5997 continue;
Lang Hames944520f2011-07-07 04:31:51 +00005998 if (N->hasPredecessorHelper(Use, Visited, Worklist))
Chris Lattner9f1794e2006-11-11 00:56:29 +00005999 return false;
6000
6001 if (!((Use->getOpcode() == ISD::LOAD &&
6002 cast<LoadSDNode>(Use)->getBasePtr() == Ptr) ||
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00006003 (Use->getOpcode() == ISD::STORE &&
6004 cast<StoreSDNode>(Use)->getBasePtr() == Ptr)))
Chris Lattner9f1794e2006-11-11 00:56:29 +00006005 RealUse = true;
6006 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006007
Chris Lattner9f1794e2006-11-11 00:56:29 +00006008 if (!RealUse)
6009 return false;
6010
Dan Gohman475871a2008-07-27 21:46:04 +00006011 SDValue Result;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006012 if (isLoad)
Bill Wendlingc0debad2009-01-30 23:27:35 +00006013 Result = DAG.getIndexedLoad(SDValue(N,0), N->getDebugLoc(),
6014 BasePtr, Offset, AM);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006015 else
Bill Wendlingc0debad2009-01-30 23:27:35 +00006016 Result = DAG.getIndexedStore(SDValue(N,0), N->getDebugLoc(),
6017 BasePtr, Offset, AM);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006018 ++PreIndexedNodes;
6019 ++NodesCombined;
David Greenef1090292010-01-05 01:25:00 +00006020 DEBUG(dbgs() << "\nReplacing.4 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006021 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006022 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006023 Result.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006024 dbgs() << '\n');
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006025 WorkListRemover DeadNodes(*this);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006026 if (isLoad) {
Dan Gohman475871a2008-07-27 21:46:04 +00006027 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0),
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006028 &DeadNodes);
Dan Gohman475871a2008-07-27 21:46:04 +00006029 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2),
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006030 &DeadNodes);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006031 } else {
Dan Gohman475871a2008-07-27 21:46:04 +00006032 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1),
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006033 &DeadNodes);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006034 }
6035
Chris Lattner9f1794e2006-11-11 00:56:29 +00006036 // Finally, since the node is now dead, remove it from the graph.
6037 DAG.DeleteNode(N);
6038
6039 // Replace the uses of Ptr with uses of the updated base value.
6040 DAG.ReplaceAllUsesOfValueWith(Ptr, Result.getValue(isLoad ? 1 : 0),
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006041 &DeadNodes);
Gabor Greifba36cb52008-08-28 21:40:38 +00006042 removeFromWorkList(Ptr.getNode());
6043 DAG.DeleteNode(Ptr.getNode());
Chris Lattner9f1794e2006-11-11 00:56:29 +00006044
6045 return true;
Chris Lattner448f2192006-11-11 00:39:41 +00006046}
6047
Duncan Sandsec87aa82008-06-15 20:12:31 +00006048/// CombineToPostIndexedLoadStore - Try to combine a load / store with a
Chris Lattner448f2192006-11-11 00:39:41 +00006049/// add / sub of the base pointer node into a post-indexed load / store.
6050/// The transformation folded the add / subtract into the new indexed
6051/// load / store effectively and all of its uses are redirected to the
6052/// new load / store.
6053bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) {
Eli Friedman50185242011-11-12 00:35:34 +00006054 if (Level < AfterLegalizeDAG)
Chris Lattner448f2192006-11-11 00:39:41 +00006055 return false;
6056
6057 bool isLoad = true;
Dan Gohman475871a2008-07-27 21:46:04 +00006058 SDValue Ptr;
Owen Andersone50ed302009-08-10 22:56:29 +00006059 EVT VT;
Chris Lattner448f2192006-11-11 00:39:41 +00006060 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00006061 if (LD->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00006062 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006063 VT = LD->getMemoryVT();
Chris Lattner448f2192006-11-11 00:39:41 +00006064 if (!TLI.isIndexedLoadLegal(ISD::POST_INC, VT) &&
6065 !TLI.isIndexedLoadLegal(ISD::POST_DEC, VT))
6066 return false;
6067 Ptr = LD->getBasePtr();
6068 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
Chris Lattnerddf89562008-01-17 19:59:44 +00006069 if (ST->isIndexed())
Evan Chenge90460e2006-12-16 06:25:23 +00006070 return false;
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006071 VT = ST->getMemoryVT();
Chris Lattner448f2192006-11-11 00:39:41 +00006072 if (!TLI.isIndexedStoreLegal(ISD::POST_INC, VT) &&
6073 !TLI.isIndexedStoreLegal(ISD::POST_DEC, VT))
6074 return false;
6075 Ptr = ST->getBasePtr();
6076 isLoad = false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00006077 } else {
Chris Lattner448f2192006-11-11 00:39:41 +00006078 return false;
Bill Wendlingc0debad2009-01-30 23:27:35 +00006079 }
Chris Lattner448f2192006-11-11 00:39:41 +00006080
Gabor Greifba36cb52008-08-28 21:40:38 +00006081 if (Ptr.getNode()->hasOneUse())
Chris Lattner9f1794e2006-11-11 00:56:29 +00006082 return false;
Scott Michelfdc40a02009-02-17 22:15:04 +00006083
Gabor Greifba36cb52008-08-28 21:40:38 +00006084 for (SDNode::use_iterator I = Ptr.getNode()->use_begin(),
6085 E = Ptr.getNode()->use_end(); I != E; ++I) {
Dan Gohman89684502008-07-27 20:43:25 +00006086 SDNode *Op = *I;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006087 if (Op == N ||
6088 (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB))
6089 continue;
6090
Dan Gohman475871a2008-07-27 21:46:04 +00006091 SDValue BasePtr;
6092 SDValue Offset;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006093 ISD::MemIndexedMode AM = ISD::UNINDEXED;
6094 if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) {
Evan Chenga7d4a042007-05-03 23:52:19 +00006095 // Don't create a indexed load / store with zero offset.
6096 if (isa<ConstantSDNode>(Offset) &&
Dan Gohman002e5d02008-03-13 22:13:53 +00006097 cast<ConstantSDNode>(Offset)->isNullValue())
Evan Chenga7d4a042007-05-03 23:52:19 +00006098 continue;
Chris Lattner448f2192006-11-11 00:39:41 +00006099
Chris Lattner9f1794e2006-11-11 00:56:29 +00006100 // Try turning it into a post-indexed load / store except when
6101 // 1) All uses are load / store ops that use it as base ptr.
6102 // 2) Op must be independent of N, i.e. Op is neither a predecessor
6103 // nor a successor of N. Otherwise, if Op is folded that would
6104 // create a cycle.
6105
Evan Chengcaab1292009-05-06 18:25:01 +00006106 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
6107 continue;
6108
Chris Lattner9f1794e2006-11-11 00:56:29 +00006109 // Check for #1.
6110 bool TryNext = false;
Gabor Greifba36cb52008-08-28 21:40:38 +00006111 for (SDNode::use_iterator II = BasePtr.getNode()->use_begin(),
6112 EE = BasePtr.getNode()->use_end(); II != EE; ++II) {
Dan Gohman89684502008-07-27 20:43:25 +00006113 SDNode *Use = *II;
Gabor Greifba36cb52008-08-28 21:40:38 +00006114 if (Use == Ptr.getNode())
Chris Lattner448f2192006-11-11 00:39:41 +00006115 continue;
6116
Chris Lattner9f1794e2006-11-11 00:56:29 +00006117 // If all the uses are load / store addresses, then don't do the
6118 // transformation.
6119 if (Use->getOpcode() == ISD::ADD || Use->getOpcode() == ISD::SUB){
6120 bool RealUse = false;
6121 for (SDNode::use_iterator III = Use->use_begin(),
6122 EEE = Use->use_end(); III != EEE; ++III) {
Dan Gohman89684502008-07-27 20:43:25 +00006123 SDNode *UseUse = *III;
Chris Lattner9f1794e2006-11-11 00:56:29 +00006124 if (!((UseUse->getOpcode() == ISD::LOAD &&
Gabor Greifba36cb52008-08-28 21:40:38 +00006125 cast<LoadSDNode>(UseUse)->getBasePtr().getNode() == Use) ||
Anton Korobeynikov4c71dfe2008-02-20 11:10:28 +00006126 (UseUse->getOpcode() == ISD::STORE &&
Gabor Greifba36cb52008-08-28 21:40:38 +00006127 cast<StoreSDNode>(UseUse)->getBasePtr().getNode() == Use)))
Chris Lattner9f1794e2006-11-11 00:56:29 +00006128 RealUse = true;
6129 }
Chris Lattner448f2192006-11-11 00:39:41 +00006130
Chris Lattner9f1794e2006-11-11 00:56:29 +00006131 if (!RealUse) {
6132 TryNext = true;
6133 break;
Chris Lattner448f2192006-11-11 00:39:41 +00006134 }
6135 }
Chris Lattner9f1794e2006-11-11 00:56:29 +00006136 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006137
Chris Lattner9f1794e2006-11-11 00:56:29 +00006138 if (TryNext)
6139 continue;
Chris Lattner448f2192006-11-11 00:39:41 +00006140
Chris Lattner9f1794e2006-11-11 00:56:29 +00006141 // Check for #2
Evan Cheng917be682008-03-04 00:41:45 +00006142 if (!Op->isPredecessorOf(N) && !N->isPredecessorOf(Op)) {
Dan Gohman475871a2008-07-27 21:46:04 +00006143 SDValue Result = isLoad
Bill Wendlingc0debad2009-01-30 23:27:35 +00006144 ? DAG.getIndexedLoad(SDValue(N,0), N->getDebugLoc(),
6145 BasePtr, Offset, AM)
6146 : DAG.getIndexedStore(SDValue(N,0), N->getDebugLoc(),
6147 BasePtr, Offset, AM);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006148 ++PostIndexedNodes;
6149 ++NodesCombined;
David Greenef1090292010-01-05 01:25:00 +00006150 DEBUG(dbgs() << "\nReplacing.5 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006151 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006152 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006153 Result.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006154 dbgs() << '\n');
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006155 WorkListRemover DeadNodes(*this);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006156 if (isLoad) {
Dan Gohman475871a2008-07-27 21:46:04 +00006157 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0),
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006158 &DeadNodes);
Dan Gohman475871a2008-07-27 21:46:04 +00006159 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2),
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006160 &DeadNodes);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006161 } else {
Dan Gohman475871a2008-07-27 21:46:04 +00006162 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1),
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006163 &DeadNodes);
Chris Lattner448f2192006-11-11 00:39:41 +00006164 }
Chris Lattner9f1794e2006-11-11 00:56:29 +00006165
Chris Lattner9f1794e2006-11-11 00:56:29 +00006166 // Finally, since the node is now dead, remove it from the graph.
6167 DAG.DeleteNode(N);
6168
6169 // Replace the uses of Use with uses of the updated base value.
Dan Gohman475871a2008-07-27 21:46:04 +00006170 DAG.ReplaceAllUsesOfValueWith(SDValue(Op, 0),
Chris Lattner9f1794e2006-11-11 00:56:29 +00006171 Result.getValue(isLoad ? 1 : 0),
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006172 &DeadNodes);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006173 removeFromWorkList(Op);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006174 DAG.DeleteNode(Op);
Chris Lattner9f1794e2006-11-11 00:56:29 +00006175 return true;
Chris Lattner448f2192006-11-11 00:39:41 +00006176 }
6177 }
6178 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006179
Chris Lattner448f2192006-11-11 00:39:41 +00006180 return false;
6181}
6182
Dan Gohman475871a2008-07-27 21:46:04 +00006183SDValue DAGCombiner::visitLOAD(SDNode *N) {
Evan Cheng466685d2006-10-09 20:57:25 +00006184 LoadSDNode *LD = cast<LoadSDNode>(N);
Dan Gohman475871a2008-07-27 21:46:04 +00006185 SDValue Chain = LD->getChain();
6186 SDValue Ptr = LD->getBasePtr();
Scott Michelfdc40a02009-02-17 22:15:04 +00006187
Evan Cheng45a7ca92007-05-01 00:38:21 +00006188 // If load is not volatile and there are no uses of the loaded value (and
6189 // the updated indexed value in case of indexed loads), change uses of the
6190 // chain value into uses of the chain input (i.e. delete the dead load).
6191 if (!LD->isVolatile()) {
Owen Anderson825b72b2009-08-11 20:47:22 +00006192 if (N->getValueType(1) == MVT::Other) {
Evan Cheng498f5592007-05-01 08:53:39 +00006193 // Unindexed loads.
Evan Cheng02c42852008-01-16 23:11:54 +00006194 if (N->hasNUsesOfValue(0, 0)) {
6195 // It's not safe to use the two value CombineTo variant here. e.g.
6196 // v1, chain2 = load chain1, loc
6197 // v2, chain3 = load chain2, loc
6198 // v3 = add v2, c
Chris Lattner125991a2008-01-24 07:57:06 +00006199 // Now we replace use of chain2 with chain1. This makes the second load
6200 // isomorphic to the one we are deleting, and thus makes this load live.
David Greenef1090292010-01-05 01:25:00 +00006201 DEBUG(dbgs() << "\nReplacing.6 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006202 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006203 dbgs() << "\nWith chain: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006204 Chain.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006205 dbgs() << "\n");
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006206 WorkListRemover DeadNodes(*this);
Dan Gohman475871a2008-07-27 21:46:04 +00006207 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain, &DeadNodes);
Bill Wendlingc0debad2009-01-30 23:27:35 +00006208
Chris Lattner125991a2008-01-24 07:57:06 +00006209 if (N->use_empty()) {
6210 removeFromWorkList(N);
6211 DAG.DeleteNode(N);
6212 }
Bill Wendlingc0debad2009-01-30 23:27:35 +00006213
Dan Gohman475871a2008-07-27 21:46:04 +00006214 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng02c42852008-01-16 23:11:54 +00006215 }
Evan Cheng498f5592007-05-01 08:53:39 +00006216 } else {
6217 // Indexed loads.
Owen Anderson825b72b2009-08-11 20:47:22 +00006218 assert(N->getValueType(2) == MVT::Other && "Malformed indexed loads?");
Evan Cheng498f5592007-05-01 08:53:39 +00006219 if (N->hasNUsesOfValue(0, 0) && N->hasNUsesOfValue(0, 1)) {
Dale Johannesene8d72302009-02-06 23:05:02 +00006220 SDValue Undef = DAG.getUNDEF(N->getValueType(0));
Evan Cheng2c755ba2010-02-27 07:36:59 +00006221 DEBUG(dbgs() << "\nReplacing.7 ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006222 N->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006223 dbgs() << "\nWith: ";
Chris Lattnerbbbfa992009-08-23 06:35:02 +00006224 Undef.getNode()->dump(&DAG);
David Greenef1090292010-01-05 01:25:00 +00006225 dbgs() << " and 2 other values\n");
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006226 WorkListRemover DeadNodes(*this);
Dan Gohman475871a2008-07-27 21:46:04 +00006227 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Undef, &DeadNodes);
6228 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1),
Dale Johannesene8d72302009-02-06 23:05:02 +00006229 DAG.getUNDEF(N->getValueType(1)),
Chris Lattnerf8dc0612008-02-03 06:49:24 +00006230 &DeadNodes);
Dan Gohman475871a2008-07-27 21:46:04 +00006231 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 2), Chain, &DeadNodes);
Evan Cheng02c42852008-01-16 23:11:54 +00006232 removeFromWorkList(N);
Evan Cheng02c42852008-01-16 23:11:54 +00006233 DAG.DeleteNode(N);
Dan Gohman475871a2008-07-27 21:46:04 +00006234 return SDValue(N, 0); // Return N so it doesn't get rechecked!
Evan Cheng45a7ca92007-05-01 00:38:21 +00006235 }
Evan Cheng45a7ca92007-05-01 00:38:21 +00006236 }
6237 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006238
Chris Lattner01a22022005-10-10 22:04:48 +00006239 // If this load is directly stored, replace the load value with the stored
6240 // value.
6241 // TODO: Handle store large -> read small portion.
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006242 // TODO: Handle TRUNCSTORE/LOADEXT
Evan Cheng9ef82ce2011-03-11 00:48:56 +00006243 if (ISD::isNormalLoad(N) && !LD->isVolatile()) {
Gabor Greifba36cb52008-08-28 21:40:38 +00006244 if (ISD::isNON_TRUNCStore(Chain.getNode())) {
Evan Cheng8b2794a2006-10-13 21:14:26 +00006245 StoreSDNode *PrevST = cast<StoreSDNode>(Chain);
6246 if (PrevST->getBasePtr() == Ptr &&
6247 PrevST->getValue().getValueType() == N->getValueType(0))
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006248 return CombineTo(N, Chain.getOperand(1), Chain);
Evan Cheng8b2794a2006-10-13 21:14:26 +00006249 }
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006250 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006251
Evan Cheng255f20f2010-04-01 06:04:33 +00006252 // Try to infer better alignment information than the load already has.
6253 if (OptLevel != CodeGenOpt::None && LD->isUnindexed()) {
Evan Chenged1c0c72011-11-28 22:37:34 +00006254 if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
6255 if (Align > LD->getAlignment())
6256 return DAG.getExtLoad(LD->getExtensionType(), N->getDebugLoc(),
6257 LD->getValueType(0),
6258 Chain, Ptr, LD->getPointerInfo(),
6259 LD->getMemoryVT(),
6260 LD->isVolatile(), LD->isNonTemporal(), Align);
Evan Cheng255f20f2010-04-01 06:04:33 +00006261 }
6262 }
6263
Jim Laskey7ca56af2006-10-11 13:47:09 +00006264 if (CombinerAA) {
Jim Laskey279f0532006-09-25 16:29:54 +00006265 // Walk up chain skipping non-aliasing memory nodes.
Dan Gohman475871a2008-07-27 21:46:04 +00006266 SDValue BetterChain = FindBetterChain(N, Chain);
Scott Michelfdc40a02009-02-17 22:15:04 +00006267
Jim Laskey6ff23e52006-10-04 16:53:27 +00006268 // If there is a better chain.
Jim Laskey279f0532006-09-25 16:29:54 +00006269 if (Chain != BetterChain) {
Dan Gohman475871a2008-07-27 21:46:04 +00006270 SDValue ReplLoad;
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006271
Jim Laskey279f0532006-09-25 16:29:54 +00006272 // Replace the chain to void dependency.
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006273 if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
Bill Wendlingc0debad2009-01-30 23:27:35 +00006274 ReplLoad = DAG.getLoad(N->getValueType(0), LD->getDebugLoc(),
Chris Lattnerfa459012010-09-21 16:08:50 +00006275 BetterChain, Ptr, LD->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00006276 LD->isVolatile(), LD->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00006277 LD->isInvariant(), LD->getAlignment());
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006278 } else {
Stuart Hastingsa9011292011-02-16 16:23:55 +00006279 ReplLoad = DAG.getExtLoad(LD->getExtensionType(), LD->getDebugLoc(),
6280 LD->getValueType(0),
Chris Lattnerfa459012010-09-21 16:08:50 +00006281 BetterChain, Ptr, LD->getPointerInfo(),
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006282 LD->getMemoryVT(),
Scott Michelfdc40a02009-02-17 22:15:04 +00006283 LD->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00006284 LD->isNonTemporal(),
Christopher Lamb95c218a2007-04-22 23:15:30 +00006285 LD->getAlignment());
Jim Laskeyc2b19f32006-10-11 17:47:52 +00006286 }
Jim Laskey279f0532006-09-25 16:29:54 +00006287
Jim Laskey6ff23e52006-10-04 16:53:27 +00006288 // Create token factor to keep old chain connected.
Bill Wendlingc0debad2009-01-30 23:27:35 +00006289 SDValue Token = DAG.getNode(ISD::TokenFactor, N->getDebugLoc(),
Owen Anderson825b72b2009-08-11 20:47:22 +00006290 MVT::Other, Chain, ReplLoad.getValue(1));
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006291
Nate Begemanb6aef5c2009-09-15 00:18:30 +00006292 // Make sure the new and old chains are cleaned up.
6293 AddToWorkList(Token.getNode());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006294
Jim Laskey274062c2006-10-13 23:32:28 +00006295 // Replace uses with load result and token factor. Don't add users
6296 // to work list.
6297 return CombineTo(N, ReplLoad.getValue(0), Token, false);
Jim Laskey279f0532006-09-25 16:29:54 +00006298 }
6299 }
6300
Evan Cheng7fc033a2006-11-03 03:06:21 +00006301 // Try transforming N to an indexed load.
Evan Chengbbd6f6e2006-11-07 09:03:05 +00006302 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
Dan Gohman475871a2008-07-27 21:46:04 +00006303 return SDValue(N, 0);
Evan Cheng7fc033a2006-11-03 03:06:21 +00006304
Dan Gohman475871a2008-07-27 21:46:04 +00006305 return SDValue();
Chris Lattner01a22022005-10-10 22:04:48 +00006306}
6307
Chris Lattner2392ae72010-04-15 04:48:01 +00006308/// CheckForMaskedLoad - Check to see if V is (and load (ptr), imm), where the
6309/// load is having specific bytes cleared out. If so, return the byte size
6310/// being masked out and the shift amount.
6311static std::pair<unsigned, unsigned>
6312CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) {
6313 std::pair<unsigned, unsigned> Result(0, 0);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006314
Chris Lattner2392ae72010-04-15 04:48:01 +00006315 // Check for the structure we're looking for.
6316 if (V->getOpcode() != ISD::AND ||
6317 !isa<ConstantSDNode>(V->getOperand(1)) ||
6318 !ISD::isNormalLoad(V->getOperand(0).getNode()))
6319 return Result;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006320
Chris Lattnere6987582010-04-15 06:10:49 +00006321 // Check the chain and pointer.
Chris Lattner2392ae72010-04-15 04:48:01 +00006322 LoadSDNode *LD = cast<LoadSDNode>(V->getOperand(0));
Chris Lattnere6987582010-04-15 06:10:49 +00006323 if (LD->getBasePtr() != Ptr) return Result; // Not from same pointer.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006324
Chris Lattnere6987582010-04-15 06:10:49 +00006325 // The store should be chained directly to the load or be an operand of a
6326 // tokenfactor.
6327 if (LD == Chain.getNode())
6328 ; // ok.
6329 else if (Chain->getOpcode() != ISD::TokenFactor)
6330 return Result; // Fail.
6331 else {
6332 bool isOk = false;
6333 for (unsigned i = 0, e = Chain->getNumOperands(); i != e; ++i)
6334 if (Chain->getOperand(i).getNode() == LD) {
6335 isOk = true;
6336 break;
6337 }
6338 if (!isOk) return Result;
6339 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006340
Chris Lattner2392ae72010-04-15 04:48:01 +00006341 // This only handles simple types.
6342 if (V.getValueType() != MVT::i16 &&
6343 V.getValueType() != MVT::i32 &&
6344 V.getValueType() != MVT::i64)
6345 return Result;
6346
6347 // Check the constant mask. Invert it so that the bits being masked out are
6348 // 0 and the bits being kept are 1. Use getSExtValue so that leading bits
6349 // follow the sign bit for uniformity.
6350 uint64_t NotMask = ~cast<ConstantSDNode>(V->getOperand(1))->getSExtValue();
6351 unsigned NotMaskLZ = CountLeadingZeros_64(NotMask);
6352 if (NotMaskLZ & 7) return Result; // Must be multiple of a byte.
6353 unsigned NotMaskTZ = CountTrailingZeros_64(NotMask);
6354 if (NotMaskTZ & 7) return Result; // Must be multiple of a byte.
6355 if (NotMaskLZ == 64) return Result; // All zero mask.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006356
Chris Lattner2392ae72010-04-15 04:48:01 +00006357 // See if we have a continuous run of bits. If so, we have 0*1+0*
6358 if (CountTrailingOnes_64(NotMask >> NotMaskTZ)+NotMaskTZ+NotMaskLZ != 64)
6359 return Result;
6360
6361 // Adjust NotMaskLZ down to be from the actual size of the int instead of i64.
6362 if (V.getValueType() != MVT::i64 && NotMaskLZ)
6363 NotMaskLZ -= 64-V.getValueSizeInBits();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006364
Chris Lattner2392ae72010-04-15 04:48:01 +00006365 unsigned MaskedBytes = (V.getValueSizeInBits()-NotMaskLZ-NotMaskTZ)/8;
6366 switch (MaskedBytes) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006367 case 1:
6368 case 2:
Chris Lattner2392ae72010-04-15 04:48:01 +00006369 case 4: break;
6370 default: return Result; // All one mask, or 5-byte mask.
6371 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006372
Chris Lattner2392ae72010-04-15 04:48:01 +00006373 // Verify that the first bit starts at a multiple of mask so that the access
6374 // is aligned the same as the access width.
6375 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes) return Result;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006376
Chris Lattner2392ae72010-04-15 04:48:01 +00006377 Result.first = MaskedBytes;
6378 Result.second = NotMaskTZ/8;
6379 return Result;
6380}
6381
6382
6383/// ShrinkLoadReplaceStoreWithStore - Check to see if IVal is something that
6384/// provides a value as specified by MaskInfo. If so, replace the specified
6385/// store with a narrower store of truncated IVal.
6386static SDNode *
6387ShrinkLoadReplaceStoreWithStore(const std::pair<unsigned, unsigned> &MaskInfo,
6388 SDValue IVal, StoreSDNode *St,
6389 DAGCombiner *DC) {
6390 unsigned NumBytes = MaskInfo.first;
6391 unsigned ByteShift = MaskInfo.second;
6392 SelectionDAG &DAG = DC->getDAG();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006393
Chris Lattner2392ae72010-04-15 04:48:01 +00006394 // Check to see if IVal is all zeros in the part being masked in by the 'or'
6395 // that uses this. If not, this is not a replacement.
6396 APInt Mask = ~APInt::getBitsSet(IVal.getValueSizeInBits(),
6397 ByteShift*8, (ByteShift+NumBytes)*8);
6398 if (!DAG.MaskedValueIsZero(IVal, Mask)) return 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006399
Chris Lattner2392ae72010-04-15 04:48:01 +00006400 // Check that it is legal on the target to do this. It is legal if the new
6401 // VT we're shrinking to (i8/i16/i32) is legal or we're still before type
6402 // legalization.
6403 MVT VT = MVT::getIntegerVT(NumBytes*8);
6404 if (!DC->isTypeLegal(VT))
6405 return 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006406
Chris Lattner2392ae72010-04-15 04:48:01 +00006407 // Okay, we can do this! Replace the 'St' store with a store of IVal that is
6408 // shifted by ByteShift and truncated down to NumBytes.
6409 if (ByteShift)
6410 IVal = DAG.getNode(ISD::SRL, IVal->getDebugLoc(), IVal.getValueType(), IVal,
Owen Anderson95771af2011-02-25 21:41:48 +00006411 DAG.getConstant(ByteShift*8,
6412 DC->getShiftAmountTy(IVal.getValueType())));
Chris Lattner2392ae72010-04-15 04:48:01 +00006413
6414 // Figure out the offset for the store and the alignment of the access.
6415 unsigned StOffset;
6416 unsigned NewAlign = St->getAlignment();
6417
6418 if (DAG.getTargetLoweringInfo().isLittleEndian())
6419 StOffset = ByteShift;
6420 else
6421 StOffset = IVal.getValueType().getStoreSize() - ByteShift - NumBytes;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006422
Chris Lattner2392ae72010-04-15 04:48:01 +00006423 SDValue Ptr = St->getBasePtr();
6424 if (StOffset) {
6425 Ptr = DAG.getNode(ISD::ADD, IVal->getDebugLoc(), Ptr.getValueType(),
6426 Ptr, DAG.getConstant(StOffset, Ptr.getValueType()));
6427 NewAlign = MinAlign(NewAlign, StOffset);
6428 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006429
Chris Lattner2392ae72010-04-15 04:48:01 +00006430 // Truncate down to the new size.
6431 IVal = DAG.getNode(ISD::TRUNCATE, IVal->getDebugLoc(), VT, IVal);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006432
Chris Lattner2392ae72010-04-15 04:48:01 +00006433 ++OpsNarrowed;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006434 return DAG.getStore(St->getChain(), St->getDebugLoc(), IVal, Ptr,
Chris Lattner6229d0a2010-09-21 18:41:36 +00006435 St->getPointerInfo().getWithOffset(StOffset),
Chris Lattner2392ae72010-04-15 04:48:01 +00006436 false, false, NewAlign).getNode();
6437}
6438
Evan Cheng8b944d32009-05-28 00:35:15 +00006439
6440/// ReduceLoadOpStoreWidth - Look for sequence of load / op / store where op is
6441/// one of 'or', 'xor', and 'and' of immediates. If 'op' is only touching some
6442/// of the loaded bits, try narrowing the load and store if it would end up
6443/// being a win for performance or code size.
6444SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
6445 StoreSDNode *ST = cast<StoreSDNode>(N);
Evan Chengcdcecc02009-05-28 18:41:02 +00006446 if (ST->isVolatile())
6447 return SDValue();
6448
Evan Cheng8b944d32009-05-28 00:35:15 +00006449 SDValue Chain = ST->getChain();
6450 SDValue Value = ST->getValue();
6451 SDValue Ptr = ST->getBasePtr();
Owen Andersone50ed302009-08-10 22:56:29 +00006452 EVT VT = Value.getValueType();
Evan Cheng8b944d32009-05-28 00:35:15 +00006453
6454 if (ST->isTruncatingStore() || VT.isVector() || !Value.hasOneUse())
Evan Chengcdcecc02009-05-28 18:41:02 +00006455 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00006456
6457 unsigned Opc = Value.getOpcode();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006458
Chris Lattner2392ae72010-04-15 04:48:01 +00006459 // If this is "store (or X, Y), P" and X is "(and (load P), cst)", where cst
6460 // is a byte mask indicating a consecutive number of bytes, check to see if
6461 // Y is known to provide just those bytes. If so, we try to replace the
6462 // load + replace + store sequence with a single (narrower) store, which makes
6463 // the load dead.
6464 if (Opc == ISD::OR) {
6465 std::pair<unsigned, unsigned> MaskedLoad;
6466 MaskedLoad = CheckForMaskedLoad(Value.getOperand(0), Ptr, Chain);
6467 if (MaskedLoad.first)
6468 if (SDNode *NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad,
6469 Value.getOperand(1), ST,this))
6470 return SDValue(NewST, 0);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006471
Chris Lattner2392ae72010-04-15 04:48:01 +00006472 // Or is commutative, so try swapping X and Y.
6473 MaskedLoad = CheckForMaskedLoad(Value.getOperand(1), Ptr, Chain);
6474 if (MaskedLoad.first)
6475 if (SDNode *NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad,
6476 Value.getOperand(0), ST,this))
6477 return SDValue(NewST, 0);
6478 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006479
Evan Cheng8b944d32009-05-28 00:35:15 +00006480 if ((Opc != ISD::OR && Opc != ISD::XOR && Opc != ISD::AND) ||
6481 Value.getOperand(1).getOpcode() != ISD::Constant)
Evan Chengcdcecc02009-05-28 18:41:02 +00006482 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00006483
6484 SDValue N0 = Value.getOperand(0);
Dan Gohman24bde5b2010-09-02 21:18:42 +00006485 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
6486 Chain == SDValue(N0.getNode(), 1)) {
Evan Cheng8b944d32009-05-28 00:35:15 +00006487 LoadSDNode *LD = cast<LoadSDNode>(N0);
Chris Lattnerfa459012010-09-21 16:08:50 +00006488 if (LD->getBasePtr() != Ptr ||
6489 LD->getPointerInfo().getAddrSpace() !=
6490 ST->getPointerInfo().getAddrSpace())
Evan Chengcdcecc02009-05-28 18:41:02 +00006491 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00006492
6493 // Find the type to narrow it the load / op / store to.
6494 SDValue N1 = Value.getOperand(1);
6495 unsigned BitWidth = N1.getValueSizeInBits();
6496 APInt Imm = cast<ConstantSDNode>(N1)->getAPIntValue();
6497 if (Opc == ISD::AND)
6498 Imm ^= APInt::getAllOnesValue(BitWidth);
Evan Chengd3c76bb2009-05-28 23:52:18 +00006499 if (Imm == 0 || Imm.isAllOnesValue())
6500 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00006501 unsigned ShAmt = Imm.countTrailingZeros();
6502 unsigned MSB = BitWidth - Imm.countLeadingZeros() - 1;
6503 unsigned NewBW = NextPowerOf2(MSB - ShAmt);
Owen Anderson23b9b192009-08-12 00:36:31 +00006504 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);
Evan Cheng8b944d32009-05-28 00:35:15 +00006505 while (NewBW < BitWidth &&
Evan Chengcdcecc02009-05-28 18:41:02 +00006506 !(TLI.isOperationLegalOrCustom(Opc, NewVT) &&
Evan Cheng8b944d32009-05-28 00:35:15 +00006507 TLI.isNarrowingProfitable(VT, NewVT))) {
6508 NewBW = NextPowerOf2(NewBW);
Owen Anderson23b9b192009-08-12 00:36:31 +00006509 NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW);
Evan Cheng8b944d32009-05-28 00:35:15 +00006510 }
Evan Chengcdcecc02009-05-28 18:41:02 +00006511 if (NewBW >= BitWidth)
6512 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00006513
6514 // If the lsb changed does not start at the type bitwidth boundary,
6515 // start at the previous one.
6516 if (ShAmt % NewBW)
6517 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW;
6518 APInt Mask = APInt::getBitsSet(BitWidth, ShAmt, ShAmt + NewBW);
6519 if ((Imm & Mask) == Imm) {
6520 APInt NewImm = (Imm & Mask).lshr(ShAmt).trunc(NewBW);
6521 if (Opc == ISD::AND)
6522 NewImm ^= APInt::getAllOnesValue(NewBW);
6523 uint64_t PtrOff = ShAmt / 8;
6524 // For big endian targets, we need to adjust the offset to the pointer to
6525 // load the correct bytes.
6526 if (TLI.isBigEndian())
Evan Chengcdcecc02009-05-28 18:41:02 +00006527 PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff;
Evan Cheng8b944d32009-05-28 00:35:15 +00006528
6529 unsigned NewAlign = MinAlign(LD->getAlignment(), PtrOff);
Chris Lattnerdb125cf2011-07-18 04:54:35 +00006530 Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext());
Chris Lattner2392ae72010-04-15 04:48:01 +00006531 if (NewAlign < TLI.getTargetData()->getABITypeAlignment(NewVTTy))
Evan Chengcdcecc02009-05-28 18:41:02 +00006532 return SDValue();
6533
Evan Cheng8b944d32009-05-28 00:35:15 +00006534 SDValue NewPtr = DAG.getNode(ISD::ADD, LD->getDebugLoc(),
6535 Ptr.getValueType(), Ptr,
6536 DAG.getConstant(PtrOff, Ptr.getValueType()));
6537 SDValue NewLD = DAG.getLoad(NewVT, N0.getDebugLoc(),
6538 LD->getChain(), NewPtr,
Chris Lattnerfa459012010-09-21 16:08:50 +00006539 LD->getPointerInfo().getWithOffset(PtrOff),
David Greene1e559442010-02-15 17:00:31 +00006540 LD->isVolatile(), LD->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00006541 LD->isInvariant(), NewAlign);
Evan Cheng8b944d32009-05-28 00:35:15 +00006542 SDValue NewVal = DAG.getNode(Opc, Value.getDebugLoc(), NewVT, NewLD,
6543 DAG.getConstant(NewImm, NewVT));
6544 SDValue NewST = DAG.getStore(Chain, N->getDebugLoc(),
6545 NewVal, NewPtr,
Chris Lattnerfa459012010-09-21 16:08:50 +00006546 ST->getPointerInfo().getWithOffset(PtrOff),
David Greene1e559442010-02-15 17:00:31 +00006547 false, false, NewAlign);
Evan Cheng8b944d32009-05-28 00:35:15 +00006548
6549 AddToWorkList(NewPtr.getNode());
6550 AddToWorkList(NewLD.getNode());
6551 AddToWorkList(NewVal.getNode());
6552 WorkListRemover DeadNodes(*this);
6553 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLD.getValue(1),
6554 &DeadNodes);
6555 ++OpsNarrowed;
6556 return NewST;
6557 }
6558 }
6559
Evan Chengcdcecc02009-05-28 18:41:02 +00006560 return SDValue();
Evan Cheng8b944d32009-05-28 00:35:15 +00006561}
6562
Evan Cheng31959b12011-02-02 01:06:55 +00006563/// TransformFPLoadStorePair - For a given floating point load / store pair,
6564/// if the load value isn't used by any other operations, then consider
6565/// transforming the pair to integer load / store operations if the target
6566/// deems the transformation profitable.
6567SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
6568 StoreSDNode *ST = cast<StoreSDNode>(N);
6569 SDValue Chain = ST->getChain();
6570 SDValue Value = ST->getValue();
6571 if (ISD::isNormalStore(ST) && ISD::isNormalLoad(Value.getNode()) &&
6572 Value.hasOneUse() &&
6573 Chain == SDValue(Value.getNode(), 1)) {
6574 LoadSDNode *LD = cast<LoadSDNode>(Value);
6575 EVT VT = LD->getMemoryVT();
6576 if (!VT.isFloatingPoint() ||
6577 VT != ST->getMemoryVT() ||
6578 LD->isNonTemporal() ||
6579 ST->isNonTemporal() ||
6580 LD->getPointerInfo().getAddrSpace() != 0 ||
6581 ST->getPointerInfo().getAddrSpace() != 0)
6582 return SDValue();
6583
6584 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
6585 if (!TLI.isOperationLegal(ISD::LOAD, IntVT) ||
6586 !TLI.isOperationLegal(ISD::STORE, IntVT) ||
6587 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) ||
6588 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT))
6589 return SDValue();
6590
6591 unsigned LDAlign = LD->getAlignment();
6592 unsigned STAlign = ST->getAlignment();
Chris Lattnerdb125cf2011-07-18 04:54:35 +00006593 Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext());
Evan Cheng31959b12011-02-02 01:06:55 +00006594 unsigned ABIAlign = TLI.getTargetData()->getABITypeAlignment(IntVTTy);
6595 if (LDAlign < ABIAlign || STAlign < ABIAlign)
6596 return SDValue();
6597
6598 SDValue NewLD = DAG.getLoad(IntVT, Value.getDebugLoc(),
6599 LD->getChain(), LD->getBasePtr(),
6600 LD->getPointerInfo(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00006601 false, false, false, LDAlign);
Evan Cheng31959b12011-02-02 01:06:55 +00006602
6603 SDValue NewST = DAG.getStore(NewLD.getValue(1), N->getDebugLoc(),
6604 NewLD, ST->getBasePtr(),
6605 ST->getPointerInfo(),
6606 false, false, STAlign);
6607
6608 AddToWorkList(NewLD.getNode());
6609 AddToWorkList(NewST.getNode());
6610 WorkListRemover DeadNodes(*this);
6611 DAG.ReplaceAllUsesOfValueWith(Value.getValue(1), NewLD.getValue(1),
6612 &DeadNodes);
6613 ++LdStFP2Int;
6614 return NewST;
6615 }
6616
6617 return SDValue();
6618}
6619
Dan Gohman475871a2008-07-27 21:46:04 +00006620SDValue DAGCombiner::visitSTORE(SDNode *N) {
Evan Cheng8b2794a2006-10-13 21:14:26 +00006621 StoreSDNode *ST = cast<StoreSDNode>(N);
Dan Gohman475871a2008-07-27 21:46:04 +00006622 SDValue Chain = ST->getChain();
6623 SDValue Value = ST->getValue();
6624 SDValue Ptr = ST->getBasePtr();
Scott Michelfdc40a02009-02-17 22:15:04 +00006625
Evan Cheng59d5b682007-05-07 21:27:48 +00006626 // If this is a store of a bit convert, store the input value if the
Evan Cheng2c4f9432007-05-09 21:49:47 +00006627 // resultant store does not need a higher alignment than the original.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006628 if (Value.getOpcode() == ISD::BITCAST && !ST->isTruncatingStore() &&
Chris Lattnerddf89562008-01-17 19:59:44 +00006629 ST->isUnindexed()) {
Dan Gohman1ba519b2009-02-20 23:29:13 +00006630 unsigned OrigAlign = ST->getAlignment();
Owen Andersone50ed302009-08-10 22:56:29 +00006631 EVT SVT = Value.getOperand(0).getValueType();
Dan Gohman1ba519b2009-02-20 23:29:13 +00006632 unsigned Align = TLI.getTargetData()->
Owen Anderson23b9b192009-08-12 00:36:31 +00006633 getABITypeAlignment(SVT.getTypeForEVT(*DAG.getContext()));
Duncan Sandsd4b9c172008-06-13 19:07:40 +00006634 if (Align <= OrigAlign &&
Duncan Sands25cf2272008-11-24 14:53:14 +00006635 ((!LegalOperations && !ST->isVolatile()) ||
Dan Gohmanf560ffa2009-01-28 17:46:25 +00006636 TLI.isOperationLegalOrCustom(ISD::STORE, SVT)))
Bill Wendlingc144a572009-01-30 23:36:47 +00006637 return DAG.getStore(Chain, N->getDebugLoc(), Value.getOperand(0),
Chris Lattner6229d0a2010-09-21 18:41:36 +00006638 Ptr, ST->getPointerInfo(), ST->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00006639 ST->isNonTemporal(), OrigAlign);
Jim Laskey279f0532006-09-25 16:29:54 +00006640 }
Owen Andersona34d9362011-04-14 17:30:49 +00006641
Chris Lattnerb3452ea2011-04-09 02:32:02 +00006642 // Turn 'store undef, Ptr' -> nothing.
6643 if (Value.getOpcode() == ISD::UNDEF && ST->isUnindexed())
6644 return Chain;
Duncan Sandsd4b9c172008-06-13 19:07:40 +00006645
Nate Begeman2cbba892006-12-11 02:23:46 +00006646 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
Nate Begeman2cbba892006-12-11 02:23:46 +00006647 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
Duncan Sandsd4b9c172008-06-13 19:07:40 +00006648 // NOTE: If the original store is volatile, this transform must not increase
6649 // the number of stores. For example, on x86-32 an f64 can be stored in one
6650 // processor operation but an i64 (which is not legal) requires two. So the
6651 // transform should not be done in this case.
Evan Cheng25ece662006-12-11 17:25:19 +00006652 if (Value.getOpcode() != ISD::TargetConstantFP) {
Dan Gohman475871a2008-07-27 21:46:04 +00006653 SDValue Tmp;
Owen Anderson825b72b2009-08-11 20:47:22 +00006654 switch (CFP->getValueType(0).getSimpleVT().SimpleTy) {
Torok Edwinc23197a2009-07-14 16:55:14 +00006655 default: llvm_unreachable("Unknown FP type");
Owen Anderson825b72b2009-08-11 20:47:22 +00006656 case MVT::f80: // We don't do this for these yet.
6657 case MVT::f128:
6658 case MVT::ppcf128:
Dale Johannesenc7b21d52007-09-18 18:36:59 +00006659 break;
Owen Anderson825b72b2009-08-11 20:47:22 +00006660 case MVT::f32:
Chris Lattner2392ae72010-04-15 04:48:01 +00006661 if ((isTypeLegal(MVT::i32) && !LegalOperations && !ST->isVolatile()) ||
Owen Anderson825b72b2009-08-11 20:47:22 +00006662 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) {
Dale Johannesen9d5f4562007-09-12 03:30:33 +00006663 Tmp = DAG.getConstant((uint32_t)CFP->getValueAPF().
Owen Anderson825b72b2009-08-11 20:47:22 +00006664 bitcastToAPInt().getZExtValue(), MVT::i32);
Bill Wendlingc144a572009-01-30 23:36:47 +00006665 return DAG.getStore(Chain, N->getDebugLoc(), Tmp,
Chris Lattner6229d0a2010-09-21 18:41:36 +00006666 Ptr, ST->getPointerInfo(), ST->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00006667 ST->isNonTemporal(), ST->getAlignment());
Chris Lattner62be1a72006-12-12 04:16:14 +00006668 }
6669 break;
Owen Anderson825b72b2009-08-11 20:47:22 +00006670 case MVT::f64:
Chris Lattner2392ae72010-04-15 04:48:01 +00006671 if ((TLI.isTypeLegal(MVT::i64) && !LegalOperations &&
Dan Gohmanf560ffa2009-01-28 17:46:25 +00006672 !ST->isVolatile()) ||
Owen Anderson825b72b2009-08-11 20:47:22 +00006673 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) {
Dale Johannesen7111b022008-10-09 18:53:47 +00006674 Tmp = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
Owen Anderson825b72b2009-08-11 20:47:22 +00006675 getZExtValue(), MVT::i64);
Bill Wendlingc144a572009-01-30 23:36:47 +00006676 return DAG.getStore(Chain, N->getDebugLoc(), Tmp,
Chris Lattner6229d0a2010-09-21 18:41:36 +00006677 Ptr, ST->getPointerInfo(), ST->isVolatile(),
David Greene1e559442010-02-15 17:00:31 +00006678 ST->isNonTemporal(), ST->getAlignment());
Chris Lattnerb3452ea2011-04-09 02:32:02 +00006679 }
Owen Andersona34d9362011-04-14 17:30:49 +00006680
Chris Lattnerb3452ea2011-04-09 02:32:02 +00006681 if (!ST->isVolatile() &&
6682 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) {
Duncan Sandsdc846502007-10-28 12:59:45 +00006683 // Many FP stores are not made apparent until after legalize, e.g. for
Chris Lattner62be1a72006-12-12 04:16:14 +00006684 // argument passing. Since this is so common, custom legalize the
6685 // 64-bit integer store into two 32-bit stores.
Dale Johannesen7111b022008-10-09 18:53:47 +00006686 uint64_t Val = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
Owen Anderson825b72b2009-08-11 20:47:22 +00006687 SDValue Lo = DAG.getConstant(Val & 0xFFFFFFFF, MVT::i32);
6688 SDValue Hi = DAG.getConstant(Val >> 32, MVT::i32);
Duncan Sands0753fc12008-02-11 10:37:04 +00006689 if (TLI.isBigEndian()) std::swap(Lo, Hi);
Chris Lattner62be1a72006-12-12 04:16:14 +00006690
Dan Gohmand6fd1bc2007-07-09 22:18:38 +00006691 unsigned Alignment = ST->getAlignment();
6692 bool isVolatile = ST->isVolatile();
David Greene1e559442010-02-15 17:00:31 +00006693 bool isNonTemporal = ST->isNonTemporal();
Dan Gohmand6fd1bc2007-07-09 22:18:38 +00006694
Bill Wendlingc144a572009-01-30 23:36:47 +00006695 SDValue St0 = DAG.getStore(Chain, ST->getDebugLoc(), Lo,
Chris Lattner6229d0a2010-09-21 18:41:36 +00006696 Ptr, ST->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00006697 isVolatile, isNonTemporal,
6698 ST->getAlignment());
Bill Wendlingc144a572009-01-30 23:36:47 +00006699 Ptr = DAG.getNode(ISD::ADD, N->getDebugLoc(), Ptr.getValueType(), Ptr,
Chris Lattner62be1a72006-12-12 04:16:14 +00006700 DAG.getConstant(4, Ptr.getValueType()));
Duncan Sandsdc846502007-10-28 12:59:45 +00006701 Alignment = MinAlign(Alignment, 4U);
Bill Wendlingc144a572009-01-30 23:36:47 +00006702 SDValue St1 = DAG.getStore(Chain, ST->getDebugLoc(), Hi,
Chris Lattner6229d0a2010-09-21 18:41:36 +00006703 Ptr, ST->getPointerInfo().getWithOffset(4),
6704 isVolatile, isNonTemporal,
David Greene1e559442010-02-15 17:00:31 +00006705 Alignment);
Owen Anderson825b72b2009-08-11 20:47:22 +00006706 return DAG.getNode(ISD::TokenFactor, N->getDebugLoc(), MVT::Other,
Bill Wendlingc144a572009-01-30 23:36:47 +00006707 St0, St1);
Chris Lattner62be1a72006-12-12 04:16:14 +00006708 }
Bill Wendlingc144a572009-01-30 23:36:47 +00006709
Chris Lattner62be1a72006-12-12 04:16:14 +00006710 break;
Evan Cheng25ece662006-12-11 17:25:19 +00006711 }
Nate Begeman2cbba892006-12-11 02:23:46 +00006712 }
Nate Begeman2cbba892006-12-11 02:23:46 +00006713 }
6714
Evan Cheng255f20f2010-04-01 06:04:33 +00006715 // Try to infer better alignment information than the store already has.
6716 if (OptLevel != CodeGenOpt::None && ST->isUnindexed()) {
Evan Chenged1c0c72011-11-28 22:37:34 +00006717 if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
6718 if (Align > ST->getAlignment())
6719 return DAG.getTruncStore(Chain, N->getDebugLoc(), Value,
6720 Ptr, ST->getPointerInfo(), ST->getMemoryVT(),
6721 ST->isVolatile(), ST->isNonTemporal(), Align);
Evan Cheng255f20f2010-04-01 06:04:33 +00006722 }
6723 }
6724
Evan Cheng31959b12011-02-02 01:06:55 +00006725 // Try transforming a pair floating point load / store ops to integer
6726 // load / store ops.
6727 SDValue NewST = TransformFPLoadStorePair(N);
6728 if (NewST.getNode())
6729 return NewST;
6730
Scott Michelfdc40a02009-02-17 22:15:04 +00006731 if (CombinerAA) {
Jim Laskey279f0532006-09-25 16:29:54 +00006732 // Walk up chain skipping non-aliasing memory nodes.
Dan Gohman475871a2008-07-27 21:46:04 +00006733 SDValue BetterChain = FindBetterChain(N, Chain);
Scott Michelfdc40a02009-02-17 22:15:04 +00006734
Jim Laskey6ff23e52006-10-04 16:53:27 +00006735 // If there is a better chain.
Jim Laskey279f0532006-09-25 16:29:54 +00006736 if (Chain != BetterChain) {
Dan Gohman475871a2008-07-27 21:46:04 +00006737 SDValue ReplStore;
Nate Begemanb6aef5c2009-09-15 00:18:30 +00006738
6739 // Replace the chain to avoid dependency.
Jim Laskeyd4edf2c2006-10-14 12:14:27 +00006740 if (ST->isTruncatingStore()) {
Bill Wendlingc144a572009-01-30 23:36:47 +00006741 ReplStore = DAG.getTruncStore(BetterChain, N->getDebugLoc(), Value, Ptr,
Chris Lattnerda2d8e12010-09-21 17:42:31 +00006742 ST->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00006743 ST->getMemoryVT(), ST->isVolatile(),
6744 ST->isNonTemporal(), ST->getAlignment());
Jim Laskeyd4edf2c2006-10-14 12:14:27 +00006745 } else {
Bill Wendlingc144a572009-01-30 23:36:47 +00006746 ReplStore = DAG.getStore(BetterChain, N->getDebugLoc(), Value, Ptr,
Chris Lattner6229d0a2010-09-21 18:41:36 +00006747 ST->getPointerInfo(),
David Greene1e559442010-02-15 17:00:31 +00006748 ST->isVolatile(), ST->isNonTemporal(),
6749 ST->getAlignment());
Jim Laskeyd4edf2c2006-10-14 12:14:27 +00006750 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006751
Jim Laskey279f0532006-09-25 16:29:54 +00006752 // Create token to keep both nodes around.
Bill Wendlingc144a572009-01-30 23:36:47 +00006753 SDValue Token = DAG.getNode(ISD::TokenFactor, N->getDebugLoc(),
Owen Anderson825b72b2009-08-11 20:47:22 +00006754 MVT::Other, Chain, ReplStore);
Bill Wendlingc144a572009-01-30 23:36:47 +00006755
Nate Begemanb6aef5c2009-09-15 00:18:30 +00006756 // Make sure the new and old chains are cleaned up.
6757 AddToWorkList(Token.getNode());
6758
Jim Laskey274062c2006-10-13 23:32:28 +00006759 // Don't add users to work list.
6760 return CombineTo(N, Token, false);
Jim Laskey279f0532006-09-25 16:29:54 +00006761 }
Jim Laskeyd1aed7a2006-09-21 16:28:59 +00006762 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006763
Evan Cheng33dbedc2006-11-05 09:31:14 +00006764 // Try transforming N to an indexed store.
Evan Chengbbd6f6e2006-11-07 09:03:05 +00006765 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
Dan Gohman475871a2008-07-27 21:46:04 +00006766 return SDValue(N, 0);
Evan Cheng33dbedc2006-11-05 09:31:14 +00006767
Chris Lattner3c872852007-12-29 06:26:16 +00006768 // FIXME: is there such a thing as a truncating indexed store?
Chris Lattnerddf89562008-01-17 19:59:44 +00006769 if (ST->isTruncatingStore() && ST->isUnindexed() &&
Nadav Rotembaff46f2011-06-15 11:19:12 +00006770 Value.getValueType().isInteger()) {
Chris Lattner2b4c2792007-10-13 06:35:54 +00006771 // See if we can simplify the input to this truncstore with knowledge that
6772 // only the low bits are being used. For example:
6773 // "truncstore (or (shl x, 8), y), i8" -> "truncstore y, i8"
Scott Michelfdc40a02009-02-17 22:15:04 +00006774 SDValue Shorter =
Dan Gohman2e68b6f2008-02-25 21:11:39 +00006775 GetDemandedBits(Value,
Nadav Rotembaff46f2011-06-15 11:19:12 +00006776 APInt::getLowBitsSet(
6777 Value.getValueType().getScalarType().getSizeInBits(),
6778 ST->getMemoryVT().getScalarType().getSizeInBits()));
Gabor Greifba36cb52008-08-28 21:40:38 +00006779 AddToWorkList(Value.getNode());
6780 if (Shorter.getNode())
Bill Wendlingc144a572009-01-30 23:36:47 +00006781 return DAG.getTruncStore(Chain, N->getDebugLoc(), Shorter,
Chris Lattnerda2d8e12010-09-21 17:42:31 +00006782 Ptr, ST->getPointerInfo(), ST->getMemoryVT(),
David Greene1e559442010-02-15 17:00:31 +00006783 ST->isVolatile(), ST->isNonTemporal(),
6784 ST->getAlignment());
Scott Michelfdc40a02009-02-17 22:15:04 +00006785
Chris Lattnere33544c2007-10-13 06:58:48 +00006786 // Otherwise, see if we can simplify the operation with
6787 // SimplifyDemandedBits, which only works if the value has a single use.
Dan Gohman7b8d4a92008-02-27 00:25:32 +00006788 if (SimplifyDemandedBits(Value,
Eric Christopher503a64d2010-12-09 04:48:06 +00006789 APInt::getLowBitsSet(
6790 Value.getValueType().getScalarType().getSizeInBits(),
6791 ST->getMemoryVT().getScalarType().getSizeInBits())))
Dan Gohman475871a2008-07-27 21:46:04 +00006792 return SDValue(N, 0);
Chris Lattner2b4c2792007-10-13 06:35:54 +00006793 }
Scott Michelfdc40a02009-02-17 22:15:04 +00006794
Chris Lattner3c872852007-12-29 06:26:16 +00006795 // If this is a load followed by a store to the same location, then the store
6796 // is dead/noop.
6797 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Value)) {
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006798 if (Ld->getBasePtr() == Ptr && ST->getMemoryVT() == Ld->getMemoryVT() &&
Chris Lattnerddf89562008-01-17 19:59:44 +00006799 ST->isUnindexed() && !ST->isVolatile() &&
Chris Lattner07649d92008-01-08 23:08:06 +00006800 // There can't be any side effects between the load and store, such as
6801 // a call or store.
Dan Gohman475871a2008-07-27 21:46:04 +00006802 Chain.reachesChainWithoutSideEffects(SDValue(Ld, 1))) {
Chris Lattner3c872852007-12-29 06:26:16 +00006803 // The store is dead, remove it.
6804 return Chain;
6805 }
6806 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00006807
Chris Lattnerddf89562008-01-17 19:59:44 +00006808 // If this is an FP_ROUND or TRUNC followed by a store, fold this into a
6809 // truncating store. We can do this even if this is already a truncstore.
6810 if ((Value.getOpcode() == ISD::FP_ROUND || Value.getOpcode() == ISD::TRUNCATE)
Gabor Greifba36cb52008-08-28 21:40:38 +00006811 && Value.getNode()->hasOneUse() && ST->isUnindexed() &&
Chris Lattnerddf89562008-01-17 19:59:44 +00006812 TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(),
Dan Gohmanb625f2f2008-01-30 00:15:11 +00006813 ST->getMemoryVT())) {
Bill Wendlingc144a572009-01-30 23:36:47 +00006814 return DAG.getTruncStore(Chain, N->getDebugLoc(), Value.getOperand(0),
Chris Lattnerda2d8e12010-09-21 17:42:31 +00006815 Ptr, ST->getPointerInfo(), ST->getMemoryVT(),
David Greene1e559442010-02-15 17:00:31 +00006816 ST->isVolatile(), ST->isNonTemporal(),
6817 ST->getAlignment());
Chris Lattnerddf89562008-01-17 19:59:44 +00006818 }
Duncan Sandsd4b9c172008-06-13 19:07:40 +00006819
Evan Cheng8b944d32009-05-28 00:35:15 +00006820 return ReduceLoadOpStoreWidth(N);
Chris Lattner87514ca2005-10-10 22:31:19 +00006821}
6822
Dan Gohman475871a2008-07-27 21:46:04 +00006823SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
6824 SDValue InVec = N->getOperand(0);
6825 SDValue InVal = N->getOperand(1);
6826 SDValue EltNo = N->getOperand(2);
Eli Friedman9db817f2011-09-09 21:04:06 +00006827 DebugLoc dl = N->getDebugLoc();
Scott Michelfdc40a02009-02-17 22:15:04 +00006828
Bob Wilson492fd452010-05-19 23:42:58 +00006829 // If the inserted element is an UNDEF, just use the input vector.
6830 if (InVal.getOpcode() == ISD::UNDEF)
6831 return InVec;
6832
Nadav Rotem609d54e2011-02-12 14:40:33 +00006833 EVT VT = InVec.getValueType();
6834
Owen Anderson95771af2011-02-25 21:41:48 +00006835 // If we can't generate a legal BUILD_VECTOR, exit
Nadav Rotem609d54e2011-02-12 14:40:33 +00006836 if (LegalOperations && !TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
6837 return SDValue();
6838
Eli Friedman9db817f2011-09-09 21:04:06 +00006839 // Check that we know which element is being inserted
6840 if (!isa<ConstantSDNode>(EltNo))
6841 return SDValue();
6842 unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00006843
Eli Friedman9db817f2011-09-09 21:04:06 +00006844 // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially
6845 // be converted to a BUILD_VECTOR). Fill in the Ops vector with the
6846 // vector elements.
6847 SmallVector<SDValue, 8> Ops;
6848 if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
6849 Ops.append(InVec.getNode()->op_begin(),
6850 InVec.getNode()->op_end());
6851 } else if (InVec.getOpcode() == ISD::UNDEF) {
6852 unsigned NElts = VT.getVectorNumElements();
6853 Ops.append(NElts, DAG.getUNDEF(InVal.getValueType()));
6854 } else {
6855 return SDValue();
Nate Begeman9008ca62009-04-27 18:41:29 +00006856 }
Eli Friedman9db817f2011-09-09 21:04:06 +00006857
6858 // Insert the element
6859 if (Elt < Ops.size()) {
6860 // All the operands of BUILD_VECTOR must have the same type;
6861 // we enforce that here.
6862 EVT OpVT = Ops[0].getValueType();
6863 if (InVal.getValueType() != OpVT)
6864 InVal = OpVT.bitsGT(InVal.getValueType()) ?
6865 DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) :
6866 DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal);
6867 Ops[Elt] = InVal;
6868 }
6869
6870 // Return the new vector
6871 return DAG.getNode(ISD::BUILD_VECTOR, dl,
6872 VT, &Ops[0], Ops.size());
Chris Lattnerca242442006-03-19 01:27:56 +00006873}
6874
Dan Gohman475871a2008-07-27 21:46:04 +00006875SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
Mon P Wang7ac9cdf2009-01-17 00:07:25 +00006876 // (vextract (scalar_to_vector val, 0) -> val
6877 SDValue InVec = N->getOperand(0);
Mon P Wang7ac9cdf2009-01-17 00:07:25 +00006878
Duncan Sandsc356f332011-05-09 08:03:33 +00006879 if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR) {
6880 // Check if the result type doesn't match the inserted element type. A
6881 // SCALAR_TO_VECTOR may truncate the inserted element and the
6882 // EXTRACT_VECTOR_ELT may widen the extracted vector.
6883 SDValue InOp = InVec.getOperand(0);
6884 EVT NVT = N->getValueType(0);
6885 if (InOp.getValueType() != NVT) {
6886 assert(InOp.getValueType().isInteger() && NVT.isInteger());
6887 return DAG.getSExtOrTrunc(InOp, InVec.getDebugLoc(), NVT);
6888 }
6889 return InOp;
6890 }
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006891
6892 // Perform only after legalization to ensure build_vector / vector_shuffle
6893 // optimizations have already been done.
Duncan Sands25cf2272008-11-24 14:53:14 +00006894 if (!LegalOperations) return SDValue();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006895
Mon P Wang7ac9cdf2009-01-17 00:07:25 +00006896 // (vextract (v4f32 load $addr), c) -> (f32 load $addr+c*size)
6897 // (vextract (v4f32 s2v (f32 load $addr)), c) -> (f32 load $addr+c*size)
6898 // (vextract (v4f32 shuffle (load $addr), <1,u,u,u>), 0) -> (f32 load $addr)
Mon P Wange3bc6ae2009-01-18 06:43:40 +00006899 SDValue EltNo = N->getOperand(1);
Evan Cheng513da432007-10-06 08:19:55 +00006900
Evan Cheng513da432007-10-06 08:19:55 +00006901 if (isa<ConstantSDNode>(EltNo)) {
Eric Christophercaebdd42010-11-03 09:36:40 +00006902 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
Evan Cheng513da432007-10-06 08:19:55 +00006903 bool NewLoad = false;
Mon P Wanga60b5232008-12-11 00:26:16 +00006904 bool BCNumEltsChanged = false;
Owen Andersone50ed302009-08-10 22:56:29 +00006905 EVT VT = InVec.getValueType();
6906 EVT ExtVT = VT.getVectorElementType();
6907 EVT LVT = ExtVT;
Bill Wendlingc144a572009-01-30 23:36:47 +00006908
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006909 if (InVec.getOpcode() == ISD::BITCAST) {
Eli Friedmand6e25602011-12-26 22:49:32 +00006910 // Don't duplicate a load with other uses.
6911 if (!InVec.hasOneUse())
6912 return SDValue();
6913
Owen Andersone50ed302009-08-10 22:56:29 +00006914 EVT BCVT = InVec.getOperand(0).getValueType();
6915 if (!BCVT.isVector() || ExtVT.bitsGT(BCVT.getVectorElementType()))
Dan Gohman475871a2008-07-27 21:46:04 +00006916 return SDValue();
Mon P Wanga60b5232008-12-11 00:26:16 +00006917 if (VT.getVectorNumElements() != BCVT.getVectorNumElements())
6918 BCNumEltsChanged = true;
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006919 InVec = InVec.getOperand(0);
Owen Andersone50ed302009-08-10 22:56:29 +00006920 ExtVT = BCVT.getVectorElementType();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006921 NewLoad = true;
6922 }
Evan Cheng513da432007-10-06 08:19:55 +00006923
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006924 LoadSDNode *LN0 = NULL;
Nate Begeman5a5ca152009-04-29 05:20:52 +00006925 const ShuffleVectorSDNode *SVN = NULL;
Bill Wendlingc144a572009-01-30 23:36:47 +00006926 if (ISD::isNormalLoad(InVec.getNode())) {
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006927 LN0 = cast<LoadSDNode>(InVec);
Bill Wendlingc144a572009-01-30 23:36:47 +00006928 } else if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR &&
Owen Andersone50ed302009-08-10 22:56:29 +00006929 InVec.getOperand(0).getValueType() == ExtVT &&
Bill Wendlingc144a572009-01-30 23:36:47 +00006930 ISD::isNormalLoad(InVec.getOperand(0).getNode())) {
Eli Friedmand6e25602011-12-26 22:49:32 +00006931 // Don't duplicate a load with other uses.
6932 if (!InVec.hasOneUse())
6933 return SDValue();
6934
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006935 LN0 = cast<LoadSDNode>(InVec.getOperand(0));
Nate Begeman5a5ca152009-04-29 05:20:52 +00006936 } else if ((SVN = dyn_cast<ShuffleVectorSDNode>(InVec))) {
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006937 // (vextract (vector_shuffle (load $addr), v2, <1, u, u, u>), 1)
6938 // =>
6939 // (load $addr+1*size)
Scott Michelfdc40a02009-02-17 22:15:04 +00006940
Eli Friedmand6e25602011-12-26 22:49:32 +00006941 // Don't duplicate a load with other uses.
6942 if (!InVec.hasOneUse())
6943 return SDValue();
6944
Mon P Wanga60b5232008-12-11 00:26:16 +00006945 // If the bit convert changed the number of elements, it is unsafe
6946 // to examine the mask.
6947 if (BCNumEltsChanged)
6948 return SDValue();
Nate Begeman5a5ca152009-04-29 05:20:52 +00006949
6950 // Select the input vector, guarding against out of range extract vector.
6951 unsigned NumElems = VT.getVectorNumElements();
Eric Christophercaebdd42010-11-03 09:36:40 +00006952 int Idx = (Elt > (int)NumElems) ? -1 : SVN->getMaskElt(Elt);
Nate Begeman5a5ca152009-04-29 05:20:52 +00006953 InVec = (Idx < (int)NumElems) ? InVec.getOperand(0) : InVec.getOperand(1);
6954
Eli Friedmand6e25602011-12-26 22:49:32 +00006955 if (InVec.getOpcode() == ISD::BITCAST) {
6956 // Don't duplicate a load with other uses.
6957 if (!InVec.hasOneUse())
6958 return SDValue();
6959
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006960 InVec = InVec.getOperand(0);
Eli Friedmand6e25602011-12-26 22:49:32 +00006961 }
Gabor Greifba36cb52008-08-28 21:40:38 +00006962 if (ISD::isNormalLoad(InVec.getNode())) {
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006963 LN0 = cast<LoadSDNode>(InVec);
Ted Kremenekd0e88f32010-04-08 18:49:30 +00006964 Elt = (Idx < (int)NumElems) ? Idx : Idx - (int)NumElems;
Evan Cheng513da432007-10-06 08:19:55 +00006965 }
6966 }
Bill Wendlingc144a572009-01-30 23:36:47 +00006967
Eli Friedmand6e25602011-12-26 22:49:32 +00006968 // Make sure we found a non-volatile load and the extractelement is
6969 // the only use.
Nadav Rotem42febc62011-05-11 14:40:50 +00006970 if (!LN0 || !LN0->hasNUsesOfValue(1,0) || LN0->isVolatile())
Dan Gohman475871a2008-07-27 21:46:04 +00006971 return SDValue();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006972
Eric Christopherd81f17a2010-11-03 20:44:42 +00006973 // If Idx was -1 above, Elt is going to be -1, so just return undef.
6974 if (Elt == -1)
Eli Friedmaned4b4272011-07-25 22:25:42 +00006975 return DAG.getUNDEF(LVT);
Eric Christopherd81f17a2010-11-03 20:44:42 +00006976
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006977 unsigned Align = LN0->getAlignment();
6978 if (NewLoad) {
6979 // Check the resultant load doesn't need a higher alignment than the
6980 // original load.
Bill Wendlingc144a572009-01-30 23:36:47 +00006981 unsigned NewAlign =
Eric Christopher503a64d2010-12-09 04:48:06 +00006982 TLI.getTargetData()
6983 ->getABITypeAlignment(LVT.getTypeForEVT(*DAG.getContext()));
Bill Wendlingc144a572009-01-30 23:36:47 +00006984
Dan Gohmanf560ffa2009-01-28 17:46:25 +00006985 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, LVT))
Dan Gohman475871a2008-07-27 21:46:04 +00006986 return SDValue();
Bill Wendlingc144a572009-01-30 23:36:47 +00006987
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006988 Align = NewAlign;
6989 }
6990
Dan Gohman475871a2008-07-27 21:46:04 +00006991 SDValue NewPtr = LN0->getBasePtr();
Chris Lattnerfa459012010-09-21 16:08:50 +00006992 unsigned PtrOff = 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00006993
Eric Christopherd81f17a2010-11-03 20:44:42 +00006994 if (Elt) {
Chris Lattnerfa459012010-09-21 16:08:50 +00006995 PtrOff = LVT.getSizeInBits() * Elt / 8;
Owen Andersone50ed302009-08-10 22:56:29 +00006996 EVT PtrType = NewPtr.getValueType();
Evan Cheng77f0b7a2008-05-13 08:35:03 +00006997 if (TLI.isBigEndian())
Duncan Sands83ec4b62008-06-06 12:08:01 +00006998 PtrOff = VT.getSizeInBits() / 8 - PtrOff;
Bill Wendlingc144a572009-01-30 23:36:47 +00006999 NewPtr = DAG.getNode(ISD::ADD, N->getDebugLoc(), PtrType, NewPtr,
Evan Cheng77f0b7a2008-05-13 08:35:03 +00007000 DAG.getConstant(PtrOff, PtrType));
7001 }
Bill Wendlingc144a572009-01-30 23:36:47 +00007002
Eli Friedman4db4add2011-11-16 23:50:22 +00007003 // The replacement we need to do here is a little tricky: we need to
7004 // replace an extractelement of a load with a load.
7005 // Use ReplaceAllUsesOfValuesWith to do the replacement.
Eli Friedmand6e25602011-12-26 22:49:32 +00007006 // Note that this replacement assumes that the extractvalue is the only
7007 // use of the load; that's okay because we don't want to perform this
7008 // transformation in other cases anyway.
Eli Friedman4db4add2011-11-16 23:50:22 +00007009 SDValue Load = DAG.getLoad(LVT, N->getDebugLoc(), LN0->getChain(), NewPtr,
7010 LN0->getPointerInfo().getWithOffset(PtrOff),
7011 LN0->isVolatile(), LN0->isNonTemporal(),
7012 LN0->isInvariant(), Align);
7013 WorkListRemover DeadNodes(*this);
7014 SDValue From[] = { SDValue(N, 0), SDValue(LN0,1) };
7015 SDValue To[] = { Load.getValue(0), Load.getValue(1) };
7016 DAG.ReplaceAllUsesOfValuesWith(From, To, 2, &DeadNodes);
7017 // Since we're explcitly calling ReplaceAllUses, add the new node to the
7018 // worklist explicitly as well.
7019 AddToWorkList(Load.getNode());
7020 // Make sure to revisit this node to clean it up; it will usually be dead.
7021 AddToWorkList(N);
7022 return SDValue(N, 0);
Evan Cheng513da432007-10-06 08:19:55 +00007023 }
Bill Wendlingc144a572009-01-30 23:36:47 +00007024
Dan Gohman475871a2008-07-27 21:46:04 +00007025 return SDValue();
Evan Cheng513da432007-10-06 08:19:55 +00007026}
Evan Cheng513da432007-10-06 08:19:55 +00007027
Dan Gohman475871a2008-07-27 21:46:04 +00007028SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
Dan Gohman7f321562007-06-25 16:23:39 +00007029 unsigned NumInScalars = N->getNumOperands();
Nadav Rotemb00418a2011-10-29 21:23:04 +00007030 DebugLoc dl = N->getDebugLoc();
Owen Andersone50ed302009-08-10 22:56:29 +00007031 EVT VT = N->getValueType(0);
Nadav Rotemb00418a2011-10-29 21:23:04 +00007032 // Check to see if this is a BUILD_VECTOR of a bunch of values
7033 // which come from any_extend or zero_extend nodes. If so, we can create
7034 // a new BUILD_VECTOR using bit-casts which may enable other BUILD_VECTOR
Nadav Rotemf47368b2011-10-31 20:08:25 +00007035 // optimizations. We do not handle sign-extend because we can't fill the sign
7036 // using shuffles.
Nadav Rotemb00418a2011-10-29 21:23:04 +00007037 EVT SourceType = MVT::Other;
Nadav Rotemb00418a2011-10-29 21:23:04 +00007038 bool allAnyExt = true;
7039 for (unsigned i = 0; i < NumInScalars; ++i) {
7040 SDValue In = N->getOperand(i);
7041 // Ignore undef inputs.
7042 if (In.getOpcode() == ISD::UNDEF) continue;
7043
7044 bool AnyExt = In.getOpcode() == ISD::ANY_EXTEND;
7045 bool ZeroExt = In.getOpcode() == ISD::ZERO_EXTEND;
7046
Nadav Rotemf47368b2011-10-31 20:08:25 +00007047 // Abort if the element is not an extension.
Nadav Rotemb00418a2011-10-29 21:23:04 +00007048 if (!ZeroExt && !AnyExt) {
Nadav Rotemf47368b2011-10-31 20:08:25 +00007049 SourceType = MVT::Other;
Nadav Rotemb00418a2011-10-29 21:23:04 +00007050 break;
7051 }
7052
7053 // The input is a ZeroExt or AnyExt. Check the original type.
7054 EVT InTy = In.getOperand(0).getValueType();
7055
7056 // Check that all of the widened source types are the same.
7057 if (SourceType == MVT::Other)
Nadav Rotemf47368b2011-10-31 20:08:25 +00007058 // First time.
Nadav Rotemb00418a2011-10-29 21:23:04 +00007059 SourceType = InTy;
7060 else if (InTy != SourceType) {
7061 // Multiple income types. Abort.
Nadav Rotemf47368b2011-10-31 20:08:25 +00007062 SourceType = MVT::Other;
Nadav Rotemb00418a2011-10-29 21:23:04 +00007063 break;
7064 }
7065
7066 // Check if all of the extends are ANY_EXTENDs.
7067 allAnyExt &= AnyExt;
7068 }
7069
Nadav Rotemf47368b2011-10-31 20:08:25 +00007070
7071 // In order to have valid types, all of the inputs must be extended from the
7072 // same source type and all of the inputs must be any or zero extend.
7073 // Scalar sizes must be a power of two.
7074 EVT OutScalarTy = N->getValueType(0).getScalarType();
7075 bool validTypes = SourceType != MVT::Other &&
7076 isPowerOf2_32(OutScalarTy.getSizeInBits()) &&
7077 isPowerOf2_32(SourceType.getSizeInBits());
7078
7079 // We perform this optimization post type-legalization because
7080 // the type-legalizer often scalarizes integer-promoted vectors.
7081 // Performing this optimization before may create bit-casts which
7082 // will be type-legalized to complex code sequences.
7083 // We perform this optimization only before the operation legalizer because we
7084 // may introduce illegal operations.
7085 if (LegalTypes && !LegalOperations && validTypes) {
Nadav Rotemb00418a2011-10-29 21:23:04 +00007086 bool isLE = TLI.isLittleEndian();
Nadav Rotemf47368b2011-10-31 20:08:25 +00007087 unsigned ElemRatio = OutScalarTy.getSizeInBits()/SourceType.getSizeInBits();
Nadav Rotemb00418a2011-10-29 21:23:04 +00007088 assert(ElemRatio > 1 && "Invalid element size ratio");
Nadav Rotemf47368b2011-10-31 20:08:25 +00007089 SDValue Filler = allAnyExt ? DAG.getUNDEF(SourceType):
7090 DAG.getConstant(0, SourceType);
Nadav Rotemb00418a2011-10-29 21:23:04 +00007091
7092 unsigned NewBVElems = ElemRatio * N->getValueType(0).getVectorNumElements();
Benjamin Kramer50bf86e2011-10-30 08:39:55 +00007093 SmallVector<SDValue, 8> Ops(NewBVElems, Filler);
Nadav Rotemb00418a2011-10-29 21:23:04 +00007094
7095 // Populate the new build_vector
7096 for (unsigned i=0; i < N->getNumOperands(); ++i) {
7097 SDValue Cast = N->getOperand(i);
Benjamin Kramer50bf86e2011-10-30 08:39:55 +00007098 assert((Cast.getOpcode() == ISD::ANY_EXTEND ||
7099 Cast.getOpcode() == ISD::ZERO_EXTEND ||
7100 Cast.getOpcode() == ISD::UNDEF) && "Invalid cast opcode");
Nadav Rotemb00418a2011-10-29 21:23:04 +00007101 SDValue In;
7102 if (Cast.getOpcode() == ISD::UNDEF)
Nadav Rotemf47368b2011-10-31 20:08:25 +00007103 In = DAG.getUNDEF(SourceType);
Nadav Rotemb00418a2011-10-29 21:23:04 +00007104 else
7105 In = Cast->getOperand(0);
7106 unsigned Index = isLE ? (i * ElemRatio) :
7107 (i * ElemRatio + (ElemRatio - 1));
7108
7109 assert(Index < Ops.size() && "Invalid index");
7110 Ops[Index] = In;
7111 }
7112
7113 // The type of the new BUILD_VECTOR node.
Nadav Rotemf47368b2011-10-31 20:08:25 +00007114 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), SourceType, NewBVElems);
Nadav Rotemb00418a2011-10-29 21:23:04 +00007115 assert(VecVT.getSizeInBits() == N->getValueType(0).getSizeInBits() &&
7116 "Invalid vector size");
Nadav Rotemf47368b2011-10-31 20:08:25 +00007117 // Check if the new vector type is legal.
7118 if (!isTypeLegal(VecVT)) return SDValue();
Nadav Rotemb00418a2011-10-29 21:23:04 +00007119
7120 // Make the new BUILD_VECTOR.
7121 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
7122 VecVT, &Ops[0], Ops.size());
7123
7124 // Bitcast to the desired type.
7125 return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), BV);
7126 }
Chris Lattnerca242442006-03-19 01:27:56 +00007127
Dan Gohman7f321562007-06-25 16:23:39 +00007128 // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT
7129 // operations. If so, and if the EXTRACT_VECTOR_ELT vector inputs come from
7130 // at most two distinct vectors, turn this into a shuffle node.
Dan Gohman475871a2008-07-27 21:46:04 +00007131 SDValue VecIn1, VecIn2;
Chris Lattnerd7648c82006-03-28 20:28:38 +00007132 for (unsigned i = 0; i != NumInScalars; ++i) {
7133 // Ignore undef inputs.
7134 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00007135
Dan Gohman7f321562007-06-25 16:23:39 +00007136 // If this input is something other than a EXTRACT_VECTOR_ELT with a
Chris Lattnerd7648c82006-03-28 20:28:38 +00007137 // constant index, bail out.
Dan Gohman7f321562007-06-25 16:23:39 +00007138 if (N->getOperand(i).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
Chris Lattnerd7648c82006-03-28 20:28:38 +00007139 !isa<ConstantSDNode>(N->getOperand(i).getOperand(1))) {
Dan Gohman475871a2008-07-27 21:46:04 +00007140 VecIn1 = VecIn2 = SDValue(0, 0);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007141 break;
7142 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007143
Dan Gohman7f321562007-06-25 16:23:39 +00007144 // If the input vector type disagrees with the result of the build_vector,
Chris Lattnerd7648c82006-03-28 20:28:38 +00007145 // we can't make a shuffle.
Dan Gohman475871a2008-07-27 21:46:04 +00007146 SDValue ExtractedFromVec = N->getOperand(i).getOperand(0);
Dan Gohman7f321562007-06-25 16:23:39 +00007147 if (ExtractedFromVec.getValueType() != VT) {
Dan Gohman475871a2008-07-27 21:46:04 +00007148 VecIn1 = VecIn2 = SDValue(0, 0);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007149 break;
7150 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007151
Chris Lattnerd7648c82006-03-28 20:28:38 +00007152 // Otherwise, remember this. We allow up to two distinct input vectors.
7153 if (ExtractedFromVec == VecIn1 || ExtractedFromVec == VecIn2)
7154 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00007155
Gabor Greifba36cb52008-08-28 21:40:38 +00007156 if (VecIn1.getNode() == 0) {
Chris Lattnerd7648c82006-03-28 20:28:38 +00007157 VecIn1 = ExtractedFromVec;
Gabor Greifba36cb52008-08-28 21:40:38 +00007158 } else if (VecIn2.getNode() == 0) {
Chris Lattnerd7648c82006-03-28 20:28:38 +00007159 VecIn2 = ExtractedFromVec;
7160 } else {
7161 // Too many inputs.
Dan Gohman475871a2008-07-27 21:46:04 +00007162 VecIn1 = VecIn2 = SDValue(0, 0);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007163 break;
7164 }
7165 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007166
Chris Lattnerd7648c82006-03-28 20:28:38 +00007167 // If everything is good, we can make a shuffle operation.
Gabor Greifba36cb52008-08-28 21:40:38 +00007168 if (VecIn1.getNode()) {
Nate Begeman9008ca62009-04-27 18:41:29 +00007169 SmallVector<int, 8> Mask;
Chris Lattnerd7648c82006-03-28 20:28:38 +00007170 for (unsigned i = 0; i != NumInScalars; ++i) {
7171 if (N->getOperand(i).getOpcode() == ISD::UNDEF) {
Nate Begeman9008ca62009-04-27 18:41:29 +00007172 Mask.push_back(-1);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007173 continue;
7174 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007175
Rafael Espindola15684b22009-04-24 12:40:33 +00007176 // If extracting from the first vector, just use the index directly.
Nate Begeman9008ca62009-04-27 18:41:29 +00007177 SDValue Extract = N->getOperand(i);
Mon P Wang93b74152009-03-17 06:33:10 +00007178 SDValue ExtVal = Extract.getOperand(1);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007179 if (Extract.getOperand(0) == VecIn1) {
Nate Begeman5a5ca152009-04-29 05:20:52 +00007180 unsigned ExtIndex = cast<ConstantSDNode>(ExtVal)->getZExtValue();
7181 if (ExtIndex > VT.getVectorNumElements())
7182 return SDValue();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007183
Nate Begeman5a5ca152009-04-29 05:20:52 +00007184 Mask.push_back(ExtIndex);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007185 continue;
7186 }
7187
7188 // Otherwise, use InIdx + VecSize
Mon P Wang93b74152009-03-17 06:33:10 +00007189 unsigned Idx = cast<ConstantSDNode>(ExtVal)->getZExtValue();
Nate Begeman9008ca62009-04-27 18:41:29 +00007190 Mask.push_back(Idx+NumInScalars);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007191 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007192
Chris Lattnerd7648c82006-03-28 20:28:38 +00007193 // Add count and size info.
Chris Lattner2392ae72010-04-15 04:48:01 +00007194 if (!isTypeLegal(VT))
Duncan Sands25cf2272008-11-24 14:53:14 +00007195 return SDValue();
7196
Dan Gohman7f321562007-06-25 16:23:39 +00007197 // Return the new VECTOR_SHUFFLE node.
Nate Begeman9008ca62009-04-27 18:41:29 +00007198 SDValue Ops[2];
Chris Lattnerbd564bf2006-08-08 02:23:42 +00007199 Ops[0] = VecIn1;
Nate Begeman9008ca62009-04-27 18:41:29 +00007200 Ops[1] = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
7201 return DAG.getVectorShuffle(VT, N->getDebugLoc(), Ops[0], Ops[1], &Mask[0]);
Chris Lattnerd7648c82006-03-28 20:28:38 +00007202 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007203
Dan Gohman475871a2008-07-27 21:46:04 +00007204 return SDValue();
Chris Lattnerd7648c82006-03-28 20:28:38 +00007205}
7206
Dan Gohman475871a2008-07-27 21:46:04 +00007207SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
Dan Gohman7f321562007-06-25 16:23:39 +00007208 // TODO: Check to see if this is a CONCAT_VECTORS of a bunch of
7209 // EXTRACT_SUBVECTOR operations. If so, and if the EXTRACT_SUBVECTOR vector
7210 // inputs come from at most two distinct vectors, turn this into a shuffle
7211 // node.
7212
7213 // If we only have one input vector, we don't need to do any concatenation.
Bill Wendlingc144a572009-01-30 23:36:47 +00007214 if (N->getNumOperands() == 1)
Dan Gohman7f321562007-06-25 16:23:39 +00007215 return N->getOperand(0);
Dan Gohman7f321562007-06-25 16:23:39 +00007216
Dan Gohman475871a2008-07-27 21:46:04 +00007217 return SDValue();
Dan Gohman7f321562007-06-25 16:23:39 +00007218}
7219
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00007220SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) {
7221 EVT NVT = N->getValueType(0);
7222 SDValue V = N->getOperand(0);
7223
7224 if (V->getOpcode() == ISD::INSERT_SUBVECTOR) {
7225 // Handle only simple case where vector being inserted and vector
7226 // being extracted are of same type, and are half size of larger vectors.
7227 EVT BigVT = V->getOperand(0).getValueType();
7228 EVT SmallVT = V->getOperand(1).getValueType();
7229 if (NVT != SmallVT || NVT.getSizeInBits()*2 != BigVT.getSizeInBits())
7230 return SDValue();
7231
Eli Friedman26323442011-12-07 00:11:56 +00007232 // Only handle cases where both indexes are constants with the same type.
7233 ConstantSDNode *InsIdx = dyn_cast<ConstantSDNode>(N->getOperand(1));
7234 ConstantSDNode *ExtIdx = dyn_cast<ConstantSDNode>(V->getOperand(2));
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00007235
Eli Friedman26323442011-12-07 00:11:56 +00007236 if (InsIdx && ExtIdx &&
7237 InsIdx->getValueType(0).getSizeInBits() <= 64 &&
7238 ExtIdx->getValueType(0).getSizeInBits() <= 64) {
7239 // Combine:
7240 // (extract_subvec (insert_subvec V1, V2, InsIdx), ExtIdx)
7241 // Into:
7242 // indices are equal => V1
7243 // otherwise => (extract_subvec V1, ExtIdx)
7244 if (InsIdx->getZExtValue() == ExtIdx->getZExtValue())
7245 return V->getOperand(1);
7246 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), NVT,
7247 V->getOperand(0), N->getOperand(1));
7248 }
Bruno Cardoso Lopese97190f2011-09-20 23:19:33 +00007249 }
7250
7251 return SDValue();
7252}
7253
Dan Gohman475871a2008-07-27 21:46:04 +00007254SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
Owen Andersone50ed302009-08-10 22:56:29 +00007255 EVT VT = N->getValueType(0);
Nate Begeman9008ca62009-04-27 18:41:29 +00007256 unsigned NumElts = VT.getVectorNumElements();
Chris Lattnerf1d0c622006-03-31 22:16:43 +00007257
Mon P Wangaeb06d22008-11-10 04:46:22 +00007258 SDValue N0 = N->getOperand(0);
Craig Topper481b79c2012-01-04 08:07:43 +00007259 SDValue N1 = N->getOperand(1);
Mon P Wangaeb06d22008-11-10 04:46:22 +00007260
7261 assert(N0.getValueType().getVectorNumElements() == NumElts &&
7262 "Vector shuffle must be normalized in DAG");
7263
Craig Topper481b79c2012-01-04 08:07:43 +00007264 // Canonicalize shuffle undef, undef -> undef
7265 if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF)
7266 return DAG.getUNDEF(VT);
7267
7268 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
7269
7270 // Canonicalize shuffle v, v -> v, undef
7271 if (N0 == N1) {
7272 SmallVector<int, 8> NewMask;
7273 for (unsigned i = 0; i != NumElts; ++i) {
7274 int Idx = SVN->getMaskElt(i);
7275 if (Idx >= (int)NumElts) Idx -= NumElts;
7276 NewMask.push_back(Idx);
7277 }
7278 return DAG.getVectorShuffle(VT, N->getDebugLoc(), N0, DAG.getUNDEF(VT),
7279 &NewMask[0]);
7280 }
7281
7282 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
7283 if (N0.getOpcode() == ISD::UNDEF) {
7284 SmallVector<int, 8> NewMask;
7285 for (unsigned i = 0; i != NumElts; ++i) {
7286 int Idx = SVN->getMaskElt(i);
7287 if (Idx < 0)
7288 NewMask.push_back(Idx);
7289 else if (Idx < (int)NumElts)
7290 NewMask.push_back(Idx + NumElts);
7291 else
7292 NewMask.push_back(Idx - NumElts);
7293 }
7294 return DAG.getVectorShuffle(VT, N->getDebugLoc(), N1, DAG.getUNDEF(VT),
7295 &NewMask[0]);
7296 }
7297
7298 // Remove references to rhs if it is undef
7299 if (N1.getOpcode() == ISD::UNDEF) {
7300 bool Changed = false;
7301 SmallVector<int, 8> NewMask;
7302 for (unsigned i = 0; i != NumElts; ++i) {
7303 int Idx = SVN->getMaskElt(i);
7304 if (Idx >= (int)NumElts) {
7305 Idx = -1;
7306 Changed = true;
7307 }
7308 NewMask.push_back(Idx);
7309 }
7310 if (Changed)
7311 return DAG.getVectorShuffle(VT, N->getDebugLoc(), N0, N1, &NewMask[0]);
7312 }
Evan Chenge7bec0d2006-07-20 22:44:41 +00007313
Bob Wilson0f1db1a2010-10-28 17:06:14 +00007314 // If it is a splat, check if the argument vector is another splat or a
7315 // build_vector with all scalar elements the same.
Bob Wilson0f1db1a2010-10-28 17:06:14 +00007316 if (SVN->isSplat() && SVN->getSplatIndex() < (int)NumElts) {
Gabor Greifba36cb52008-08-28 21:40:38 +00007317 SDNode *V = N0.getNode();
Evan Cheng917ec982006-07-21 08:25:53 +00007318
Dan Gohman7f321562007-06-25 16:23:39 +00007319 // If this is a bit convert that changes the element type of the vector but
Evan Cheng59569222006-10-16 22:49:37 +00007320 // not the number of vector elements, look through it. Be careful not to
7321 // look though conversions that change things like v4f32 to v2f64.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007322 if (V->getOpcode() == ISD::BITCAST) {
Dan Gohman475871a2008-07-27 21:46:04 +00007323 SDValue ConvInput = V->getOperand(0);
Evan Cheng29257862008-07-22 20:42:56 +00007324 if (ConvInput.getValueType().isVector() &&
7325 ConvInput.getValueType().getVectorNumElements() == NumElts)
Gabor Greifba36cb52008-08-28 21:40:38 +00007326 V = ConvInput.getNode();
Evan Cheng59569222006-10-16 22:49:37 +00007327 }
7328
Dan Gohman7f321562007-06-25 16:23:39 +00007329 if (V->getOpcode() == ISD::BUILD_VECTOR) {
Bob Wilson0f1db1a2010-10-28 17:06:14 +00007330 assert(V->getNumOperands() == NumElts &&
7331 "BUILD_VECTOR has wrong number of operands");
7332 SDValue Base;
7333 bool AllSame = true;
7334 for (unsigned i = 0; i != NumElts; ++i) {
7335 if (V->getOperand(i).getOpcode() != ISD::UNDEF) {
7336 Base = V->getOperand(i);
7337 break;
Evan Cheng917ec982006-07-21 08:25:53 +00007338 }
Evan Cheng917ec982006-07-21 08:25:53 +00007339 }
Bob Wilson0f1db1a2010-10-28 17:06:14 +00007340 // Splat of <u, u, u, u>, return <u, u, u, u>
7341 if (!Base.getNode())
7342 return N0;
7343 for (unsigned i = 0; i != NumElts; ++i) {
7344 if (V->getOperand(i) != Base) {
7345 AllSame = false;
7346 break;
7347 }
7348 }
7349 // Splat of <x, x, x, x>, return <x, x, x, x>
7350 if (AllSame)
7351 return N0;
Evan Cheng917ec982006-07-21 08:25:53 +00007352 }
7353 }
Dan Gohman475871a2008-07-27 21:46:04 +00007354 return SDValue();
Chris Lattnerf1d0c622006-03-31 22:16:43 +00007355}
7356
Jim Grosbach9a526492010-06-23 16:07:42 +00007357SDValue DAGCombiner::visitMEMBARRIER(SDNode* N) {
7358 if (!TLI.getShouldFoldAtomicFences())
7359 return SDValue();
7360
7361 SDValue atomic = N->getOperand(0);
7362 switch (atomic.getOpcode()) {
7363 case ISD::ATOMIC_CMP_SWAP:
7364 case ISD::ATOMIC_SWAP:
7365 case ISD::ATOMIC_LOAD_ADD:
7366 case ISD::ATOMIC_LOAD_SUB:
7367 case ISD::ATOMIC_LOAD_AND:
7368 case ISD::ATOMIC_LOAD_OR:
7369 case ISD::ATOMIC_LOAD_XOR:
7370 case ISD::ATOMIC_LOAD_NAND:
7371 case ISD::ATOMIC_LOAD_MIN:
7372 case ISD::ATOMIC_LOAD_MAX:
7373 case ISD::ATOMIC_LOAD_UMIN:
7374 case ISD::ATOMIC_LOAD_UMAX:
7375 break;
7376 default:
7377 return SDValue();
7378 }
7379
7380 SDValue fence = atomic.getOperand(0);
7381 if (fence.getOpcode() != ISD::MEMBARRIER)
7382 return SDValue();
7383
7384 switch (atomic.getOpcode()) {
7385 case ISD::ATOMIC_CMP_SWAP:
7386 return SDValue(DAG.UpdateNodeOperands(atomic.getNode(),
7387 fence.getOperand(0),
7388 atomic.getOperand(1), atomic.getOperand(2),
7389 atomic.getOperand(3)), atomic.getResNo());
7390 case ISD::ATOMIC_SWAP:
7391 case ISD::ATOMIC_LOAD_ADD:
7392 case ISD::ATOMIC_LOAD_SUB:
7393 case ISD::ATOMIC_LOAD_AND:
7394 case ISD::ATOMIC_LOAD_OR:
7395 case ISD::ATOMIC_LOAD_XOR:
7396 case ISD::ATOMIC_LOAD_NAND:
7397 case ISD::ATOMIC_LOAD_MIN:
7398 case ISD::ATOMIC_LOAD_MAX:
7399 case ISD::ATOMIC_LOAD_UMIN:
7400 case ISD::ATOMIC_LOAD_UMAX:
7401 return SDValue(DAG.UpdateNodeOperands(atomic.getNode(),
7402 fence.getOperand(0),
7403 atomic.getOperand(1), atomic.getOperand(2)),
7404 atomic.getResNo());
7405 default:
7406 return SDValue();
7407 }
7408}
7409
Evan Cheng44f1f092006-04-20 08:56:16 +00007410/// XformToShuffleWithZero - Returns a vector_shuffle if it able to transform
Dan Gohman7f321562007-06-25 16:23:39 +00007411/// an AND to a vector_shuffle with the destination vector and a zero vector.
7412/// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==>
Evan Cheng44f1f092006-04-20 08:56:16 +00007413/// vector_shuffle V, Zero, <0, 4, 2, 4>
Dan Gohman475871a2008-07-27 21:46:04 +00007414SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) {
Owen Andersone50ed302009-08-10 22:56:29 +00007415 EVT VT = N->getValueType(0);
Nate Begeman9008ca62009-04-27 18:41:29 +00007416 DebugLoc dl = N->getDebugLoc();
Dan Gohman475871a2008-07-27 21:46:04 +00007417 SDValue LHS = N->getOperand(0);
7418 SDValue RHS = N->getOperand(1);
Dan Gohman7f321562007-06-25 16:23:39 +00007419 if (N->getOpcode() == ISD::AND) {
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007420 if (RHS.getOpcode() == ISD::BITCAST)
Evan Cheng44f1f092006-04-20 08:56:16 +00007421 RHS = RHS.getOperand(0);
Dan Gohman7f321562007-06-25 16:23:39 +00007422 if (RHS.getOpcode() == ISD::BUILD_VECTOR) {
Nate Begeman9008ca62009-04-27 18:41:29 +00007423 SmallVector<int, 8> Indices;
7424 unsigned NumElts = RHS.getNumOperands();
Evan Cheng44f1f092006-04-20 08:56:16 +00007425 for (unsigned i = 0; i != NumElts; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00007426 SDValue Elt = RHS.getOperand(i);
Evan Cheng44f1f092006-04-20 08:56:16 +00007427 if (!isa<ConstantSDNode>(Elt))
Dan Gohman475871a2008-07-27 21:46:04 +00007428 return SDValue();
Evan Cheng44f1f092006-04-20 08:56:16 +00007429 else if (cast<ConstantSDNode>(Elt)->isAllOnesValue())
Nate Begeman9008ca62009-04-27 18:41:29 +00007430 Indices.push_back(i);
Evan Cheng44f1f092006-04-20 08:56:16 +00007431 else if (cast<ConstantSDNode>(Elt)->isNullValue())
Nate Begeman9008ca62009-04-27 18:41:29 +00007432 Indices.push_back(NumElts);
Evan Cheng44f1f092006-04-20 08:56:16 +00007433 else
Dan Gohman475871a2008-07-27 21:46:04 +00007434 return SDValue();
Evan Cheng44f1f092006-04-20 08:56:16 +00007435 }
7436
7437 // Let's see if the target supports this vector_shuffle.
Owen Andersone50ed302009-08-10 22:56:29 +00007438 EVT RVT = RHS.getValueType();
Nate Begeman9008ca62009-04-27 18:41:29 +00007439 if (!TLI.isVectorClearMaskLegal(Indices, RVT))
Dan Gohman475871a2008-07-27 21:46:04 +00007440 return SDValue();
Evan Cheng44f1f092006-04-20 08:56:16 +00007441
Dan Gohman7f321562007-06-25 16:23:39 +00007442 // Return the new VECTOR_SHUFFLE node.
Dan Gohman8a55ce42009-09-23 21:02:20 +00007443 EVT EltVT = RVT.getVectorElementType();
Nate Begeman9008ca62009-04-27 18:41:29 +00007444 SmallVector<SDValue,8> ZeroOps(RVT.getVectorNumElements(),
Dan Gohman8a55ce42009-09-23 21:02:20 +00007445 DAG.getConstant(0, EltVT));
Nate Begeman9008ca62009-04-27 18:41:29 +00007446 SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
7447 RVT, &ZeroOps[0], ZeroOps.size());
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007448 LHS = DAG.getNode(ISD::BITCAST, dl, RVT, LHS);
Nate Begeman9008ca62009-04-27 18:41:29 +00007449 SDValue Shuf = DAG.getVectorShuffle(RVT, dl, LHS, Zero, &Indices[0]);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007450 return DAG.getNode(ISD::BITCAST, dl, VT, Shuf);
Evan Cheng44f1f092006-04-20 08:56:16 +00007451 }
7452 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00007453
Dan Gohman475871a2008-07-27 21:46:04 +00007454 return SDValue();
Evan Cheng44f1f092006-04-20 08:56:16 +00007455}
7456
Dan Gohman7f321562007-06-25 16:23:39 +00007457/// SimplifyVBinOp - Visit a binary vector operation, like ADD.
Dan Gohman475871a2008-07-27 21:46:04 +00007458SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) {
Dan Gohman7f321562007-06-25 16:23:39 +00007459 // After legalize, the target may be depending on adds and other
7460 // binary ops to provide legal ways to construct constants or other
7461 // things. Simplifying them may result in a loss of legality.
Duncan Sands25cf2272008-11-24 14:53:14 +00007462 if (LegalOperations) return SDValue();
Dan Gohman7f321562007-06-25 16:23:39 +00007463
Bob Wilsond7273432010-12-17 23:06:49 +00007464 assert(N->getValueType(0).isVector() &&
7465 "SimplifyVBinOp only works on vectors!");
Dan Gohman7f321562007-06-25 16:23:39 +00007466
Dan Gohman475871a2008-07-27 21:46:04 +00007467 SDValue LHS = N->getOperand(0);
7468 SDValue RHS = N->getOperand(1);
7469 SDValue Shuffle = XformToShuffleWithZero(N);
Gabor Greifba36cb52008-08-28 21:40:38 +00007470 if (Shuffle.getNode()) return Shuffle;
Evan Cheng44f1f092006-04-20 08:56:16 +00007471
Dan Gohman7f321562007-06-25 16:23:39 +00007472 // If the LHS and RHS are BUILD_VECTOR nodes, see if we can constant fold
Chris Lattneredab1b92006-04-02 03:25:57 +00007473 // this operation.
Scott Michelfdc40a02009-02-17 22:15:04 +00007474 if (LHS.getOpcode() == ISD::BUILD_VECTOR &&
Dan Gohman7f321562007-06-25 16:23:39 +00007475 RHS.getOpcode() == ISD::BUILD_VECTOR) {
Dan Gohman475871a2008-07-27 21:46:04 +00007476 SmallVector<SDValue, 8> Ops;
Dan Gohman7f321562007-06-25 16:23:39 +00007477 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
Dan Gohman475871a2008-07-27 21:46:04 +00007478 SDValue LHSOp = LHS.getOperand(i);
7479 SDValue RHSOp = RHS.getOperand(i);
Chris Lattneredab1b92006-04-02 03:25:57 +00007480 // If these two elements can't be folded, bail out.
7481 if ((LHSOp.getOpcode() != ISD::UNDEF &&
7482 LHSOp.getOpcode() != ISD::Constant &&
7483 LHSOp.getOpcode() != ISD::ConstantFP) ||
7484 (RHSOp.getOpcode() != ISD::UNDEF &&
7485 RHSOp.getOpcode() != ISD::Constant &&
7486 RHSOp.getOpcode() != ISD::ConstantFP))
7487 break;
Bill Wendling836ca7d2009-01-30 23:59:18 +00007488
Evan Cheng7b336a82006-05-31 06:08:35 +00007489 // Can't fold divide by zero.
Dan Gohman7f321562007-06-25 16:23:39 +00007490 if (N->getOpcode() == ISD::SDIV || N->getOpcode() == ISD::UDIV ||
7491 N->getOpcode() == ISD::FDIV) {
Evan Cheng7b336a82006-05-31 06:08:35 +00007492 if ((RHSOp.getOpcode() == ISD::Constant &&
Gabor Greifba36cb52008-08-28 21:40:38 +00007493 cast<ConstantSDNode>(RHSOp.getNode())->isNullValue()) ||
Evan Cheng7b336a82006-05-31 06:08:35 +00007494 (RHSOp.getOpcode() == ISD::ConstantFP &&
Gabor Greifba36cb52008-08-28 21:40:38 +00007495 cast<ConstantFPSDNode>(RHSOp.getNode())->getValueAPF().isZero()))
Evan Cheng7b336a82006-05-31 06:08:35 +00007496 break;
7497 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00007498
Bob Wilsond7273432010-12-17 23:06:49 +00007499 EVT VT = LHSOp.getValueType();
Bob Wilsondb2b18f2011-10-18 17:34:47 +00007500 EVT RVT = RHSOp.getValueType();
7501 if (RVT != VT) {
7502 // Integer BUILD_VECTOR operands may have types larger than the element
7503 // size (e.g., when the element type is not legal). Prior to type
7504 // legalization, the types may not match between the two BUILD_VECTORS.
7505 // Truncate one of the operands to make them match.
7506 if (RVT.getSizeInBits() > VT.getSizeInBits()) {
7507 RHSOp = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, RHSOp);
7508 } else {
7509 LHSOp = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), RVT, LHSOp);
7510 VT = RVT;
7511 }
7512 }
Bob Wilsond7273432010-12-17 23:06:49 +00007513 SDValue FoldOp = DAG.getNode(N->getOpcode(), LHS.getDebugLoc(), VT,
Evan Chenga0839882010-05-18 00:03:40 +00007514 LHSOp, RHSOp);
7515 if (FoldOp.getOpcode() != ISD::UNDEF &&
7516 FoldOp.getOpcode() != ISD::Constant &&
7517 FoldOp.getOpcode() != ISD::ConstantFP)
7518 break;
7519 Ops.push_back(FoldOp);
7520 AddToWorkList(FoldOp.getNode());
Chris Lattneredab1b92006-04-02 03:25:57 +00007521 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007522
Bob Wilsond7273432010-12-17 23:06:49 +00007523 if (Ops.size() == LHS.getNumOperands())
7524 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
7525 LHS.getValueType(), &Ops[0], Ops.size());
Chris Lattneredab1b92006-04-02 03:25:57 +00007526 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007527
Dan Gohman475871a2008-07-27 21:46:04 +00007528 return SDValue();
Chris Lattneredab1b92006-04-02 03:25:57 +00007529}
7530
Bill Wendling836ca7d2009-01-30 23:59:18 +00007531SDValue DAGCombiner::SimplifySelect(DebugLoc DL, SDValue N0,
7532 SDValue N1, SDValue N2){
Nate Begemanf845b452005-10-08 00:29:44 +00007533 assert(N0.getOpcode() ==ISD::SETCC && "First argument must be a SetCC node!");
Scott Michelfdc40a02009-02-17 22:15:04 +00007534
Bill Wendling836ca7d2009-01-30 23:59:18 +00007535 SDValue SCC = SimplifySelectCC(DL, N0.getOperand(0), N0.getOperand(1), N1, N2,
Nate Begemanf845b452005-10-08 00:29:44 +00007536 cast<CondCodeSDNode>(N0.getOperand(2))->get());
Bill Wendling836ca7d2009-01-30 23:59:18 +00007537
Nate Begemanf845b452005-10-08 00:29:44 +00007538 // If we got a simplified select_cc node back from SimplifySelectCC, then
7539 // break it down into a new SETCC node, and a new SELECT node, and then return
7540 // the SELECT node, since we were called with a SELECT node.
Gabor Greifba36cb52008-08-28 21:40:38 +00007541 if (SCC.getNode()) {
Nate Begemanf845b452005-10-08 00:29:44 +00007542 // Check to see if we got a select_cc back (to turn into setcc/select).
7543 // Otherwise, just return whatever node we got back, like fabs.
7544 if (SCC.getOpcode() == ISD::SELECT_CC) {
Bill Wendling836ca7d2009-01-30 23:59:18 +00007545 SDValue SETCC = DAG.getNode(ISD::SETCC, N0.getDebugLoc(),
7546 N0.getValueType(),
Scott Michelfdc40a02009-02-17 22:15:04 +00007547 SCC.getOperand(0), SCC.getOperand(1),
Bill Wendling836ca7d2009-01-30 23:59:18 +00007548 SCC.getOperand(4));
Gabor Greifba36cb52008-08-28 21:40:38 +00007549 AddToWorkList(SETCC.getNode());
Bill Wendling836ca7d2009-01-30 23:59:18 +00007550 return DAG.getNode(ISD::SELECT, SCC.getDebugLoc(), SCC.getValueType(),
7551 SCC.getOperand(2), SCC.getOperand(3), SETCC);
Nate Begemanf845b452005-10-08 00:29:44 +00007552 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00007553
Nate Begemanf845b452005-10-08 00:29:44 +00007554 return SCC;
7555 }
Dan Gohman475871a2008-07-27 21:46:04 +00007556 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00007557}
7558
Chris Lattner40c62d52005-10-18 06:04:22 +00007559/// SimplifySelectOps - Given a SELECT or a SELECT_CC node, where LHS and RHS
7560/// are the two values being selected between, see if we can simplify the
Chris Lattner729c6d12006-05-27 00:43:02 +00007561/// select. Callers of this should assume that TheSelect is deleted if this
7562/// returns true. As such, they should return the appropriate thing (e.g. the
7563/// node) back to the top-level of the DAG combiner loop to avoid it being
7564/// looked at.
Scott Michelfdc40a02009-02-17 22:15:04 +00007565bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
Dan Gohman475871a2008-07-27 21:46:04 +00007566 SDValue RHS) {
Scott Michelfdc40a02009-02-17 22:15:04 +00007567
Nadav Rotemf94fdb62011-02-11 19:57:47 +00007568 // Cannot simplify select with vector condition
7569 if (TheSelect->getOperand(0).getValueType().isVector()) return false;
7570
Chris Lattner40c62d52005-10-18 06:04:22 +00007571 // If this is a select from two identical things, try to pull the operation
7572 // through the select.
Chris Lattner18061612010-09-21 15:46:59 +00007573 if (LHS.getOpcode() != RHS.getOpcode() ||
7574 !LHS.hasOneUse() || !RHS.hasOneUse())
7575 return false;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007576
Chris Lattner18061612010-09-21 15:46:59 +00007577 // If this is a load and the token chain is identical, replace the select
7578 // of two loads with a load through a select of the address to load from.
7579 // This triggers in things like "select bool X, 10.0, 123.0" after the FP
7580 // constants have been dropped into the constant pool.
7581 if (LHS.getOpcode() == ISD::LOAD) {
7582 LoadSDNode *LLD = cast<LoadSDNode>(LHS);
7583 LoadSDNode *RLD = cast<LoadSDNode>(RHS);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007584
Chris Lattner18061612010-09-21 15:46:59 +00007585 // Token chains must be identical.
7586 if (LHS.getOperand(0) != RHS.getOperand(0) ||
Duncan Sandsd4b9c172008-06-13 19:07:40 +00007587 // Do not let this transformation reduce the number of volatile loads.
Chris Lattner18061612010-09-21 15:46:59 +00007588 LLD->isVolatile() || RLD->isVolatile() ||
7589 // If this is an EXTLOAD, the VT's must match.
7590 LLD->getMemoryVT() != RLD->getMemoryVT() ||
Duncan Sandsdcfd3a72010-11-18 20:05:18 +00007591 // If this is an EXTLOAD, the kind of extension must match.
7592 (LLD->getExtensionType() != RLD->getExtensionType() &&
7593 // The only exception is if one of the extensions is anyext.
7594 LLD->getExtensionType() != ISD::EXTLOAD &&
7595 RLD->getExtensionType() != ISD::EXTLOAD) ||
Dan Gohman75832d72009-10-31 14:14:04 +00007596 // FIXME: this discards src value information. This is
7597 // over-conservative. It would be beneficial to be able to remember
Mon P Wangfe240b12010-01-11 20:12:49 +00007598 // both potential memory locations. Since we are discarding
7599 // src value info, don't do the transformation if the memory
7600 // locations are not in the default address space.
Chris Lattner18061612010-09-21 15:46:59 +00007601 LLD->getPointerInfo().getAddrSpace() != 0 ||
7602 RLD->getPointerInfo().getAddrSpace() != 0)
7603 return false;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007604
Chris Lattnerf1658062010-09-21 15:58:55 +00007605 // Check that the select condition doesn't reach either load. If so,
7606 // folding this will induce a cycle into the DAG. If not, this is safe to
7607 // xform, so create a select of the addresses.
Chris Lattner18061612010-09-21 15:46:59 +00007608 SDValue Addr;
7609 if (TheSelect->getOpcode() == ISD::SELECT) {
Chris Lattnerf1658062010-09-21 15:58:55 +00007610 SDNode *CondNode = TheSelect->getOperand(0).getNode();
7611 if ((LLD->hasAnyUseOfValue(1) && LLD->isPredecessorOf(CondNode)) ||
7612 (RLD->hasAnyUseOfValue(1) && RLD->isPredecessorOf(CondNode)))
7613 return false;
7614 Addr = DAG.getNode(ISD::SELECT, TheSelect->getDebugLoc(),
7615 LLD->getBasePtr().getValueType(),
7616 TheSelect->getOperand(0), LLD->getBasePtr(),
7617 RLD->getBasePtr());
Chris Lattner18061612010-09-21 15:46:59 +00007618 } else { // Otherwise SELECT_CC
Chris Lattnerf1658062010-09-21 15:58:55 +00007619 SDNode *CondLHS = TheSelect->getOperand(0).getNode();
7620 SDNode *CondRHS = TheSelect->getOperand(1).getNode();
7621
7622 if ((LLD->hasAnyUseOfValue(1) &&
7623 (LLD->isPredecessorOf(CondLHS) || LLD->isPredecessorOf(CondRHS))) ||
7624 (LLD->hasAnyUseOfValue(1) &&
7625 (LLD->isPredecessorOf(CondLHS) || LLD->isPredecessorOf(CondRHS))))
7626 return false;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007627
Chris Lattnerf1658062010-09-21 15:58:55 +00007628 Addr = DAG.getNode(ISD::SELECT_CC, TheSelect->getDebugLoc(),
7629 LLD->getBasePtr().getValueType(),
7630 TheSelect->getOperand(0),
7631 TheSelect->getOperand(1),
7632 LLD->getBasePtr(), RLD->getBasePtr(),
7633 TheSelect->getOperand(4));
Chris Lattner18061612010-09-21 15:46:59 +00007634 }
7635
Chris Lattnerf1658062010-09-21 15:58:55 +00007636 SDValue Load;
7637 if (LLD->getExtensionType() == ISD::NON_EXTLOAD) {
7638 Load = DAG.getLoad(TheSelect->getValueType(0),
7639 TheSelect->getDebugLoc(),
7640 // FIXME: Discards pointer info.
7641 LLD->getChain(), Addr, MachinePointerInfo(),
7642 LLD->isVolatile(), LLD->isNonTemporal(),
Pete Cooperd752e0f2011-11-08 18:42:53 +00007643 LLD->isInvariant(), LLD->getAlignment());
Chris Lattnerf1658062010-09-21 15:58:55 +00007644 } else {
Duncan Sandsb9064bb2010-11-18 21:16:28 +00007645 Load = DAG.getExtLoad(LLD->getExtensionType() == ISD::EXTLOAD ?
7646 RLD->getExtensionType() : LLD->getExtensionType(),
Chris Lattnerf1658062010-09-21 15:58:55 +00007647 TheSelect->getDebugLoc(),
Stuart Hastingsa9011292011-02-16 16:23:55 +00007648 TheSelect->getValueType(0),
Chris Lattnerf1658062010-09-21 15:58:55 +00007649 // FIXME: Discards pointer info.
7650 LLD->getChain(), Addr, MachinePointerInfo(),
7651 LLD->getMemoryVT(), LLD->isVolatile(),
7652 LLD->isNonTemporal(), LLD->getAlignment());
Chris Lattner40c62d52005-10-18 06:04:22 +00007653 }
Chris Lattnerf1658062010-09-21 15:58:55 +00007654
7655 // Users of the select now use the result of the load.
7656 CombineTo(TheSelect, Load);
7657
7658 // Users of the old loads now use the new load's chain. We know the
7659 // old-load value is dead now.
7660 CombineTo(LHS.getNode(), Load.getValue(0), Load.getValue(1));
7661 CombineTo(RHS.getNode(), Load.getValue(0), Load.getValue(1));
7662 return true;
Chris Lattner40c62d52005-10-18 06:04:22 +00007663 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007664
Chris Lattner40c62d52005-10-18 06:04:22 +00007665 return false;
7666}
7667
Chris Lattner600fec32009-03-11 05:08:08 +00007668/// SimplifySelectCC - Simplify an expression of the form (N0 cond N1) ? N2 : N3
7669/// where 'cond' is the comparison specified by CC.
Scott Michelfdc40a02009-02-17 22:15:04 +00007670SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1,
Dan Gohman475871a2008-07-27 21:46:04 +00007671 SDValue N2, SDValue N3,
7672 ISD::CondCode CC, bool NotExtCompare) {
Chris Lattner600fec32009-03-11 05:08:08 +00007673 // (x ? y : y) -> y.
7674 if (N2 == N3) return N2;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007675
Owen Andersone50ed302009-08-10 22:56:29 +00007676 EVT VT = N2.getValueType();
Gabor Greifba36cb52008-08-28 21:40:38 +00007677 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
7678 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
7679 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00007680
7681 // Determine if the condition we're dealing with is constant
Duncan Sands5480c042009-01-01 15:52:00 +00007682 SDValue SCC = SimplifySetCC(TLI.getSetCCResultType(N0.getValueType()),
Dale Johannesenff97d4f2009-02-03 00:47:48 +00007683 N0, N1, CC, DL, false);
Gabor Greifba36cb52008-08-28 21:40:38 +00007684 if (SCC.getNode()) AddToWorkList(SCC.getNode());
7685 ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00007686
7687 // fold select_cc true, x, y -> x
Dan Gohman002e5d02008-03-13 22:13:53 +00007688 if (SCCC && !SCCC->isNullValue())
Nate Begemanf845b452005-10-08 00:29:44 +00007689 return N2;
7690 // fold select_cc false, x, y -> y
Dan Gohman002e5d02008-03-13 22:13:53 +00007691 if (SCCC && SCCC->isNullValue())
Nate Begemanf845b452005-10-08 00:29:44 +00007692 return N3;
Scott Michelfdc40a02009-02-17 22:15:04 +00007693
Nate Begemanf845b452005-10-08 00:29:44 +00007694 // Check to see if we can simplify the select into an fabs node
7695 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) {
7696 // Allow either -0.0 or 0.0
Dale Johannesen87503a62007-08-25 22:10:57 +00007697 if (CFP->getValueAPF().isZero()) {
Nate Begemanf845b452005-10-08 00:29:44 +00007698 // select (setg[te] X, +/-0.0), X, fneg(X) -> fabs
7699 if ((CC == ISD::SETGE || CC == ISD::SETGT) &&
7700 N0 == N2 && N3.getOpcode() == ISD::FNEG &&
7701 N2 == N3.getOperand(0))
Bill Wendling836ca7d2009-01-30 23:59:18 +00007702 return DAG.getNode(ISD::FABS, DL, VT, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00007703
Nate Begemanf845b452005-10-08 00:29:44 +00007704 // select (setl[te] X, +/-0.0), fneg(X), X -> fabs
7705 if ((CC == ISD::SETLT || CC == ISD::SETLE) &&
7706 N0 == N3 && N2.getOpcode() == ISD::FNEG &&
7707 N2.getOperand(0) == N3)
Bill Wendling836ca7d2009-01-30 23:59:18 +00007708 return DAG.getNode(ISD::FABS, DL, VT, N3);
Nate Begemanf845b452005-10-08 00:29:44 +00007709 }
7710 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007711
Chris Lattner600fec32009-03-11 05:08:08 +00007712 // Turn "(a cond b) ? 1.0f : 2.0f" into "load (tmp + ((a cond b) ? 0 : 4)"
7713 // where "tmp" is a constant pool entry containing an array with 1.0 and 2.0
7714 // in it. This is a win when the constant is not otherwise available because
7715 // it replaces two constant pool loads with one. We only do this if the FP
7716 // type is known to be legal, because if it isn't, then we are before legalize
7717 // types an we want the other legalization to happen first (e.g. to avoid
Mon P Wang0b7a7862009-03-14 00:25:19 +00007718 // messing with soft float) and if the ConstantFP is not legal, because if
7719 // it is legal, we may not need to store the FP constant in a constant pool.
Chris Lattner600fec32009-03-11 05:08:08 +00007720 if (ConstantFPSDNode *TV = dyn_cast<ConstantFPSDNode>(N2))
7721 if (ConstantFPSDNode *FV = dyn_cast<ConstantFPSDNode>(N3)) {
7722 if (TLI.isTypeLegal(N2.getValueType()) &&
Mon P Wang0b7a7862009-03-14 00:25:19 +00007723 (TLI.getOperationAction(ISD::ConstantFP, N2.getValueType()) !=
7724 TargetLowering::Legal) &&
Chris Lattner600fec32009-03-11 05:08:08 +00007725 // If both constants have multiple uses, then we won't need to do an
7726 // extra load, they are likely around in registers for other users.
7727 (TV->hasOneUse() || FV->hasOneUse())) {
7728 Constant *Elts[] = {
7729 const_cast<ConstantFP*>(FV->getConstantFPValue()),
7730 const_cast<ConstantFP*>(TV->getConstantFPValue())
7731 };
Chris Lattnerdb125cf2011-07-18 04:54:35 +00007732 Type *FPTy = Elts[0]->getType();
Chris Lattner600fec32009-03-11 05:08:08 +00007733 const TargetData &TD = *TLI.getTargetData();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007734
Chris Lattner600fec32009-03-11 05:08:08 +00007735 // Create a ConstantArray of the two constants.
Jay Foad26701082011-06-22 09:24:39 +00007736 Constant *CA = ConstantArray::get(ArrayType::get(FPTy, 2), Elts);
Chris Lattner600fec32009-03-11 05:08:08 +00007737 SDValue CPIdx = DAG.getConstantPool(CA, TLI.getPointerTy(),
7738 TD.getPrefTypeAlignment(FPTy));
Evan Cheng1606e8e2009-03-13 07:51:59 +00007739 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
Chris Lattner600fec32009-03-11 05:08:08 +00007740
7741 // Get the offsets to the 0 and 1 element of the array so that we can
7742 // select between them.
7743 SDValue Zero = DAG.getIntPtrConstant(0);
Duncan Sands777d2302009-05-09 07:06:46 +00007744 unsigned EltSize = (unsigned)TD.getTypeAllocSize(Elts[0]->getType());
Chris Lattner600fec32009-03-11 05:08:08 +00007745 SDValue One = DAG.getIntPtrConstant(EltSize);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007746
Chris Lattner600fec32009-03-11 05:08:08 +00007747 SDValue Cond = DAG.getSetCC(DL,
7748 TLI.getSetCCResultType(N0.getValueType()),
7749 N0, N1, CC);
Dan Gohman7b316c92011-09-22 23:01:29 +00007750 AddToWorkList(Cond.getNode());
Chris Lattner600fec32009-03-11 05:08:08 +00007751 SDValue CstOffset = DAG.getNode(ISD::SELECT, DL, Zero.getValueType(),
7752 Cond, One, Zero);
Dan Gohman7b316c92011-09-22 23:01:29 +00007753 AddToWorkList(CstOffset.getNode());
Chris Lattner600fec32009-03-11 05:08:08 +00007754 CPIdx = DAG.getNode(ISD::ADD, DL, TLI.getPointerTy(), CPIdx,
7755 CstOffset);
Dan Gohman7b316c92011-09-22 23:01:29 +00007756 AddToWorkList(CPIdx.getNode());
Chris Lattner600fec32009-03-11 05:08:08 +00007757 return DAG.getLoad(TV->getValueType(0), DL, DAG.getEntryNode(), CPIdx,
Chris Lattner85ca1062010-09-21 07:32:19 +00007758 MachinePointerInfo::getConstantPool(), false,
Pete Cooperd752e0f2011-11-08 18:42:53 +00007759 false, false, Alignment);
Chris Lattner600fec32009-03-11 05:08:08 +00007760
7761 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007762 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007763
Nate Begemanf845b452005-10-08 00:29:44 +00007764 // Check to see if we can perform the "gzip trick", transforming
Bill Wendling836ca7d2009-01-30 23:59:18 +00007765 // (select_cc setlt X, 0, A, 0) -> (and (sra X, (sub size(X), 1), A)
Chris Lattnere3152e52006-09-20 06:41:35 +00007766 if (N1C && N3C && N3C->isNullValue() && CC == ISD::SETLT &&
Dan Gohman002e5d02008-03-13 22:13:53 +00007767 (N1C->isNullValue() || // (a < 0) ? b : 0
7768 (N1C->getAPIntValue() == 1 && N0 == N2))) { // (a < 1) ? a : 0
Owen Andersone50ed302009-08-10 22:56:29 +00007769 EVT XType = N0.getValueType();
7770 EVT AType = N2.getValueType();
Duncan Sands8e4eb092008-06-08 20:54:56 +00007771 if (XType.bitsGE(AType)) {
Nate Begemanf845b452005-10-08 00:29:44 +00007772 // and (sra X, size(X)-1, A) -> "and (srl X, C2), A" iff A is a
Nate Begeman07ed4172005-10-10 21:26:48 +00007773 // single-bit constant.
Dan Gohman002e5d02008-03-13 22:13:53 +00007774 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue()-1)) == 0)) {
7775 unsigned ShCtV = N2C->getAPIntValue().logBase2();
Duncan Sands83ec4b62008-06-06 12:08:01 +00007776 ShCtV = XType.getSizeInBits()-ShCtV-1;
Owen Anderson95771af2011-02-25 21:41:48 +00007777 SDValue ShCt = DAG.getConstant(ShCtV,
7778 getShiftAmountTy(N0.getValueType()));
Bill Wendling9729c5a2009-01-31 03:12:48 +00007779 SDValue Shift = DAG.getNode(ISD::SRL, N0.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00007780 XType, N0, ShCt);
Gabor Greifba36cb52008-08-28 21:40:38 +00007781 AddToWorkList(Shift.getNode());
Bill Wendling836ca7d2009-01-30 23:59:18 +00007782
Duncan Sands8e4eb092008-06-08 20:54:56 +00007783 if (XType.bitsGT(AType)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00007784 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
Gabor Greifba36cb52008-08-28 21:40:38 +00007785 AddToWorkList(Shift.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00007786 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00007787
7788 return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
Nate Begemanf845b452005-10-08 00:29:44 +00007789 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00007790
Bill Wendling9729c5a2009-01-31 03:12:48 +00007791 SDValue Shift = DAG.getNode(ISD::SRA, N0.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00007792 XType, N0,
7793 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00007794 getShiftAmountTy(N0.getValueType())));
Gabor Greifba36cb52008-08-28 21:40:38 +00007795 AddToWorkList(Shift.getNode());
Bill Wendling836ca7d2009-01-30 23:59:18 +00007796
Duncan Sands8e4eb092008-06-08 20:54:56 +00007797 if (XType.bitsGT(AType)) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00007798 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
Gabor Greifba36cb52008-08-28 21:40:38 +00007799 AddToWorkList(Shift.getNode());
Nate Begemanf845b452005-10-08 00:29:44 +00007800 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00007801
7802 return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
Nate Begemanf845b452005-10-08 00:29:44 +00007803 }
7804 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007805
Owen Andersoned1088a2010-09-22 22:58:22 +00007806 // fold (select_cc seteq (and x, y), 0, 0, A) -> (and (shr (shl x)) A)
7807 // where y is has a single bit set.
7808 // A plaintext description would be, we can turn the SELECT_CC into an AND
7809 // when the condition can be materialized as an all-ones register. Any
7810 // single bit-test can be materialized as an all-ones register with
7811 // shift-left and shift-right-arith.
7812 if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND &&
7813 N0->getValueType(0) == VT &&
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007814 N1C && N1C->isNullValue() &&
Owen Andersoned1088a2010-09-22 22:58:22 +00007815 N2C && N2C->isNullValue()) {
7816 SDValue AndLHS = N0->getOperand(0);
7817 ConstantSDNode *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7818 if (ConstAndRHS && ConstAndRHS->getAPIntValue().countPopulation() == 1) {
7819 // Shift the tested bit over the sign bit.
7820 APInt AndMask = ConstAndRHS->getAPIntValue();
7821 SDValue ShlAmt =
Owen Anderson95771af2011-02-25 21:41:48 +00007822 DAG.getConstant(AndMask.countLeadingZeros(),
7823 getShiftAmountTy(AndLHS.getValueType()));
Owen Andersoned1088a2010-09-22 22:58:22 +00007824 SDValue Shl = DAG.getNode(ISD::SHL, N0.getDebugLoc(), VT, AndLHS, ShlAmt);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007825
Owen Andersoned1088a2010-09-22 22:58:22 +00007826 // Now arithmetic right shift it all the way over, so the result is either
7827 // all-ones, or zero.
7828 SDValue ShrAmt =
Owen Anderson95771af2011-02-25 21:41:48 +00007829 DAG.getConstant(AndMask.getBitWidth()-1,
7830 getShiftAmountTy(Shl.getValueType()));
Owen Andersoned1088a2010-09-22 22:58:22 +00007831 SDValue Shr = DAG.getNode(ISD::SRA, N0.getDebugLoc(), VT, Shl, ShrAmt);
Wesley Peckbf17cfa2010-11-23 03:31:01 +00007832
Owen Andersoned1088a2010-09-22 22:58:22 +00007833 return DAG.getNode(ISD::AND, DL, VT, Shr, N3);
7834 }
7835 }
7836
Nate Begeman07ed4172005-10-10 21:26:48 +00007837 // fold select C, 16, 0 -> shl C, 4
Dan Gohman002e5d02008-03-13 22:13:53 +00007838 if (N2C && N3C && N3C->isNullValue() && N2C->getAPIntValue().isPowerOf2() &&
Duncan Sands28b77e92011-09-06 19:07:46 +00007839 TLI.getBooleanContents(N0.getValueType().isVector()) ==
7840 TargetLowering::ZeroOrOneBooleanContent) {
Scott Michelfdc40a02009-02-17 22:15:04 +00007841
Chris Lattner1eba01e2007-04-11 06:50:51 +00007842 // If the caller doesn't want us to simplify this into a zext of a compare,
7843 // don't do it.
Dan Gohman002e5d02008-03-13 22:13:53 +00007844 if (NotExtCompare && N2C->getAPIntValue() == 1)
Dan Gohman475871a2008-07-27 21:46:04 +00007845 return SDValue();
Scott Michelfdc40a02009-02-17 22:15:04 +00007846
Nate Begeman07ed4172005-10-10 21:26:48 +00007847 // Get a SetCC of the condition
7848 // FIXME: Should probably make sure that setcc is legal if we ever have a
7849 // target where it isn't.
Dan Gohman475871a2008-07-27 21:46:04 +00007850 SDValue Temp, SCC;
Nate Begeman07ed4172005-10-10 21:26:48 +00007851 // cast from setcc result type to select result type
Duncan Sands25cf2272008-11-24 14:53:14 +00007852 if (LegalTypes) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00007853 SCC = DAG.getSetCC(DL, TLI.getSetCCResultType(N0.getValueType()),
Duncan Sands5480c042009-01-01 15:52:00 +00007854 N0, N1, CC);
Duncan Sands8e4eb092008-06-08 20:54:56 +00007855 if (N2.getValueType().bitsLT(SCC.getValueType()))
Bill Wendling9729c5a2009-01-31 03:12:48 +00007856 Temp = DAG.getZeroExtendInReg(SCC, N2.getDebugLoc(), N2.getValueType());
Chris Lattner555d8d62006-12-07 22:36:47 +00007857 else
Bill Wendling9729c5a2009-01-31 03:12:48 +00007858 Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00007859 N2.getValueType(), SCC);
Nate Begemanb0d04a72006-02-18 02:40:58 +00007860 } else {
Owen Anderson825b72b2009-08-11 20:47:22 +00007861 SCC = DAG.getSetCC(N0.getDebugLoc(), MVT::i1, N0, N1, CC);
Bill Wendling9729c5a2009-01-31 03:12:48 +00007862 Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getDebugLoc(),
Bill Wendling836ca7d2009-01-30 23:59:18 +00007863 N2.getValueType(), SCC);
Nate Begemanb0d04a72006-02-18 02:40:58 +00007864 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00007865
Gabor Greifba36cb52008-08-28 21:40:38 +00007866 AddToWorkList(SCC.getNode());
7867 AddToWorkList(Temp.getNode());
Scott Michelfdc40a02009-02-17 22:15:04 +00007868
Dan Gohman002e5d02008-03-13 22:13:53 +00007869 if (N2C->getAPIntValue() == 1)
Chris Lattnerc56a81d2007-04-11 06:43:25 +00007870 return Temp;
Bill Wendling836ca7d2009-01-30 23:59:18 +00007871
Nate Begeman07ed4172005-10-10 21:26:48 +00007872 // shl setcc result by log2 n2c
Bill Wendling836ca7d2009-01-30 23:59:18 +00007873 return DAG.getNode(ISD::SHL, DL, N2.getValueType(), Temp,
Dan Gohman002e5d02008-03-13 22:13:53 +00007874 DAG.getConstant(N2C->getAPIntValue().logBase2(),
Owen Anderson95771af2011-02-25 21:41:48 +00007875 getShiftAmountTy(Temp.getValueType())));
Nate Begeman07ed4172005-10-10 21:26:48 +00007876 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007877
Nate Begemanf845b452005-10-08 00:29:44 +00007878 // Check to see if this is the equivalent of setcc
7879 // FIXME: Turn all of these into setcc if setcc if setcc is legal
7880 // otherwise, go ahead with the folds.
Dan Gohman002e5d02008-03-13 22:13:53 +00007881 if (0 && N3C && N3C->isNullValue() && N2C && (N2C->getAPIntValue() == 1ULL)) {
Owen Andersone50ed302009-08-10 22:56:29 +00007882 EVT XType = N0.getValueType();
Duncan Sands25cf2272008-11-24 14:53:14 +00007883 if (!LegalOperations ||
Duncan Sands5480c042009-01-01 15:52:00 +00007884 TLI.isOperationLegal(ISD::SETCC, TLI.getSetCCResultType(XType))) {
Bill Wendling836ca7d2009-01-30 23:59:18 +00007885 SDValue Res = DAG.getSetCC(DL, TLI.getSetCCResultType(XType), N0, N1, CC);
Nate Begemanf845b452005-10-08 00:29:44 +00007886 if (Res.getValueType() != VT)
Bill Wendling836ca7d2009-01-30 23:59:18 +00007887 Res = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Res);
Nate Begemanf845b452005-10-08 00:29:44 +00007888 return Res;
7889 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007890
Bill Wendling836ca7d2009-01-30 23:59:18 +00007891 // fold (seteq X, 0) -> (srl (ctlz X, log2(size(X))))
Scott Michelfdc40a02009-02-17 22:15:04 +00007892 if (N1C && N1C->isNullValue() && CC == ISD::SETEQ &&
Duncan Sands25cf2272008-11-24 14:53:14 +00007893 (!LegalOperations ||
Duncan Sands184a8762008-06-14 17:48:34 +00007894 TLI.isOperationLegal(ISD::CTLZ, XType))) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00007895 SDValue Ctlz = DAG.getNode(ISD::CTLZ, N0.getDebugLoc(), XType, N0);
Scott Michelfdc40a02009-02-17 22:15:04 +00007896 return DAG.getNode(ISD::SRL, DL, XType, Ctlz,
Duncan Sands83ec4b62008-06-06 12:08:01 +00007897 DAG.getConstant(Log2_32(XType.getSizeInBits()),
Owen Anderson95771af2011-02-25 21:41:48 +00007898 getShiftAmountTy(Ctlz.getValueType())));
Nate Begemanf845b452005-10-08 00:29:44 +00007899 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00007900 // fold (setgt X, 0) -> (srl (and (-X, ~X), size(X)-1))
Scott Michelfdc40a02009-02-17 22:15:04 +00007901 if (N1C && N1C->isNullValue() && CC == ISD::SETGT) {
Bill Wendling836ca7d2009-01-30 23:59:18 +00007902 SDValue NegN0 = DAG.getNode(ISD::SUB, N0.getDebugLoc(),
7903 XType, DAG.getConstant(0, XType), N0);
Bill Wendling7581bfa2009-01-30 23:03:19 +00007904 SDValue NotN0 = DAG.getNOT(N0.getDebugLoc(), N0, XType);
Bill Wendling836ca7d2009-01-30 23:59:18 +00007905 return DAG.getNode(ISD::SRL, DL, XType,
Bill Wendlingfc4b6772009-02-01 11:19:36 +00007906 DAG.getNode(ISD::AND, DL, XType, NegN0, NotN0),
Duncan Sands83ec4b62008-06-06 12:08:01 +00007907 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00007908 getShiftAmountTy(XType)));
Nate Begemanf845b452005-10-08 00:29:44 +00007909 }
Bill Wendling836ca7d2009-01-30 23:59:18 +00007910 // fold (setgt X, -1) -> (xor (srl (X, size(X)-1), 1))
Nate Begemanf845b452005-10-08 00:29:44 +00007911 if (N1C && N1C->isAllOnesValue() && CC == ISD::SETGT) {
Bill Wendling9729c5a2009-01-31 03:12:48 +00007912 SDValue Sign = DAG.getNode(ISD::SRL, N0.getDebugLoc(), XType, N0,
Bill Wendling836ca7d2009-01-30 23:59:18 +00007913 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00007914 getShiftAmountTy(N0.getValueType())));
Bill Wendling836ca7d2009-01-30 23:59:18 +00007915 return DAG.getNode(ISD::XOR, DL, XType, Sign, DAG.getConstant(1, XType));
Nate Begemanf845b452005-10-08 00:29:44 +00007916 }
7917 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007918
Benjamin Kramercde51102010-07-08 12:09:56 +00007919 // Check to see if this is an integer abs.
7920 // select_cc setg[te] X, 0, X, -X ->
7921 // select_cc setgt X, -1, X, -X ->
7922 // select_cc setl[te] X, 0, -X, X ->
7923 // select_cc setlt X, 1, -X, X ->
Nate Begemanf845b452005-10-08 00:29:44 +00007924 // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
Benjamin Kramercde51102010-07-08 12:09:56 +00007925 if (N1C) {
7926 ConstantSDNode *SubC = NULL;
7927 if (((N1C->isNullValue() && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
7928 (N1C->isAllOnesValue() && CC == ISD::SETGT)) &&
7929 N0 == N2 && N3.getOpcode() == ISD::SUB && N0 == N3.getOperand(1))
7930 SubC = dyn_cast<ConstantSDNode>(N3.getOperand(0));
7931 else if (((N1C->isNullValue() && (CC == ISD::SETLT || CC == ISD::SETLE)) ||
7932 (N1C->isOne() && CC == ISD::SETLT)) &&
7933 N0 == N3 && N2.getOpcode() == ISD::SUB && N0 == N2.getOperand(1))
7934 SubC = dyn_cast<ConstantSDNode>(N2.getOperand(0));
7935
Owen Andersone50ed302009-08-10 22:56:29 +00007936 EVT XType = N0.getValueType();
Benjamin Kramercde51102010-07-08 12:09:56 +00007937 if (SubC && SubC->isNullValue() && XType.isInteger()) {
7938 SDValue Shift = DAG.getNode(ISD::SRA, N0.getDebugLoc(), XType,
7939 N0,
7940 DAG.getConstant(XType.getSizeInBits()-1,
Owen Anderson95771af2011-02-25 21:41:48 +00007941 getShiftAmountTy(N0.getValueType())));
Benjamin Kramercde51102010-07-08 12:09:56 +00007942 SDValue Add = DAG.getNode(ISD::ADD, N0.getDebugLoc(),
7943 XType, N0, Shift);
7944 AddToWorkList(Shift.getNode());
7945 AddToWorkList(Add.getNode());
7946 return DAG.getNode(ISD::XOR, DL, XType, Add, Shift);
Nate Begemanf845b452005-10-08 00:29:44 +00007947 }
7948 }
Scott Michelfdc40a02009-02-17 22:15:04 +00007949
Dan Gohman475871a2008-07-27 21:46:04 +00007950 return SDValue();
Nate Begeman44728a72005-09-19 22:34:01 +00007951}
7952
Evan Chengfa1eb272007-02-08 22:13:59 +00007953/// SimplifySetCC - This is a stub for TargetLowering::SimplifySetCC.
Owen Andersone50ed302009-08-10 22:56:29 +00007954SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0,
Dan Gohman475871a2008-07-27 21:46:04 +00007955 SDValue N1, ISD::CondCode Cond,
Dale Johannesenff97d4f2009-02-03 00:47:48 +00007956 DebugLoc DL, bool foldBooleans) {
Scott Michelfdc40a02009-02-17 22:15:04 +00007957 TargetLowering::DAGCombinerInfo
Jakob Stoklund Olesen78d12642009-07-24 18:22:59 +00007958 DagCombineInfo(DAG, !LegalTypes, !LegalOperations, false, this);
Dale Johannesenff97d4f2009-02-03 00:47:48 +00007959 return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL);
Nate Begeman452d7beb2005-09-16 00:54:12 +00007960}
7961
Nate Begeman69575232005-10-20 02:15:44 +00007962/// BuildSDIVSequence - Given an ISD::SDIV node expressing a divide by constant,
7963/// return a DAG expression to select that will generate the same value by
7964/// multiplying by a magic number. See:
7965/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html>
Dan Gohman475871a2008-07-27 21:46:04 +00007966SDValue DAGCombiner::BuildSDIV(SDNode *N) {
Andrew Lenharth232c9102006-06-12 16:07:18 +00007967 std::vector<SDNode*> Built;
Richard Osborne19a4daf2011-11-07 17:09:05 +00007968 SDValue S = TLI.BuildSDIV(N, DAG, LegalOperations, &Built);
Andrew Lenharthdae9cbe2006-05-16 17:42:15 +00007969
Andrew Lenharth232c9102006-06-12 16:07:18 +00007970 for (std::vector<SDNode*>::iterator ii = Built.begin(), ee = Built.end();
Andrew Lenharthdae9cbe2006-05-16 17:42:15 +00007971 ii != ee; ++ii)
7972 AddToWorkList(*ii);
7973 return S;
Nate Begeman69575232005-10-20 02:15:44 +00007974}
7975
7976/// BuildUDIVSequence - Given an ISD::UDIV node expressing a divide by constant,
7977/// return a DAG expression to select that will generate the same value by
7978/// multiplying by a magic number. See:
7979/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html>
Dan Gohman475871a2008-07-27 21:46:04 +00007980SDValue DAGCombiner::BuildUDIV(SDNode *N) {
Andrew Lenharth232c9102006-06-12 16:07:18 +00007981 std::vector<SDNode*> Built;
Richard Osborne19a4daf2011-11-07 17:09:05 +00007982 SDValue S = TLI.BuildUDIV(N, DAG, LegalOperations, &Built);
Nate Begeman69575232005-10-20 02:15:44 +00007983
Andrew Lenharth232c9102006-06-12 16:07:18 +00007984 for (std::vector<SDNode*>::iterator ii = Built.begin(), ee = Built.end();
Andrew Lenharthdae9cbe2006-05-16 17:42:15 +00007985 ii != ee; ++ii)
7986 AddToWorkList(*ii);
7987 return S;
Nate Begeman69575232005-10-20 02:15:44 +00007988}
7989
Nate Begemancc66cdd2009-09-25 06:05:26 +00007990/// FindBaseOffset - Return true if base is a frame index, which is known not
Eric Christopher503a64d2010-12-09 04:48:06 +00007991// to alias with anything but itself. Provides base object and offset as
7992// results.
Nate Begemancc66cdd2009-09-25 06:05:26 +00007993static bool FindBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset,
Dan Gohman46510a72010-04-15 01:51:59 +00007994 const GlobalValue *&GV, void *&CV) {
Jim Laskey71382342006-10-07 23:37:56 +00007995 // Assume it is a primitive operation.
Nate Begemancc66cdd2009-09-25 06:05:26 +00007996 Base = Ptr; Offset = 0; GV = 0; CV = 0;
Scott Michelfdc40a02009-02-17 22:15:04 +00007997
Jim Laskey71382342006-10-07 23:37:56 +00007998 // If it's an adding a simple constant then integrate the offset.
7999 if (Base.getOpcode() == ISD::ADD) {
8000 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Base.getOperand(1))) {
8001 Base = Base.getOperand(0);
Dan Gohmanf5aeb1a2008-09-12 16:56:44 +00008002 Offset += C->getZExtValue();
Jim Laskey71382342006-10-07 23:37:56 +00008003 }
8004 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008005
Nate Begemancc66cdd2009-09-25 06:05:26 +00008006 // Return the underlying GlobalValue, and update the Offset. Return false
8007 // for GlobalAddressSDNode since the same GlobalAddress may be represented
8008 // by multiple nodes with different offsets.
8009 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Base)) {
8010 GV = G->getGlobal();
8011 Offset += G->getOffset();
8012 return false;
8013 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008014
Nate Begemancc66cdd2009-09-25 06:05:26 +00008015 // Return the underlying Constant value, and update the Offset. Return false
8016 // for ConstantSDNodes since the same constant pool entry may be represented
8017 // by multiple nodes with different offsets.
8018 if (ConstantPoolSDNode *C = dyn_cast<ConstantPoolSDNode>(Base)) {
8019 CV = C->isMachineConstantPoolEntry() ? (void *)C->getMachineCPVal()
8020 : (void *)C->getConstVal();
8021 Offset += C->getOffset();
8022 return false;
8023 }
Jim Laskey71382342006-10-07 23:37:56 +00008024 // If it's any of the following then it can't alias with anything but itself.
Nate Begemancc66cdd2009-09-25 06:05:26 +00008025 return isa<FrameIndexSDNode>(Base);
Jim Laskey71382342006-10-07 23:37:56 +00008026}
8027
8028/// isAlias - Return true if there is any possibility that the two addresses
8029/// overlap.
Dan Gohman475871a2008-07-27 21:46:04 +00008030bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1,
Jim Laskey096c22e2006-10-18 12:29:57 +00008031 const Value *SrcValue1, int SrcValueOffset1,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008032 unsigned SrcValueAlign1,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008033 const MDNode *TBAAInfo1,
Dan Gohman475871a2008-07-27 21:46:04 +00008034 SDValue Ptr2, int64_t Size2,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008035 const Value *SrcValue2, int SrcValueOffset2,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008036 unsigned SrcValueAlign2,
8037 const MDNode *TBAAInfo2) const {
Jim Laskey71382342006-10-07 23:37:56 +00008038 // If they are the same then they must be aliases.
8039 if (Ptr1 == Ptr2) return true;
Scott Michelfdc40a02009-02-17 22:15:04 +00008040
Jim Laskey71382342006-10-07 23:37:56 +00008041 // Gather base node and offset information.
Dan Gohman475871a2008-07-27 21:46:04 +00008042 SDValue Base1, Base2;
Jim Laskey71382342006-10-07 23:37:56 +00008043 int64_t Offset1, Offset2;
Dan Gohman46510a72010-04-15 01:51:59 +00008044 const GlobalValue *GV1, *GV2;
Nate Begemancc66cdd2009-09-25 06:05:26 +00008045 void *CV1, *CV2;
8046 bool isFrameIndex1 = FindBaseOffset(Ptr1, Base1, Offset1, GV1, CV1);
8047 bool isFrameIndex2 = FindBaseOffset(Ptr2, Base2, Offset2, GV2, CV2);
Scott Michelfdc40a02009-02-17 22:15:04 +00008048
Nate Begemancc66cdd2009-09-25 06:05:26 +00008049 // If they have a same base address then check to see if they overlap.
8050 if (Base1 == Base2 || (GV1 && (GV1 == GV2)) || (CV1 && (CV1 == CV2)))
Bill Wendling836ca7d2009-01-30 23:59:18 +00008051 return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1);
Scott Michelfdc40a02009-02-17 22:15:04 +00008052
Owen Anderson4a9f1502010-09-20 20:39:59 +00008053 // It is possible for different frame indices to alias each other, mostly
8054 // when tail call optimization reuses return address slots for arguments.
8055 // To catch this case, look up the actual index of frame indices to compute
8056 // the real alias relationship.
8057 if (isFrameIndex1 && isFrameIndex2) {
8058 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
8059 Offset1 += MFI->getObjectOffset(cast<FrameIndexSDNode>(Base1)->getIndex());
8060 Offset2 += MFI->getObjectOffset(cast<FrameIndexSDNode>(Base2)->getIndex());
8061 return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1);
8062 }
8063
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008064 // Otherwise, if we know what the bases are, and they aren't identical, then
Owen Anderson4a9f1502010-09-20 20:39:59 +00008065 // we know they cannot alias.
Nate Begemancc66cdd2009-09-25 06:05:26 +00008066 if ((isFrameIndex1 || CV1 || GV1) && (isFrameIndex2 || CV2 || GV2))
8067 return false;
Jim Laskey096c22e2006-10-18 12:29:57 +00008068
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008069 // If we know required SrcValue1 and SrcValue2 have relatively large alignment
8070 // compared to the size and offset of the access, we may be able to prove they
8071 // do not alias. This check is conservative for now to catch cases created by
8072 // splitting vector types.
8073 if ((SrcValueAlign1 == SrcValueAlign2) &&
8074 (SrcValueOffset1 != SrcValueOffset2) &&
8075 (Size1 == Size2) && (SrcValueAlign1 > Size1)) {
8076 int64_t OffAlign1 = SrcValueOffset1 % SrcValueAlign1;
8077 int64_t OffAlign2 = SrcValueOffset2 % SrcValueAlign1;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008078
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008079 // There is no overlap between these relatively aligned accesses of similar
8080 // size, return no alias.
8081 if ((OffAlign1 + Size1) <= OffAlign2 || (OffAlign2 + Size2) <= OffAlign1)
8082 return false;
8083 }
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008084
Jim Laskey07a27092006-10-18 19:08:31 +00008085 if (CombinerGlobalAA) {
8086 // Use alias analysis information.
Dan Gohmane9c8fa02007-08-27 16:32:11 +00008087 int64_t MinOffset = std::min(SrcValueOffset1, SrcValueOffset2);
8088 int64_t Overlap1 = Size1 + SrcValueOffset1 - MinOffset;
8089 int64_t Overlap2 = Size2 + SrcValueOffset2 - MinOffset;
Scott Michelfdc40a02009-02-17 22:15:04 +00008090 AliasAnalysis::AliasResult AAResult =
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008091 AA.alias(AliasAnalysis::Location(SrcValue1, Overlap1, TBAAInfo1),
8092 AliasAnalysis::Location(SrcValue2, Overlap2, TBAAInfo2));
Jim Laskey07a27092006-10-18 19:08:31 +00008093 if (AAResult == AliasAnalysis::NoAlias)
8094 return false;
8095 }
Jim Laskey096c22e2006-10-18 12:29:57 +00008096
8097 // Otherwise we have to assume they alias.
8098 return true;
Jim Laskey71382342006-10-07 23:37:56 +00008099}
8100
8101/// FindAliasInfo - Extracts the relevant alias information from the memory
8102/// node. Returns true if the operand was a load.
Jim Laskey7ca56af2006-10-11 13:47:09 +00008103bool DAGCombiner::FindAliasInfo(SDNode *N,
Dan Gohman475871a2008-07-27 21:46:04 +00008104 SDValue &Ptr, int64_t &Size,
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008105 const Value *&SrcValue,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008106 int &SrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008107 unsigned &SrcValueAlign,
8108 const MDNode *&TBAAInfo) const {
Jim Laskey7ca56af2006-10-11 13:47:09 +00008109 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
8110 Ptr = LD->getBasePtr();
Duncan Sands83ec4b62008-06-06 12:08:01 +00008111 Size = LD->getMemoryVT().getSizeInBits() >> 3;
Jim Laskey7ca56af2006-10-11 13:47:09 +00008112 SrcValue = LD->getSrcValue();
Jim Laskey096c22e2006-10-18 12:29:57 +00008113 SrcValueOffset = LD->getSrcValueOffset();
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008114 SrcValueAlign = LD->getOriginalAlignment();
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008115 TBAAInfo = LD->getTBAAInfo();
Jim Laskey71382342006-10-07 23:37:56 +00008116 return true;
Dan Gohman71dc7c92011-05-17 22:20:36 +00008117 }
8118 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
Jim Laskey7ca56af2006-10-11 13:47:09 +00008119 Ptr = ST->getBasePtr();
Duncan Sands83ec4b62008-06-06 12:08:01 +00008120 Size = ST->getMemoryVT().getSizeInBits() >> 3;
Jim Laskey7ca56af2006-10-11 13:47:09 +00008121 SrcValue = ST->getSrcValue();
Jim Laskey096c22e2006-10-18 12:29:57 +00008122 SrcValueOffset = ST->getSrcValueOffset();
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008123 SrcValueAlign = ST->getOriginalAlignment();
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008124 TBAAInfo = ST->getTBAAInfo();
Dan Gohman71dc7c92011-05-17 22:20:36 +00008125 return false;
Jim Laskey71382342006-10-07 23:37:56 +00008126 }
Dan Gohman71dc7c92011-05-17 22:20:36 +00008127 llvm_unreachable("FindAliasInfo expected a memory operand");
Jim Laskey71382342006-10-07 23:37:56 +00008128}
8129
Jim Laskey6ff23e52006-10-04 16:53:27 +00008130/// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
8131/// looking for aliasing nodes and adding them to the Aliases vector.
Dan Gohman475871a2008-07-27 21:46:04 +00008132void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
8133 SmallVector<SDValue, 8> &Aliases) {
8134 SmallVector<SDValue, 8> Chains; // List of chains to visit.
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008135 SmallPtrSet<SDNode *, 16> Visited; // Visited node set.
Scott Michelfdc40a02009-02-17 22:15:04 +00008136
Jim Laskey279f0532006-09-25 16:29:54 +00008137 // Get alias information for node.
Dan Gohman475871a2008-07-27 21:46:04 +00008138 SDValue Ptr;
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008139 int64_t Size;
8140 const Value *SrcValue;
8141 int SrcValueOffset;
8142 unsigned SrcValueAlign;
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008143 const MDNode *SrcTBAAInfo;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008144 bool IsLoad = FindAliasInfo(N, Ptr, Size, SrcValue, SrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008145 SrcValueAlign, SrcTBAAInfo);
Jim Laskey279f0532006-09-25 16:29:54 +00008146
Jim Laskey6ff23e52006-10-04 16:53:27 +00008147 // Starting off.
Jim Laskeybc588b82006-10-05 15:07:25 +00008148 Chains.push_back(OriginalChain);
Nate Begeman677c89d2009-10-12 05:53:58 +00008149 unsigned Depth = 0;
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008150
Jim Laskeybc588b82006-10-05 15:07:25 +00008151 // Look at each chain and determine if it is an alias. If so, add it to the
8152 // aliases list. If not, then continue up the chain looking for the next
Scott Michelfdc40a02009-02-17 22:15:04 +00008153 // candidate.
Jim Laskeybc588b82006-10-05 15:07:25 +00008154 while (!Chains.empty()) {
Dan Gohman475871a2008-07-27 21:46:04 +00008155 SDValue Chain = Chains.back();
Jim Laskeybc588b82006-10-05 15:07:25 +00008156 Chains.pop_back();
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008157
8158 // For TokenFactor nodes, look at each operand and only continue up the
8159 // chain until we find two aliases. If we've seen two aliases, assume we'll
Nate Begeman677c89d2009-10-12 05:53:58 +00008160 // find more and revert to original chain since the xform is unlikely to be
8161 // profitable.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008162 //
8163 // FIXME: The depth check could be made to return the last non-aliasing
Nate Begeman677c89d2009-10-12 05:53:58 +00008164 // chain we found before we hit a tokenfactor rather than the original
8165 // chain.
8166 if (Depth > 6 || Aliases.size() == 2) {
8167 Aliases.clear();
8168 Aliases.push_back(OriginalChain);
8169 break;
8170 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008171
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008172 // Don't bother if we've been before.
8173 if (!Visited.insert(Chain.getNode()))
8174 continue;
Scott Michelfdc40a02009-02-17 22:15:04 +00008175
Jim Laskeybc588b82006-10-05 15:07:25 +00008176 switch (Chain.getOpcode()) {
8177 case ISD::EntryToken:
8178 // Entry token is ideal chain operand, but handled in FindBetterChain.
8179 break;
Scott Michelfdc40a02009-02-17 22:15:04 +00008180
Jim Laskeybc588b82006-10-05 15:07:25 +00008181 case ISD::LOAD:
8182 case ISD::STORE: {
8183 // Get alias information for Chain.
Dan Gohman475871a2008-07-27 21:46:04 +00008184 SDValue OpPtr;
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008185 int64_t OpSize;
8186 const Value *OpSrcValue;
8187 int OpSrcValueOffset;
8188 unsigned OpSrcValueAlign;
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008189 const MDNode *OpSrcTBAAInfo;
Gabor Greifba36cb52008-08-28 21:40:38 +00008190 bool IsOpLoad = FindAliasInfo(Chain.getNode(), OpPtr, OpSize,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008191 OpSrcValue, OpSrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008192 OpSrcValueAlign,
8193 OpSrcTBAAInfo);
Scott Michelfdc40a02009-02-17 22:15:04 +00008194
Jim Laskeybc588b82006-10-05 15:07:25 +00008195 // If chain is alias then stop here.
8196 if (!(IsLoad && IsOpLoad) &&
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008197 isAlias(Ptr, Size, SrcValue, SrcValueOffset, SrcValueAlign,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008198 SrcTBAAInfo,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008199 OpPtr, OpSize, OpSrcValue, OpSrcValueOffset,
Dan Gohmanf96e4bd2010-10-20 00:31:05 +00008200 OpSrcValueAlign, OpSrcTBAAInfo)) {
Jim Laskeybc588b82006-10-05 15:07:25 +00008201 Aliases.push_back(Chain);
8202 } else {
8203 // Look further up the chain.
Scott Michelfdc40a02009-02-17 22:15:04 +00008204 Chains.push_back(Chain.getOperand(0));
Nate Begeman677c89d2009-10-12 05:53:58 +00008205 ++Depth;
Jim Laskey279f0532006-09-25 16:29:54 +00008206 }
Jim Laskeybc588b82006-10-05 15:07:25 +00008207 break;
8208 }
Scott Michelfdc40a02009-02-17 22:15:04 +00008209
Jim Laskeybc588b82006-10-05 15:07:25 +00008210 case ISD::TokenFactor:
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008211 // We have to check each of the operands of the token factor for "small"
8212 // token factors, so we queue them up. Adding the operands to the queue
8213 // (stack) in reverse order maintains the original order and increases the
8214 // likelihood that getNode will find a matching token factor (CSE.)
8215 if (Chain.getNumOperands() > 16) {
8216 Aliases.push_back(Chain);
8217 break;
8218 }
Jim Laskeybc588b82006-10-05 15:07:25 +00008219 for (unsigned n = Chain.getNumOperands(); n;)
8220 Chains.push_back(Chain.getOperand(--n));
Nate Begeman677c89d2009-10-12 05:53:58 +00008221 ++Depth;
Jim Laskeybc588b82006-10-05 15:07:25 +00008222 break;
Scott Michelfdc40a02009-02-17 22:15:04 +00008223
Jim Laskeybc588b82006-10-05 15:07:25 +00008224 default:
8225 // For all other instructions we will just have to take what we can get.
8226 Aliases.push_back(Chain);
8227 break;
Jim Laskey279f0532006-09-25 16:29:54 +00008228 }
8229 }
Jim Laskey6ff23e52006-10-04 16:53:27 +00008230}
8231
8232/// FindBetterChain - Walk up chain skipping non-aliasing memory nodes, looking
8233/// for a better chain (aliasing node.)
Dan Gohman475871a2008-07-27 21:46:04 +00008234SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) {
8235 SmallVector<SDValue, 8> Aliases; // Ops for replacing token factor.
Scott Michelfdc40a02009-02-17 22:15:04 +00008236
Jim Laskey6ff23e52006-10-04 16:53:27 +00008237 // Accumulate all the aliases to this node.
8238 GatherAllAliases(N, OldChain, Aliases);
Scott Michelfdc40a02009-02-17 22:15:04 +00008239
Dan Gohman71dc7c92011-05-17 22:20:36 +00008240 // If no operands then chain to entry token.
8241 if (Aliases.size() == 0)
Jim Laskey6ff23e52006-10-04 16:53:27 +00008242 return DAG.getEntryNode();
Dan Gohman71dc7c92011-05-17 22:20:36 +00008243
8244 // If a single operand then chain to it. We don't need to revisit it.
8245 if (Aliases.size() == 1)
Jim Laskey6ff23e52006-10-04 16:53:27 +00008246 return Aliases[0];
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008247
Jim Laskey6ff23e52006-10-04 16:53:27 +00008248 // Construct a custom tailored token factor.
Wesley Peckbf17cfa2010-11-23 03:31:01 +00008249 return DAG.getNode(ISD::TokenFactor, N->getDebugLoc(), MVT::Other,
Nate Begemanb6aef5c2009-09-15 00:18:30 +00008250 &Aliases[0], Aliases.size());
Jim Laskey279f0532006-09-25 16:29:54 +00008251}
8252
Nate Begeman1d4d4142005-09-01 00:19:25 +00008253// SelectionDAG::Combine - This is the entry point for the file.
8254//
Bill Wendlingbe8cc2a2009-04-29 00:15:41 +00008255void SelectionDAG::Combine(CombineLevel Level, AliasAnalysis &AA,
Bill Wendling98a366d2009-04-29 23:29:43 +00008256 CodeGenOpt::Level OptLevel) {
Nate Begeman1d4d4142005-09-01 00:19:25 +00008257 /// run - This is the main entry point to this class.
8258 ///
Bill Wendlingbe8cc2a2009-04-29 00:15:41 +00008259 DAGCombiner(*this, AA, OptLevel).Run(Level);
Nate Begeman1d4d4142005-09-01 00:19:25 +00008260}