blob: 7155972724a86216d7e15dc72a69359b71c7a77f [file] [log] [blame]
Chandler Carruthd3e73552013-01-07 03:08:10 +00001//===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===//
Nadav Rotem5dc203e2012-10-18 23:22:48 +00002//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Nadav Rotem5dc203e2012-10-18 23:22:48 +00006//
7//===----------------------------------------------------------------------===//
8
Chandler Carruthd3e73552013-01-07 03:08:10 +00009#include "llvm/Analysis/TargetTransformInfo.h"
Chandler Carruth705b1852015-01-31 03:43:40 +000010#include "llvm/Analysis/TargetTransformInfoImpl.h"
Chandler Carruth219b89b2014-03-04 11:01:28 +000011#include "llvm/IR/CallSite.h"
Chandler Carruth511aa762013-01-21 01:27:39 +000012#include "llvm/IR/DataLayout.h"
Chandler Carruth511aa762013-01-21 01:27:39 +000013#include "llvm/IR/Instruction.h"
Chandler Carruth511aa762013-01-21 01:27:39 +000014#include "llvm/IR/Instructions.h"
Chandler Carruth8a8cd2b2014-01-07 11:48:04 +000015#include "llvm/IR/IntrinsicInst.h"
Chandler Carruthe0385522015-02-01 10:11:22 +000016#include "llvm/IR/Module.h"
Chandler Carruth8a8cd2b2014-01-07 11:48:04 +000017#include "llvm/IR/Operator.h"
Guozhi Wei62d64142017-09-08 22:29:17 +000018#include "llvm/IR/PatternMatch.h"
Sean Fertile9cd1cdf2017-07-07 02:00:06 +000019#include "llvm/Support/CommandLine.h"
Nadav Rotem5dc203e2012-10-18 23:22:48 +000020#include "llvm/Support/ErrorHandling.h"
Benjamin Kramer82de7d32016-05-27 14:27:24 +000021#include <utility>
Nadav Rotem5dc203e2012-10-18 23:22:48 +000022
23using namespace llvm;
Guozhi Wei62d64142017-09-08 22:29:17 +000024using namespace PatternMatch;
Nadav Rotem5dc203e2012-10-18 23:22:48 +000025
Chandler Carruthf1221bd2014-04-22 02:48:03 +000026#define DEBUG_TYPE "tti"
27
Guozhi Wei62d64142017-09-08 22:29:17 +000028static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false),
29 cl::Hidden,
30 cl::desc("Recognize reduction patterns."));
31
Chandler Carruth93dcdc42015-01-31 11:17:59 +000032namespace {
Adrian Prantl5f8f34e42018-05-01 15:54:18 +000033/// No-op implementation of the TTI interface using the utility base
Chandler Carruth93dcdc42015-01-31 11:17:59 +000034/// classes.
35///
36/// This is used when no target specific information is available.
37struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> {
Mehdi Amini5010ebf2015-07-09 02:08:42 +000038 explicit NoTTIImpl(const DataLayout &DL)
Chandler Carruth93dcdc42015-01-31 11:17:59 +000039 : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {}
40};
41}
42
Mehdi Amini5010ebf2015-07-09 02:08:42 +000043TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)
Chandler Carruth93dcdc42015-01-31 11:17:59 +000044 : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {}
45
Chandler Carruth705b1852015-01-31 03:43:40 +000046TargetTransformInfo::~TargetTransformInfo() {}
Nadav Rotem5dc203e2012-10-18 23:22:48 +000047
Chandler Carruth705b1852015-01-31 03:43:40 +000048TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg)
49 : TTIImpl(std::move(Arg.TTIImpl)) {}
Chandler Carruth539edf42013-01-05 11:43:11 +000050
Chandler Carruth705b1852015-01-31 03:43:40 +000051TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) {
52 TTIImpl = std::move(RHS.TTIImpl);
53 return *this;
Chandler Carruth539edf42013-01-05 11:43:11 +000054}
55
Chandler Carruth93205eb2015-08-05 18:08:10 +000056int TargetTransformInfo::getOperationCost(unsigned Opcode, Type *Ty,
57 Type *OpTy) const {
58 int Cost = TTIImpl->getOperationCost(Opcode, Ty, OpTy);
59 assert(Cost >= 0 && "TTI should not produce negative costs!");
60 return Cost;
Chandler Carruth511aa762013-01-21 01:27:39 +000061}
62
Chandler Carruth93205eb2015-08-05 18:08:10 +000063int TargetTransformInfo::getCallCost(FunctionType *FTy, int NumArgs) const {
64 int Cost = TTIImpl->getCallCost(FTy, NumArgs);
65 assert(Cost >= 0 && "TTI should not produce negative costs!");
66 return Cost;
Chandler Carruth0ba8db42013-01-22 11:26:02 +000067}
68
Chandler Carruth93205eb2015-08-05 18:08:10 +000069int TargetTransformInfo::getCallCost(const Function *F,
70 ArrayRef<const Value *> Arguments) const {
71 int Cost = TTIImpl->getCallCost(F, Arguments);
72 assert(Cost >= 0 && "TTI should not produce negative costs!");
73 return Cost;
Chandler Carruth0ba8db42013-01-22 11:26:02 +000074}
75
Justin Lebar8650a4d2016-04-15 01:38:48 +000076unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {
77 return TTIImpl->getInliningThresholdMultiplier();
78}
79
Jingyue Wu15f3e822016-07-08 21:48:05 +000080int TargetTransformInfo::getGEPCost(Type *PointeeType, const Value *Ptr,
81 ArrayRef<const Value *> Operands) const {
82 return TTIImpl->getGEPCost(PointeeType, Ptr, Operands);
83}
84
Haicheng Wuabdef9e2017-07-15 02:12:16 +000085int TargetTransformInfo::getExtCost(const Instruction *I,
86 const Value *Src) const {
87 return TTIImpl->getExtCost(I, Src);
88}
89
Chandler Carruth93205eb2015-08-05 18:08:10 +000090int TargetTransformInfo::getIntrinsicCost(
91 Intrinsic::ID IID, Type *RetTy, ArrayRef<const Value *> Arguments) const {
92 int Cost = TTIImpl->getIntrinsicCost(IID, RetTy, Arguments);
93 assert(Cost >= 0 && "TTI should not produce negative costs!");
94 return Cost;
Chandler Carruth0ba8db42013-01-22 11:26:02 +000095}
96
Jun Bum Lim919f9e82017-04-28 16:04:03 +000097unsigned
98TargetTransformInfo::getEstimatedNumberOfCaseClusters(const SwitchInst &SI,
99 unsigned &JTSize) const {
100 return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize);
101}
102
Evgeny Astigeevich70ed78e2017-06-29 13:42:12 +0000103int TargetTransformInfo::getUserCost(const User *U,
104 ArrayRef<const Value *> Operands) const {
105 int Cost = TTIImpl->getUserCost(U, Operands);
Chandler Carruth93205eb2015-08-05 18:08:10 +0000106 assert(Cost >= 0 && "TTI should not produce negative costs!");
107 return Cost;
Chandler Carruth511aa762013-01-21 01:27:39 +0000108}
109
Tom Stellard8b1e0212013-07-27 00:01:07 +0000110bool TargetTransformInfo::hasBranchDivergence() const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000111 return TTIImpl->hasBranchDivergence();
Tom Stellard8b1e0212013-07-27 00:01:07 +0000112}
113
Jingyue Wu5da831c2015-04-10 05:03:50 +0000114bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const {
115 return TTIImpl->isSourceOfDivergence(V);
116}
117
Alexander Timofeev0f9c84c2017-06-15 19:33:10 +0000118bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const {
119 return TTIImpl->isAlwaysUniform(V);
120}
121
Matt Arsenault42b64782017-01-30 23:02:12 +0000122unsigned TargetTransformInfo::getFlatAddressSpace() const {
123 return TTIImpl->getFlatAddressSpace();
124}
125
Chandler Carruth0ba8db42013-01-22 11:26:02 +0000126bool TargetTransformInfo::isLoweredToCall(const Function *F) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000127 return TTIImpl->isLoweredToCall(F);
Chandler Carruth0ba8db42013-01-22 11:26:02 +0000128}
129
Chandler Carruth705b1852015-01-31 03:43:40 +0000130void TargetTransformInfo::getUnrollingPreferences(
Geoff Berry66d9bdb2017-06-28 15:53:17 +0000131 Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP) const {
132 return TTIImpl->getUnrollingPreferences(L, SE, UP);
Hal Finkel8f2e7002013-09-11 19:25:43 +0000133}
134
Chandler Carruth539edf42013-01-05 11:43:11 +0000135bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000136 return TTIImpl->isLegalAddImmediate(Imm);
Chandler Carruth539edf42013-01-05 11:43:11 +0000137}
138
139bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000140 return TTIImpl->isLegalICmpImmediate(Imm);
Chandler Carruth539edf42013-01-05 11:43:11 +0000141}
142
143bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
144 int64_t BaseOffset,
145 bool HasBaseReg,
Matt Arsenaulte83379e2015-06-07 20:12:03 +0000146 int64_t Scale,
Jonas Paulsson024e3192017-07-21 11:59:37 +0000147 unsigned AddrSpace,
148 Instruction *I) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000149 return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,
Jonas Paulsson024e3192017-07-21 11:59:37 +0000150 Scale, AddrSpace, I);
Chandler Carruth539edf42013-01-05 11:43:11 +0000151}
152
Evgeny Stupachenkof2b3b462017-06-05 23:37:00 +0000153bool TargetTransformInfo::isLSRCostLess(LSRCost &C1, LSRCost &C2) const {
154 return TTIImpl->isLSRCostLess(C1, C2);
155}
156
Sanjay Pateld7c702b2018-02-05 23:43:05 +0000157bool TargetTransformInfo::canMacroFuseCmp() const {
158 return TTIImpl->canMacroFuseCmp();
159}
160
Krzysztof Parzyszek0b377e02018-03-26 13:10:09 +0000161bool TargetTransformInfo::shouldFavorPostInc() const {
162 return TTIImpl->shouldFavorPostInc();
163}
164
Elena Demikhovsky20662e32015-10-19 07:43:38 +0000165bool TargetTransformInfo::isLegalMaskedStore(Type *DataType) const {
166 return TTIImpl->isLegalMaskedStore(DataType);
Chandler Carruth705b1852015-01-31 03:43:40 +0000167}
168
Elena Demikhovsky20662e32015-10-19 07:43:38 +0000169bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType) const {
170 return TTIImpl->isLegalMaskedLoad(DataType);
Chandler Carruth705b1852015-01-31 03:43:40 +0000171}
172
Elena Demikhovsky09285852015-10-25 15:37:55 +0000173bool TargetTransformInfo::isLegalMaskedGather(Type *DataType) const {
174 return TTIImpl->isLegalMaskedGather(DataType);
175}
176
177bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType) const {
Mohammed Agabariacef53dc2017-07-27 10:28:16 +0000178 return TTIImpl->isLegalMaskedScatter(DataType);
Elena Demikhovsky09285852015-10-25 15:37:55 +0000179}
180
Sanjay Patel6fd43912017-09-09 13:38:18 +0000181bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const {
182 return TTIImpl->hasDivRemOp(DataType, IsSigned);
183}
184
Artem Belevichcb8f6322017-10-24 20:31:44 +0000185bool TargetTransformInfo::hasVolatileVariant(Instruction *I,
186 unsigned AddrSpace) const {
187 return TTIImpl->hasVolatileVariant(I, AddrSpace);
188}
189
Jonas Paulsson8624b7e2017-05-24 13:42:56 +0000190bool TargetTransformInfo::prefersVectorizedAddressing() const {
191 return TTIImpl->prefersVectorizedAddressing();
192}
193
Quentin Colombetbf490d42013-05-31 21:29:03 +0000194int TargetTransformInfo::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
195 int64_t BaseOffset,
196 bool HasBaseReg,
Matt Arsenaulte83379e2015-06-07 20:12:03 +0000197 int64_t Scale,
198 unsigned AddrSpace) const {
Chandler Carruth93205eb2015-08-05 18:08:10 +0000199 int Cost = TTIImpl->getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg,
200 Scale, AddrSpace);
201 assert(Cost >= 0 && "TTI should not produce negative costs!");
202 return Cost;
Quentin Colombetbf490d42013-05-31 21:29:03 +0000203}
204
Jonas Paulsson024e3192017-07-21 11:59:37 +0000205bool TargetTransformInfo::LSRWithInstrQueries() const {
206 return TTIImpl->LSRWithInstrQueries();
207}
208
Chandler Carruth539edf42013-01-05 11:43:11 +0000209bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000210 return TTIImpl->isTruncateFree(Ty1, Ty2);
Chandler Carruth539edf42013-01-05 11:43:11 +0000211}
212
Chad Rosier54390052015-02-23 19:15:16 +0000213bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const {
214 return TTIImpl->isProfitableToHoist(I);
215}
216
David Blaikie8ad9a972018-03-28 22:28:50 +0000217bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); }
218
Chandler Carruth539edf42013-01-05 11:43:11 +0000219bool TargetTransformInfo::isTypeLegal(Type *Ty) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000220 return TTIImpl->isTypeLegal(Ty);
Chandler Carruth539edf42013-01-05 11:43:11 +0000221}
222
223unsigned TargetTransformInfo::getJumpBufAlignment() const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000224 return TTIImpl->getJumpBufAlignment();
Chandler Carruth539edf42013-01-05 11:43:11 +0000225}
226
227unsigned TargetTransformInfo::getJumpBufSize() const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000228 return TTIImpl->getJumpBufSize();
Chandler Carruth539edf42013-01-05 11:43:11 +0000229}
230
231bool TargetTransformInfo::shouldBuildLookupTables() const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000232 return TTIImpl->shouldBuildLookupTables();
Chandler Carruth539edf42013-01-05 11:43:11 +0000233}
Oliver Stannard4df1cc02016-10-07 08:48:24 +0000234bool TargetTransformInfo::shouldBuildLookupTablesForConstant(Constant *C) const {
235 return TTIImpl->shouldBuildLookupTablesForConstant(C);
236}
Chandler Carruth539edf42013-01-05 11:43:11 +0000237
Zaara Syeda1f59ae32018-01-30 16:17:22 +0000238bool TargetTransformInfo::useColdCCForColdCall(Function &F) const {
239 return TTIImpl->useColdCCForColdCall(F);
240}
241
Jonas Paulsson8e2f9482017-01-26 07:03:25 +0000242unsigned TargetTransformInfo::
243getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) const {
244 return TTIImpl->getScalarizationOverhead(Ty, Insert, Extract);
245}
246
247unsigned TargetTransformInfo::
248getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
249 unsigned VF) const {
250 return TTIImpl->getOperandsScalarizationOverhead(Args, VF);
251}
252
Jonas Paulssonda74ed42017-04-12 12:41:37 +0000253bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {
254 return TTIImpl->supportsEfficientVectorElementLoadStore();
255}
256
Olivier Sallenave049d8032015-03-06 23:12:04 +0000257bool TargetTransformInfo::enableAggressiveInterleaving(bool LoopHasReductions) const {
258 return TTIImpl->enableAggressiveInterleaving(LoopHasReductions);
259}
260
Clement Courbetb2c3eb82017-10-30 14:19:33 +0000261const TargetTransformInfo::MemCmpExpansionOptions *
262TargetTransformInfo::enableMemCmpExpansion(bool IsZeroCmp) const {
263 return TTIImpl->enableMemCmpExpansion(IsZeroCmp);
Zaara Syeda3a7578c2017-05-31 17:12:38 +0000264}
265
Silviu Baranga61bdc512015-08-10 14:50:54 +0000266bool TargetTransformInfo::enableInterleavedAccessVectorization() const {
267 return TTIImpl->enableInterleavedAccessVectorization();
268}
269
Dorit Nuzman38bbf812018-10-14 08:50:06 +0000270bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const {
271 return TTIImpl->enableMaskedInterleavedAccessVectorization();
272}
273
Renato Golin5cb666a2016-04-14 20:42:18 +0000274bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const {
275 return TTIImpl->isFPVectorizationPotentiallyUnsafe();
276}
277
Alina Sbirlea6f937b12016-08-04 16:38:44 +0000278bool TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context,
279 unsigned BitWidth,
Alina Sbirlea327955e2016-07-11 20:46:17 +0000280 unsigned AddressSpace,
281 unsigned Alignment,
282 bool *Fast) const {
Alina Sbirlea6f937b12016-08-04 16:38:44 +0000283 return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth, AddressSpace,
Alina Sbirlea327955e2016-07-11 20:46:17 +0000284 Alignment, Fast);
285}
286
Chandler Carruth50a36cd2013-01-07 03:16:03 +0000287TargetTransformInfo::PopcntSupportKind
288TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000289 return TTIImpl->getPopcntSupport(IntTyWidthInBit);
Chandler Carruth539edf42013-01-05 11:43:11 +0000290}
291
Richard Sandiford37cd6cf2013-08-23 10:27:02 +0000292bool TargetTransformInfo::haveFastSqrt(Type *Ty) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000293 return TTIImpl->haveFastSqrt(Ty);
Richard Sandiford37cd6cf2013-08-23 10:27:02 +0000294}
295
Sanjay Patel0de1a4b2017-11-27 21:15:43 +0000296bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const {
297 return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty);
298}
299
Chandler Carruth93205eb2015-08-05 18:08:10 +0000300int TargetTransformInfo::getFPOpCost(Type *Ty) const {
301 int Cost = TTIImpl->getFPOpCost(Ty);
302 assert(Cost >= 0 && "TTI should not produce negative costs!");
303 return Cost;
Cameron Esfahani17177d12015-02-05 02:09:33 +0000304}
305
Sjoerd Meijer38c2cd02016-07-14 07:44:20 +0000306int TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx,
307 const APInt &Imm,
308 Type *Ty) const {
309 int Cost = TTIImpl->getIntImmCodeSizeCost(Opcode, Idx, Imm, Ty);
310 assert(Cost >= 0 && "TTI should not produce negative costs!");
311 return Cost;
312}
313
Chandler Carruth93205eb2015-08-05 18:08:10 +0000314int TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty) const {
315 int Cost = TTIImpl->getIntImmCost(Imm, Ty);
316 assert(Cost >= 0 && "TTI should not produce negative costs!");
317 return Cost;
Chandler Carruth539edf42013-01-05 11:43:11 +0000318}
319
Chandler Carruth93205eb2015-08-05 18:08:10 +0000320int TargetTransformInfo::getIntImmCost(unsigned Opcode, unsigned Idx,
321 const APInt &Imm, Type *Ty) const {
322 int Cost = TTIImpl->getIntImmCost(Opcode, Idx, Imm, Ty);
323 assert(Cost >= 0 && "TTI should not produce negative costs!");
324 return Cost;
Juergen Ributzkaf26beda2014-01-25 02:02:55 +0000325}
326
Chandler Carruth93205eb2015-08-05 18:08:10 +0000327int TargetTransformInfo::getIntImmCost(Intrinsic::ID IID, unsigned Idx,
328 const APInt &Imm, Type *Ty) const {
329 int Cost = TTIImpl->getIntImmCost(IID, Idx, Imm, Ty);
330 assert(Cost >= 0 && "TTI should not produce negative costs!");
331 return Cost;
Juergen Ributzkaf26beda2014-01-25 02:02:55 +0000332}
333
Chandler Carruth539edf42013-01-05 11:43:11 +0000334unsigned TargetTransformInfo::getNumberOfRegisters(bool Vector) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000335 return TTIImpl->getNumberOfRegisters(Vector);
Chandler Carruth539edf42013-01-05 11:43:11 +0000336}
337
Nadav Rotemb1791a72013-01-09 22:29:00 +0000338unsigned TargetTransformInfo::getRegisterBitWidth(bool Vector) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000339 return TTIImpl->getRegisterBitWidth(Vector);
Nadav Rotemb1791a72013-01-09 22:29:00 +0000340}
341
Adam Nemete29686e2017-05-15 21:15:01 +0000342unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {
343 return TTIImpl->getMinVectorRegisterBitWidth();
344}
345
Krzysztof Parzyszek5d93fdf2018-03-27 16:14:11 +0000346bool TargetTransformInfo::shouldMaximizeVectorBandwidth(bool OptSize) const {
347 return TTIImpl->shouldMaximizeVectorBandwidth(OptSize);
348}
349
Krzysztof Parzyszekdfed9412018-04-13 20:16:32 +0000350unsigned TargetTransformInfo::getMinimumVF(unsigned ElemWidth) const {
351 return TTIImpl->getMinimumVF(ElemWidth);
352}
353
Jun Bum Limdee55652017-04-03 19:20:07 +0000354bool TargetTransformInfo::shouldConsiderAddressTypePromotion(
355 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
356 return TTIImpl->shouldConsiderAddressTypePromotion(
357 I, AllowPromotionWithoutCommonHeader);
358}
359
Adam Nemetaf761102016-01-21 18:28:36 +0000360unsigned TargetTransformInfo::getCacheLineSize() const {
361 return TTIImpl->getCacheLineSize();
362}
363
Tobias Grosserd7eb6192017-08-24 09:46:25 +0000364llvm::Optional<unsigned> TargetTransformInfo::getCacheSize(CacheLevel Level)
365 const {
366 return TTIImpl->getCacheSize(Level);
367}
368
369llvm::Optional<unsigned> TargetTransformInfo::getCacheAssociativity(
370 CacheLevel Level) const {
371 return TTIImpl->getCacheAssociativity(Level);
372}
373
Adam Nemetdadfbb52016-01-27 22:21:25 +0000374unsigned TargetTransformInfo::getPrefetchDistance() const {
375 return TTIImpl->getPrefetchDistance();
376}
377
Adam Nemet6d8beec2016-03-18 00:27:38 +0000378unsigned TargetTransformInfo::getMinPrefetchStride() const {
379 return TTIImpl->getMinPrefetchStride();
380}
381
Adam Nemet709e3042016-03-18 00:27:43 +0000382unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const {
383 return TTIImpl->getMaxPrefetchIterationsAhead();
384}
385
Wei Mi062c7442015-05-06 17:12:25 +0000386unsigned TargetTransformInfo::getMaxInterleaveFactor(unsigned VF) const {
387 return TTIImpl->getMaxInterleaveFactor(VF);
Nadav Rotemb696c362013-01-09 01:15:42 +0000388}
389
Jonas Paulsson29d80f02018-10-05 14:34:04 +0000390TargetTransformInfo::OperandValueKind
Simon Pilgrim077a42c2018-11-13 13:45:10 +0000391TargetTransformInfo::getOperandInfo(Value *V, OperandValueProperties &OpProps) {
Jonas Paulsson29d80f02018-10-05 14:34:04 +0000392 OperandValueKind OpInfo = OK_AnyValue;
393 OpProps = OP_None;
394
395 if (auto *CI = dyn_cast<ConstantInt>(V)) {
396 if (CI->getValue().isPowerOf2())
397 OpProps = OP_PowerOf2;
398 return OK_UniformConstantValue;
399 }
400
Simon Pilgrim2b166c52018-11-14 15:04:08 +0000401 // A broadcast shuffle creates a uniform value.
402 // TODO: Add support for non-zero index broadcasts.
403 // TODO: Add support for different source vector width.
404 if (auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V))
405 if (ShuffleInst->isZeroEltSplat())
406 OpInfo = OK_UniformValue;
407
Jonas Paulsson29d80f02018-10-05 14:34:04 +0000408 const Value *Splat = getSplatValue(V);
409
410 // Check for a splat of a constant or for a non uniform vector of constants
411 // and check if the constant(s) are all powers of two.
412 if (isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) {
413 OpInfo = OK_NonUniformConstantValue;
414 if (Splat) {
415 OpInfo = OK_UniformConstantValue;
416 if (auto *CI = dyn_cast<ConstantInt>(Splat))
417 if (CI->getValue().isPowerOf2())
418 OpProps = OP_PowerOf2;
419 } else if (auto *CDS = dyn_cast<ConstantDataSequential>(V)) {
420 OpProps = OP_PowerOf2;
421 for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) {
422 if (auto *CI = dyn_cast<ConstantInt>(CDS->getElementAsConstant(I)))
423 if (CI->getValue().isPowerOf2())
424 continue;
425 OpProps = OP_None;
426 break;
427 }
428 }
429 }
430
431 // Check for a splat of a uniform value. This is not loop aware, so return
432 // true only for the obviously uniform cases (argument, globalvalue)
433 if (Splat && (isa<Argument>(Splat) || isa<GlobalValue>(Splat)))
434 OpInfo = OK_UniformValue;
435
436 return OpInfo;
437}
438
Chandler Carruth93205eb2015-08-05 18:08:10 +0000439int TargetTransformInfo::getArithmeticInstrCost(
Chandler Carruth705b1852015-01-31 03:43:40 +0000440 unsigned Opcode, Type *Ty, OperandValueKind Opd1Info,
441 OperandValueKind Opd2Info, OperandValueProperties Opd1PropInfo,
Mohammed Agabaria2c96c432017-01-11 08:23:37 +0000442 OperandValueProperties Opd2PropInfo,
443 ArrayRef<const Value *> Args) const {
Chandler Carruth93205eb2015-08-05 18:08:10 +0000444 int Cost = TTIImpl->getArithmeticInstrCost(Opcode, Ty, Opd1Info, Opd2Info,
Mohammed Agabaria2c96c432017-01-11 08:23:37 +0000445 Opd1PropInfo, Opd2PropInfo, Args);
Chandler Carruth93205eb2015-08-05 18:08:10 +0000446 assert(Cost >= 0 && "TTI should not produce negative costs!");
447 return Cost;
Chandler Carruth539edf42013-01-05 11:43:11 +0000448}
449
Chandler Carruth93205eb2015-08-05 18:08:10 +0000450int TargetTransformInfo::getShuffleCost(ShuffleKind Kind, Type *Ty, int Index,
451 Type *SubTp) const {
452 int Cost = TTIImpl->getShuffleCost(Kind, Ty, Index, SubTp);
453 assert(Cost >= 0 && "TTI should not produce negative costs!");
454 return Cost;
Chandler Carruth539edf42013-01-05 11:43:11 +0000455}
456
Chandler Carruth93205eb2015-08-05 18:08:10 +0000457int TargetTransformInfo::getCastInstrCost(unsigned Opcode, Type *Dst,
Jonas Paulssonfccc7d62017-04-12 11:49:08 +0000458 Type *Src, const Instruction *I) const {
459 assert ((I == nullptr || I->getOpcode() == Opcode) &&
460 "Opcode should reflect passed instruction.");
461 int Cost = TTIImpl->getCastInstrCost(Opcode, Dst, Src, I);
Chandler Carruth93205eb2015-08-05 18:08:10 +0000462 assert(Cost >= 0 && "TTI should not produce negative costs!");
463 return Cost;
Chandler Carruth539edf42013-01-05 11:43:11 +0000464}
465
Matthew Simpsone5dfb082016-04-27 15:20:21 +0000466int TargetTransformInfo::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
467 VectorType *VecTy,
468 unsigned Index) const {
469 int Cost = TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index);
470 assert(Cost >= 0 && "TTI should not produce negative costs!");
471 return Cost;
472}
473
Chandler Carruth93205eb2015-08-05 18:08:10 +0000474int TargetTransformInfo::getCFInstrCost(unsigned Opcode) const {
475 int Cost = TTIImpl->getCFInstrCost(Opcode);
476 assert(Cost >= 0 && "TTI should not produce negative costs!");
477 return Cost;
Chandler Carruth539edf42013-01-05 11:43:11 +0000478}
479
Chandler Carruth93205eb2015-08-05 18:08:10 +0000480int TargetTransformInfo::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
Jonas Paulssonfccc7d62017-04-12 11:49:08 +0000481 Type *CondTy, const Instruction *I) const {
482 assert ((I == nullptr || I->getOpcode() == Opcode) &&
483 "Opcode should reflect passed instruction.");
484 int Cost = TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, I);
Chandler Carruth93205eb2015-08-05 18:08:10 +0000485 assert(Cost >= 0 && "TTI should not produce negative costs!");
486 return Cost;
Chandler Carruth539edf42013-01-05 11:43:11 +0000487}
488
Chandler Carruth93205eb2015-08-05 18:08:10 +0000489int TargetTransformInfo::getVectorInstrCost(unsigned Opcode, Type *Val,
490 unsigned Index) const {
491 int Cost = TTIImpl->getVectorInstrCost(Opcode, Val, Index);
492 assert(Cost >= 0 && "TTI should not produce negative costs!");
493 return Cost;
Chandler Carruth539edf42013-01-05 11:43:11 +0000494}
495
Chandler Carruth93205eb2015-08-05 18:08:10 +0000496int TargetTransformInfo::getMemoryOpCost(unsigned Opcode, Type *Src,
497 unsigned Alignment,
Jonas Paulssonfccc7d62017-04-12 11:49:08 +0000498 unsigned AddressSpace,
499 const Instruction *I) const {
500 assert ((I == nullptr || I->getOpcode() == Opcode) &&
501 "Opcode should reflect passed instruction.");
502 int Cost = TTIImpl->getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, I);
Chandler Carruth93205eb2015-08-05 18:08:10 +0000503 assert(Cost >= 0 && "TTI should not produce negative costs!");
504 return Cost;
Elena Demikhovskya3232f72015-01-25 08:44:46 +0000505}
506
Chandler Carruth93205eb2015-08-05 18:08:10 +0000507int TargetTransformInfo::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
508 unsigned Alignment,
509 unsigned AddressSpace) const {
510 int Cost =
511 TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace);
512 assert(Cost >= 0 && "TTI should not produce negative costs!");
513 return Cost;
Chandler Carruth705b1852015-01-31 03:43:40 +0000514}
515
Elena Demikhovsky54946982015-12-28 20:10:59 +0000516int TargetTransformInfo::getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
517 Value *Ptr, bool VariableMask,
518 unsigned Alignment) const {
519 int Cost = TTIImpl->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
520 Alignment);
521 assert(Cost >= 0 && "TTI should not produce negative costs!");
522 return Cost;
523}
524
Chandler Carruth93205eb2015-08-05 18:08:10 +0000525int TargetTransformInfo::getInterleavedMemoryOpCost(
Hao Liu32c05392015-06-08 06:39:56 +0000526 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Dorit Nuzman34da6dd2018-10-31 09:57:56 +0000527 unsigned Alignment, unsigned AddressSpace, bool UseMaskForCond,
528 bool UseMaskForGaps) const {
529 int Cost = TTIImpl->getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
530 Alignment, AddressSpace,
531 UseMaskForCond,
532 UseMaskForGaps);
Chandler Carruth93205eb2015-08-05 18:08:10 +0000533 assert(Cost >= 0 && "TTI should not produce negative costs!");
534 return Cost;
Hao Liu32c05392015-06-08 06:39:56 +0000535}
536
Chandler Carruth93205eb2015-08-05 18:08:10 +0000537int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
Jonas Paulssona48ea232017-03-14 06:35:36 +0000538 ArrayRef<Type *> Tys, FastMathFlags FMF,
539 unsigned ScalarizationCostPassed) const {
540 int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Tys, FMF,
541 ScalarizationCostPassed);
Chandler Carruth93205eb2015-08-05 18:08:10 +0000542 assert(Cost >= 0 && "TTI should not produce negative costs!");
543 return Cost;
544}
545
Elena Demikhovsky54946982015-12-28 20:10:59 +0000546int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
Jonas Paulssona48ea232017-03-14 06:35:36 +0000547 ArrayRef<Value *> Args, FastMathFlags FMF, unsigned VF) const {
548 int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Args, FMF, VF);
Elena Demikhovsky54946982015-12-28 20:10:59 +0000549 assert(Cost >= 0 && "TTI should not produce negative costs!");
550 return Cost;
551}
552
Chandler Carruth93205eb2015-08-05 18:08:10 +0000553int TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy,
554 ArrayRef<Type *> Tys) const {
555 int Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys);
556 assert(Cost >= 0 && "TTI should not produce negative costs!");
557 return Cost;
Michael Zolotukhin7ed84a82015-03-17 19:26:23 +0000558}
559
Chandler Carruth539edf42013-01-05 11:43:11 +0000560unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000561 return TTIImpl->getNumberOfParts(Tp);
Chandler Carruth539edf42013-01-05 11:43:11 +0000562}
563
Chandler Carruth93205eb2015-08-05 18:08:10 +0000564int TargetTransformInfo::getAddressComputationCost(Type *Tp,
Mohammed Agabaria23599ba2017-01-05 14:03:41 +0000565 ScalarEvolution *SE,
566 const SCEV *Ptr) const {
567 int Cost = TTIImpl->getAddressComputationCost(Tp, SE, Ptr);
Chandler Carruth93205eb2015-08-05 18:08:10 +0000568 assert(Cost >= 0 && "TTI should not produce negative costs!");
569 return Cost;
Arnold Schwaighofer594fa2d2013-02-08 14:50:48 +0000570}
Chandler Carruth539edf42013-01-05 11:43:11 +0000571
Alexey Bataev3e9b3eb2017-07-31 14:19:32 +0000572int TargetTransformInfo::getArithmeticReductionCost(unsigned Opcode, Type *Ty,
573 bool IsPairwiseForm) const {
574 int Cost = TTIImpl->getArithmeticReductionCost(Opcode, Ty, IsPairwiseForm);
Chandler Carruth93205eb2015-08-05 18:08:10 +0000575 assert(Cost >= 0 && "TTI should not produce negative costs!");
576 return Cost;
Arnold Schwaighofercae87352013-09-17 18:06:50 +0000577}
578
Alexey Bataev6dd29fc2017-09-08 13:49:36 +0000579int TargetTransformInfo::getMinMaxReductionCost(Type *Ty, Type *CondTy,
580 bool IsPairwiseForm,
581 bool IsUnsigned) const {
582 int Cost =
583 TTIImpl->getMinMaxReductionCost(Ty, CondTy, IsPairwiseForm, IsUnsigned);
584 assert(Cost >= 0 && "TTI should not produce negative costs!");
585 return Cost;
586}
587
Chandler Carruth705b1852015-01-31 03:43:40 +0000588unsigned
589TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
590 return TTIImpl->getCostOfKeepingLiveOverCall(Tys);
Chad Rosierf9327d62015-01-26 22:51:15 +0000591}
592
593bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,
594 MemIntrinsicInfo &Info) const {
Chandler Carruth705b1852015-01-31 03:43:40 +0000595 return TTIImpl->getTgtMemIntrinsic(Inst, Info);
Chad Rosierf9327d62015-01-26 22:51:15 +0000596}
597
Anna Thomasb2a212c2017-06-06 16:45:25 +0000598unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
599 return TTIImpl->getAtomicMemIntrinsicMaxElementSize();
600}
601
Chandler Carruth705b1852015-01-31 03:43:40 +0000602Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
603 IntrinsicInst *Inst, Type *ExpectedType) const {
604 return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
605}
606
Sean Fertile9cd1cdf2017-07-07 02:00:06 +0000607Type *TargetTransformInfo::getMemcpyLoopLoweringType(LLVMContext &Context,
608 Value *Length,
609 unsigned SrcAlign,
610 unsigned DestAlign) const {
611 return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAlign,
612 DestAlign);
613}
614
615void TargetTransformInfo::getMemcpyLoopResidualLoweringType(
616 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
617 unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const {
618 TTIImpl->getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes,
619 SrcAlign, DestAlign);
620}
621
Eric Christopherd566fb12015-07-29 22:09:48 +0000622bool TargetTransformInfo::areInlineCompatible(const Function *Caller,
623 const Function *Callee) const {
624 return TTIImpl->areInlineCompatible(Caller, Callee);
Eric Christopher4371b132015-07-02 01:11:47 +0000625}
626
Tom Stellard3d36e5c2019-01-16 05:15:31 +0000627bool TargetTransformInfo::areFunctionArgsABICompatible(
628 const Function *Caller, const Function *Callee,
629 SmallPtrSetImpl<Argument *> &Args) const {
630 return TTIImpl->areFunctionArgsABICompatible(Caller, Callee, Args);
631}
632
Krzysztof Parzyszek0b377e02018-03-26 13:10:09 +0000633bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode,
634 Type *Ty) const {
635 return TTIImpl->isIndexedLoadLegal(Mode, Ty);
636}
637
638bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode,
639 Type *Ty) const {
640 return TTIImpl->isIndexedStoreLegal(Mode, Ty);
641}
642
Volkan Keles1c386812016-10-03 10:31:34 +0000643unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const {
644 return TTIImpl->getLoadStoreVecRegBitWidth(AS);
645}
646
647bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const {
648 return TTIImpl->isLegalToVectorizeLoad(LI);
649}
650
651bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const {
652 return TTIImpl->isLegalToVectorizeStore(SI);
653}
654
655bool TargetTransformInfo::isLegalToVectorizeLoadChain(
656 unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const {
657 return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment,
658 AddrSpace);
659}
660
661bool TargetTransformInfo::isLegalToVectorizeStoreChain(
662 unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const {
663 return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment,
664 AddrSpace);
665}
666
667unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF,
668 unsigned LoadSize,
669 unsigned ChainSizeInBytes,
670 VectorType *VecTy) const {
671 return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy);
672}
673
674unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,
675 unsigned StoreSize,
676 unsigned ChainSizeInBytes,
677 VectorType *VecTy) const {
678 return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
679}
680
Amara Emersoncf9daa32017-05-09 10:43:25 +0000681bool TargetTransformInfo::useReductionIntrinsic(unsigned Opcode,
682 Type *Ty, ReductionFlags Flags) const {
683 return TTIImpl->useReductionIntrinsic(Opcode, Ty, Flags);
684}
685
Amara Emerson836b0f42017-05-10 09:42:49 +0000686bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {
687 return TTIImpl->shouldExpandReduction(II);
688}
Amara Emersoncf9daa32017-05-09 10:43:25 +0000689
Guozhi Wei62d64142017-09-08 22:29:17 +0000690int TargetTransformInfo::getInstructionLatency(const Instruction *I) const {
691 return TTIImpl->getInstructionLatency(I);
692}
693
Guozhi Wei62d64142017-09-08 22:29:17 +0000694static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft,
695 unsigned Level) {
696 // We don't need a shuffle if we just want to have element 0 in position 0 of
697 // the vector.
698 if (!SI && Level == 0 && IsLeft)
699 return true;
700 else if (!SI)
701 return false;
702
703 SmallVector<int, 32> Mask(SI->getType()->getVectorNumElements(), -1);
704
705 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right) depending on whether
706 // we look at the left or right side.
707 for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
708 Mask[i] = val;
709
710 SmallVector<int, 16> ActualMask = SI->getShuffleMask();
711 return Mask == ActualMask;
712}
713
714namespace {
715/// Kind of the reduction data.
716enum ReductionKind {
717 RK_None, /// Not a reduction.
718 RK_Arithmetic, /// Binary reduction data.
719 RK_MinMax, /// Min/max reduction data.
720 RK_UnsignedMinMax, /// Unsigned min/max reduction data.
721};
722/// Contains opcode + LHS/RHS parts of the reduction operations.
723struct ReductionData {
724 ReductionData() = delete;
725 ReductionData(ReductionKind Kind, unsigned Opcode, Value *LHS, Value *RHS)
726 : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind) {
727 assert(Kind != RK_None && "expected binary or min/max reduction only.");
728 }
729 unsigned Opcode = 0;
730 Value *LHS = nullptr;
731 Value *RHS = nullptr;
732 ReductionKind Kind = RK_None;
733 bool hasSameData(ReductionData &RD) const {
734 return Kind == RD.Kind && Opcode == RD.Opcode;
735 }
736};
737} // namespace
738
739static Optional<ReductionData> getReductionData(Instruction *I) {
740 Value *L, *R;
741 if (m_BinOp(m_Value(L), m_Value(R)).match(I))
Fangrui Songf78650a2018-07-30 19:41:25 +0000742 return ReductionData(RK_Arithmetic, I->getOpcode(), L, R);
Guozhi Wei62d64142017-09-08 22:29:17 +0000743 if (auto *SI = dyn_cast<SelectInst>(I)) {
744 if (m_SMin(m_Value(L), m_Value(R)).match(SI) ||
745 m_SMax(m_Value(L), m_Value(R)).match(SI) ||
746 m_OrdFMin(m_Value(L), m_Value(R)).match(SI) ||
747 m_OrdFMax(m_Value(L), m_Value(R)).match(SI) ||
748 m_UnordFMin(m_Value(L), m_Value(R)).match(SI) ||
749 m_UnordFMax(m_Value(L), m_Value(R)).match(SI)) {
750 auto *CI = cast<CmpInst>(SI->getCondition());
Fangrui Songf78650a2018-07-30 19:41:25 +0000751 return ReductionData(RK_MinMax, CI->getOpcode(), L, R);
752 }
Guozhi Wei62d64142017-09-08 22:29:17 +0000753 if (m_UMin(m_Value(L), m_Value(R)).match(SI) ||
754 m_UMax(m_Value(L), m_Value(R)).match(SI)) {
755 auto *CI = cast<CmpInst>(SI->getCondition());
756 return ReductionData(RK_UnsignedMinMax, CI->getOpcode(), L, R);
757 }
758 }
759 return llvm::None;
760}
761
762static ReductionKind matchPairwiseReductionAtLevel(Instruction *I,
763 unsigned Level,
764 unsigned NumLevels) {
765 // Match one level of pairwise operations.
766 // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
767 // <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
768 // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef,
769 // <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
770 // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
771 if (!I)
772 return RK_None;
773
774 assert(I->getType()->isVectorTy() && "Expecting a vector type");
775
776 Optional<ReductionData> RD = getReductionData(I);
777 if (!RD)
778 return RK_None;
779
780 ShuffleVectorInst *LS = dyn_cast<ShuffleVectorInst>(RD->LHS);
781 if (!LS && Level)
782 return RK_None;
783 ShuffleVectorInst *RS = dyn_cast<ShuffleVectorInst>(RD->RHS);
784 if (!RS && Level)
785 return RK_None;
786
787 // On level 0 we can omit one shufflevector instruction.
788 if (!Level && !RS && !LS)
789 return RK_None;
790
791 // Shuffle inputs must match.
792 Value *NextLevelOpL = LS ? LS->getOperand(0) : nullptr;
793 Value *NextLevelOpR = RS ? RS->getOperand(0) : nullptr;
794 Value *NextLevelOp = nullptr;
795 if (NextLevelOpR && NextLevelOpL) {
796 // If we have two shuffles their operands must match.
797 if (NextLevelOpL != NextLevelOpR)
798 return RK_None;
799
800 NextLevelOp = NextLevelOpL;
801 } else if (Level == 0 && (NextLevelOpR || NextLevelOpL)) {
802 // On the first level we can omit the shufflevector <0, undef,...>. So the
803 // input to the other shufflevector <1, undef> must match with one of the
804 // inputs to the current binary operation.
805 // Example:
806 // %NextLevelOpL = shufflevector %R, <1, undef ...>
807 // %BinOp = fadd %NextLevelOpL, %R
808 if (NextLevelOpL && NextLevelOpL != RD->RHS)
809 return RK_None;
810 else if (NextLevelOpR && NextLevelOpR != RD->LHS)
811 return RK_None;
812
813 NextLevelOp = NextLevelOpL ? RD->RHS : RD->LHS;
814 } else
815 return RK_None;
816
817 // Check that the next levels binary operation exists and matches with the
818 // current one.
819 if (Level + 1 != NumLevels) {
820 Optional<ReductionData> NextLevelRD =
821 getReductionData(cast<Instruction>(NextLevelOp));
822 if (!NextLevelRD || !RD->hasSameData(*NextLevelRD))
823 return RK_None;
824 }
825
826 // Shuffle mask for pairwise operation must match.
827 if (matchPairwiseShuffleMask(LS, /*IsLeft=*/true, Level)) {
828 if (!matchPairwiseShuffleMask(RS, /*IsLeft=*/false, Level))
829 return RK_None;
830 } else if (matchPairwiseShuffleMask(RS, /*IsLeft=*/true, Level)) {
831 if (!matchPairwiseShuffleMask(LS, /*IsLeft=*/false, Level))
832 return RK_None;
833 } else {
834 return RK_None;
835 }
836
837 if (++Level == NumLevels)
838 return RD->Kind;
839
840 // Match next level.
841 return matchPairwiseReductionAtLevel(cast<Instruction>(NextLevelOp), Level,
842 NumLevels);
843}
844
845static ReductionKind matchPairwiseReduction(const ExtractElementInst *ReduxRoot,
846 unsigned &Opcode, Type *&Ty) {
847 if (!EnableReduxCost)
848 return RK_None;
849
850 // Need to extract the first element.
851 ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
852 unsigned Idx = ~0u;
853 if (CI)
854 Idx = CI->getZExtValue();
855 if (Idx != 0)
856 return RK_None;
857
858 auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0));
859 if (!RdxStart)
860 return RK_None;
861 Optional<ReductionData> RD = getReductionData(RdxStart);
862 if (!RD)
863 return RK_None;
864
865 Type *VecTy = RdxStart->getType();
866 unsigned NumVecElems = VecTy->getVectorNumElements();
867 if (!isPowerOf2_32(NumVecElems))
868 return RK_None;
869
870 // We look for a sequence of shuffle,shuffle,add triples like the following
871 // that builds a pairwise reduction tree.
Fangrui Songf78650a2018-07-30 19:41:25 +0000872 //
Guozhi Wei62d64142017-09-08 22:29:17 +0000873 // (X0, X1, X2, X3)
874 // (X0 + X1, X2 + X3, undef, undef)
875 // ((X0 + X1) + (X2 + X3), undef, undef, undef)
Fangrui Songf78650a2018-07-30 19:41:25 +0000876 //
Guozhi Wei62d64142017-09-08 22:29:17 +0000877 // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
878 // <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
879 // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef,
880 // <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
881 // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
882 // %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef,
883 // <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
884 // %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef,
885 // <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
886 // %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
887 // %r = extractelement <4 x float> %bin.rdx8, i32 0
888 if (matchPairwiseReductionAtLevel(RdxStart, 0, Log2_32(NumVecElems)) ==
889 RK_None)
890 return RK_None;
891
892 Opcode = RD->Opcode;
893 Ty = VecTy;
894
895 return RD->Kind;
896}
897
898static std::pair<Value *, ShuffleVectorInst *>
899getShuffleAndOtherOprd(Value *L, Value *R) {
900 ShuffleVectorInst *S = nullptr;
901
902 if ((S = dyn_cast<ShuffleVectorInst>(L)))
903 return std::make_pair(R, S);
904
905 S = dyn_cast<ShuffleVectorInst>(R);
906 return std::make_pair(L, S);
907}
908
909static ReductionKind
910matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot,
911 unsigned &Opcode, Type *&Ty) {
912 if (!EnableReduxCost)
913 return RK_None;
914
915 // Need to extract the first element.
916 ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
917 unsigned Idx = ~0u;
918 if (CI)
919 Idx = CI->getZExtValue();
920 if (Idx != 0)
921 return RK_None;
922
923 auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0));
924 if (!RdxStart)
925 return RK_None;
926 Optional<ReductionData> RD = getReductionData(RdxStart);
927 if (!RD)
928 return RK_None;
929
930 Type *VecTy = ReduxRoot->getOperand(0)->getType();
931 unsigned NumVecElems = VecTy->getVectorNumElements();
932 if (!isPowerOf2_32(NumVecElems))
933 return RK_None;
934
935 // We look for a sequence of shuffles and adds like the following matching one
936 // fadd, shuffle vector pair at a time.
Fangrui Songf78650a2018-07-30 19:41:25 +0000937 //
Guozhi Wei62d64142017-09-08 22:29:17 +0000938 // %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef,
939 // <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
940 // %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
941 // %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef,
942 // <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
943 // %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
944 // %r = extractelement <4 x float> %bin.rdx8, i32 0
945
946 unsigned MaskStart = 1;
947 Instruction *RdxOp = RdxStart;
Fangrui Songf78650a2018-07-30 19:41:25 +0000948 SmallVector<int, 32> ShuffleMask(NumVecElems, 0);
Guozhi Wei62d64142017-09-08 22:29:17 +0000949 unsigned NumVecElemsRemain = NumVecElems;
950 while (NumVecElemsRemain - 1) {
951 // Check for the right reduction operation.
952 if (!RdxOp)
953 return RK_None;
954 Optional<ReductionData> RDLevel = getReductionData(RdxOp);
955 if (!RDLevel || !RDLevel->hasSameData(*RD))
956 return RK_None;
957
958 Value *NextRdxOp;
959 ShuffleVectorInst *Shuffle;
960 std::tie(NextRdxOp, Shuffle) =
961 getShuffleAndOtherOprd(RDLevel->LHS, RDLevel->RHS);
962
963 // Check the current reduction operation and the shuffle use the same value.
964 if (Shuffle == nullptr)
965 return RK_None;
966 if (Shuffle->getOperand(0) != NextRdxOp)
967 return RK_None;
968
969 // Check that shuffle masks matches.
970 for (unsigned j = 0; j != MaskStart; ++j)
971 ShuffleMask[j] = MaskStart + j;
972 // Fill the rest of the mask with -1 for undef.
973 std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
974
975 SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
976 if (ShuffleMask != Mask)
977 return RK_None;
978
979 RdxOp = dyn_cast<Instruction>(NextRdxOp);
980 NumVecElemsRemain /= 2;
981 MaskStart *= 2;
982 }
983
984 Opcode = RD->Opcode;
985 Ty = VecTy;
986 return RD->Kind;
987}
988
989int TargetTransformInfo::getInstructionThroughput(const Instruction *I) const {
990 switch (I->getOpcode()) {
991 case Instruction::GetElementPtr:
992 return getUserCost(I);
993
994 case Instruction::Ret:
995 case Instruction::PHI:
996 case Instruction::Br: {
997 return getCFInstrCost(I->getOpcode());
998 }
999 case Instruction::Add:
1000 case Instruction::FAdd:
1001 case Instruction::Sub:
1002 case Instruction::FSub:
1003 case Instruction::Mul:
1004 case Instruction::FMul:
1005 case Instruction::UDiv:
1006 case Instruction::SDiv:
1007 case Instruction::FDiv:
1008 case Instruction::URem:
1009 case Instruction::SRem:
1010 case Instruction::FRem:
1011 case Instruction::Shl:
1012 case Instruction::LShr:
1013 case Instruction::AShr:
1014 case Instruction::And:
1015 case Instruction::Or:
1016 case Instruction::Xor: {
Simon Pilgrim4162d772018-05-22 10:40:09 +00001017 TargetTransformInfo::OperandValueKind Op1VK, Op2VK;
1018 TargetTransformInfo::OperandValueProperties Op1VP, Op2VP;
1019 Op1VK = getOperandInfo(I->getOperand(0), Op1VP);
1020 Op2VK = getOperandInfo(I->getOperand(1), Op2VP);
1021 SmallVector<const Value *, 2> Operands(I->operand_values());
1022 return getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK, Op2VK,
1023 Op1VP, Op2VP, Operands);
Guozhi Wei62d64142017-09-08 22:29:17 +00001024 }
1025 case Instruction::Select: {
1026 const SelectInst *SI = cast<SelectInst>(I);
1027 Type *CondTy = SI->getCondition()->getType();
1028 return getCmpSelInstrCost(I->getOpcode(), I->getType(), CondTy, I);
1029 }
1030 case Instruction::ICmp:
1031 case Instruction::FCmp: {
1032 Type *ValTy = I->getOperand(0)->getType();
1033 return getCmpSelInstrCost(I->getOpcode(), ValTy, I->getType(), I);
1034 }
1035 case Instruction::Store: {
1036 const StoreInst *SI = cast<StoreInst>(I);
1037 Type *ValTy = SI->getValueOperand()->getType();
1038 return getMemoryOpCost(I->getOpcode(), ValTy,
1039 SI->getAlignment(),
1040 SI->getPointerAddressSpace(), I);
1041 }
1042 case Instruction::Load: {
1043 const LoadInst *LI = cast<LoadInst>(I);
1044 return getMemoryOpCost(I->getOpcode(), I->getType(),
1045 LI->getAlignment(),
1046 LI->getPointerAddressSpace(), I);
1047 }
1048 case Instruction::ZExt:
1049 case Instruction::SExt:
1050 case Instruction::FPToUI:
1051 case Instruction::FPToSI:
1052 case Instruction::FPExt:
1053 case Instruction::PtrToInt:
1054 case Instruction::IntToPtr:
1055 case Instruction::SIToFP:
1056 case Instruction::UIToFP:
1057 case Instruction::Trunc:
1058 case Instruction::FPTrunc:
1059 case Instruction::BitCast:
1060 case Instruction::AddrSpaceCast: {
1061 Type *SrcTy = I->getOperand(0)->getType();
1062 return getCastInstrCost(I->getOpcode(), I->getType(), SrcTy, I);
1063 }
1064 case Instruction::ExtractElement: {
1065 const ExtractElementInst * EEI = cast<ExtractElementInst>(I);
1066 ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1));
1067 unsigned Idx = -1;
1068 if (CI)
1069 Idx = CI->getZExtValue();
1070
1071 // Try to match a reduction sequence (series of shufflevector and vector
1072 // adds followed by a extractelement).
1073 unsigned ReduxOpCode;
1074 Type *ReduxType;
1075
1076 switch (matchVectorSplittingReduction(EEI, ReduxOpCode, ReduxType)) {
1077 case RK_Arithmetic:
1078 return getArithmeticReductionCost(ReduxOpCode, ReduxType,
1079 /*IsPairwiseForm=*/false);
1080 case RK_MinMax:
1081 return getMinMaxReductionCost(
1082 ReduxType, CmpInst::makeCmpResultType(ReduxType),
1083 /*IsPairwiseForm=*/false, /*IsUnsigned=*/false);
1084 case RK_UnsignedMinMax:
1085 return getMinMaxReductionCost(
1086 ReduxType, CmpInst::makeCmpResultType(ReduxType),
1087 /*IsPairwiseForm=*/false, /*IsUnsigned=*/true);
1088 case RK_None:
1089 break;
1090 }
1091
1092 switch (matchPairwiseReduction(EEI, ReduxOpCode, ReduxType)) {
1093 case RK_Arithmetic:
1094 return getArithmeticReductionCost(ReduxOpCode, ReduxType,
1095 /*IsPairwiseForm=*/true);
1096 case RK_MinMax:
1097 return getMinMaxReductionCost(
1098 ReduxType, CmpInst::makeCmpResultType(ReduxType),
1099 /*IsPairwiseForm=*/true, /*IsUnsigned=*/false);
1100 case RK_UnsignedMinMax:
1101 return getMinMaxReductionCost(
1102 ReduxType, CmpInst::makeCmpResultType(ReduxType),
1103 /*IsPairwiseForm=*/true, /*IsUnsigned=*/true);
1104 case RK_None:
1105 break;
1106 }
1107
1108 return getVectorInstrCost(I->getOpcode(),
1109 EEI->getOperand(0)->getType(), Idx);
1110 }
1111 case Instruction::InsertElement: {
1112 const InsertElementInst * IE = cast<InsertElementInst>(I);
1113 ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
Fangrui Songf78650a2018-07-30 19:41:25 +00001114 unsigned Idx = -1;
Guozhi Wei62d64142017-09-08 22:29:17 +00001115 if (CI)
1116 Idx = CI->getZExtValue();
1117 return getVectorInstrCost(I->getOpcode(),
1118 IE->getType(), Idx);
1119 }
1120 case Instruction::ShuffleVector: {
1121 const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I);
Simon Pilgrimd0c71602018-11-09 16:28:19 +00001122 Type *Ty = Shuffle->getType();
1123 Type *SrcTy = Shuffle->getOperand(0)->getType();
1124
1125 // TODO: Identify and add costs for insert subvector, etc.
1126 int SubIndex;
1127 if (Shuffle->isExtractSubvectorMask(SubIndex))
Simon Pilgrim26e1c882018-11-09 18:30:59 +00001128 return TTIImpl->getShuffleCost(SK_ExtractSubvector, SrcTy, SubIndex, Ty);
Simon Pilgrimd0c71602018-11-09 16:28:19 +00001129
Sanjay Patel2ca33602018-06-19 18:44:00 +00001130 if (Shuffle->changesLength())
1131 return -1;
Fangrui Songf78650a2018-07-30 19:41:25 +00001132
Sanjay Patel2ca33602018-06-19 18:44:00 +00001133 if (Shuffle->isIdentity())
1134 return 0;
Guozhi Wei62d64142017-09-08 22:29:17 +00001135
Sanjay Patel2ca33602018-06-19 18:44:00 +00001136 if (Shuffle->isReverse())
1137 return TTIImpl->getShuffleCost(SK_Reverse, Ty, 0, nullptr);
Simon Pilgrim07839212018-06-12 14:47:13 +00001138
Sanjay Patel2ca33602018-06-19 18:44:00 +00001139 if (Shuffle->isSelect())
1140 return TTIImpl->getShuffleCost(SK_Select, Ty, 0, nullptr);
Simon Pilgrim07839212018-06-12 14:47:13 +00001141
Sanjay Patel2ca33602018-06-19 18:44:00 +00001142 if (Shuffle->isTranspose())
1143 return TTIImpl->getShuffleCost(SK_Transpose, Ty, 0, nullptr);
Matthew Simpsonb4096eb2018-04-26 13:48:33 +00001144
Sanjay Patel2ca33602018-06-19 18:44:00 +00001145 if (Shuffle->isZeroEltSplat())
1146 return TTIImpl->getShuffleCost(SK_Broadcast, Ty, 0, nullptr);
Guozhi Wei62d64142017-09-08 22:29:17 +00001147
Sanjay Patel2ca33602018-06-19 18:44:00 +00001148 if (Shuffle->isSingleSource())
1149 return TTIImpl->getShuffleCost(SK_PermuteSingleSrc, Ty, 0, nullptr);
Guozhi Wei62d64142017-09-08 22:29:17 +00001150
Sanjay Patel2ca33602018-06-19 18:44:00 +00001151 return TTIImpl->getShuffleCost(SK_PermuteTwoSrc, Ty, 0, nullptr);
Guozhi Wei62d64142017-09-08 22:29:17 +00001152 }
1153 case Instruction::Call:
1154 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
1155 SmallVector<Value *, 4> Args(II->arg_operands());
1156
1157 FastMathFlags FMF;
1158 if (auto *FPMO = dyn_cast<FPMathOperator>(II))
1159 FMF = FPMO->getFastMathFlags();
1160
1161 return getIntrinsicInstrCost(II->getIntrinsicID(), II->getType(),
1162 Args, FMF);
1163 }
1164 return -1;
1165 default:
1166 // We don't have any information on this instruction.
1167 return -1;
1168 }
1169}
1170
Chandler Carruth705b1852015-01-31 03:43:40 +00001171TargetTransformInfo::Concept::~Concept() {}
1172
Chandler Carruthe0385522015-02-01 10:11:22 +00001173TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
1174
1175TargetIRAnalysis::TargetIRAnalysis(
Eric Christophera4e5d3c2015-09-16 23:38:13 +00001176 std::function<Result(const Function &)> TTICallback)
Benjamin Kramer82de7d32016-05-27 14:27:24 +00001177 : TTICallback(std::move(TTICallback)) {}
Chandler Carruthe0385522015-02-01 10:11:22 +00001178
Chandler Carruth164a2aa62016-06-17 00:11:01 +00001179TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F,
Sean Silva36e0d012016-08-09 00:28:15 +00001180 FunctionAnalysisManager &) {
Chandler Carruthe0385522015-02-01 10:11:22 +00001181 return TTICallback(F);
1182}
1183
Chandler Carruthdab4eae2016-11-23 17:53:26 +00001184AnalysisKey TargetIRAnalysis::Key;
NAKAMURA Takumidf0cd722016-02-28 17:17:00 +00001185
Eric Christophera4e5d3c2015-09-16 23:38:13 +00001186TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) {
Mehdi Amini5010ebf2015-07-09 02:08:42 +00001187 return Result(F.getParent()->getDataLayout());
Chandler Carruthe0385522015-02-01 10:11:22 +00001188}
1189
Chandler Carruth705b1852015-01-31 03:43:40 +00001190// Register the basic pass.
1191INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti",
1192 "Target Transform Information", false, true)
1193char TargetTransformInfoWrapperPass::ID = 0;
Chandler Carruth539edf42013-01-05 11:43:11 +00001194
Chandler Carruth705b1852015-01-31 03:43:40 +00001195void TargetTransformInfoWrapperPass::anchor() {}
Chandler Carruth539edf42013-01-05 11:43:11 +00001196
Chandler Carruth705b1852015-01-31 03:43:40 +00001197TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass()
Chandler Carruth5ec2b1d2015-02-01 12:26:09 +00001198 : ImmutablePass(ID) {
Chandler Carruth705b1852015-01-31 03:43:40 +00001199 initializeTargetTransformInfoWrapperPassPass(
1200 *PassRegistry::getPassRegistry());
1201}
1202
1203TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass(
Chandler Carruth5ec2b1d2015-02-01 12:26:09 +00001204 TargetIRAnalysis TIRA)
1205 : ImmutablePass(ID), TIRA(std::move(TIRA)) {
Chandler Carruth705b1852015-01-31 03:43:40 +00001206 initializeTargetTransformInfoWrapperPassPass(
1207 *PassRegistry::getPassRegistry());
1208}
1209
Eric Christophera4e5d3c2015-09-16 23:38:13 +00001210TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) {
Sean Silva36e0d012016-08-09 00:28:15 +00001211 FunctionAnalysisManager DummyFAM;
Chandler Carruth164a2aa62016-06-17 00:11:01 +00001212 TTI = TIRA.run(F, DummyFAM);
Chandler Carruth5ec2b1d2015-02-01 12:26:09 +00001213 return *TTI;
1214}
1215
Chandler Carruth93dcdc42015-01-31 11:17:59 +00001216ImmutablePass *
Chandler Carruth5ec2b1d2015-02-01 12:26:09 +00001217llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) {
1218 return new TargetTransformInfoWrapperPass(std::move(TIRA));
Chandler Carruth539edf42013-01-05 11:43:11 +00001219}