Chandler Carruth | d3e7355 | 2013-01-07 03:08:10 +0000 | [diff] [blame] | 1 | //===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===// |
Nadav Rotem | 5dc203e | 2012-10-18 23:22:48 +0000 | [diff] [blame] | 2 | // |
Chandler Carruth | 2946cd7 | 2019-01-19 08:50:56 +0000 | [diff] [blame^] | 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
Nadav Rotem | 5dc203e | 2012-10-18 23:22:48 +0000 | [diff] [blame] | 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
Chandler Carruth | d3e7355 | 2013-01-07 03:08:10 +0000 | [diff] [blame] | 9 | #include "llvm/Analysis/TargetTransformInfo.h" |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 10 | #include "llvm/Analysis/TargetTransformInfoImpl.h" |
Chandler Carruth | 219b89b | 2014-03-04 11:01:28 +0000 | [diff] [blame] | 11 | #include "llvm/IR/CallSite.h" |
Chandler Carruth | 511aa76 | 2013-01-21 01:27:39 +0000 | [diff] [blame] | 12 | #include "llvm/IR/DataLayout.h" |
Chandler Carruth | 511aa76 | 2013-01-21 01:27:39 +0000 | [diff] [blame] | 13 | #include "llvm/IR/Instruction.h" |
Chandler Carruth | 511aa76 | 2013-01-21 01:27:39 +0000 | [diff] [blame] | 14 | #include "llvm/IR/Instructions.h" |
Chandler Carruth | 8a8cd2b | 2014-01-07 11:48:04 +0000 | [diff] [blame] | 15 | #include "llvm/IR/IntrinsicInst.h" |
Chandler Carruth | e038552 | 2015-02-01 10:11:22 +0000 | [diff] [blame] | 16 | #include "llvm/IR/Module.h" |
Chandler Carruth | 8a8cd2b | 2014-01-07 11:48:04 +0000 | [diff] [blame] | 17 | #include "llvm/IR/Operator.h" |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 18 | #include "llvm/IR/PatternMatch.h" |
Sean Fertile | 9cd1cdf | 2017-07-07 02:00:06 +0000 | [diff] [blame] | 19 | #include "llvm/Support/CommandLine.h" |
Nadav Rotem | 5dc203e | 2012-10-18 23:22:48 +0000 | [diff] [blame] | 20 | #include "llvm/Support/ErrorHandling.h" |
Benjamin Kramer | 82de7d3 | 2016-05-27 14:27:24 +0000 | [diff] [blame] | 21 | #include <utility> |
Nadav Rotem | 5dc203e | 2012-10-18 23:22:48 +0000 | [diff] [blame] | 22 | |
| 23 | using namespace llvm; |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 24 | using namespace PatternMatch; |
Nadav Rotem | 5dc203e | 2012-10-18 23:22:48 +0000 | [diff] [blame] | 25 | |
Chandler Carruth | f1221bd | 2014-04-22 02:48:03 +0000 | [diff] [blame] | 26 | #define DEBUG_TYPE "tti" |
| 27 | |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 28 | static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false), |
| 29 | cl::Hidden, |
| 30 | cl::desc("Recognize reduction patterns.")); |
| 31 | |
Chandler Carruth | 93dcdc4 | 2015-01-31 11:17:59 +0000 | [diff] [blame] | 32 | namespace { |
Adrian Prantl | 5f8f34e4 | 2018-05-01 15:54:18 +0000 | [diff] [blame] | 33 | /// No-op implementation of the TTI interface using the utility base |
Chandler Carruth | 93dcdc4 | 2015-01-31 11:17:59 +0000 | [diff] [blame] | 34 | /// classes. |
| 35 | /// |
| 36 | /// This is used when no target specific information is available. |
| 37 | struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> { |
Mehdi Amini | 5010ebf | 2015-07-09 02:08:42 +0000 | [diff] [blame] | 38 | explicit NoTTIImpl(const DataLayout &DL) |
Chandler Carruth | 93dcdc4 | 2015-01-31 11:17:59 +0000 | [diff] [blame] | 39 | : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {} |
| 40 | }; |
| 41 | } |
| 42 | |
Mehdi Amini | 5010ebf | 2015-07-09 02:08:42 +0000 | [diff] [blame] | 43 | TargetTransformInfo::TargetTransformInfo(const DataLayout &DL) |
Chandler Carruth | 93dcdc4 | 2015-01-31 11:17:59 +0000 | [diff] [blame] | 44 | : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {} |
| 45 | |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 46 | TargetTransformInfo::~TargetTransformInfo() {} |
Nadav Rotem | 5dc203e | 2012-10-18 23:22:48 +0000 | [diff] [blame] | 47 | |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 48 | TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg) |
| 49 | : TTIImpl(std::move(Arg.TTIImpl)) {} |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 50 | |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 51 | TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) { |
| 52 | TTIImpl = std::move(RHS.TTIImpl); |
| 53 | return *this; |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 54 | } |
| 55 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 56 | int TargetTransformInfo::getOperationCost(unsigned Opcode, Type *Ty, |
| 57 | Type *OpTy) const { |
| 58 | int Cost = TTIImpl->getOperationCost(Opcode, Ty, OpTy); |
| 59 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 60 | return Cost; |
Chandler Carruth | 511aa76 | 2013-01-21 01:27:39 +0000 | [diff] [blame] | 61 | } |
| 62 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 63 | int TargetTransformInfo::getCallCost(FunctionType *FTy, int NumArgs) const { |
| 64 | int Cost = TTIImpl->getCallCost(FTy, NumArgs); |
| 65 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 66 | return Cost; |
Chandler Carruth | 0ba8db4 | 2013-01-22 11:26:02 +0000 | [diff] [blame] | 67 | } |
| 68 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 69 | int TargetTransformInfo::getCallCost(const Function *F, |
| 70 | ArrayRef<const Value *> Arguments) const { |
| 71 | int Cost = TTIImpl->getCallCost(F, Arguments); |
| 72 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 73 | return Cost; |
Chandler Carruth | 0ba8db4 | 2013-01-22 11:26:02 +0000 | [diff] [blame] | 74 | } |
| 75 | |
Justin Lebar | 8650a4d | 2016-04-15 01:38:48 +0000 | [diff] [blame] | 76 | unsigned TargetTransformInfo::getInliningThresholdMultiplier() const { |
| 77 | return TTIImpl->getInliningThresholdMultiplier(); |
| 78 | } |
| 79 | |
Jingyue Wu | 15f3e82 | 2016-07-08 21:48:05 +0000 | [diff] [blame] | 80 | int TargetTransformInfo::getGEPCost(Type *PointeeType, const Value *Ptr, |
| 81 | ArrayRef<const Value *> Operands) const { |
| 82 | return TTIImpl->getGEPCost(PointeeType, Ptr, Operands); |
| 83 | } |
| 84 | |
Haicheng Wu | abdef9e | 2017-07-15 02:12:16 +0000 | [diff] [blame] | 85 | int TargetTransformInfo::getExtCost(const Instruction *I, |
| 86 | const Value *Src) const { |
| 87 | return TTIImpl->getExtCost(I, Src); |
| 88 | } |
| 89 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 90 | int TargetTransformInfo::getIntrinsicCost( |
| 91 | Intrinsic::ID IID, Type *RetTy, ArrayRef<const Value *> Arguments) const { |
| 92 | int Cost = TTIImpl->getIntrinsicCost(IID, RetTy, Arguments); |
| 93 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 94 | return Cost; |
Chandler Carruth | 0ba8db4 | 2013-01-22 11:26:02 +0000 | [diff] [blame] | 95 | } |
| 96 | |
Jun Bum Lim | 919f9e8 | 2017-04-28 16:04:03 +0000 | [diff] [blame] | 97 | unsigned |
| 98 | TargetTransformInfo::getEstimatedNumberOfCaseClusters(const SwitchInst &SI, |
| 99 | unsigned &JTSize) const { |
| 100 | return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize); |
| 101 | } |
| 102 | |
Evgeny Astigeevich | 70ed78e | 2017-06-29 13:42:12 +0000 | [diff] [blame] | 103 | int TargetTransformInfo::getUserCost(const User *U, |
| 104 | ArrayRef<const Value *> Operands) const { |
| 105 | int Cost = TTIImpl->getUserCost(U, Operands); |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 106 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 107 | return Cost; |
Chandler Carruth | 511aa76 | 2013-01-21 01:27:39 +0000 | [diff] [blame] | 108 | } |
| 109 | |
Tom Stellard | 8b1e021 | 2013-07-27 00:01:07 +0000 | [diff] [blame] | 110 | bool TargetTransformInfo::hasBranchDivergence() const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 111 | return TTIImpl->hasBranchDivergence(); |
Tom Stellard | 8b1e021 | 2013-07-27 00:01:07 +0000 | [diff] [blame] | 112 | } |
| 113 | |
Jingyue Wu | 5da831c | 2015-04-10 05:03:50 +0000 | [diff] [blame] | 114 | bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const { |
| 115 | return TTIImpl->isSourceOfDivergence(V); |
| 116 | } |
| 117 | |
Alexander Timofeev | 0f9c84c | 2017-06-15 19:33:10 +0000 | [diff] [blame] | 118 | bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const { |
| 119 | return TTIImpl->isAlwaysUniform(V); |
| 120 | } |
| 121 | |
Matt Arsenault | 42b6478 | 2017-01-30 23:02:12 +0000 | [diff] [blame] | 122 | unsigned TargetTransformInfo::getFlatAddressSpace() const { |
| 123 | return TTIImpl->getFlatAddressSpace(); |
| 124 | } |
| 125 | |
Chandler Carruth | 0ba8db4 | 2013-01-22 11:26:02 +0000 | [diff] [blame] | 126 | bool TargetTransformInfo::isLoweredToCall(const Function *F) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 127 | return TTIImpl->isLoweredToCall(F); |
Chandler Carruth | 0ba8db4 | 2013-01-22 11:26:02 +0000 | [diff] [blame] | 128 | } |
| 129 | |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 130 | void TargetTransformInfo::getUnrollingPreferences( |
Geoff Berry | 66d9bdb | 2017-06-28 15:53:17 +0000 | [diff] [blame] | 131 | Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP) const { |
| 132 | return TTIImpl->getUnrollingPreferences(L, SE, UP); |
Hal Finkel | 8f2e700 | 2013-09-11 19:25:43 +0000 | [diff] [blame] | 133 | } |
| 134 | |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 135 | bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 136 | return TTIImpl->isLegalAddImmediate(Imm); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 137 | } |
| 138 | |
| 139 | bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 140 | return TTIImpl->isLegalICmpImmediate(Imm); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 141 | } |
| 142 | |
| 143 | bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, |
| 144 | int64_t BaseOffset, |
| 145 | bool HasBaseReg, |
Matt Arsenault | e83379e | 2015-06-07 20:12:03 +0000 | [diff] [blame] | 146 | int64_t Scale, |
Jonas Paulsson | 024e319 | 2017-07-21 11:59:37 +0000 | [diff] [blame] | 147 | unsigned AddrSpace, |
| 148 | Instruction *I) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 149 | return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, |
Jonas Paulsson | 024e319 | 2017-07-21 11:59:37 +0000 | [diff] [blame] | 150 | Scale, AddrSpace, I); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 151 | } |
| 152 | |
Evgeny Stupachenko | f2b3b46 | 2017-06-05 23:37:00 +0000 | [diff] [blame] | 153 | bool TargetTransformInfo::isLSRCostLess(LSRCost &C1, LSRCost &C2) const { |
| 154 | return TTIImpl->isLSRCostLess(C1, C2); |
| 155 | } |
| 156 | |
Sanjay Patel | d7c702b | 2018-02-05 23:43:05 +0000 | [diff] [blame] | 157 | bool TargetTransformInfo::canMacroFuseCmp() const { |
| 158 | return TTIImpl->canMacroFuseCmp(); |
| 159 | } |
| 160 | |
Krzysztof Parzyszek | 0b377e0 | 2018-03-26 13:10:09 +0000 | [diff] [blame] | 161 | bool TargetTransformInfo::shouldFavorPostInc() const { |
| 162 | return TTIImpl->shouldFavorPostInc(); |
| 163 | } |
| 164 | |
Elena Demikhovsky | 20662e3 | 2015-10-19 07:43:38 +0000 | [diff] [blame] | 165 | bool TargetTransformInfo::isLegalMaskedStore(Type *DataType) const { |
| 166 | return TTIImpl->isLegalMaskedStore(DataType); |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 167 | } |
| 168 | |
Elena Demikhovsky | 20662e3 | 2015-10-19 07:43:38 +0000 | [diff] [blame] | 169 | bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType) const { |
| 170 | return TTIImpl->isLegalMaskedLoad(DataType); |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 171 | } |
| 172 | |
Elena Demikhovsky | 0928585 | 2015-10-25 15:37:55 +0000 | [diff] [blame] | 173 | bool TargetTransformInfo::isLegalMaskedGather(Type *DataType) const { |
| 174 | return TTIImpl->isLegalMaskedGather(DataType); |
| 175 | } |
| 176 | |
| 177 | bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType) const { |
Mohammed Agabaria | cef53dc | 2017-07-27 10:28:16 +0000 | [diff] [blame] | 178 | return TTIImpl->isLegalMaskedScatter(DataType); |
Elena Demikhovsky | 0928585 | 2015-10-25 15:37:55 +0000 | [diff] [blame] | 179 | } |
| 180 | |
Sanjay Patel | 6fd4391 | 2017-09-09 13:38:18 +0000 | [diff] [blame] | 181 | bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const { |
| 182 | return TTIImpl->hasDivRemOp(DataType, IsSigned); |
| 183 | } |
| 184 | |
Artem Belevich | cb8f632 | 2017-10-24 20:31:44 +0000 | [diff] [blame] | 185 | bool TargetTransformInfo::hasVolatileVariant(Instruction *I, |
| 186 | unsigned AddrSpace) const { |
| 187 | return TTIImpl->hasVolatileVariant(I, AddrSpace); |
| 188 | } |
| 189 | |
Jonas Paulsson | 8624b7e | 2017-05-24 13:42:56 +0000 | [diff] [blame] | 190 | bool TargetTransformInfo::prefersVectorizedAddressing() const { |
| 191 | return TTIImpl->prefersVectorizedAddressing(); |
| 192 | } |
| 193 | |
Quentin Colombet | bf490d4 | 2013-05-31 21:29:03 +0000 | [diff] [blame] | 194 | int TargetTransformInfo::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, |
| 195 | int64_t BaseOffset, |
| 196 | bool HasBaseReg, |
Matt Arsenault | e83379e | 2015-06-07 20:12:03 +0000 | [diff] [blame] | 197 | int64_t Scale, |
| 198 | unsigned AddrSpace) const { |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 199 | int Cost = TTIImpl->getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, |
| 200 | Scale, AddrSpace); |
| 201 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 202 | return Cost; |
Quentin Colombet | bf490d4 | 2013-05-31 21:29:03 +0000 | [diff] [blame] | 203 | } |
| 204 | |
Jonas Paulsson | 024e319 | 2017-07-21 11:59:37 +0000 | [diff] [blame] | 205 | bool TargetTransformInfo::LSRWithInstrQueries() const { |
| 206 | return TTIImpl->LSRWithInstrQueries(); |
| 207 | } |
| 208 | |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 209 | bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 210 | return TTIImpl->isTruncateFree(Ty1, Ty2); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 211 | } |
| 212 | |
Chad Rosier | 5439005 | 2015-02-23 19:15:16 +0000 | [diff] [blame] | 213 | bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const { |
| 214 | return TTIImpl->isProfitableToHoist(I); |
| 215 | } |
| 216 | |
David Blaikie | 8ad9a97 | 2018-03-28 22:28:50 +0000 | [diff] [blame] | 217 | bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); } |
| 218 | |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 219 | bool TargetTransformInfo::isTypeLegal(Type *Ty) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 220 | return TTIImpl->isTypeLegal(Ty); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 221 | } |
| 222 | |
| 223 | unsigned TargetTransformInfo::getJumpBufAlignment() const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 224 | return TTIImpl->getJumpBufAlignment(); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 225 | } |
| 226 | |
| 227 | unsigned TargetTransformInfo::getJumpBufSize() const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 228 | return TTIImpl->getJumpBufSize(); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 229 | } |
| 230 | |
| 231 | bool TargetTransformInfo::shouldBuildLookupTables() const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 232 | return TTIImpl->shouldBuildLookupTables(); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 233 | } |
Oliver Stannard | 4df1cc0 | 2016-10-07 08:48:24 +0000 | [diff] [blame] | 234 | bool TargetTransformInfo::shouldBuildLookupTablesForConstant(Constant *C) const { |
| 235 | return TTIImpl->shouldBuildLookupTablesForConstant(C); |
| 236 | } |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 237 | |
Zaara Syeda | 1f59ae3 | 2018-01-30 16:17:22 +0000 | [diff] [blame] | 238 | bool TargetTransformInfo::useColdCCForColdCall(Function &F) const { |
| 239 | return TTIImpl->useColdCCForColdCall(F); |
| 240 | } |
| 241 | |
Jonas Paulsson | 8e2f948 | 2017-01-26 07:03:25 +0000 | [diff] [blame] | 242 | unsigned TargetTransformInfo:: |
| 243 | getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) const { |
| 244 | return TTIImpl->getScalarizationOverhead(Ty, Insert, Extract); |
| 245 | } |
| 246 | |
| 247 | unsigned TargetTransformInfo:: |
| 248 | getOperandsScalarizationOverhead(ArrayRef<const Value *> Args, |
| 249 | unsigned VF) const { |
| 250 | return TTIImpl->getOperandsScalarizationOverhead(Args, VF); |
| 251 | } |
| 252 | |
Jonas Paulsson | da74ed4 | 2017-04-12 12:41:37 +0000 | [diff] [blame] | 253 | bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const { |
| 254 | return TTIImpl->supportsEfficientVectorElementLoadStore(); |
| 255 | } |
| 256 | |
Olivier Sallenave | 049d803 | 2015-03-06 23:12:04 +0000 | [diff] [blame] | 257 | bool TargetTransformInfo::enableAggressiveInterleaving(bool LoopHasReductions) const { |
| 258 | return TTIImpl->enableAggressiveInterleaving(LoopHasReductions); |
| 259 | } |
| 260 | |
Clement Courbet | b2c3eb8 | 2017-10-30 14:19:33 +0000 | [diff] [blame] | 261 | const TargetTransformInfo::MemCmpExpansionOptions * |
| 262 | TargetTransformInfo::enableMemCmpExpansion(bool IsZeroCmp) const { |
| 263 | return TTIImpl->enableMemCmpExpansion(IsZeroCmp); |
Zaara Syeda | 3a7578c | 2017-05-31 17:12:38 +0000 | [diff] [blame] | 264 | } |
| 265 | |
Silviu Baranga | 61bdc51 | 2015-08-10 14:50:54 +0000 | [diff] [blame] | 266 | bool TargetTransformInfo::enableInterleavedAccessVectorization() const { |
| 267 | return TTIImpl->enableInterleavedAccessVectorization(); |
| 268 | } |
| 269 | |
Dorit Nuzman | 38bbf81 | 2018-10-14 08:50:06 +0000 | [diff] [blame] | 270 | bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const { |
| 271 | return TTIImpl->enableMaskedInterleavedAccessVectorization(); |
| 272 | } |
| 273 | |
Renato Golin | 5cb666a | 2016-04-14 20:42:18 +0000 | [diff] [blame] | 274 | bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const { |
| 275 | return TTIImpl->isFPVectorizationPotentiallyUnsafe(); |
| 276 | } |
| 277 | |
Alina Sbirlea | 6f937b1 | 2016-08-04 16:38:44 +0000 | [diff] [blame] | 278 | bool TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context, |
| 279 | unsigned BitWidth, |
Alina Sbirlea | 327955e | 2016-07-11 20:46:17 +0000 | [diff] [blame] | 280 | unsigned AddressSpace, |
| 281 | unsigned Alignment, |
| 282 | bool *Fast) const { |
Alina Sbirlea | 6f937b1 | 2016-08-04 16:38:44 +0000 | [diff] [blame] | 283 | return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth, AddressSpace, |
Alina Sbirlea | 327955e | 2016-07-11 20:46:17 +0000 | [diff] [blame] | 284 | Alignment, Fast); |
| 285 | } |
| 286 | |
Chandler Carruth | 50a36cd | 2013-01-07 03:16:03 +0000 | [diff] [blame] | 287 | TargetTransformInfo::PopcntSupportKind |
| 288 | TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 289 | return TTIImpl->getPopcntSupport(IntTyWidthInBit); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 290 | } |
| 291 | |
Richard Sandiford | 37cd6cf | 2013-08-23 10:27:02 +0000 | [diff] [blame] | 292 | bool TargetTransformInfo::haveFastSqrt(Type *Ty) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 293 | return TTIImpl->haveFastSqrt(Ty); |
Richard Sandiford | 37cd6cf | 2013-08-23 10:27:02 +0000 | [diff] [blame] | 294 | } |
| 295 | |
Sanjay Patel | 0de1a4b | 2017-11-27 21:15:43 +0000 | [diff] [blame] | 296 | bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const { |
| 297 | return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty); |
| 298 | } |
| 299 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 300 | int TargetTransformInfo::getFPOpCost(Type *Ty) const { |
| 301 | int Cost = TTIImpl->getFPOpCost(Ty); |
| 302 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 303 | return Cost; |
Cameron Esfahani | 17177d1 | 2015-02-05 02:09:33 +0000 | [diff] [blame] | 304 | } |
| 305 | |
Sjoerd Meijer | 38c2cd0 | 2016-07-14 07:44:20 +0000 | [diff] [blame] | 306 | int TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx, |
| 307 | const APInt &Imm, |
| 308 | Type *Ty) const { |
| 309 | int Cost = TTIImpl->getIntImmCodeSizeCost(Opcode, Idx, Imm, Ty); |
| 310 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 311 | return Cost; |
| 312 | } |
| 313 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 314 | int TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty) const { |
| 315 | int Cost = TTIImpl->getIntImmCost(Imm, Ty); |
| 316 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 317 | return Cost; |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 318 | } |
| 319 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 320 | int TargetTransformInfo::getIntImmCost(unsigned Opcode, unsigned Idx, |
| 321 | const APInt &Imm, Type *Ty) const { |
| 322 | int Cost = TTIImpl->getIntImmCost(Opcode, Idx, Imm, Ty); |
| 323 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 324 | return Cost; |
Juergen Ributzka | f26beda | 2014-01-25 02:02:55 +0000 | [diff] [blame] | 325 | } |
| 326 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 327 | int TargetTransformInfo::getIntImmCost(Intrinsic::ID IID, unsigned Idx, |
| 328 | const APInt &Imm, Type *Ty) const { |
| 329 | int Cost = TTIImpl->getIntImmCost(IID, Idx, Imm, Ty); |
| 330 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 331 | return Cost; |
Juergen Ributzka | f26beda | 2014-01-25 02:02:55 +0000 | [diff] [blame] | 332 | } |
| 333 | |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 334 | unsigned TargetTransformInfo::getNumberOfRegisters(bool Vector) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 335 | return TTIImpl->getNumberOfRegisters(Vector); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 336 | } |
| 337 | |
Nadav Rotem | b1791a7 | 2013-01-09 22:29:00 +0000 | [diff] [blame] | 338 | unsigned TargetTransformInfo::getRegisterBitWidth(bool Vector) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 339 | return TTIImpl->getRegisterBitWidth(Vector); |
Nadav Rotem | b1791a7 | 2013-01-09 22:29:00 +0000 | [diff] [blame] | 340 | } |
| 341 | |
Adam Nemet | e29686e | 2017-05-15 21:15:01 +0000 | [diff] [blame] | 342 | unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const { |
| 343 | return TTIImpl->getMinVectorRegisterBitWidth(); |
| 344 | } |
| 345 | |
Krzysztof Parzyszek | 5d93fdf | 2018-03-27 16:14:11 +0000 | [diff] [blame] | 346 | bool TargetTransformInfo::shouldMaximizeVectorBandwidth(bool OptSize) const { |
| 347 | return TTIImpl->shouldMaximizeVectorBandwidth(OptSize); |
| 348 | } |
| 349 | |
Krzysztof Parzyszek | dfed941 | 2018-04-13 20:16:32 +0000 | [diff] [blame] | 350 | unsigned TargetTransformInfo::getMinimumVF(unsigned ElemWidth) const { |
| 351 | return TTIImpl->getMinimumVF(ElemWidth); |
| 352 | } |
| 353 | |
Jun Bum Lim | dee5565 | 2017-04-03 19:20:07 +0000 | [diff] [blame] | 354 | bool TargetTransformInfo::shouldConsiderAddressTypePromotion( |
| 355 | const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const { |
| 356 | return TTIImpl->shouldConsiderAddressTypePromotion( |
| 357 | I, AllowPromotionWithoutCommonHeader); |
| 358 | } |
| 359 | |
Adam Nemet | af76110 | 2016-01-21 18:28:36 +0000 | [diff] [blame] | 360 | unsigned TargetTransformInfo::getCacheLineSize() const { |
| 361 | return TTIImpl->getCacheLineSize(); |
| 362 | } |
| 363 | |
Tobias Grosser | d7eb619 | 2017-08-24 09:46:25 +0000 | [diff] [blame] | 364 | llvm::Optional<unsigned> TargetTransformInfo::getCacheSize(CacheLevel Level) |
| 365 | const { |
| 366 | return TTIImpl->getCacheSize(Level); |
| 367 | } |
| 368 | |
| 369 | llvm::Optional<unsigned> TargetTransformInfo::getCacheAssociativity( |
| 370 | CacheLevel Level) const { |
| 371 | return TTIImpl->getCacheAssociativity(Level); |
| 372 | } |
| 373 | |
Adam Nemet | dadfbb5 | 2016-01-27 22:21:25 +0000 | [diff] [blame] | 374 | unsigned TargetTransformInfo::getPrefetchDistance() const { |
| 375 | return TTIImpl->getPrefetchDistance(); |
| 376 | } |
| 377 | |
Adam Nemet | 6d8beec | 2016-03-18 00:27:38 +0000 | [diff] [blame] | 378 | unsigned TargetTransformInfo::getMinPrefetchStride() const { |
| 379 | return TTIImpl->getMinPrefetchStride(); |
| 380 | } |
| 381 | |
Adam Nemet | 709e304 | 2016-03-18 00:27:43 +0000 | [diff] [blame] | 382 | unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const { |
| 383 | return TTIImpl->getMaxPrefetchIterationsAhead(); |
| 384 | } |
| 385 | |
Wei Mi | 062c744 | 2015-05-06 17:12:25 +0000 | [diff] [blame] | 386 | unsigned TargetTransformInfo::getMaxInterleaveFactor(unsigned VF) const { |
| 387 | return TTIImpl->getMaxInterleaveFactor(VF); |
Nadav Rotem | b696c36 | 2013-01-09 01:15:42 +0000 | [diff] [blame] | 388 | } |
| 389 | |
Jonas Paulsson | 29d80f0 | 2018-10-05 14:34:04 +0000 | [diff] [blame] | 390 | TargetTransformInfo::OperandValueKind |
Simon Pilgrim | 077a42c | 2018-11-13 13:45:10 +0000 | [diff] [blame] | 391 | TargetTransformInfo::getOperandInfo(Value *V, OperandValueProperties &OpProps) { |
Jonas Paulsson | 29d80f0 | 2018-10-05 14:34:04 +0000 | [diff] [blame] | 392 | OperandValueKind OpInfo = OK_AnyValue; |
| 393 | OpProps = OP_None; |
| 394 | |
| 395 | if (auto *CI = dyn_cast<ConstantInt>(V)) { |
| 396 | if (CI->getValue().isPowerOf2()) |
| 397 | OpProps = OP_PowerOf2; |
| 398 | return OK_UniformConstantValue; |
| 399 | } |
| 400 | |
Simon Pilgrim | 2b166c5 | 2018-11-14 15:04:08 +0000 | [diff] [blame] | 401 | // A broadcast shuffle creates a uniform value. |
| 402 | // TODO: Add support for non-zero index broadcasts. |
| 403 | // TODO: Add support for different source vector width. |
| 404 | if (auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V)) |
| 405 | if (ShuffleInst->isZeroEltSplat()) |
| 406 | OpInfo = OK_UniformValue; |
| 407 | |
Jonas Paulsson | 29d80f0 | 2018-10-05 14:34:04 +0000 | [diff] [blame] | 408 | const Value *Splat = getSplatValue(V); |
| 409 | |
| 410 | // Check for a splat of a constant or for a non uniform vector of constants |
| 411 | // and check if the constant(s) are all powers of two. |
| 412 | if (isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) { |
| 413 | OpInfo = OK_NonUniformConstantValue; |
| 414 | if (Splat) { |
| 415 | OpInfo = OK_UniformConstantValue; |
| 416 | if (auto *CI = dyn_cast<ConstantInt>(Splat)) |
| 417 | if (CI->getValue().isPowerOf2()) |
| 418 | OpProps = OP_PowerOf2; |
| 419 | } else if (auto *CDS = dyn_cast<ConstantDataSequential>(V)) { |
| 420 | OpProps = OP_PowerOf2; |
| 421 | for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) { |
| 422 | if (auto *CI = dyn_cast<ConstantInt>(CDS->getElementAsConstant(I))) |
| 423 | if (CI->getValue().isPowerOf2()) |
| 424 | continue; |
| 425 | OpProps = OP_None; |
| 426 | break; |
| 427 | } |
| 428 | } |
| 429 | } |
| 430 | |
| 431 | // Check for a splat of a uniform value. This is not loop aware, so return |
| 432 | // true only for the obviously uniform cases (argument, globalvalue) |
| 433 | if (Splat && (isa<Argument>(Splat) || isa<GlobalValue>(Splat))) |
| 434 | OpInfo = OK_UniformValue; |
| 435 | |
| 436 | return OpInfo; |
| 437 | } |
| 438 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 439 | int TargetTransformInfo::getArithmeticInstrCost( |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 440 | unsigned Opcode, Type *Ty, OperandValueKind Opd1Info, |
| 441 | OperandValueKind Opd2Info, OperandValueProperties Opd1PropInfo, |
Mohammed Agabaria | 2c96c43 | 2017-01-11 08:23:37 +0000 | [diff] [blame] | 442 | OperandValueProperties Opd2PropInfo, |
| 443 | ArrayRef<const Value *> Args) const { |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 444 | int Cost = TTIImpl->getArithmeticInstrCost(Opcode, Ty, Opd1Info, Opd2Info, |
Mohammed Agabaria | 2c96c43 | 2017-01-11 08:23:37 +0000 | [diff] [blame] | 445 | Opd1PropInfo, Opd2PropInfo, Args); |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 446 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 447 | return Cost; |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 448 | } |
| 449 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 450 | int TargetTransformInfo::getShuffleCost(ShuffleKind Kind, Type *Ty, int Index, |
| 451 | Type *SubTp) const { |
| 452 | int Cost = TTIImpl->getShuffleCost(Kind, Ty, Index, SubTp); |
| 453 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 454 | return Cost; |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 455 | } |
| 456 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 457 | int TargetTransformInfo::getCastInstrCost(unsigned Opcode, Type *Dst, |
Jonas Paulsson | fccc7d6 | 2017-04-12 11:49:08 +0000 | [diff] [blame] | 458 | Type *Src, const Instruction *I) const { |
| 459 | assert ((I == nullptr || I->getOpcode() == Opcode) && |
| 460 | "Opcode should reflect passed instruction."); |
| 461 | int Cost = TTIImpl->getCastInstrCost(Opcode, Dst, Src, I); |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 462 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 463 | return Cost; |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 464 | } |
| 465 | |
Matthew Simpson | e5dfb08 | 2016-04-27 15:20:21 +0000 | [diff] [blame] | 466 | int TargetTransformInfo::getExtractWithExtendCost(unsigned Opcode, Type *Dst, |
| 467 | VectorType *VecTy, |
| 468 | unsigned Index) const { |
| 469 | int Cost = TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index); |
| 470 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 471 | return Cost; |
| 472 | } |
| 473 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 474 | int TargetTransformInfo::getCFInstrCost(unsigned Opcode) const { |
| 475 | int Cost = TTIImpl->getCFInstrCost(Opcode); |
| 476 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 477 | return Cost; |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 478 | } |
| 479 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 480 | int TargetTransformInfo::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, |
Jonas Paulsson | fccc7d6 | 2017-04-12 11:49:08 +0000 | [diff] [blame] | 481 | Type *CondTy, const Instruction *I) const { |
| 482 | assert ((I == nullptr || I->getOpcode() == Opcode) && |
| 483 | "Opcode should reflect passed instruction."); |
| 484 | int Cost = TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, I); |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 485 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 486 | return Cost; |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 487 | } |
| 488 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 489 | int TargetTransformInfo::getVectorInstrCost(unsigned Opcode, Type *Val, |
| 490 | unsigned Index) const { |
| 491 | int Cost = TTIImpl->getVectorInstrCost(Opcode, Val, Index); |
| 492 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 493 | return Cost; |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 494 | } |
| 495 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 496 | int TargetTransformInfo::getMemoryOpCost(unsigned Opcode, Type *Src, |
| 497 | unsigned Alignment, |
Jonas Paulsson | fccc7d6 | 2017-04-12 11:49:08 +0000 | [diff] [blame] | 498 | unsigned AddressSpace, |
| 499 | const Instruction *I) const { |
| 500 | assert ((I == nullptr || I->getOpcode() == Opcode) && |
| 501 | "Opcode should reflect passed instruction."); |
| 502 | int Cost = TTIImpl->getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, I); |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 503 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 504 | return Cost; |
Elena Demikhovsky | a3232f7 | 2015-01-25 08:44:46 +0000 | [diff] [blame] | 505 | } |
| 506 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 507 | int TargetTransformInfo::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, |
| 508 | unsigned Alignment, |
| 509 | unsigned AddressSpace) const { |
| 510 | int Cost = |
| 511 | TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace); |
| 512 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 513 | return Cost; |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 514 | } |
| 515 | |
Elena Demikhovsky | 5494698 | 2015-12-28 20:10:59 +0000 | [diff] [blame] | 516 | int TargetTransformInfo::getGatherScatterOpCost(unsigned Opcode, Type *DataTy, |
| 517 | Value *Ptr, bool VariableMask, |
| 518 | unsigned Alignment) const { |
| 519 | int Cost = TTIImpl->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, |
| 520 | Alignment); |
| 521 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 522 | return Cost; |
| 523 | } |
| 524 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 525 | int TargetTransformInfo::getInterleavedMemoryOpCost( |
Hao Liu | 32c0539 | 2015-06-08 06:39:56 +0000 | [diff] [blame] | 526 | unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, |
Dorit Nuzman | 34da6dd | 2018-10-31 09:57:56 +0000 | [diff] [blame] | 527 | unsigned Alignment, unsigned AddressSpace, bool UseMaskForCond, |
| 528 | bool UseMaskForGaps) const { |
| 529 | int Cost = TTIImpl->getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, |
| 530 | Alignment, AddressSpace, |
| 531 | UseMaskForCond, |
| 532 | UseMaskForGaps); |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 533 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 534 | return Cost; |
Hao Liu | 32c0539 | 2015-06-08 06:39:56 +0000 | [diff] [blame] | 535 | } |
| 536 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 537 | int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, |
Jonas Paulsson | a48ea23 | 2017-03-14 06:35:36 +0000 | [diff] [blame] | 538 | ArrayRef<Type *> Tys, FastMathFlags FMF, |
| 539 | unsigned ScalarizationCostPassed) const { |
| 540 | int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Tys, FMF, |
| 541 | ScalarizationCostPassed); |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 542 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 543 | return Cost; |
| 544 | } |
| 545 | |
Elena Demikhovsky | 5494698 | 2015-12-28 20:10:59 +0000 | [diff] [blame] | 546 | int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, |
Jonas Paulsson | a48ea23 | 2017-03-14 06:35:36 +0000 | [diff] [blame] | 547 | ArrayRef<Value *> Args, FastMathFlags FMF, unsigned VF) const { |
| 548 | int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Args, FMF, VF); |
Elena Demikhovsky | 5494698 | 2015-12-28 20:10:59 +0000 | [diff] [blame] | 549 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 550 | return Cost; |
| 551 | } |
| 552 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 553 | int TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy, |
| 554 | ArrayRef<Type *> Tys) const { |
| 555 | int Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys); |
| 556 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 557 | return Cost; |
Michael Zolotukhin | 7ed84a8 | 2015-03-17 19:26:23 +0000 | [diff] [blame] | 558 | } |
| 559 | |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 560 | unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 561 | return TTIImpl->getNumberOfParts(Tp); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 562 | } |
| 563 | |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 564 | int TargetTransformInfo::getAddressComputationCost(Type *Tp, |
Mohammed Agabaria | 23599ba | 2017-01-05 14:03:41 +0000 | [diff] [blame] | 565 | ScalarEvolution *SE, |
| 566 | const SCEV *Ptr) const { |
| 567 | int Cost = TTIImpl->getAddressComputationCost(Tp, SE, Ptr); |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 568 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 569 | return Cost; |
Arnold Schwaighofer | 594fa2d | 2013-02-08 14:50:48 +0000 | [diff] [blame] | 570 | } |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 571 | |
Alexey Bataev | 3e9b3eb | 2017-07-31 14:19:32 +0000 | [diff] [blame] | 572 | int TargetTransformInfo::getArithmeticReductionCost(unsigned Opcode, Type *Ty, |
| 573 | bool IsPairwiseForm) const { |
| 574 | int Cost = TTIImpl->getArithmeticReductionCost(Opcode, Ty, IsPairwiseForm); |
Chandler Carruth | 93205eb | 2015-08-05 18:08:10 +0000 | [diff] [blame] | 575 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 576 | return Cost; |
Arnold Schwaighofer | cae8735 | 2013-09-17 18:06:50 +0000 | [diff] [blame] | 577 | } |
| 578 | |
Alexey Bataev | 6dd29fc | 2017-09-08 13:49:36 +0000 | [diff] [blame] | 579 | int TargetTransformInfo::getMinMaxReductionCost(Type *Ty, Type *CondTy, |
| 580 | bool IsPairwiseForm, |
| 581 | bool IsUnsigned) const { |
| 582 | int Cost = |
| 583 | TTIImpl->getMinMaxReductionCost(Ty, CondTy, IsPairwiseForm, IsUnsigned); |
| 584 | assert(Cost >= 0 && "TTI should not produce negative costs!"); |
| 585 | return Cost; |
| 586 | } |
| 587 | |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 588 | unsigned |
| 589 | TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const { |
| 590 | return TTIImpl->getCostOfKeepingLiveOverCall(Tys); |
Chad Rosier | f9327d6 | 2015-01-26 22:51:15 +0000 | [diff] [blame] | 591 | } |
| 592 | |
| 593 | bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst, |
| 594 | MemIntrinsicInfo &Info) const { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 595 | return TTIImpl->getTgtMemIntrinsic(Inst, Info); |
Chad Rosier | f9327d6 | 2015-01-26 22:51:15 +0000 | [diff] [blame] | 596 | } |
| 597 | |
Anna Thomas | b2a212c | 2017-06-06 16:45:25 +0000 | [diff] [blame] | 598 | unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const { |
| 599 | return TTIImpl->getAtomicMemIntrinsicMaxElementSize(); |
| 600 | } |
| 601 | |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 602 | Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic( |
| 603 | IntrinsicInst *Inst, Type *ExpectedType) const { |
| 604 | return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType); |
| 605 | } |
| 606 | |
Sean Fertile | 9cd1cdf | 2017-07-07 02:00:06 +0000 | [diff] [blame] | 607 | Type *TargetTransformInfo::getMemcpyLoopLoweringType(LLVMContext &Context, |
| 608 | Value *Length, |
| 609 | unsigned SrcAlign, |
| 610 | unsigned DestAlign) const { |
| 611 | return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAlign, |
| 612 | DestAlign); |
| 613 | } |
| 614 | |
| 615 | void TargetTransformInfo::getMemcpyLoopResidualLoweringType( |
| 616 | SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, |
| 617 | unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const { |
| 618 | TTIImpl->getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes, |
| 619 | SrcAlign, DestAlign); |
| 620 | } |
| 621 | |
Eric Christopher | d566fb1 | 2015-07-29 22:09:48 +0000 | [diff] [blame] | 622 | bool TargetTransformInfo::areInlineCompatible(const Function *Caller, |
| 623 | const Function *Callee) const { |
| 624 | return TTIImpl->areInlineCompatible(Caller, Callee); |
Eric Christopher | 4371b13 | 2015-07-02 01:11:47 +0000 | [diff] [blame] | 625 | } |
| 626 | |
Tom Stellard | 3d36e5c | 2019-01-16 05:15:31 +0000 | [diff] [blame] | 627 | bool TargetTransformInfo::areFunctionArgsABICompatible( |
| 628 | const Function *Caller, const Function *Callee, |
| 629 | SmallPtrSetImpl<Argument *> &Args) const { |
| 630 | return TTIImpl->areFunctionArgsABICompatible(Caller, Callee, Args); |
| 631 | } |
| 632 | |
Krzysztof Parzyszek | 0b377e0 | 2018-03-26 13:10:09 +0000 | [diff] [blame] | 633 | bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode, |
| 634 | Type *Ty) const { |
| 635 | return TTIImpl->isIndexedLoadLegal(Mode, Ty); |
| 636 | } |
| 637 | |
| 638 | bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode, |
| 639 | Type *Ty) const { |
| 640 | return TTIImpl->isIndexedStoreLegal(Mode, Ty); |
| 641 | } |
| 642 | |
Volkan Keles | 1c38681 | 2016-10-03 10:31:34 +0000 | [diff] [blame] | 643 | unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const { |
| 644 | return TTIImpl->getLoadStoreVecRegBitWidth(AS); |
| 645 | } |
| 646 | |
| 647 | bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const { |
| 648 | return TTIImpl->isLegalToVectorizeLoad(LI); |
| 649 | } |
| 650 | |
| 651 | bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const { |
| 652 | return TTIImpl->isLegalToVectorizeStore(SI); |
| 653 | } |
| 654 | |
| 655 | bool TargetTransformInfo::isLegalToVectorizeLoadChain( |
| 656 | unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const { |
| 657 | return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment, |
| 658 | AddrSpace); |
| 659 | } |
| 660 | |
| 661 | bool TargetTransformInfo::isLegalToVectorizeStoreChain( |
| 662 | unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const { |
| 663 | return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment, |
| 664 | AddrSpace); |
| 665 | } |
| 666 | |
| 667 | unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF, |
| 668 | unsigned LoadSize, |
| 669 | unsigned ChainSizeInBytes, |
| 670 | VectorType *VecTy) const { |
| 671 | return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy); |
| 672 | } |
| 673 | |
| 674 | unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF, |
| 675 | unsigned StoreSize, |
| 676 | unsigned ChainSizeInBytes, |
| 677 | VectorType *VecTy) const { |
| 678 | return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy); |
| 679 | } |
| 680 | |
Amara Emerson | cf9daa3 | 2017-05-09 10:43:25 +0000 | [diff] [blame] | 681 | bool TargetTransformInfo::useReductionIntrinsic(unsigned Opcode, |
| 682 | Type *Ty, ReductionFlags Flags) const { |
| 683 | return TTIImpl->useReductionIntrinsic(Opcode, Ty, Flags); |
| 684 | } |
| 685 | |
Amara Emerson | 836b0f4 | 2017-05-10 09:42:49 +0000 | [diff] [blame] | 686 | bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const { |
| 687 | return TTIImpl->shouldExpandReduction(II); |
| 688 | } |
Amara Emerson | cf9daa3 | 2017-05-09 10:43:25 +0000 | [diff] [blame] | 689 | |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 690 | int TargetTransformInfo::getInstructionLatency(const Instruction *I) const { |
| 691 | return TTIImpl->getInstructionLatency(I); |
| 692 | } |
| 693 | |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 694 | static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft, |
| 695 | unsigned Level) { |
| 696 | // We don't need a shuffle if we just want to have element 0 in position 0 of |
| 697 | // the vector. |
| 698 | if (!SI && Level == 0 && IsLeft) |
| 699 | return true; |
| 700 | else if (!SI) |
| 701 | return false; |
| 702 | |
| 703 | SmallVector<int, 32> Mask(SI->getType()->getVectorNumElements(), -1); |
| 704 | |
| 705 | // Build a mask of 0, 2, ... (left) or 1, 3, ... (right) depending on whether |
| 706 | // we look at the left or right side. |
| 707 | for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2) |
| 708 | Mask[i] = val; |
| 709 | |
| 710 | SmallVector<int, 16> ActualMask = SI->getShuffleMask(); |
| 711 | return Mask == ActualMask; |
| 712 | } |
| 713 | |
| 714 | namespace { |
| 715 | /// Kind of the reduction data. |
| 716 | enum ReductionKind { |
| 717 | RK_None, /// Not a reduction. |
| 718 | RK_Arithmetic, /// Binary reduction data. |
| 719 | RK_MinMax, /// Min/max reduction data. |
| 720 | RK_UnsignedMinMax, /// Unsigned min/max reduction data. |
| 721 | }; |
| 722 | /// Contains opcode + LHS/RHS parts of the reduction operations. |
| 723 | struct ReductionData { |
| 724 | ReductionData() = delete; |
| 725 | ReductionData(ReductionKind Kind, unsigned Opcode, Value *LHS, Value *RHS) |
| 726 | : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind) { |
| 727 | assert(Kind != RK_None && "expected binary or min/max reduction only."); |
| 728 | } |
| 729 | unsigned Opcode = 0; |
| 730 | Value *LHS = nullptr; |
| 731 | Value *RHS = nullptr; |
| 732 | ReductionKind Kind = RK_None; |
| 733 | bool hasSameData(ReductionData &RD) const { |
| 734 | return Kind == RD.Kind && Opcode == RD.Opcode; |
| 735 | } |
| 736 | }; |
| 737 | } // namespace |
| 738 | |
| 739 | static Optional<ReductionData> getReductionData(Instruction *I) { |
| 740 | Value *L, *R; |
| 741 | if (m_BinOp(m_Value(L), m_Value(R)).match(I)) |
Fangrui Song | f78650a | 2018-07-30 19:41:25 +0000 | [diff] [blame] | 742 | return ReductionData(RK_Arithmetic, I->getOpcode(), L, R); |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 743 | if (auto *SI = dyn_cast<SelectInst>(I)) { |
| 744 | if (m_SMin(m_Value(L), m_Value(R)).match(SI) || |
| 745 | m_SMax(m_Value(L), m_Value(R)).match(SI) || |
| 746 | m_OrdFMin(m_Value(L), m_Value(R)).match(SI) || |
| 747 | m_OrdFMax(m_Value(L), m_Value(R)).match(SI) || |
| 748 | m_UnordFMin(m_Value(L), m_Value(R)).match(SI) || |
| 749 | m_UnordFMax(m_Value(L), m_Value(R)).match(SI)) { |
| 750 | auto *CI = cast<CmpInst>(SI->getCondition()); |
Fangrui Song | f78650a | 2018-07-30 19:41:25 +0000 | [diff] [blame] | 751 | return ReductionData(RK_MinMax, CI->getOpcode(), L, R); |
| 752 | } |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 753 | if (m_UMin(m_Value(L), m_Value(R)).match(SI) || |
| 754 | m_UMax(m_Value(L), m_Value(R)).match(SI)) { |
| 755 | auto *CI = cast<CmpInst>(SI->getCondition()); |
| 756 | return ReductionData(RK_UnsignedMinMax, CI->getOpcode(), L, R); |
| 757 | } |
| 758 | } |
| 759 | return llvm::None; |
| 760 | } |
| 761 | |
| 762 | static ReductionKind matchPairwiseReductionAtLevel(Instruction *I, |
| 763 | unsigned Level, |
| 764 | unsigned NumLevels) { |
| 765 | // Match one level of pairwise operations. |
| 766 | // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, |
| 767 | // <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef> |
| 768 | // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, |
| 769 | // <4 x i32> <i32 1, i32 3, i32 undef, i32 undef> |
| 770 | // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1 |
| 771 | if (!I) |
| 772 | return RK_None; |
| 773 | |
| 774 | assert(I->getType()->isVectorTy() && "Expecting a vector type"); |
| 775 | |
| 776 | Optional<ReductionData> RD = getReductionData(I); |
| 777 | if (!RD) |
| 778 | return RK_None; |
| 779 | |
| 780 | ShuffleVectorInst *LS = dyn_cast<ShuffleVectorInst>(RD->LHS); |
| 781 | if (!LS && Level) |
| 782 | return RK_None; |
| 783 | ShuffleVectorInst *RS = dyn_cast<ShuffleVectorInst>(RD->RHS); |
| 784 | if (!RS && Level) |
| 785 | return RK_None; |
| 786 | |
| 787 | // On level 0 we can omit one shufflevector instruction. |
| 788 | if (!Level && !RS && !LS) |
| 789 | return RK_None; |
| 790 | |
| 791 | // Shuffle inputs must match. |
| 792 | Value *NextLevelOpL = LS ? LS->getOperand(0) : nullptr; |
| 793 | Value *NextLevelOpR = RS ? RS->getOperand(0) : nullptr; |
| 794 | Value *NextLevelOp = nullptr; |
| 795 | if (NextLevelOpR && NextLevelOpL) { |
| 796 | // If we have two shuffles their operands must match. |
| 797 | if (NextLevelOpL != NextLevelOpR) |
| 798 | return RK_None; |
| 799 | |
| 800 | NextLevelOp = NextLevelOpL; |
| 801 | } else if (Level == 0 && (NextLevelOpR || NextLevelOpL)) { |
| 802 | // On the first level we can omit the shufflevector <0, undef,...>. So the |
| 803 | // input to the other shufflevector <1, undef> must match with one of the |
| 804 | // inputs to the current binary operation. |
| 805 | // Example: |
| 806 | // %NextLevelOpL = shufflevector %R, <1, undef ...> |
| 807 | // %BinOp = fadd %NextLevelOpL, %R |
| 808 | if (NextLevelOpL && NextLevelOpL != RD->RHS) |
| 809 | return RK_None; |
| 810 | else if (NextLevelOpR && NextLevelOpR != RD->LHS) |
| 811 | return RK_None; |
| 812 | |
| 813 | NextLevelOp = NextLevelOpL ? RD->RHS : RD->LHS; |
| 814 | } else |
| 815 | return RK_None; |
| 816 | |
| 817 | // Check that the next levels binary operation exists and matches with the |
| 818 | // current one. |
| 819 | if (Level + 1 != NumLevels) { |
| 820 | Optional<ReductionData> NextLevelRD = |
| 821 | getReductionData(cast<Instruction>(NextLevelOp)); |
| 822 | if (!NextLevelRD || !RD->hasSameData(*NextLevelRD)) |
| 823 | return RK_None; |
| 824 | } |
| 825 | |
| 826 | // Shuffle mask for pairwise operation must match. |
| 827 | if (matchPairwiseShuffleMask(LS, /*IsLeft=*/true, Level)) { |
| 828 | if (!matchPairwiseShuffleMask(RS, /*IsLeft=*/false, Level)) |
| 829 | return RK_None; |
| 830 | } else if (matchPairwiseShuffleMask(RS, /*IsLeft=*/true, Level)) { |
| 831 | if (!matchPairwiseShuffleMask(LS, /*IsLeft=*/false, Level)) |
| 832 | return RK_None; |
| 833 | } else { |
| 834 | return RK_None; |
| 835 | } |
| 836 | |
| 837 | if (++Level == NumLevels) |
| 838 | return RD->Kind; |
| 839 | |
| 840 | // Match next level. |
| 841 | return matchPairwiseReductionAtLevel(cast<Instruction>(NextLevelOp), Level, |
| 842 | NumLevels); |
| 843 | } |
| 844 | |
| 845 | static ReductionKind matchPairwiseReduction(const ExtractElementInst *ReduxRoot, |
| 846 | unsigned &Opcode, Type *&Ty) { |
| 847 | if (!EnableReduxCost) |
| 848 | return RK_None; |
| 849 | |
| 850 | // Need to extract the first element. |
| 851 | ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1)); |
| 852 | unsigned Idx = ~0u; |
| 853 | if (CI) |
| 854 | Idx = CI->getZExtValue(); |
| 855 | if (Idx != 0) |
| 856 | return RK_None; |
| 857 | |
| 858 | auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0)); |
| 859 | if (!RdxStart) |
| 860 | return RK_None; |
| 861 | Optional<ReductionData> RD = getReductionData(RdxStart); |
| 862 | if (!RD) |
| 863 | return RK_None; |
| 864 | |
| 865 | Type *VecTy = RdxStart->getType(); |
| 866 | unsigned NumVecElems = VecTy->getVectorNumElements(); |
| 867 | if (!isPowerOf2_32(NumVecElems)) |
| 868 | return RK_None; |
| 869 | |
| 870 | // We look for a sequence of shuffle,shuffle,add triples like the following |
| 871 | // that builds a pairwise reduction tree. |
Fangrui Song | f78650a | 2018-07-30 19:41:25 +0000 | [diff] [blame] | 872 | // |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 873 | // (X0, X1, X2, X3) |
| 874 | // (X0 + X1, X2 + X3, undef, undef) |
| 875 | // ((X0 + X1) + (X2 + X3), undef, undef, undef) |
Fangrui Song | f78650a | 2018-07-30 19:41:25 +0000 | [diff] [blame] | 876 | // |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 877 | // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, |
| 878 | // <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef> |
| 879 | // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, |
| 880 | // <4 x i32> <i32 1, i32 3, i32 undef, i32 undef> |
| 881 | // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1 |
| 882 | // %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, |
| 883 | // <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef> |
| 884 | // %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, |
| 885 | // <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> |
| 886 | // %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1 |
| 887 | // %r = extractelement <4 x float> %bin.rdx8, i32 0 |
| 888 | if (matchPairwiseReductionAtLevel(RdxStart, 0, Log2_32(NumVecElems)) == |
| 889 | RK_None) |
| 890 | return RK_None; |
| 891 | |
| 892 | Opcode = RD->Opcode; |
| 893 | Ty = VecTy; |
| 894 | |
| 895 | return RD->Kind; |
| 896 | } |
| 897 | |
| 898 | static std::pair<Value *, ShuffleVectorInst *> |
| 899 | getShuffleAndOtherOprd(Value *L, Value *R) { |
| 900 | ShuffleVectorInst *S = nullptr; |
| 901 | |
| 902 | if ((S = dyn_cast<ShuffleVectorInst>(L))) |
| 903 | return std::make_pair(R, S); |
| 904 | |
| 905 | S = dyn_cast<ShuffleVectorInst>(R); |
| 906 | return std::make_pair(L, S); |
| 907 | } |
| 908 | |
| 909 | static ReductionKind |
| 910 | matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot, |
| 911 | unsigned &Opcode, Type *&Ty) { |
| 912 | if (!EnableReduxCost) |
| 913 | return RK_None; |
| 914 | |
| 915 | // Need to extract the first element. |
| 916 | ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1)); |
| 917 | unsigned Idx = ~0u; |
| 918 | if (CI) |
| 919 | Idx = CI->getZExtValue(); |
| 920 | if (Idx != 0) |
| 921 | return RK_None; |
| 922 | |
| 923 | auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0)); |
| 924 | if (!RdxStart) |
| 925 | return RK_None; |
| 926 | Optional<ReductionData> RD = getReductionData(RdxStart); |
| 927 | if (!RD) |
| 928 | return RK_None; |
| 929 | |
| 930 | Type *VecTy = ReduxRoot->getOperand(0)->getType(); |
| 931 | unsigned NumVecElems = VecTy->getVectorNumElements(); |
| 932 | if (!isPowerOf2_32(NumVecElems)) |
| 933 | return RK_None; |
| 934 | |
| 935 | // We look for a sequence of shuffles and adds like the following matching one |
| 936 | // fadd, shuffle vector pair at a time. |
Fangrui Song | f78650a | 2018-07-30 19:41:25 +0000 | [diff] [blame] | 937 | // |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 938 | // %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, |
| 939 | // <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> |
| 940 | // %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf |
| 941 | // %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, |
| 942 | // <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> |
| 943 | // %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7 |
| 944 | // %r = extractelement <4 x float> %bin.rdx8, i32 0 |
| 945 | |
| 946 | unsigned MaskStart = 1; |
| 947 | Instruction *RdxOp = RdxStart; |
Fangrui Song | f78650a | 2018-07-30 19:41:25 +0000 | [diff] [blame] | 948 | SmallVector<int, 32> ShuffleMask(NumVecElems, 0); |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 949 | unsigned NumVecElemsRemain = NumVecElems; |
| 950 | while (NumVecElemsRemain - 1) { |
| 951 | // Check for the right reduction operation. |
| 952 | if (!RdxOp) |
| 953 | return RK_None; |
| 954 | Optional<ReductionData> RDLevel = getReductionData(RdxOp); |
| 955 | if (!RDLevel || !RDLevel->hasSameData(*RD)) |
| 956 | return RK_None; |
| 957 | |
| 958 | Value *NextRdxOp; |
| 959 | ShuffleVectorInst *Shuffle; |
| 960 | std::tie(NextRdxOp, Shuffle) = |
| 961 | getShuffleAndOtherOprd(RDLevel->LHS, RDLevel->RHS); |
| 962 | |
| 963 | // Check the current reduction operation and the shuffle use the same value. |
| 964 | if (Shuffle == nullptr) |
| 965 | return RK_None; |
| 966 | if (Shuffle->getOperand(0) != NextRdxOp) |
| 967 | return RK_None; |
| 968 | |
| 969 | // Check that shuffle masks matches. |
| 970 | for (unsigned j = 0; j != MaskStart; ++j) |
| 971 | ShuffleMask[j] = MaskStart + j; |
| 972 | // Fill the rest of the mask with -1 for undef. |
| 973 | std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1); |
| 974 | |
| 975 | SmallVector<int, 16> Mask = Shuffle->getShuffleMask(); |
| 976 | if (ShuffleMask != Mask) |
| 977 | return RK_None; |
| 978 | |
| 979 | RdxOp = dyn_cast<Instruction>(NextRdxOp); |
| 980 | NumVecElemsRemain /= 2; |
| 981 | MaskStart *= 2; |
| 982 | } |
| 983 | |
| 984 | Opcode = RD->Opcode; |
| 985 | Ty = VecTy; |
| 986 | return RD->Kind; |
| 987 | } |
| 988 | |
| 989 | int TargetTransformInfo::getInstructionThroughput(const Instruction *I) const { |
| 990 | switch (I->getOpcode()) { |
| 991 | case Instruction::GetElementPtr: |
| 992 | return getUserCost(I); |
| 993 | |
| 994 | case Instruction::Ret: |
| 995 | case Instruction::PHI: |
| 996 | case Instruction::Br: { |
| 997 | return getCFInstrCost(I->getOpcode()); |
| 998 | } |
| 999 | case Instruction::Add: |
| 1000 | case Instruction::FAdd: |
| 1001 | case Instruction::Sub: |
| 1002 | case Instruction::FSub: |
| 1003 | case Instruction::Mul: |
| 1004 | case Instruction::FMul: |
| 1005 | case Instruction::UDiv: |
| 1006 | case Instruction::SDiv: |
| 1007 | case Instruction::FDiv: |
| 1008 | case Instruction::URem: |
| 1009 | case Instruction::SRem: |
| 1010 | case Instruction::FRem: |
| 1011 | case Instruction::Shl: |
| 1012 | case Instruction::LShr: |
| 1013 | case Instruction::AShr: |
| 1014 | case Instruction::And: |
| 1015 | case Instruction::Or: |
| 1016 | case Instruction::Xor: { |
Simon Pilgrim | 4162d77 | 2018-05-22 10:40:09 +0000 | [diff] [blame] | 1017 | TargetTransformInfo::OperandValueKind Op1VK, Op2VK; |
| 1018 | TargetTransformInfo::OperandValueProperties Op1VP, Op2VP; |
| 1019 | Op1VK = getOperandInfo(I->getOperand(0), Op1VP); |
| 1020 | Op2VK = getOperandInfo(I->getOperand(1), Op2VP); |
| 1021 | SmallVector<const Value *, 2> Operands(I->operand_values()); |
| 1022 | return getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK, Op2VK, |
| 1023 | Op1VP, Op2VP, Operands); |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 1024 | } |
| 1025 | case Instruction::Select: { |
| 1026 | const SelectInst *SI = cast<SelectInst>(I); |
| 1027 | Type *CondTy = SI->getCondition()->getType(); |
| 1028 | return getCmpSelInstrCost(I->getOpcode(), I->getType(), CondTy, I); |
| 1029 | } |
| 1030 | case Instruction::ICmp: |
| 1031 | case Instruction::FCmp: { |
| 1032 | Type *ValTy = I->getOperand(0)->getType(); |
| 1033 | return getCmpSelInstrCost(I->getOpcode(), ValTy, I->getType(), I); |
| 1034 | } |
| 1035 | case Instruction::Store: { |
| 1036 | const StoreInst *SI = cast<StoreInst>(I); |
| 1037 | Type *ValTy = SI->getValueOperand()->getType(); |
| 1038 | return getMemoryOpCost(I->getOpcode(), ValTy, |
| 1039 | SI->getAlignment(), |
| 1040 | SI->getPointerAddressSpace(), I); |
| 1041 | } |
| 1042 | case Instruction::Load: { |
| 1043 | const LoadInst *LI = cast<LoadInst>(I); |
| 1044 | return getMemoryOpCost(I->getOpcode(), I->getType(), |
| 1045 | LI->getAlignment(), |
| 1046 | LI->getPointerAddressSpace(), I); |
| 1047 | } |
| 1048 | case Instruction::ZExt: |
| 1049 | case Instruction::SExt: |
| 1050 | case Instruction::FPToUI: |
| 1051 | case Instruction::FPToSI: |
| 1052 | case Instruction::FPExt: |
| 1053 | case Instruction::PtrToInt: |
| 1054 | case Instruction::IntToPtr: |
| 1055 | case Instruction::SIToFP: |
| 1056 | case Instruction::UIToFP: |
| 1057 | case Instruction::Trunc: |
| 1058 | case Instruction::FPTrunc: |
| 1059 | case Instruction::BitCast: |
| 1060 | case Instruction::AddrSpaceCast: { |
| 1061 | Type *SrcTy = I->getOperand(0)->getType(); |
| 1062 | return getCastInstrCost(I->getOpcode(), I->getType(), SrcTy, I); |
| 1063 | } |
| 1064 | case Instruction::ExtractElement: { |
| 1065 | const ExtractElementInst * EEI = cast<ExtractElementInst>(I); |
| 1066 | ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1)); |
| 1067 | unsigned Idx = -1; |
| 1068 | if (CI) |
| 1069 | Idx = CI->getZExtValue(); |
| 1070 | |
| 1071 | // Try to match a reduction sequence (series of shufflevector and vector |
| 1072 | // adds followed by a extractelement). |
| 1073 | unsigned ReduxOpCode; |
| 1074 | Type *ReduxType; |
| 1075 | |
| 1076 | switch (matchVectorSplittingReduction(EEI, ReduxOpCode, ReduxType)) { |
| 1077 | case RK_Arithmetic: |
| 1078 | return getArithmeticReductionCost(ReduxOpCode, ReduxType, |
| 1079 | /*IsPairwiseForm=*/false); |
| 1080 | case RK_MinMax: |
| 1081 | return getMinMaxReductionCost( |
| 1082 | ReduxType, CmpInst::makeCmpResultType(ReduxType), |
| 1083 | /*IsPairwiseForm=*/false, /*IsUnsigned=*/false); |
| 1084 | case RK_UnsignedMinMax: |
| 1085 | return getMinMaxReductionCost( |
| 1086 | ReduxType, CmpInst::makeCmpResultType(ReduxType), |
| 1087 | /*IsPairwiseForm=*/false, /*IsUnsigned=*/true); |
| 1088 | case RK_None: |
| 1089 | break; |
| 1090 | } |
| 1091 | |
| 1092 | switch (matchPairwiseReduction(EEI, ReduxOpCode, ReduxType)) { |
| 1093 | case RK_Arithmetic: |
| 1094 | return getArithmeticReductionCost(ReduxOpCode, ReduxType, |
| 1095 | /*IsPairwiseForm=*/true); |
| 1096 | case RK_MinMax: |
| 1097 | return getMinMaxReductionCost( |
| 1098 | ReduxType, CmpInst::makeCmpResultType(ReduxType), |
| 1099 | /*IsPairwiseForm=*/true, /*IsUnsigned=*/false); |
| 1100 | case RK_UnsignedMinMax: |
| 1101 | return getMinMaxReductionCost( |
| 1102 | ReduxType, CmpInst::makeCmpResultType(ReduxType), |
| 1103 | /*IsPairwiseForm=*/true, /*IsUnsigned=*/true); |
| 1104 | case RK_None: |
| 1105 | break; |
| 1106 | } |
| 1107 | |
| 1108 | return getVectorInstrCost(I->getOpcode(), |
| 1109 | EEI->getOperand(0)->getType(), Idx); |
| 1110 | } |
| 1111 | case Instruction::InsertElement: { |
| 1112 | const InsertElementInst * IE = cast<InsertElementInst>(I); |
| 1113 | ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2)); |
Fangrui Song | f78650a | 2018-07-30 19:41:25 +0000 | [diff] [blame] | 1114 | unsigned Idx = -1; |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 1115 | if (CI) |
| 1116 | Idx = CI->getZExtValue(); |
| 1117 | return getVectorInstrCost(I->getOpcode(), |
| 1118 | IE->getType(), Idx); |
| 1119 | } |
| 1120 | case Instruction::ShuffleVector: { |
| 1121 | const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I); |
Simon Pilgrim | d0c7160 | 2018-11-09 16:28:19 +0000 | [diff] [blame] | 1122 | Type *Ty = Shuffle->getType(); |
| 1123 | Type *SrcTy = Shuffle->getOperand(0)->getType(); |
| 1124 | |
| 1125 | // TODO: Identify and add costs for insert subvector, etc. |
| 1126 | int SubIndex; |
| 1127 | if (Shuffle->isExtractSubvectorMask(SubIndex)) |
Simon Pilgrim | 26e1c88 | 2018-11-09 18:30:59 +0000 | [diff] [blame] | 1128 | return TTIImpl->getShuffleCost(SK_ExtractSubvector, SrcTy, SubIndex, Ty); |
Simon Pilgrim | d0c7160 | 2018-11-09 16:28:19 +0000 | [diff] [blame] | 1129 | |
Sanjay Patel | 2ca3360 | 2018-06-19 18:44:00 +0000 | [diff] [blame] | 1130 | if (Shuffle->changesLength()) |
| 1131 | return -1; |
Fangrui Song | f78650a | 2018-07-30 19:41:25 +0000 | [diff] [blame] | 1132 | |
Sanjay Patel | 2ca3360 | 2018-06-19 18:44:00 +0000 | [diff] [blame] | 1133 | if (Shuffle->isIdentity()) |
| 1134 | return 0; |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 1135 | |
Sanjay Patel | 2ca3360 | 2018-06-19 18:44:00 +0000 | [diff] [blame] | 1136 | if (Shuffle->isReverse()) |
| 1137 | return TTIImpl->getShuffleCost(SK_Reverse, Ty, 0, nullptr); |
Simon Pilgrim | 0783921 | 2018-06-12 14:47:13 +0000 | [diff] [blame] | 1138 | |
Sanjay Patel | 2ca3360 | 2018-06-19 18:44:00 +0000 | [diff] [blame] | 1139 | if (Shuffle->isSelect()) |
| 1140 | return TTIImpl->getShuffleCost(SK_Select, Ty, 0, nullptr); |
Simon Pilgrim | 0783921 | 2018-06-12 14:47:13 +0000 | [diff] [blame] | 1141 | |
Sanjay Patel | 2ca3360 | 2018-06-19 18:44:00 +0000 | [diff] [blame] | 1142 | if (Shuffle->isTranspose()) |
| 1143 | return TTIImpl->getShuffleCost(SK_Transpose, Ty, 0, nullptr); |
Matthew Simpson | b4096eb | 2018-04-26 13:48:33 +0000 | [diff] [blame] | 1144 | |
Sanjay Patel | 2ca3360 | 2018-06-19 18:44:00 +0000 | [diff] [blame] | 1145 | if (Shuffle->isZeroEltSplat()) |
| 1146 | return TTIImpl->getShuffleCost(SK_Broadcast, Ty, 0, nullptr); |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 1147 | |
Sanjay Patel | 2ca3360 | 2018-06-19 18:44:00 +0000 | [diff] [blame] | 1148 | if (Shuffle->isSingleSource()) |
| 1149 | return TTIImpl->getShuffleCost(SK_PermuteSingleSrc, Ty, 0, nullptr); |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 1150 | |
Sanjay Patel | 2ca3360 | 2018-06-19 18:44:00 +0000 | [diff] [blame] | 1151 | return TTIImpl->getShuffleCost(SK_PermuteTwoSrc, Ty, 0, nullptr); |
Guozhi Wei | 62d6414 | 2017-09-08 22:29:17 +0000 | [diff] [blame] | 1152 | } |
| 1153 | case Instruction::Call: |
| 1154 | if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) { |
| 1155 | SmallVector<Value *, 4> Args(II->arg_operands()); |
| 1156 | |
| 1157 | FastMathFlags FMF; |
| 1158 | if (auto *FPMO = dyn_cast<FPMathOperator>(II)) |
| 1159 | FMF = FPMO->getFastMathFlags(); |
| 1160 | |
| 1161 | return getIntrinsicInstrCost(II->getIntrinsicID(), II->getType(), |
| 1162 | Args, FMF); |
| 1163 | } |
| 1164 | return -1; |
| 1165 | default: |
| 1166 | // We don't have any information on this instruction. |
| 1167 | return -1; |
| 1168 | } |
| 1169 | } |
| 1170 | |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 1171 | TargetTransformInfo::Concept::~Concept() {} |
| 1172 | |
Chandler Carruth | e038552 | 2015-02-01 10:11:22 +0000 | [diff] [blame] | 1173 | TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {} |
| 1174 | |
| 1175 | TargetIRAnalysis::TargetIRAnalysis( |
Eric Christopher | a4e5d3c | 2015-09-16 23:38:13 +0000 | [diff] [blame] | 1176 | std::function<Result(const Function &)> TTICallback) |
Benjamin Kramer | 82de7d3 | 2016-05-27 14:27:24 +0000 | [diff] [blame] | 1177 | : TTICallback(std::move(TTICallback)) {} |
Chandler Carruth | e038552 | 2015-02-01 10:11:22 +0000 | [diff] [blame] | 1178 | |
Chandler Carruth | 164a2aa6 | 2016-06-17 00:11:01 +0000 | [diff] [blame] | 1179 | TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F, |
Sean Silva | 36e0d01 | 2016-08-09 00:28:15 +0000 | [diff] [blame] | 1180 | FunctionAnalysisManager &) { |
Chandler Carruth | e038552 | 2015-02-01 10:11:22 +0000 | [diff] [blame] | 1181 | return TTICallback(F); |
| 1182 | } |
| 1183 | |
Chandler Carruth | dab4eae | 2016-11-23 17:53:26 +0000 | [diff] [blame] | 1184 | AnalysisKey TargetIRAnalysis::Key; |
NAKAMURA Takumi | df0cd72 | 2016-02-28 17:17:00 +0000 | [diff] [blame] | 1185 | |
Eric Christopher | a4e5d3c | 2015-09-16 23:38:13 +0000 | [diff] [blame] | 1186 | TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) { |
Mehdi Amini | 5010ebf | 2015-07-09 02:08:42 +0000 | [diff] [blame] | 1187 | return Result(F.getParent()->getDataLayout()); |
Chandler Carruth | e038552 | 2015-02-01 10:11:22 +0000 | [diff] [blame] | 1188 | } |
| 1189 | |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 1190 | // Register the basic pass. |
| 1191 | INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti", |
| 1192 | "Target Transform Information", false, true) |
| 1193 | char TargetTransformInfoWrapperPass::ID = 0; |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 1194 | |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 1195 | void TargetTransformInfoWrapperPass::anchor() {} |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 1196 | |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 1197 | TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass() |
Chandler Carruth | 5ec2b1d | 2015-02-01 12:26:09 +0000 | [diff] [blame] | 1198 | : ImmutablePass(ID) { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 1199 | initializeTargetTransformInfoWrapperPassPass( |
| 1200 | *PassRegistry::getPassRegistry()); |
| 1201 | } |
| 1202 | |
| 1203 | TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass( |
Chandler Carruth | 5ec2b1d | 2015-02-01 12:26:09 +0000 | [diff] [blame] | 1204 | TargetIRAnalysis TIRA) |
| 1205 | : ImmutablePass(ID), TIRA(std::move(TIRA)) { |
Chandler Carruth | 705b185 | 2015-01-31 03:43:40 +0000 | [diff] [blame] | 1206 | initializeTargetTransformInfoWrapperPassPass( |
| 1207 | *PassRegistry::getPassRegistry()); |
| 1208 | } |
| 1209 | |
Eric Christopher | a4e5d3c | 2015-09-16 23:38:13 +0000 | [diff] [blame] | 1210 | TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) { |
Sean Silva | 36e0d01 | 2016-08-09 00:28:15 +0000 | [diff] [blame] | 1211 | FunctionAnalysisManager DummyFAM; |
Chandler Carruth | 164a2aa6 | 2016-06-17 00:11:01 +0000 | [diff] [blame] | 1212 | TTI = TIRA.run(F, DummyFAM); |
Chandler Carruth | 5ec2b1d | 2015-02-01 12:26:09 +0000 | [diff] [blame] | 1213 | return *TTI; |
| 1214 | } |
| 1215 | |
Chandler Carruth | 93dcdc4 | 2015-01-31 11:17:59 +0000 | [diff] [blame] | 1216 | ImmutablePass * |
Chandler Carruth | 5ec2b1d | 2015-02-01 12:26:09 +0000 | [diff] [blame] | 1217 | llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) { |
| 1218 | return new TargetTransformInfoWrapperPass(std::move(TIRA)); |
Chandler Carruth | 539edf4 | 2013-01-05 11:43:11 +0000 | [diff] [blame] | 1219 | } |