blob: 23a52814ab4d48a4b064b6bc82f3de855fda1a4b [file] [log] [blame]
Tobias Grosser75805372011-04-29 06:27:02 +00001//===--------- ScopInfo.cpp - Create Scops from LLVM IR ------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Create a polyhedral description for a static control flow region.
11//
12// The pass creates a polyhedral description of the Scops detected by the Scop
13// detection derived from their LLVM-IR code.
14//
Tobias Grossera5605d32014-10-29 19:58:28 +000015// This representation is shared among several tools in the polyhedral
Tobias Grosser75805372011-04-29 06:27:02 +000016// community, which are e.g. Cloog, Pluto, Loopo, Graphite.
17//
18//===----------------------------------------------------------------------===//
19
Sebastian Pop27c10c62013-03-22 22:07:43 +000020#include "polly/CodeGen/BlockGenerators.h"
Tobias Grosser75805372011-04-29 06:27:02 +000021#include "polly/LinkAllPasses.h"
Sebastian Pop27c10c62013-03-22 22:07:43 +000022#include "polly/ScopInfo.h"
Johannes Doerfert0ee1f212014-06-17 17:31:36 +000023#include "polly/Options.h"
Tobias Grosser75805372011-04-29 06:27:02 +000024#include "polly/Support/GICHelper.h"
Tobias Grosser60b54f12011-11-08 15:41:28 +000025#include "polly/Support/SCEVValidator.h"
Tobias Grosser83628182013-05-07 08:11:54 +000026#include "polly/Support/ScopHelper.h"
Sebastian Pop27c10c62013-03-22 22:07:43 +000027#include "polly/TempScopInfo.h"
Tobias Grosser75805372011-04-29 06:27:02 +000028#include "llvm/ADT/SetVector.h"
Tobias Grosser83628182013-05-07 08:11:54 +000029#include "llvm/ADT/Statistic.h"
Hongbin Zheng86a37742012-04-25 08:01:38 +000030#include "llvm/ADT/StringExtras.h"
Tobias Grosser83628182013-05-07 08:11:54 +000031#include "llvm/Analysis/LoopInfo.h"
Johannes Doerfertb164c792014-09-18 11:17:17 +000032#include "llvm/Analysis/AliasAnalysis.h"
Tobias Grosser83628182013-05-07 08:11:54 +000033#include "llvm/Analysis/RegionIterator.h"
34#include "llvm/Analysis/ScalarEvolutionExpressions.h"
Tobias Grosser75805372011-04-29 06:27:02 +000035#include "llvm/Support/Debug.h"
36
37#include "isl/constraint.h"
38#include "isl/set.h"
39#include "isl/map.h"
Tobias Grosser37eb4222014-02-20 21:43:54 +000040#include "isl/union_map.h"
Tobias Grosser33ba62ad2011-08-18 06:31:50 +000041#include "isl/aff.h"
42#include "isl/printer.h"
Tobias Grosserf5338802011-10-06 00:03:35 +000043#include "isl/local_space.h"
Tobias Grosser4a8e3562011-12-07 07:42:51 +000044#include "isl/options.h"
Tobias Grosseredab1352013-06-21 06:41:31 +000045#include "isl/val.h"
Chandler Carruth95fef942014-04-22 03:30:19 +000046
Tobias Grosser75805372011-04-29 06:27:02 +000047#include <sstream>
48#include <string>
49#include <vector>
50
51using namespace llvm;
52using namespace polly;
53
Chandler Carruth95fef942014-04-22 03:30:19 +000054#define DEBUG_TYPE "polly-scops"
55
Tobias Grosser74394f02013-01-14 22:40:23 +000056STATISTIC(ScopFound, "Number of valid Scops");
57STATISTIC(RichScopFound, "Number of Scops containing a loop");
Tobias Grosser75805372011-04-29 06:27:02 +000058
Johannes Doerfert9e7b17b2014-08-18 00:40:13 +000059// Multiplicative reductions can be disabled separately as these kind of
Johannes Doerfert0ee1f212014-06-17 17:31:36 +000060// operations can overflow easily. Additive reductions and bit operations
61// are in contrast pretty stable.
Tobias Grosser483a90d2014-07-09 10:50:10 +000062static cl::opt<bool> DisableMultiplicativeReductions(
63 "polly-disable-multiplicative-reductions",
64 cl::desc("Disable multiplicative reductions"), cl::Hidden, cl::ZeroOrMore,
65 cl::init(false), cl::cat(PollyCategory));
Johannes Doerfert0ee1f212014-06-17 17:31:36 +000066
Johannes Doerfert9143d672014-09-27 11:02:39 +000067static cl::opt<unsigned> RunTimeChecksMaxParameters(
68 "polly-rtc-max-parameters",
69 cl::desc("The maximal number of parameters allowed in RTCs."), cl::Hidden,
70 cl::ZeroOrMore, cl::init(8), cl::cat(PollyCategory));
71
Tobias Grosser0695ee42013-09-17 03:30:31 +000072/// Translate a 'const SCEV *' expression in an isl_pw_aff.
Tobias Grosserabfbe632013-02-05 12:09:06 +000073struct SCEVAffinator : public SCEVVisitor<SCEVAffinator, isl_pw_aff *> {
Tobias Grosser0695ee42013-09-17 03:30:31 +000074public:
Tobias Grosser0695ee42013-09-17 03:30:31 +000075 /// @brief Translate a 'const SCEV *' to an isl_pw_aff.
76 ///
77 /// @param Stmt The location at which the scalar evolution expression
78 /// is evaluated.
79 /// @param Expr The expression that is translated.
80 static __isl_give isl_pw_aff *getPwAff(ScopStmt *Stmt, const SCEV *Expr);
81
Tobias Grosser33ba62ad2011-08-18 06:31:50 +000082private:
Tobias Grosser3cc99742012-06-06 16:33:15 +000083 isl_ctx *Ctx;
Tobias Grosserf5338802011-10-06 00:03:35 +000084 int NbLoopSpaces;
Tobias Grosser3cc99742012-06-06 16:33:15 +000085 const Scop *S;
Tobias Grosser33ba62ad2011-08-18 06:31:50 +000086
Tobias Grosser0695ee42013-09-17 03:30:31 +000087 SCEVAffinator(const ScopStmt *Stmt);
88 int getLoopDepth(const Loop *L);
Tobias Grosser60b54f12011-11-08 15:41:28 +000089
Tobias Grosser0695ee42013-09-17 03:30:31 +000090 __isl_give isl_pw_aff *visit(const SCEV *Expr);
91 __isl_give isl_pw_aff *visitConstant(const SCEVConstant *Expr);
92 __isl_give isl_pw_aff *visitTruncateExpr(const SCEVTruncateExpr *Expr);
93 __isl_give isl_pw_aff *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr);
94 __isl_give isl_pw_aff *visitSignExtendExpr(const SCEVSignExtendExpr *Expr);
95 __isl_give isl_pw_aff *visitAddExpr(const SCEVAddExpr *Expr);
96 __isl_give isl_pw_aff *visitMulExpr(const SCEVMulExpr *Expr);
97 __isl_give isl_pw_aff *visitUDivExpr(const SCEVUDivExpr *Expr);
98 __isl_give isl_pw_aff *visitAddRecExpr(const SCEVAddRecExpr *Expr);
99 __isl_give isl_pw_aff *visitSMaxExpr(const SCEVSMaxExpr *Expr);
100 __isl_give isl_pw_aff *visitUMaxExpr(const SCEVUMaxExpr *Expr);
101 __isl_give isl_pw_aff *visitUnknown(const SCEVUnknown *Expr);
Tobias Grosser60b54f12011-11-08 15:41:28 +0000102
Tobias Grosser0695ee42013-09-17 03:30:31 +0000103 friend struct SCEVVisitor<SCEVAffinator, isl_pw_aff *>;
104};
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000105
Tobias Grosser0695ee42013-09-17 03:30:31 +0000106SCEVAffinator::SCEVAffinator(const ScopStmt *Stmt)
107 : Ctx(Stmt->getIslCtx()), NbLoopSpaces(Stmt->getNumIterators()),
108 S(Stmt->getParent()) {}
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000109
Tobias Grosser0695ee42013-09-17 03:30:31 +0000110__isl_give isl_pw_aff *SCEVAffinator::getPwAff(ScopStmt *Stmt,
111 const SCEV *Scev) {
112 Scop *S = Stmt->getParent();
113 const Region *Reg = &S->getRegion();
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000114
Tobias Grosser0695ee42013-09-17 03:30:31 +0000115 S->addParams(getParamsInAffineExpr(Reg, Scev, *S->getSE()));
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000116
Tobias Grosser0695ee42013-09-17 03:30:31 +0000117 SCEVAffinator Affinator(Stmt);
118 return Affinator.visit(Scev);
119}
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000120
Tobias Grosser0695ee42013-09-17 03:30:31 +0000121__isl_give isl_pw_aff *SCEVAffinator::visit(const SCEV *Expr) {
122 // In case the scev is a valid parameter, we do not further analyze this
123 // expression, but create a new parameter in the isl_pw_aff. This allows us
124 // to treat subexpressions that we cannot translate into an piecewise affine
125 // expression, as constant parameters of the piecewise affine expression.
126 if (isl_id *Id = S->getIdForParam(Expr)) {
127 isl_space *Space = isl_space_set_alloc(Ctx, 1, NbLoopSpaces);
128 Space = isl_space_set_dim_id(Space, isl_dim_param, 0, Id);
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000129
Tobias Grosser0695ee42013-09-17 03:30:31 +0000130 isl_set *Domain = isl_set_universe(isl_space_copy(Space));
131 isl_aff *Affine = isl_aff_zero_on_domain(isl_local_space_from_space(Space));
132 Affine = isl_aff_add_coefficient_si(Affine, isl_dim_param, 0, 1);
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000133
134 return isl_pw_aff_alloc(Domain, Affine);
135 }
136
Tobias Grosser0695ee42013-09-17 03:30:31 +0000137 return SCEVVisitor<SCEVAffinator, isl_pw_aff *>::visit(Expr);
138}
139
Tobias Grosser0d170132013-10-03 13:09:19 +0000140__isl_give isl_pw_aff *SCEVAffinator::visitConstant(const SCEVConstant *Expr) {
Tobias Grosser0695ee42013-09-17 03:30:31 +0000141 ConstantInt *Value = Expr->getValue();
142 isl_val *v;
143
144 // LLVM does not define if an integer value is interpreted as a signed or
145 // unsigned value. Hence, without further information, it is unknown how
146 // this value needs to be converted to GMP. At the moment, we only support
147 // signed operations. So we just interpret it as signed. Later, there are
148 // two options:
149 //
150 // 1. We always interpret any value as signed and convert the values on
151 // demand.
152 // 2. We pass down the signedness of the calculation and use it to interpret
153 // this constant correctly.
154 v = isl_valFromAPInt(Ctx, Value->getValue(), /* isSigned */ true);
155
156 isl_space *Space = isl_space_set_alloc(Ctx, 0, NbLoopSpaces);
Johannes Doerfert9c147372014-11-19 15:36:59 +0000157 isl_local_space *ls = isl_local_space_from_space(Space);
158 return isl_pw_aff_from_aff(isl_aff_val_on_domain(ls, v));
Tobias Grosser0695ee42013-09-17 03:30:31 +0000159}
160
161__isl_give isl_pw_aff *
162SCEVAffinator::visitTruncateExpr(const SCEVTruncateExpr *Expr) {
163 llvm_unreachable("SCEVTruncateExpr not yet supported");
164}
165
166__isl_give isl_pw_aff *
167SCEVAffinator::visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
168 llvm_unreachable("SCEVZeroExtendExpr not yet supported");
169}
170
171__isl_give isl_pw_aff *
172SCEVAffinator::visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
173 // Assuming the value is signed, a sign extension is basically a noop.
174 // TODO: Reconsider this as soon as we support unsigned values.
175 return visit(Expr->getOperand());
176}
177
178__isl_give isl_pw_aff *SCEVAffinator::visitAddExpr(const SCEVAddExpr *Expr) {
179 isl_pw_aff *Sum = visit(Expr->getOperand(0));
180
181 for (int i = 1, e = Expr->getNumOperands(); i < e; ++i) {
182 isl_pw_aff *NextSummand = visit(Expr->getOperand(i));
183 Sum = isl_pw_aff_add(Sum, NextSummand);
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000184 }
185
Tobias Grosser0695ee42013-09-17 03:30:31 +0000186 // TODO: Check for NSW and NUW.
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000187
Tobias Grosser0695ee42013-09-17 03:30:31 +0000188 return Sum;
189}
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000190
Tobias Grosser0695ee42013-09-17 03:30:31 +0000191__isl_give isl_pw_aff *SCEVAffinator::visitMulExpr(const SCEVMulExpr *Expr) {
192 isl_pw_aff *Product = visit(Expr->getOperand(0));
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000193
Tobias Grosser0695ee42013-09-17 03:30:31 +0000194 for (int i = 1, e = Expr->getNumOperands(); i < e; ++i) {
195 isl_pw_aff *NextOperand = visit(Expr->getOperand(i));
196
197 if (!isl_pw_aff_is_cst(Product) && !isl_pw_aff_is_cst(NextOperand)) {
198 isl_pw_aff_free(Product);
199 isl_pw_aff_free(NextOperand);
Tobias Grosser5a56cbf2014-04-16 07:33:47 +0000200 return nullptr;
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000201 }
202
Tobias Grosser0695ee42013-09-17 03:30:31 +0000203 Product = isl_pw_aff_mul(Product, NextOperand);
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000204 }
205
Tobias Grosser0695ee42013-09-17 03:30:31 +0000206 // TODO: Check for NSW and NUW.
207 return Product;
208}
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000209
Tobias Grosser0695ee42013-09-17 03:30:31 +0000210__isl_give isl_pw_aff *SCEVAffinator::visitUDivExpr(const SCEVUDivExpr *Expr) {
211 llvm_unreachable("SCEVUDivExpr not yet supported");
212}
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000213
Tobias Grosser0695ee42013-09-17 03:30:31 +0000214__isl_give isl_pw_aff *
215SCEVAffinator::visitAddRecExpr(const SCEVAddRecExpr *Expr) {
216 assert(Expr->isAffine() && "Only affine AddRecurrences allowed");
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000217
Tobias Grosser0695ee42013-09-17 03:30:31 +0000218 // Directly generate isl_pw_aff for Expr if 'start' is zero.
219 if (Expr->getStart()->isZero()) {
220 assert(S->getRegion().contains(Expr->getLoop()) &&
221 "Scop does not contain the loop referenced in this AddRec");
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000222
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000223 isl_pw_aff *Start = visit(Expr->getStart());
Tobias Grosser0695ee42013-09-17 03:30:31 +0000224 isl_pw_aff *Step = visit(Expr->getOperand(1));
225 isl_space *Space = isl_space_set_alloc(Ctx, 0, NbLoopSpaces);
226 isl_local_space *LocalSpace = isl_local_space_from_space(Space);
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000227
Tobias Grosser0695ee42013-09-17 03:30:31 +0000228 int loopDimension = getLoopDepth(Expr->getLoop());
229
230 isl_aff *LAff = isl_aff_set_coefficient_si(
231 isl_aff_zero_on_domain(LocalSpace), isl_dim_in, loopDimension, 1);
232 isl_pw_aff *LPwAff = isl_pw_aff_from_aff(LAff);
233
234 // TODO: Do we need to check for NSW and NUW?
235 return isl_pw_aff_add(Start, isl_pw_aff_mul(Step, LPwAff));
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000236 }
237
Tobias Grosser0695ee42013-09-17 03:30:31 +0000238 // Translate AddRecExpr from '{start, +, inc}' into 'start + {0, +, inc}'
239 // if 'start' is not zero.
240 ScalarEvolution &SE = *S->getSE();
241 const SCEV *ZeroStartExpr = SE.getAddRecExpr(
242 SE.getConstant(Expr->getStart()->getType(), 0),
243 Expr->getStepRecurrence(SE), Expr->getLoop(), SCEV::FlagAnyWrap);
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000244
Tobias Grosser0695ee42013-09-17 03:30:31 +0000245 isl_pw_aff *ZeroStartResult = visit(ZeroStartExpr);
246 isl_pw_aff *Start = visit(Expr->getStart());
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000247
Tobias Grosser0695ee42013-09-17 03:30:31 +0000248 return isl_pw_aff_add(ZeroStartResult, Start);
249}
250
251__isl_give isl_pw_aff *SCEVAffinator::visitSMaxExpr(const SCEVSMaxExpr *Expr) {
252 isl_pw_aff *Max = visit(Expr->getOperand(0));
253
254 for (int i = 1, e = Expr->getNumOperands(); i < e; ++i) {
255 isl_pw_aff *NextOperand = visit(Expr->getOperand(i));
256 Max = isl_pw_aff_max(Max, NextOperand);
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000257 }
258
Tobias Grosser0695ee42013-09-17 03:30:31 +0000259 return Max;
260}
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000261
Tobias Grosser0695ee42013-09-17 03:30:31 +0000262__isl_give isl_pw_aff *SCEVAffinator::visitUMaxExpr(const SCEVUMaxExpr *Expr) {
263 llvm_unreachable("SCEVUMaxExpr not yet supported");
264}
265
266__isl_give isl_pw_aff *SCEVAffinator::visitUnknown(const SCEVUnknown *Expr) {
Tobias Grosserf4daf342014-08-16 09:08:55 +0000267 llvm_unreachable("Unknowns are always parameters");
Tobias Grosser0695ee42013-09-17 03:30:31 +0000268}
269
270int SCEVAffinator::getLoopDepth(const Loop *L) {
271 Loop *outerLoop = S->getRegion().outermostLoopInRegion(const_cast<Loop *>(L));
272 assert(outerLoop && "Scop does not contain this loop");
273 return L->getLoopDepth() - outerLoop->getLoopDepth();
274}
Tobias Grosser33ba62ad2011-08-18 06:31:50 +0000275
Johannes Doerfert1a28a892014-10-05 11:32:18 +0000276ScopArrayInfo::ScopArrayInfo(Value *BasePtr, Type *AccessType, isl_ctx *Ctx,
277 const SmallVector<const SCEV *, 4> &DimensionSizes)
278 : BasePtr(BasePtr), AccessType(AccessType), DimensionSizes(DimensionSizes) {
279 const std::string BasePtrName = getIslCompatibleName("MemRef_", BasePtr, "");
280 Id = isl_id_alloc(Ctx, BasePtrName.c_str(), this);
281}
282
283ScopArrayInfo::~ScopArrayInfo() { isl_id_free(Id); }
284
285isl_id *ScopArrayInfo::getBasePtrId() const { return isl_id_copy(Id); }
286
287void ScopArrayInfo::dump() const { print(errs()); }
288
289void ScopArrayInfo::print(raw_ostream &OS) const {
290 OS << "ScopArrayInfo:\n";
291 OS << " Base: " << *getBasePtr() << "\n";
292 OS << " Type: " << *getType() << "\n";
293 OS << " Dimension Sizes:\n";
294 for (unsigned u = 0; u < getNumberOfDimensions(); u++)
295 OS << " " << u << ") " << *DimensionSizes[u] << "\n";
296 OS << "\n";
297}
298
299const ScopArrayInfo *
300ScopArrayInfo::getFromAccessFunction(__isl_keep isl_pw_multi_aff *PMA) {
301 isl_id *Id = isl_pw_multi_aff_get_tuple_id(PMA, isl_dim_out);
302 assert(Id && "Output dimension didn't have an ID");
303 return getFromId(Id);
304}
305
306const ScopArrayInfo *ScopArrayInfo::getFromId(isl_id *Id) {
307 void *User = isl_id_get_user(Id);
308 const ScopArrayInfo *SAI = static_cast<ScopArrayInfo *>(User);
309 isl_id_free(Id);
310 return SAI;
311}
312
Johannes Doerfert32868bf2014-08-01 08:13:25 +0000313const std::string
314MemoryAccess::getReductionOperatorStr(MemoryAccess::ReductionType RT) {
315 switch (RT) {
316 case MemoryAccess::RT_NONE:
317 llvm_unreachable("Requested a reduction operator string for a memory "
318 "access which isn't a reduction");
319 case MemoryAccess::RT_ADD:
320 return "+";
321 case MemoryAccess::RT_MUL:
322 return "*";
323 case MemoryAccess::RT_BOR:
324 return "|";
325 case MemoryAccess::RT_BXOR:
326 return "^";
327 case MemoryAccess::RT_BAND:
328 return "&";
329 }
330 llvm_unreachable("Unknown reduction type");
331 return "";
332}
333
Johannes Doerfertf6183392014-07-01 20:52:51 +0000334/// @brief Return the reduction type for a given binary operator
335static MemoryAccess::ReductionType getReductionType(const BinaryOperator *BinOp,
336 const Instruction *Load) {
337 if (!BinOp)
338 return MemoryAccess::RT_NONE;
339 switch (BinOp->getOpcode()) {
340 case Instruction::FAdd:
341 if (!BinOp->hasUnsafeAlgebra())
342 return MemoryAccess::RT_NONE;
343 // Fall through
344 case Instruction::Add:
345 return MemoryAccess::RT_ADD;
346 case Instruction::Or:
347 return MemoryAccess::RT_BOR;
348 case Instruction::Xor:
349 return MemoryAccess::RT_BXOR;
350 case Instruction::And:
351 return MemoryAccess::RT_BAND;
352 case Instruction::FMul:
353 if (!BinOp->hasUnsafeAlgebra())
354 return MemoryAccess::RT_NONE;
355 // Fall through
356 case Instruction::Mul:
357 if (DisableMultiplicativeReductions)
358 return MemoryAccess::RT_NONE;
359 return MemoryAccess::RT_MUL;
360 default:
361 return MemoryAccess::RT_NONE;
362 }
363}
Tobias Grosser75805372011-04-29 06:27:02 +0000364//===----------------------------------------------------------------------===//
365
366MemoryAccess::~MemoryAccess() {
Tobias Grosser54a86e62011-08-18 06:31:46 +0000367 isl_map_free(AccessRelation);
Raghesh Aloor129e8672011-08-15 02:33:39 +0000368 isl_map_free(newAccessRelation);
Tobias Grosser75805372011-04-29 06:27:02 +0000369}
370
Johannes Doerfert8f7124c2014-09-12 11:00:49 +0000371static MemoryAccess::AccessType getMemoryAccessType(const IRAccess &Access) {
372 switch (Access.getType()) {
373 case IRAccess::READ:
374 return MemoryAccess::READ;
375 case IRAccess::MUST_WRITE:
376 return MemoryAccess::MUST_WRITE;
377 case IRAccess::MAY_WRITE:
378 return MemoryAccess::MAY_WRITE;
379 }
380 llvm_unreachable("Unknown IRAccess type!");
381}
382
Johannes Doerfert1a28a892014-10-05 11:32:18 +0000383const ScopArrayInfo *MemoryAccess::getScopArrayInfo() const {
384 isl_id *ArrayId = getArrayId();
385 void *User = isl_id_get_user(ArrayId);
386 const ScopArrayInfo *SAI = static_cast<ScopArrayInfo *>(User);
387 isl_id_free(ArrayId);
388 return SAI;
389}
390
Johannes Doerfert5d83f092014-07-29 08:37:55 +0000391isl_id *MemoryAccess::getArrayId() const {
392 return isl_map_get_tuple_id(AccessRelation, isl_dim_out);
393}
394
Johannes Doerferta99130f2014-10-13 12:58:03 +0000395isl_pw_multi_aff *
396MemoryAccess::applyScheduleToAccessRelation(isl_union_map *USchedule) const {
397 isl_map *Schedule, *ScheduledAccRel;
398 isl_union_set *UDomain;
399
400 UDomain = isl_union_set_from_set(getStatement()->getDomain());
401 USchedule = isl_union_map_intersect_domain(USchedule, UDomain);
402 Schedule = isl_map_from_union_map(USchedule);
403 ScheduledAccRel = isl_map_apply_domain(getAccessRelation(), Schedule);
404 return isl_pw_multi_aff_from_map(ScheduledAccRel);
405}
406
407isl_map *MemoryAccess::getOriginalAccessRelation() const {
Tobias Grosser5d453812011-10-06 00:04:11 +0000408 return isl_map_copy(AccessRelation);
409}
410
Johannes Doerferta99130f2014-10-13 12:58:03 +0000411std::string MemoryAccess::getOriginalAccessRelationStr() const {
Tobias Grosser5d453812011-10-06 00:04:11 +0000412 return stringFromIslObj(AccessRelation);
413}
414
Johannes Doerferta99130f2014-10-13 12:58:03 +0000415__isl_give isl_space *MemoryAccess::getOriginalAccessRelationSpace() const {
Tobias Grosser5e6813d2014-07-02 17:47:48 +0000416 return isl_map_get_space(AccessRelation);
417}
418
Tobias Grosser5d453812011-10-06 00:04:11 +0000419isl_map *MemoryAccess::getNewAccessRelation() const {
420 return isl_map_copy(newAccessRelation);
Tobias Grosser75805372011-04-29 06:27:02 +0000421}
422
423isl_basic_map *MemoryAccess::createBasicAccessMap(ScopStmt *Statement) {
Tobias Grosser084d8f72012-05-29 09:29:44 +0000424 isl_space *Space = isl_space_set_alloc(Statement->getIslCtx(), 0, 1);
Tobias Grossered295662012-09-11 13:50:21 +0000425 Space = isl_space_align_params(Space, Statement->getDomainSpace());
Tobias Grosser75805372011-04-29 06:27:02 +0000426
Tobias Grosser084d8f72012-05-29 09:29:44 +0000427 return isl_basic_map_from_domain_and_range(
Tobias Grosserabfbe632013-02-05 12:09:06 +0000428 isl_basic_set_universe(Statement->getDomainSpace()),
429 isl_basic_set_universe(Space));
Tobias Grosser75805372011-04-29 06:27:02 +0000430}
431
Tobias Grosser5e6813d2014-07-02 17:47:48 +0000432// Formalize no out-of-bound access assumption
433//
434// When delinearizing array accesses we optimistically assume that the
435// delinearized accesses do not access out of bound locations (the subscript
436// expression of each array evaluates for each statement instance that is
437// executed to a value that is larger than zero and strictly smaller than the
438// size of the corresponding dimension). The only exception is the outermost
Tobias Grosserf57d63f2014-08-03 21:07:30 +0000439// dimension for which we do not need to assume any upper bound. At this point
440// we formalize this assumption to ensure that at code generation time the
441// relevant run-time checks can be generated.
Tobias Grosser5e6813d2014-07-02 17:47:48 +0000442//
443// To find the set of constraints necessary to avoid out of bound accesses, we
444// first build the set of data locations that are not within array bounds. We
445// then apply the reverse access relation to obtain the set of iterations that
446// may contain invalid accesses and reduce this set of iterations to the ones
447// that are actually executed by intersecting them with the domain of the
448// statement. If we now project out all loop dimensions, we obtain a set of
449// parameters that may cause statement instances to be executed that may
450// possibly yield out of bound memory accesses. The complement of these
451// constraints is the set of constraints that needs to be assumed to ensure such
452// statement instances are never executed.
453void MemoryAccess::assumeNoOutOfBound(const IRAccess &Access) {
Johannes Doerferta99130f2014-10-13 12:58:03 +0000454 isl_space *Space = isl_space_range(getOriginalAccessRelationSpace());
Tobias Grosser5e6813d2014-07-02 17:47:48 +0000455 isl_set *Outside = isl_set_empty(isl_space_copy(Space));
Tobias Grosserf57d63f2014-08-03 21:07:30 +0000456 for (int i = 1, Size = Access.Subscripts.size(); i < Size; ++i) {
Tobias Grosser5e6813d2014-07-02 17:47:48 +0000457 isl_local_space *LS = isl_local_space_from_space(isl_space_copy(Space));
458 isl_pw_aff *Var =
459 isl_pw_aff_var_on_domain(isl_local_space_copy(LS), isl_dim_set, i);
460 isl_pw_aff *Zero = isl_pw_aff_zero_on_domain(LS);
461
462 isl_set *DimOutside;
463
Tobias Grosserf57d63f2014-08-03 21:07:30 +0000464 DimOutside = isl_pw_aff_lt_set(isl_pw_aff_copy(Var), Zero);
465 isl_pw_aff *SizeE = SCEVAffinator::getPwAff(Statement, Access.Sizes[i - 1]);
Tobias Grosser5e6813d2014-07-02 17:47:48 +0000466
Tobias Grosserf57d63f2014-08-03 21:07:30 +0000467 SizeE = isl_pw_aff_drop_dims(SizeE, isl_dim_in, 0,
468 Statement->getNumIterators());
469 SizeE = isl_pw_aff_add_dims(SizeE, isl_dim_in,
470 isl_space_dim(Space, isl_dim_set));
471 SizeE = isl_pw_aff_set_tuple_id(SizeE, isl_dim_in,
472 isl_space_get_tuple_id(Space, isl_dim_set));
Tobias Grosser5e6813d2014-07-02 17:47:48 +0000473
Tobias Grosserf57d63f2014-08-03 21:07:30 +0000474 DimOutside = isl_set_union(DimOutside, isl_pw_aff_le_set(SizeE, Var));
Tobias Grosser5e6813d2014-07-02 17:47:48 +0000475
476 Outside = isl_set_union(Outside, DimOutside);
477 }
478
479 Outside = isl_set_apply(Outside, isl_map_reverse(getAccessRelation()));
480 Outside = isl_set_intersect(Outside, Statement->getDomain());
481 Outside = isl_set_params(Outside);
482 Outside = isl_set_complement(Outside);
483 Statement->getParent()->addAssumption(Outside);
484 isl_space_free(Space);
485}
486
Johannes Doerfert13c8cf22014-08-10 08:09:38 +0000487MemoryAccess::MemoryAccess(const IRAccess &Access, Instruction *AccInst,
Johannes Doerfert1a28a892014-10-05 11:32:18 +0000488 ScopStmt *Statement, const ScopArrayInfo *SAI)
Johannes Doerfert4c7ce472014-10-08 10:11:33 +0000489 : AccType(getMemoryAccessType(Access)), Statement(Statement), Inst(AccInst),
Johannes Doerfert8f7124c2014-09-12 11:00:49 +0000490 newAccessRelation(nullptr) {
Tobias Grosser75805372011-04-29 06:27:02 +0000491
Johannes Doerfert5d83f092014-07-29 08:37:55 +0000492 isl_ctx *Ctx = Statement->getIslCtx();
Tobias Grosser9759f852011-11-10 12:44:55 +0000493 BaseAddr = Access.getBase();
Johannes Doerfert79fc23f2014-07-24 23:48:02 +0000494 BaseName = getIslCompatibleName("MemRef_", getBaseAddr(), "");
Johannes Doerfert1a28a892014-10-05 11:32:18 +0000495
496 isl_id *BaseAddrId = SAI->getBasePtrId();
Tobias Grosser5683df42011-11-09 22:34:34 +0000497
Tobias Grossera1879642011-12-20 10:43:14 +0000498 if (!Access.isAffine()) {
Tobias Grosser4f967492013-06-23 05:21:18 +0000499 // We overapproximate non-affine accesses with a possible access to the
500 // whole array. For read accesses it does not make a difference, if an
501 // access must or may happen. However, for write accesses it is important to
502 // differentiate between writes that must happen and writes that may happen.
Tobias Grosser04d6ae62013-06-23 06:04:54 +0000503 AccessRelation = isl_map_from_basic_map(createBasicAccessMap(Statement));
Johannes Doerfert5d83f092014-07-29 08:37:55 +0000504 AccessRelation =
505 isl_map_set_tuple_id(AccessRelation, isl_dim_out, BaseAddrId);
Tobias Grossera1879642011-12-20 10:43:14 +0000506 return;
507 }
508
Johannes Doerfert5d83f092014-07-29 08:37:55 +0000509 isl_space *Space = isl_space_alloc(Ctx, 0, Statement->getNumIterators(), 0);
Tobias Grosser79baa212014-04-10 08:38:02 +0000510 AccessRelation = isl_map_universe(Space);
Tobias Grossera1879642011-12-20 10:43:14 +0000511
Tobias Grosser79baa212014-04-10 08:38:02 +0000512 for (int i = 0, Size = Access.Subscripts.size(); i < Size; ++i) {
Sebastian Pop18016682014-04-08 21:20:44 +0000513 isl_pw_aff *Affine =
514 SCEVAffinator::getPwAff(Statement, Access.Subscripts[i]);
Tobias Grosser75805372011-04-29 06:27:02 +0000515
Sebastian Pop422e33f2014-06-03 18:16:31 +0000516 if (Size == 1) {
517 // For the non delinearized arrays, divide the access function of the last
518 // subscript by the size of the elements in the array.
Sebastian Pop18016682014-04-08 21:20:44 +0000519 //
520 // A stride one array access in C expressed as A[i] is expressed in
521 // LLVM-IR as something like A[i * elementsize]. This hides the fact that
522 // two subsequent values of 'i' index two values that are stored next to
523 // each other in memory. By this division we make this characteristic
524 // obvious again.
Johannes Doerfert5d83f092014-07-29 08:37:55 +0000525 isl_val *v = isl_val_int_from_si(Ctx, Access.getElemSizeInBytes());
Sebastian Pop18016682014-04-08 21:20:44 +0000526 Affine = isl_pw_aff_scale_down_val(Affine, v);
527 }
528
529 isl_map *SubscriptMap = isl_map_from_pw_aff(Affine);
530
Tobias Grosser79baa212014-04-10 08:38:02 +0000531 AccessRelation = isl_map_flat_range_product(AccessRelation, SubscriptMap);
Sebastian Pop18016682014-04-08 21:20:44 +0000532 }
533
Tobias Grosser79baa212014-04-10 08:38:02 +0000534 Space = Statement->getDomainSpace();
Tobias Grosserabfbe632013-02-05 12:09:06 +0000535 AccessRelation = isl_map_set_tuple_id(
536 AccessRelation, isl_dim_in, isl_space_get_tuple_id(Space, isl_dim_set));
Johannes Doerfert5d83f092014-07-29 08:37:55 +0000537 AccessRelation =
538 isl_map_set_tuple_id(AccessRelation, isl_dim_out, BaseAddrId);
539
Tobias Grosser5e6813d2014-07-02 17:47:48 +0000540 assumeNoOutOfBound(Access);
Johannes Doerfert5d83f092014-07-29 08:37:55 +0000541 isl_space_free(Space);
Tobias Grosser8cae72f2011-11-08 15:41:08 +0000542}
Tobias Grosser30b8a092011-08-18 07:51:37 +0000543
Tobias Grosser8cae72f2011-11-08 15:41:08 +0000544void MemoryAccess::realignParams() {
Tobias Grosser6defb5b2014-04-10 08:37:44 +0000545 isl_space *ParamSpace = Statement->getParent()->getParamSpace();
Tobias Grosser37487052011-10-06 00:03:42 +0000546 AccessRelation = isl_map_align_params(AccessRelation, ParamSpace);
Tobias Grosser75805372011-04-29 06:27:02 +0000547}
548
Johannes Doerfert32868bf2014-08-01 08:13:25 +0000549const std::string MemoryAccess::getReductionOperatorStr() const {
550 return MemoryAccess::getReductionOperatorStr(getReductionType());
551}
552
Johannes Doerfertf6183392014-07-01 20:52:51 +0000553raw_ostream &polly::operator<<(raw_ostream &OS,
554 MemoryAccess::ReductionType RT) {
Johannes Doerfert32868bf2014-08-01 08:13:25 +0000555 if (RT == MemoryAccess::RT_NONE)
Johannes Doerfertf6183392014-07-01 20:52:51 +0000556 OS << "NONE";
Johannes Doerfert32868bf2014-08-01 08:13:25 +0000557 else
558 OS << MemoryAccess::getReductionOperatorStr(RT);
Johannes Doerfertf6183392014-07-01 20:52:51 +0000559 return OS;
560}
561
Tobias Grosser75805372011-04-29 06:27:02 +0000562void MemoryAccess::print(raw_ostream &OS) const {
Johannes Doerfert4c7ce472014-10-08 10:11:33 +0000563 switch (AccType) {
Tobias Grosserb58f6a42013-07-13 20:41:24 +0000564 case READ:
Johannes Doerfert6780bc32014-06-26 18:47:03 +0000565 OS.indent(12) << "ReadAccess :=\t";
Tobias Grosser4f967492013-06-23 05:21:18 +0000566 break;
Tobias Grosserb58f6a42013-07-13 20:41:24 +0000567 case MUST_WRITE:
Johannes Doerfert6780bc32014-06-26 18:47:03 +0000568 OS.indent(12) << "MustWriteAccess :=\t";
Tobias Grosser4f967492013-06-23 05:21:18 +0000569 break;
Tobias Grosserb58f6a42013-07-13 20:41:24 +0000570 case MAY_WRITE:
Johannes Doerfert6780bc32014-06-26 18:47:03 +0000571 OS.indent(12) << "MayWriteAccess :=\t";
Tobias Grosser4f967492013-06-23 05:21:18 +0000572 break;
573 }
Johannes Doerfertf6183392014-07-01 20:52:51 +0000574 OS << "[Reduction Type: " << getReductionType() << "]\n";
Johannes Doerferta99130f2014-10-13 12:58:03 +0000575 OS.indent(16) << getOriginalAccessRelationStr() << ";\n";
Tobias Grosser75805372011-04-29 06:27:02 +0000576}
577
Tobias Grosser74394f02013-01-14 22:40:23 +0000578void MemoryAccess::dump() const { print(errs()); }
Tobias Grosser75805372011-04-29 06:27:02 +0000579
580// Create a map in the size of the provided set domain, that maps from the
581// one element of the provided set domain to another element of the provided
582// set domain.
583// The mapping is limited to all points that are equal in all but the last
584// dimension and for which the last dimension of the input is strict smaller
585// than the last dimension of the output.
586//
587// getEqualAndLarger(set[i0, i1, ..., iX]):
588//
589// set[i0, i1, ..., iX] -> set[o0, o1, ..., oX]
590// : i0 = o0, i1 = o1, ..., i(X-1) = o(X-1), iX < oX
591//
Tobias Grosserf5338802011-10-06 00:03:35 +0000592static isl_map *getEqualAndLarger(isl_space *setDomain) {
Tobias Grosserc327932c2012-02-01 14:23:36 +0000593 isl_space *Space = isl_space_map_from_set(setDomain);
594 isl_map *Map = isl_map_universe(isl_space_copy(Space));
595 isl_local_space *MapLocalSpace = isl_local_space_from_space(Space);
Sebastian Pop40408762013-10-04 17:14:53 +0000596 unsigned lastDimension = isl_map_dim(Map, isl_dim_in) - 1;
Tobias Grosser75805372011-04-29 06:27:02 +0000597
598 // Set all but the last dimension to be equal for the input and output
599 //
600 // input[i0, i1, ..., iX] -> output[o0, o1, ..., oX]
601 // : i0 = o0, i1 = o1, ..., i(X-1) = o(X-1)
Sebastian Pop40408762013-10-04 17:14:53 +0000602 for (unsigned i = 0; i < lastDimension; ++i)
Tobias Grosserc327932c2012-02-01 14:23:36 +0000603 Map = isl_map_equate(Map, isl_dim_in, i, isl_dim_out, i);
Tobias Grosser75805372011-04-29 06:27:02 +0000604
605 // Set the last dimension of the input to be strict smaller than the
606 // last dimension of the output.
607 //
608 // input[?,?,?,...,iX] -> output[?,?,?,...,oX] : iX < oX
609 //
Tobias Grosseredab1352013-06-21 06:41:31 +0000610 isl_val *v;
611 isl_ctx *Ctx = isl_map_get_ctx(Map);
Tobias Grosserf5338802011-10-06 00:03:35 +0000612 isl_constraint *c = isl_inequality_alloc(isl_local_space_copy(MapLocalSpace));
Tobias Grosseredab1352013-06-21 06:41:31 +0000613 v = isl_val_int_from_si(Ctx, -1);
614 c = isl_constraint_set_coefficient_val(c, isl_dim_in, lastDimension, v);
615 v = isl_val_int_from_si(Ctx, 1);
616 c = isl_constraint_set_coefficient_val(c, isl_dim_out, lastDimension, v);
617 v = isl_val_int_from_si(Ctx, -1);
618 c = isl_constraint_set_constant_val(c, v);
Tobias Grosser75805372011-04-29 06:27:02 +0000619
Tobias Grosserc327932c2012-02-01 14:23:36 +0000620 Map = isl_map_add_constraint(Map, c);
Tobias Grosser75805372011-04-29 06:27:02 +0000621
Tobias Grosser23b36662011-10-17 08:32:36 +0000622 isl_local_space_free(MapLocalSpace);
Tobias Grosserc327932c2012-02-01 14:23:36 +0000623 return Map;
Tobias Grosser75805372011-04-29 06:27:02 +0000624}
625
Sebastian Popa00a0292012-12-18 07:46:06 +0000626isl_set *MemoryAccess::getStride(__isl_take const isl_map *Schedule) const {
Tobias Grosserabfbe632013-02-05 12:09:06 +0000627 isl_map *S = const_cast<isl_map *>(Schedule);
Johannes Doerferta99130f2014-10-13 12:58:03 +0000628 isl_map *AccessRelation = getAccessRelation();
Sebastian Popa00a0292012-12-18 07:46:06 +0000629 isl_space *Space = isl_space_range(isl_map_get_space(S));
630 isl_map *NextScatt = getEqualAndLarger(Space);
Tobias Grosser75805372011-04-29 06:27:02 +0000631
Sebastian Popa00a0292012-12-18 07:46:06 +0000632 S = isl_map_reverse(S);
633 NextScatt = isl_map_lexmin(NextScatt);
Tobias Grosser75805372011-04-29 06:27:02 +0000634
Sebastian Popa00a0292012-12-18 07:46:06 +0000635 NextScatt = isl_map_apply_range(NextScatt, isl_map_copy(S));
636 NextScatt = isl_map_apply_range(NextScatt, isl_map_copy(AccessRelation));
637 NextScatt = isl_map_apply_domain(NextScatt, S);
638 NextScatt = isl_map_apply_domain(NextScatt, AccessRelation);
Tobias Grosser75805372011-04-29 06:27:02 +0000639
Sebastian Popa00a0292012-12-18 07:46:06 +0000640 isl_set *Deltas = isl_map_deltas(NextScatt);
641 return Deltas;
Tobias Grosser75805372011-04-29 06:27:02 +0000642}
643
Sebastian Popa00a0292012-12-18 07:46:06 +0000644bool MemoryAccess::isStrideX(__isl_take const isl_map *Schedule,
Tobias Grosser28dd4862012-01-24 16:42:16 +0000645 int StrideWidth) const {
646 isl_set *Stride, *StrideX;
647 bool IsStrideX;
Tobias Grosser75805372011-04-29 06:27:02 +0000648
Sebastian Popa00a0292012-12-18 07:46:06 +0000649 Stride = getStride(Schedule);
Tobias Grosser28dd4862012-01-24 16:42:16 +0000650 StrideX = isl_set_universe(isl_set_get_space(Stride));
651 StrideX = isl_set_fix_si(StrideX, isl_dim_set, 0, StrideWidth);
652 IsStrideX = isl_set_is_equal(Stride, StrideX);
Tobias Grosser75805372011-04-29 06:27:02 +0000653
Tobias Grosser28dd4862012-01-24 16:42:16 +0000654 isl_set_free(StrideX);
Tobias Grosserdea98232012-01-17 20:34:27 +0000655 isl_set_free(Stride);
Tobias Grosserb76f38532011-08-20 11:11:25 +0000656
Tobias Grosser28dd4862012-01-24 16:42:16 +0000657 return IsStrideX;
658}
659
Sebastian Popa00a0292012-12-18 07:46:06 +0000660bool MemoryAccess::isStrideZero(const isl_map *Schedule) const {
661 return isStrideX(Schedule, 0);
Tobias Grosser75805372011-04-29 06:27:02 +0000662}
663
Tobias Grosser79baa212014-04-10 08:38:02 +0000664bool MemoryAccess::isScalar() const {
665 return isl_map_n_out(AccessRelation) == 0;
666}
667
Sebastian Popa00a0292012-12-18 07:46:06 +0000668bool MemoryAccess::isStrideOne(const isl_map *Schedule) const {
669 return isStrideX(Schedule, 1);
Tobias Grosser75805372011-04-29 06:27:02 +0000670}
671
Tobias Grosser5d453812011-10-06 00:04:11 +0000672void MemoryAccess::setNewAccessRelation(isl_map *newAccess) {
Tobias Grosserb76f38532011-08-20 11:11:25 +0000673 isl_map_free(newAccessRelation);
Raghesh Aloor7a04f4f2011-08-03 13:47:59 +0000674 newAccessRelation = newAccess;
Raghesh Aloor3cb66282011-07-12 17:14:03 +0000675}
Tobias Grosser75805372011-04-29 06:27:02 +0000676
677//===----------------------------------------------------------------------===//
Tobias Grossercf3942d2011-10-06 00:04:05 +0000678
Tobias Grosser74394f02013-01-14 22:40:23 +0000679isl_map *ScopStmt::getScattering() const { return isl_map_copy(Scattering); }
Tobias Grossercf3942d2011-10-06 00:04:05 +0000680
Tobias Grosser37eb4222014-02-20 21:43:54 +0000681void ScopStmt::restrictDomain(__isl_take isl_set *NewDomain) {
682 assert(isl_set_is_subset(NewDomain, Domain) &&
683 "New domain is not a subset of old domain!");
684 isl_set_free(Domain);
685 Domain = NewDomain;
686 Scattering = isl_map_intersect_domain(Scattering, isl_set_copy(Domain));
687}
688
Tobias Grossercf3942d2011-10-06 00:04:05 +0000689void ScopStmt::setScattering(isl_map *NewScattering) {
Tobias Grosser5a56cbf2014-04-16 07:33:47 +0000690 assert(NewScattering && "New scattering is nullptr");
Tobias Grosserb76f38532011-08-20 11:11:25 +0000691 isl_map_free(Scattering);
Tobias Grossercf3942d2011-10-06 00:04:05 +0000692 Scattering = NewScattering;
Tobias Grosserb76f38532011-08-20 11:11:25 +0000693}
694
Tobias Grosser75805372011-04-29 06:27:02 +0000695void ScopStmt::buildScattering(SmallVectorImpl<unsigned> &Scatter) {
Tobias Grosser78d8a3d2012-01-17 20:34:23 +0000696 unsigned NbIterators = getNumIterators();
697 unsigned NbScatteringDims = Parent.getMaxLoopDepth() * 2 + 1;
698
Tobias Grosser084d8f72012-05-29 09:29:44 +0000699 isl_space *Space = isl_space_set_alloc(getIslCtx(), 0, NbScatteringDims);
Tobias Grosserf5338802011-10-06 00:03:35 +0000700 Space = isl_space_set_tuple_name(Space, isl_dim_out, "scattering");
Tobias Grosser78d8a3d2012-01-17 20:34:23 +0000701
Tobias Grosser084d8f72012-05-29 09:29:44 +0000702 Scattering = isl_map_from_domain_and_range(isl_set_universe(getDomainSpace()),
703 isl_set_universe(Space));
Tobias Grosser75805372011-04-29 06:27:02 +0000704
705 // Loop dimensions.
Tobias Grosser78d8a3d2012-01-17 20:34:23 +0000706 for (unsigned i = 0; i < NbIterators; ++i)
Tobias Grosserabfbe632013-02-05 12:09:06 +0000707 Scattering =
708 isl_map_equate(Scattering, isl_dim_out, 2 * i + 1, isl_dim_in, i);
Tobias Grosser75805372011-04-29 06:27:02 +0000709
710 // Constant dimensions
Tobias Grosser78d8a3d2012-01-17 20:34:23 +0000711 for (unsigned i = 0; i < NbIterators + 1; ++i)
712 Scattering = isl_map_fix_si(Scattering, isl_dim_out, 2 * i, Scatter[i]);
Tobias Grosser75805372011-04-29 06:27:02 +0000713
714 // Fill scattering dimensions.
Tobias Grosser78d8a3d2012-01-17 20:34:23 +0000715 for (unsigned i = 2 * NbIterators + 1; i < NbScatteringDims; ++i)
716 Scattering = isl_map_fix_si(Scattering, isl_dim_out, i, 0);
Tobias Grosser75805372011-04-29 06:27:02 +0000717
Tobias Grosser37487052011-10-06 00:03:42 +0000718 Scattering = isl_map_align_params(Scattering, Parent.getParamSpace());
Tobias Grosser75805372011-04-29 06:27:02 +0000719}
720
Johannes Doerfert75bd66e2014-10-31 23:16:02 +0000721void ScopStmt::buildAccesses(TempScop &tempScop) {
Johannes Doerfert1a28a892014-10-05 11:32:18 +0000722 for (const auto &AccessPair : *tempScop.getAccessFunctions(BB)) {
723 const IRAccess &Access = AccessPair.first;
724 Instruction *AccessInst = AccessPair.second;
725
Johannes Doerfert80ef1102014-11-07 08:31:31 +0000726 Type *AccessType = getAccessInstType(AccessInst)->getPointerTo();
727 const ScopArrayInfo *SAI = getParent()->getOrCreateScopArrayInfo(
728 Access.getBase(), AccessType, Access.Sizes);
729
Johannes Doerfert1a28a892014-10-05 11:32:18 +0000730 MemAccs.push_back(new MemoryAccess(Access, AccessInst, this, SAI));
Tobias Grosserd6aafa72014-02-20 21:29:09 +0000731
732 // We do not track locations for scalar memory accesses at the moment.
733 //
734 // We do not have a use for this information at the moment. If we need this
735 // at some point, the "instruction -> access" mapping needs to be enhanced
736 // as a single instruction could then possibly perform multiple accesses.
Johannes Doerfert1a28a892014-10-05 11:32:18 +0000737 if (!Access.isScalar()) {
738 assert(!InstructionToAccess.count(AccessInst) &&
Tobias Grosser3fc91542014-02-20 21:43:45 +0000739 "Unexpected 1-to-N mapping on instruction to access map!");
Johannes Doerfert1a28a892014-10-05 11:32:18 +0000740 InstructionToAccess[AccessInst] = MemAccs.back();
Tobias Grosserd6aafa72014-02-20 21:29:09 +0000741 }
Tobias Grosser75805372011-04-29 06:27:02 +0000742 }
743}
744
Tobias Grosser8cae72f2011-11-08 15:41:08 +0000745void ScopStmt::realignParams() {
Johannes Doerfertf6752892014-06-13 18:01:45 +0000746 for (MemoryAccess *MA : *this)
747 MA->realignParams();
Tobias Grosser8cae72f2011-11-08 15:41:08 +0000748
749 Domain = isl_set_align_params(Domain, Parent.getParamSpace());
750 Scattering = isl_map_align_params(Scattering, Parent.getParamSpace());
751}
752
Tobias Grosser65b00582011-11-08 15:41:19 +0000753__isl_give isl_set *ScopStmt::buildConditionSet(const Comparison &Comp) {
Tobias Grossera601fbd2011-11-09 22:34:44 +0000754 isl_pw_aff *L = SCEVAffinator::getPwAff(this, Comp.getLHS());
755 isl_pw_aff *R = SCEVAffinator::getPwAff(this, Comp.getRHS());
Tobias Grosser75805372011-04-29 06:27:02 +0000756
Tobias Grosserd2795d02011-08-18 07:51:40 +0000757 switch (Comp.getPred()) {
Tobias Grosser75805372011-04-29 06:27:02 +0000758 case ICmpInst::ICMP_EQ:
Tobias Grosser048c8792011-10-23 20:59:20 +0000759 return isl_pw_aff_eq_set(L, R);
Tobias Grosser75805372011-04-29 06:27:02 +0000760 case ICmpInst::ICMP_NE:
Tobias Grosser048c8792011-10-23 20:59:20 +0000761 return isl_pw_aff_ne_set(L, R);
Tobias Grosser75805372011-04-29 06:27:02 +0000762 case ICmpInst::ICMP_SLT:
Tobias Grosser048c8792011-10-23 20:59:20 +0000763 return isl_pw_aff_lt_set(L, R);
Tobias Grosser75805372011-04-29 06:27:02 +0000764 case ICmpInst::ICMP_SLE:
Tobias Grosser048c8792011-10-23 20:59:20 +0000765 return isl_pw_aff_le_set(L, R);
Tobias Grosserd2795d02011-08-18 07:51:40 +0000766 case ICmpInst::ICMP_SGT:
Tobias Grosser048c8792011-10-23 20:59:20 +0000767 return isl_pw_aff_gt_set(L, R);
Tobias Grosser75805372011-04-29 06:27:02 +0000768 case ICmpInst::ICMP_SGE:
Tobias Grosser048c8792011-10-23 20:59:20 +0000769 return isl_pw_aff_ge_set(L, R);
Tobias Grosserd2795d02011-08-18 07:51:40 +0000770 case ICmpInst::ICMP_ULT:
771 case ICmpInst::ICMP_UGT:
772 case ICmpInst::ICMP_ULE:
Tobias Grosser75805372011-04-29 06:27:02 +0000773 case ICmpInst::ICMP_UGE:
Tobias Grosserd2795d02011-08-18 07:51:40 +0000774 llvm_unreachable("Unsigned comparisons not yet supported");
Tobias Grosser75805372011-04-29 06:27:02 +0000775 default:
776 llvm_unreachable("Non integer predicate not supported");
777 }
Tobias Grosser75805372011-04-29 06:27:02 +0000778}
779
Tobias Grossere19661e2011-10-07 08:46:57 +0000780__isl_give isl_set *ScopStmt::addLoopBoundsToDomain(__isl_take isl_set *Domain,
Tobias Grosser60b54f12011-11-08 15:41:28 +0000781 TempScop &tempScop) {
Tobias Grossere19661e2011-10-07 08:46:57 +0000782 isl_space *Space;
783 isl_local_space *LocalSpace;
Tobias Grosser75805372011-04-29 06:27:02 +0000784
Tobias Grossere19661e2011-10-07 08:46:57 +0000785 Space = isl_set_get_space(Domain);
786 LocalSpace = isl_local_space_from_space(Space);
Tobias Grosserf5338802011-10-06 00:03:35 +0000787
Johannes Doerfert5ad8a6a2014-11-01 01:14:56 +0000788 ScalarEvolution *SE = getParent()->getSE();
Tobias Grosser75805372011-04-29 06:27:02 +0000789 for (int i = 0, e = getNumIterators(); i != e; ++i) {
Tobias Grosser9b13d3d2011-10-06 22:32:58 +0000790 isl_aff *Zero = isl_aff_zero_on_domain(isl_local_space_copy(LocalSpace));
Tobias Grosserabfbe632013-02-05 12:09:06 +0000791 isl_pw_aff *IV =
792 isl_pw_aff_from_aff(isl_aff_set_coefficient_si(Zero, isl_dim_in, i, 1));
Tobias Grosser75805372011-04-29 06:27:02 +0000793
Tobias Grosser9b13d3d2011-10-06 22:32:58 +0000794 // 0 <= IV.
795 isl_set *LowerBound = isl_pw_aff_nonneg_set(isl_pw_aff_copy(IV));
796 Domain = isl_set_intersect(Domain, LowerBound);
797
798 // IV <= LatchExecutions.
Hongbin Zheng27f3afb2011-04-30 03:26:51 +0000799 const Loop *L = getLoopForDimension(i);
Johannes Doerfert5ad8a6a2014-11-01 01:14:56 +0000800 const SCEV *LatchExecutions = SE->getBackedgeTakenCount(L);
Tobias Grosser9b13d3d2011-10-06 22:32:58 +0000801 isl_pw_aff *UpperBound = SCEVAffinator::getPwAff(this, LatchExecutions);
802 isl_set *UpperBoundSet = isl_pw_aff_le_set(IV, UpperBound);
Tobias Grosser75805372011-04-29 06:27:02 +0000803 Domain = isl_set_intersect(Domain, UpperBoundSet);
804 }
805
Tobias Grosserf5338802011-10-06 00:03:35 +0000806 isl_local_space_free(LocalSpace);
Tobias Grossere19661e2011-10-07 08:46:57 +0000807 return Domain;
Tobias Grosser75805372011-04-29 06:27:02 +0000808}
809
Tobias Grossere602a072013-05-07 07:30:56 +0000810__isl_give isl_set *ScopStmt::addConditionsToDomain(__isl_take isl_set *Domain,
811 TempScop &tempScop,
812 const Region &CurRegion) {
Tobias Grossere19661e2011-10-07 08:46:57 +0000813 const Region *TopRegion = tempScop.getMaxRegion().getParent(),
Tobias Grosserd7e58642013-04-10 06:55:45 +0000814 *CurrentRegion = &CurRegion;
Tobias Grossere19661e2011-10-07 08:46:57 +0000815 const BasicBlock *BranchingBB = BB;
Tobias Grosser75805372011-04-29 06:27:02 +0000816
Tobias Grosser75805372011-04-29 06:27:02 +0000817 do {
Tobias Grossere19661e2011-10-07 08:46:57 +0000818 if (BranchingBB != CurrentRegion->getEntry()) {
819 if (const BBCond *Condition = tempScop.getBBCond(BranchingBB))
Tobias Grosser083d3d32014-06-28 08:59:45 +0000820 for (const auto &C : *Condition) {
821 isl_set *ConditionSet = buildConditionSet(C);
Tobias Grossere19661e2011-10-07 08:46:57 +0000822 Domain = isl_set_intersect(Domain, ConditionSet);
Tobias Grosser75805372011-04-29 06:27:02 +0000823 }
824 }
Tobias Grossere19661e2011-10-07 08:46:57 +0000825 BranchingBB = CurrentRegion->getEntry();
826 CurrentRegion = CurrentRegion->getParent();
827 } while (TopRegion != CurrentRegion);
Tobias Grosser75805372011-04-29 06:27:02 +0000828
Tobias Grossere19661e2011-10-07 08:46:57 +0000829 return Domain;
Tobias Grosser75805372011-04-29 06:27:02 +0000830}
831
Tobias Grossere602a072013-05-07 07:30:56 +0000832__isl_give isl_set *ScopStmt::buildDomain(TempScop &tempScop,
833 const Region &CurRegion) {
Tobias Grossere19661e2011-10-07 08:46:57 +0000834 isl_space *Space;
835 isl_set *Domain;
Tobias Grosser084d8f72012-05-29 09:29:44 +0000836 isl_id *Id;
Tobias Grossere19661e2011-10-07 08:46:57 +0000837
838 Space = isl_space_set_alloc(getIslCtx(), 0, getNumIterators());
839
Tobias Grosser084d8f72012-05-29 09:29:44 +0000840 Id = isl_id_alloc(getIslCtx(), getBaseName(), this);
841
Tobias Grossere19661e2011-10-07 08:46:57 +0000842 Domain = isl_set_universe(Space);
Tobias Grossere19661e2011-10-07 08:46:57 +0000843 Domain = addLoopBoundsToDomain(Domain, tempScop);
844 Domain = addConditionsToDomain(Domain, tempScop, CurRegion);
Tobias Grosser084d8f72012-05-29 09:29:44 +0000845 Domain = isl_set_set_tuple_id(Domain, Id);
Tobias Grossere19661e2011-10-07 08:46:57 +0000846
847 return Domain;
Tobias Grosser75805372011-04-29 06:27:02 +0000848}
849
Tobias Grosser7b50bee2014-11-25 10:51:12 +0000850void ScopStmt::deriveAssumptionsFromGEP(GetElementPtrInst *GEP) {
851 int Dimension = 0;
852 isl_ctx *Ctx = Parent.getIslCtx();
853 isl_local_space *LSpace = isl_local_space_from_space(getDomainSpace());
854 Type *Ty = GEP->getPointerOperandType();
855 ScalarEvolution &SE = *Parent.getSE();
856
857 if (auto *PtrTy = dyn_cast<PointerType>(Ty)) {
858 Dimension = 1;
859 Ty = PtrTy->getElementType();
860 }
861
862 while (auto ArrayTy = dyn_cast<ArrayType>(Ty)) {
863 unsigned int Operand = 1 + Dimension;
864
865 if (GEP->getNumOperands() <= Operand)
866 break;
867
868 const SCEV *Expr = SE.getSCEV(GEP->getOperand(Operand));
869
870 if (isAffineExpr(&Parent.getRegion(), Expr, SE)) {
871 isl_pw_aff *AccessOffset = SCEVAffinator::getPwAff(this, Expr);
872 AccessOffset =
873 isl_pw_aff_set_tuple_id(AccessOffset, isl_dim_in, getDomainId());
874
875 isl_pw_aff *DimSize = isl_pw_aff_from_aff(isl_aff_val_on_domain(
876 isl_local_space_copy(LSpace),
877 isl_val_int_from_si(Ctx, ArrayTy->getNumElements())));
878
879 isl_set *OutOfBound = isl_pw_aff_ge_set(AccessOffset, DimSize);
880 OutOfBound = isl_set_intersect(getDomain(), OutOfBound);
881 OutOfBound = isl_set_params(OutOfBound);
882 isl_set *InBound = isl_set_complement(OutOfBound);
883 isl_set *Executed = isl_set_params(getDomain());
884
885 // A => B == !A or B
886 isl_set *InBoundIfExecuted =
887 isl_set_union(isl_set_complement(Executed), InBound);
888
889 Parent.addAssumption(InBoundIfExecuted);
890 }
891
892 Dimension += 1;
893 Ty = ArrayTy->getElementType();
894 }
895
896 isl_local_space_free(LSpace);
897}
898
899void ScopStmt::deriveAssumptions() {
900 for (Instruction &Inst : *BB)
901 if (auto *GEP = dyn_cast<GetElementPtrInst>(&Inst))
902 deriveAssumptionsFromGEP(GEP);
903}
904
Tobias Grosser74394f02013-01-14 22:40:23 +0000905ScopStmt::ScopStmt(Scop &parent, TempScop &tempScop, const Region &CurRegion,
Sebastian Pop860e0212013-02-15 21:26:44 +0000906 BasicBlock &bb, SmallVectorImpl<Loop *> &Nest,
Tobias Grosser75805372011-04-29 06:27:02 +0000907 SmallVectorImpl<unsigned> &Scatter)
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000908 : Parent(parent), BB(&bb), IVS(Nest.size()), NestLoops(Nest.size()) {
Tobias Grosser75805372011-04-29 06:27:02 +0000909 // Setup the induction variables.
Sebastian Pop860e0212013-02-15 21:26:44 +0000910 for (unsigned i = 0, e = Nest.size(); i < e; ++i) {
Sebastian Pop27c10c62013-03-22 22:07:43 +0000911 if (!SCEVCodegen) {
912 PHINode *PN = Nest[i]->getCanonicalInductionVariable();
913 assert(PN && "Non canonical IV in Scop!");
Tobias Grosser826b2af2013-03-21 16:14:50 +0000914 IVS[i] = PN;
Sebastian Pop27c10c62013-03-22 22:07:43 +0000915 }
Sebastian Pop860e0212013-02-15 21:26:44 +0000916 NestLoops[i] = Nest[i];
Tobias Grosser75805372011-04-29 06:27:02 +0000917 }
918
Johannes Doerfert79fc23f2014-07-24 23:48:02 +0000919 BaseName = getIslCompatibleName("Stmt_", &bb, "");
Tobias Grosser75805372011-04-29 06:27:02 +0000920
Tobias Grossere19661e2011-10-07 08:46:57 +0000921 Domain = buildDomain(tempScop, CurRegion);
Tobias Grosser75805372011-04-29 06:27:02 +0000922 buildScattering(Scatter);
Johannes Doerfert75bd66e2014-10-31 23:16:02 +0000923 buildAccesses(tempScop);
Johannes Doerferte58a0122014-06-27 20:31:28 +0000924 checkForReductions();
Tobias Grosser7b50bee2014-11-25 10:51:12 +0000925 deriveAssumptions();
Johannes Doerfert0ee1f212014-06-17 17:31:36 +0000926}
927
Johannes Doerferte58a0122014-06-27 20:31:28 +0000928/// @brief Collect loads which might form a reduction chain with @p StoreMA
929///
930/// Check if the stored value for @p StoreMA is a binary operator with one or
931/// two loads as operands. If the binary operand is commutative & associative,
932/// used only once (by @p StoreMA) and its load operands are also used only
933/// once, we have found a possible reduction chain. It starts at an operand
934/// load and includes the binary operator and @p StoreMA.
935///
936/// Note: We allow only one use to ensure the load and binary operator cannot
937/// escape this block or into any other store except @p StoreMA.
938void ScopStmt::collectCandiateReductionLoads(
939 MemoryAccess *StoreMA, SmallVectorImpl<MemoryAccess *> &Loads) {
940 auto *Store = dyn_cast<StoreInst>(StoreMA->getAccessInstruction());
941 if (!Store)
Johannes Doerfert0ee1f212014-06-17 17:31:36 +0000942 return;
943
944 // Skip if there is not one binary operator between the load and the store
945 auto *BinOp = dyn_cast<BinaryOperator>(Store->getValueOperand());
Johannes Doerferte58a0122014-06-27 20:31:28 +0000946 if (!BinOp)
947 return;
948
949 // Skip if the binary operators has multiple uses
950 if (BinOp->getNumUses() != 1)
Johannes Doerfert0ee1f212014-06-17 17:31:36 +0000951 return;
952
953 // Skip if the opcode of the binary operator is not commutative/associative
954 if (!BinOp->isCommutative() || !BinOp->isAssociative())
955 return;
956
Johannes Doerfert9890a052014-07-01 00:32:29 +0000957 // Skip if the binary operator is outside the current SCoP
958 if (BinOp->getParent() != Store->getParent())
959 return;
960
Johannes Doerfert0ee1f212014-06-17 17:31:36 +0000961 // Skip if it is a multiplicative reduction and we disabled them
962 if (DisableMultiplicativeReductions &&
963 (BinOp->getOpcode() == Instruction::Mul ||
964 BinOp->getOpcode() == Instruction::FMul))
965 return;
966
Johannes Doerferte58a0122014-06-27 20:31:28 +0000967 // Check the binary operator operands for a candidate load
968 auto *PossibleLoad0 = dyn_cast<LoadInst>(BinOp->getOperand(0));
969 auto *PossibleLoad1 = dyn_cast<LoadInst>(BinOp->getOperand(1));
970 if (!PossibleLoad0 && !PossibleLoad1)
971 return;
972
973 // A load is only a candidate if it cannot escape (thus has only this use)
974 if (PossibleLoad0 && PossibleLoad0->getNumUses() == 1)
Johannes Doerfert9890a052014-07-01 00:32:29 +0000975 if (PossibleLoad0->getParent() == Store->getParent())
976 Loads.push_back(lookupAccessFor(PossibleLoad0));
Johannes Doerferte58a0122014-06-27 20:31:28 +0000977 if (PossibleLoad1 && PossibleLoad1->getNumUses() == 1)
Johannes Doerfert9890a052014-07-01 00:32:29 +0000978 if (PossibleLoad1->getParent() == Store->getParent())
979 Loads.push_back(lookupAccessFor(PossibleLoad1));
Johannes Doerferte58a0122014-06-27 20:31:28 +0000980}
981
982/// @brief Check for reductions in this ScopStmt
983///
984/// Iterate over all store memory accesses and check for valid binary reduction
985/// like chains. For all candidates we check if they have the same base address
986/// and there are no other accesses which overlap with them. The base address
987/// check rules out impossible reductions candidates early. The overlap check,
988/// together with the "only one user" check in collectCandiateReductionLoads,
989/// guarantees that none of the intermediate results will escape during
990/// execution of the loop nest. We basically check here that no other memory
991/// access can access the same memory as the potential reduction.
992void ScopStmt::checkForReductions() {
993 SmallVector<MemoryAccess *, 2> Loads;
994 SmallVector<std::pair<MemoryAccess *, MemoryAccess *>, 4> Candidates;
995
996 // First collect candidate load-store reduction chains by iterating over all
997 // stores and collecting possible reduction loads.
998 for (MemoryAccess *StoreMA : MemAccs) {
999 if (StoreMA->isRead())
1000 continue;
1001
1002 Loads.clear();
1003 collectCandiateReductionLoads(StoreMA, Loads);
1004 for (MemoryAccess *LoadMA : Loads)
1005 Candidates.push_back(std::make_pair(LoadMA, StoreMA));
1006 }
1007
1008 // Then check each possible candidate pair.
1009 for (const auto &CandidatePair : Candidates) {
1010 bool Valid = true;
1011 isl_map *LoadAccs = CandidatePair.first->getAccessRelation();
1012 isl_map *StoreAccs = CandidatePair.second->getAccessRelation();
1013
1014 // Skip those with obviously unequal base addresses.
1015 if (!isl_map_has_equal_space(LoadAccs, StoreAccs)) {
1016 isl_map_free(LoadAccs);
1017 isl_map_free(StoreAccs);
1018 continue;
1019 }
1020
1021 // And check if the remaining for overlap with other memory accesses.
1022 isl_map *AllAccsRel = isl_map_union(LoadAccs, StoreAccs);
1023 AllAccsRel = isl_map_intersect_domain(AllAccsRel, getDomain());
1024 isl_set *AllAccs = isl_map_range(AllAccsRel);
1025
1026 for (MemoryAccess *MA : MemAccs) {
1027 if (MA == CandidatePair.first || MA == CandidatePair.second)
1028 continue;
1029
1030 isl_map *AccRel =
1031 isl_map_intersect_domain(MA->getAccessRelation(), getDomain());
1032 isl_set *Accs = isl_map_range(AccRel);
1033
1034 if (isl_set_has_equal_space(AllAccs, Accs) || isl_set_free(Accs)) {
1035 isl_set *OverlapAccs = isl_set_intersect(Accs, isl_set_copy(AllAccs));
1036 Valid = Valid && isl_set_is_empty(OverlapAccs);
1037 isl_set_free(OverlapAccs);
1038 }
1039 }
1040
1041 isl_set_free(AllAccs);
1042 if (!Valid)
1043 continue;
1044
Johannes Doerfertf6183392014-07-01 20:52:51 +00001045 const LoadInst *Load =
1046 dyn_cast<const LoadInst>(CandidatePair.first->getAccessInstruction());
1047 MemoryAccess::ReductionType RT =
1048 getReductionType(dyn_cast<BinaryOperator>(Load->user_back()), Load);
1049
Johannes Doerferte58a0122014-06-27 20:31:28 +00001050 // If no overlapping access was found we mark the load and store as
1051 // reduction like.
Johannes Doerfertf6183392014-07-01 20:52:51 +00001052 CandidatePair.first->markAsReductionLike(RT);
1053 CandidatePair.second->markAsReductionLike(RT);
Johannes Doerferte58a0122014-06-27 20:31:28 +00001054 }
Tobias Grosser75805372011-04-29 06:27:02 +00001055}
1056
Tobias Grosser74394f02013-01-14 22:40:23 +00001057std::string ScopStmt::getDomainStr() const { return stringFromIslObj(Domain); }
Tobias Grosser75805372011-04-29 06:27:02 +00001058
1059std::string ScopStmt::getScatteringStr() const {
Tobias Grossercf3942d2011-10-06 00:04:05 +00001060 return stringFromIslObj(Scattering);
Tobias Grosser75805372011-04-29 06:27:02 +00001061}
1062
Tobias Grosser74394f02013-01-14 22:40:23 +00001063unsigned ScopStmt::getNumParams() const { return Parent.getNumParams(); }
Tobias Grosser75805372011-04-29 06:27:02 +00001064
1065unsigned ScopStmt::getNumIterators() const {
1066 // The final read has one dimension with one element.
1067 if (!BB)
1068 return 1;
1069
Sebastian Pop860e0212013-02-15 21:26:44 +00001070 return NestLoops.size();
Tobias Grosser75805372011-04-29 06:27:02 +00001071}
1072
1073unsigned ScopStmt::getNumScattering() const {
1074 return isl_map_dim(Scattering, isl_dim_out);
1075}
1076
1077const char *ScopStmt::getBaseName() const { return BaseName.c_str(); }
1078
Tobias Grosserabfbe632013-02-05 12:09:06 +00001079const PHINode *
1080ScopStmt::getInductionVariableForDimension(unsigned Dimension) const {
Sebastian Popf30d3b22013-02-15 21:26:48 +00001081 return IVS[Dimension];
Hongbin Zheng27f3afb2011-04-30 03:26:51 +00001082}
1083
1084const Loop *ScopStmt::getLoopForDimension(unsigned Dimension) const {
Sebastian Pop860e0212013-02-15 21:26:44 +00001085 return NestLoops[Dimension];
Tobias Grosser75805372011-04-29 06:27:02 +00001086}
1087
Tobias Grosser74394f02013-01-14 22:40:23 +00001088isl_ctx *ScopStmt::getIslCtx() const { return Parent.getIslCtx(); }
Tobias Grosser75805372011-04-29 06:27:02 +00001089
Tobias Grosser74394f02013-01-14 22:40:23 +00001090isl_set *ScopStmt::getDomain() const { return isl_set_copy(Domain); }
Tobias Grosserd5a7bfc2011-05-06 19:52:19 +00001091
Tobias Grosser78d8a3d2012-01-17 20:34:23 +00001092isl_space *ScopStmt::getDomainSpace() const {
1093 return isl_set_get_space(Domain);
1094}
1095
Tobias Grosser74394f02013-01-14 22:40:23 +00001096isl_id *ScopStmt::getDomainId() const { return isl_set_get_tuple_id(Domain); }
Tobias Grossercd95b772012-08-30 11:49:38 +00001097
Tobias Grosser75805372011-04-29 06:27:02 +00001098ScopStmt::~ScopStmt() {
1099 while (!MemAccs.empty()) {
1100 delete MemAccs.back();
1101 MemAccs.pop_back();
1102 }
1103
1104 isl_set_free(Domain);
1105 isl_map_free(Scattering);
1106}
1107
1108void ScopStmt::print(raw_ostream &OS) const {
1109 OS << "\t" << getBaseName() << "\n";
Tobias Grosser75805372011-04-29 06:27:02 +00001110 OS.indent(12) << "Domain :=\n";
1111
1112 if (Domain) {
1113 OS.indent(16) << getDomainStr() << ";\n";
1114 } else
1115 OS.indent(16) << "n/a\n";
1116
1117 OS.indent(12) << "Scattering :=\n";
1118
1119 if (Domain) {
1120 OS.indent(16) << getScatteringStr() << ";\n";
1121 } else
1122 OS.indent(16) << "n/a\n";
1123
Tobias Grosser083d3d32014-06-28 08:59:45 +00001124 for (MemoryAccess *Access : MemAccs)
1125 Access->print(OS);
Tobias Grosser75805372011-04-29 06:27:02 +00001126}
1127
1128void ScopStmt::dump() const { print(dbgs()); }
1129
1130//===----------------------------------------------------------------------===//
1131/// Scop class implement
Tobias Grosser60b54f12011-11-08 15:41:28 +00001132
Tobias Grosser7ffe4e82011-11-17 12:56:10 +00001133void Scop::setContext(__isl_take isl_set *NewContext) {
Tobias Grosserff9b54d2011-11-15 11:38:44 +00001134 NewContext = isl_set_align_params(NewContext, isl_set_get_space(Context));
1135 isl_set_free(Context);
1136 Context = NewContext;
1137}
1138
Tobias Grosserabfbe632013-02-05 12:09:06 +00001139void Scop::addParams(std::vector<const SCEV *> NewParameters) {
Tobias Grosser083d3d32014-06-28 08:59:45 +00001140 for (const SCEV *Parameter : NewParameters) {
Tobias Grosser60b54f12011-11-08 15:41:28 +00001141 if (ParameterIds.find(Parameter) != ParameterIds.end())
1142 continue;
1143
1144 int dimension = Parameters.size();
1145
1146 Parameters.push_back(Parameter);
1147 ParameterIds[Parameter] = dimension;
1148 }
1149}
1150
Tobias Grosser9a38ab82011-11-08 15:41:03 +00001151__isl_give isl_id *Scop::getIdForParam(const SCEV *Parameter) const {
1152 ParamIdType::const_iterator IdIter = ParameterIds.find(Parameter);
Tobias Grosser76c2e322011-11-07 12:58:59 +00001153
Tobias Grosser9a38ab82011-11-08 15:41:03 +00001154 if (IdIter == ParameterIds.end())
Tobias Grosser5a56cbf2014-04-16 07:33:47 +00001155 return nullptr;
Tobias Grosser76c2e322011-11-07 12:58:59 +00001156
Tobias Grosser8f99c162011-11-15 11:38:55 +00001157 std::string ParameterName;
1158
1159 if (const SCEVUnknown *ValueParameter = dyn_cast<SCEVUnknown>(Parameter)) {
1160 Value *Val = ValueParameter->getValue();
Tobias Grosser29ee0b12011-11-17 14:52:36 +00001161 ParameterName = Val->getName();
Tobias Grosser8f99c162011-11-15 11:38:55 +00001162 }
1163
1164 if (ParameterName == "" || ParameterName.substr(0, 2) == "p_")
Hongbin Zheng86a37742012-04-25 08:01:38 +00001165 ParameterName = "p_" + utostr_32(IdIter->second);
Tobias Grosser8f99c162011-11-15 11:38:55 +00001166
Tobias Grosser20532b82014-04-11 17:56:49 +00001167 return isl_id_alloc(getIslCtx(), ParameterName.c_str(),
1168 const_cast<void *>((const void *)Parameter));
Tobias Grosser76c2e322011-11-07 12:58:59 +00001169}
Tobias Grosser75805372011-04-29 06:27:02 +00001170
Tobias Grosser6be480c2011-11-08 15:41:13 +00001171void Scop::buildContext() {
1172 isl_space *Space = isl_space_params_alloc(IslCtx, 0);
Tobias Grossere86109f2013-10-29 21:05:49 +00001173 Context = isl_set_universe(isl_space_copy(Space));
1174 AssumedContext = isl_set_universe(Space);
Tobias Grosser0e27e242011-10-06 00:03:48 +00001175}
1176
Tobias Grosser18daaca2012-05-22 10:47:27 +00001177void Scop::addParameterBounds() {
1178 for (unsigned i = 0; i < isl_set_dim(Context, isl_dim_param); ++i) {
Tobias Grosseredab1352013-06-21 06:41:31 +00001179 isl_val *V;
Tobias Grosser18daaca2012-05-22 10:47:27 +00001180 isl_id *Id;
1181 const SCEV *Scev;
1182 const IntegerType *T;
1183
1184 Id = isl_set_get_dim_id(Context, isl_dim_param, i);
Tobias Grosserabfbe632013-02-05 12:09:06 +00001185 Scev = (const SCEV *)isl_id_get_user(Id);
Tobias Grosser18daaca2012-05-22 10:47:27 +00001186 T = dyn_cast<IntegerType>(Scev->getType());
1187 isl_id_free(Id);
1188
1189 assert(T && "Not an integer type");
1190 int Width = T->getBitWidth();
1191
Tobias Grosseredab1352013-06-21 06:41:31 +00001192 V = isl_val_int_from_si(IslCtx, Width - 1);
1193 V = isl_val_2exp(V);
1194 V = isl_val_neg(V);
1195 Context = isl_set_lower_bound_val(Context, isl_dim_param, i, V);
Tobias Grosser18daaca2012-05-22 10:47:27 +00001196
Tobias Grosseredab1352013-06-21 06:41:31 +00001197 V = isl_val_int_from_si(IslCtx, Width - 1);
1198 V = isl_val_2exp(V);
1199 V = isl_val_sub_ui(V, 1);
1200 Context = isl_set_upper_bound_val(Context, isl_dim_param, i, V);
Tobias Grosser18daaca2012-05-22 10:47:27 +00001201 }
1202}
1203
Tobias Grosser8cae72f2011-11-08 15:41:08 +00001204void Scop::realignParams() {
Tobias Grosser6be480c2011-11-08 15:41:13 +00001205 // Add all parameters into a common model.
Tobias Grosser60b54f12011-11-08 15:41:28 +00001206 isl_space *Space = isl_space_params_alloc(IslCtx, ParameterIds.size());
Tobias Grosser6be480c2011-11-08 15:41:13 +00001207
Tobias Grosser083d3d32014-06-28 08:59:45 +00001208 for (const auto &ParamID : ParameterIds) {
1209 const SCEV *Parameter = ParamID.first;
Tobias Grosser6be480c2011-11-08 15:41:13 +00001210 isl_id *id = getIdForParam(Parameter);
Tobias Grosser083d3d32014-06-28 08:59:45 +00001211 Space = isl_space_set_dim_id(Space, isl_dim_param, ParamID.second, id);
Tobias Grosser6be480c2011-11-08 15:41:13 +00001212 }
1213
1214 // Align the parameters of all data structures to the model.
1215 Context = isl_set_align_params(Context, Space);
1216
Tobias Grosser083d3d32014-06-28 08:59:45 +00001217 for (ScopStmt *Stmt : *this)
1218 Stmt->realignParams();
Tobias Grosser8cae72f2011-11-08 15:41:08 +00001219}
1220
Tobias Grosser5e6813d2014-07-02 17:47:48 +00001221void Scop::simplifyAssumedContext() {
1222 // The parameter constraints of the iteration domains give us a set of
1223 // constraints that need to hold for all cases where at least a single
1224 // statement iteration is executed in the whole scop. We now simplify the
1225 // assumed context under the assumption that such constraints hold and at
1226 // least a single statement iteration is executed. For cases where no
1227 // statement instances are executed, the assumptions we have taken about
1228 // the executed code do not matter and can be changed.
1229 //
1230 // WARNING: This only holds if the assumptions we have taken do not reduce
1231 // the set of statement instances that are executed. Otherwise we
1232 // may run into a case where the iteration domains suggest that
1233 // for a certain set of parameter constraints no code is executed,
1234 // but in the original program some computation would have been
1235 // performed. In such a case, modifying the run-time conditions and
1236 // possibly influencing the run-time check may cause certain scops
1237 // to not be executed.
1238 //
1239 // Example:
1240 //
1241 // When delinearizing the following code:
1242 //
1243 // for (long i = 0; i < 100; i++)
1244 // for (long j = 0; j < m; j++)
1245 // A[i+p][j] = 1.0;
1246 //
1247 // we assume that the condition m <= 0 or (m >= 1 and p >= 0) holds as
1248 // otherwise we would access out of bound data. Now, knowing that code is
1249 // only executed for the case m >= 0, it is sufficient to assume p >= 0.
1250 AssumedContext =
1251 isl_set_gist_params(AssumedContext, isl_union_set_params(getDomains()));
1252}
1253
Johannes Doerfertb164c792014-09-18 11:17:17 +00001254/// @brief Add the minimal/maximal access in @p Set to @p User.
1255static int buildMinMaxAccess(__isl_take isl_set *Set, void *User) {
1256 Scop::MinMaxVectorTy *MinMaxAccesses = (Scop::MinMaxVectorTy *)User;
1257 isl_pw_multi_aff *MinPMA, *MaxPMA;
1258 isl_pw_aff *LastDimAff;
1259 isl_aff *OneAff;
1260 unsigned Pos;
1261
Johannes Doerfert9143d672014-09-27 11:02:39 +00001262 // Restrict the number of parameters involved in the access as the lexmin/
1263 // lexmax computation will take too long if this number is high.
1264 //
1265 // Experiments with a simple test case using an i7 4800MQ:
1266 //
1267 // #Parameters involved | Time (in sec)
1268 // 6 | 0.01
1269 // 7 | 0.04
1270 // 8 | 0.12
1271 // 9 | 0.40
1272 // 10 | 1.54
1273 // 11 | 6.78
1274 // 12 | 30.38
1275 //
1276 if (isl_set_n_param(Set) > RunTimeChecksMaxParameters) {
1277 unsigned InvolvedParams = 0;
1278 for (unsigned u = 0, e = isl_set_n_param(Set); u < e; u++)
1279 if (isl_set_involves_dims(Set, isl_dim_param, u, 1))
1280 InvolvedParams++;
1281
1282 if (InvolvedParams > RunTimeChecksMaxParameters) {
1283 isl_set_free(Set);
1284 return -1;
1285 }
1286 }
1287
Johannes Doerfertb164c792014-09-18 11:17:17 +00001288 MinPMA = isl_set_lexmin_pw_multi_aff(isl_set_copy(Set));
1289 MaxPMA = isl_set_lexmax_pw_multi_aff(isl_set_copy(Set));
1290
Johannes Doerfert219b20e2014-10-07 14:37:59 +00001291 MinPMA = isl_pw_multi_aff_coalesce(MinPMA);
1292 MaxPMA = isl_pw_multi_aff_coalesce(MaxPMA);
1293
Johannes Doerfertb164c792014-09-18 11:17:17 +00001294 // Adjust the last dimension of the maximal access by one as we want to
1295 // enclose the accessed memory region by MinPMA and MaxPMA. The pointer
1296 // we test during code generation might now point after the end of the
1297 // allocated array but we will never dereference it anyway.
1298 assert(isl_pw_multi_aff_dim(MaxPMA, isl_dim_out) &&
1299 "Assumed at least one output dimension");
1300 Pos = isl_pw_multi_aff_dim(MaxPMA, isl_dim_out) - 1;
1301 LastDimAff = isl_pw_multi_aff_get_pw_aff(MaxPMA, Pos);
1302 OneAff = isl_aff_zero_on_domain(
1303 isl_local_space_from_space(isl_pw_aff_get_domain_space(LastDimAff)));
1304 OneAff = isl_aff_add_constant_si(OneAff, 1);
1305 LastDimAff = isl_pw_aff_add(LastDimAff, isl_pw_aff_from_aff(OneAff));
1306 MaxPMA = isl_pw_multi_aff_set_pw_aff(MaxPMA, Pos, LastDimAff);
1307
1308 MinMaxAccesses->push_back(std::make_pair(MinPMA, MaxPMA));
1309
1310 isl_set_free(Set);
1311 return 0;
1312}
1313
Johannes Doerferteeab05a2014-10-01 12:42:37 +00001314static __isl_give isl_set *getAccessDomain(MemoryAccess *MA) {
1315 isl_set *Domain = MA->getStatement()->getDomain();
1316 Domain = isl_set_project_out(Domain, isl_dim_set, 0, isl_set_n_dim(Domain));
1317 return isl_set_reset_tuple_id(Domain);
1318}
1319
Johannes Doerfert9143d672014-09-27 11:02:39 +00001320bool Scop::buildAliasGroups(AliasAnalysis &AA) {
Johannes Doerfertb164c792014-09-18 11:17:17 +00001321 // To create sound alias checks we perform the following steps:
1322 // o) Use the alias analysis and an alias set tracker to build alias sets
1323 // for all memory accesses inside the SCoP.
1324 // o) For each alias set we then map the aliasing pointers back to the
1325 // memory accesses we know, thus obtain groups of memory accesses which
1326 // might alias.
Johannes Doerferteeab05a2014-10-01 12:42:37 +00001327 // o) We divide each group based on the domains of the minimal/maximal
1328 // accesses. That means two minimal/maximal accesses are only in a group
1329 // if their access domains intersect, otherwise they are in different
1330 // ones.
Johannes Doerfert13771732014-10-01 12:40:46 +00001331 // o) We split groups such that they contain at most one read only base
1332 // address.
1333 // o) For each group with more than one base pointer we then compute minimal
Johannes Doerfertb164c792014-09-18 11:17:17 +00001334 // and maximal accesses to each array in this group.
1335 using AliasGroupTy = SmallVector<MemoryAccess *, 4>;
1336
1337 AliasSetTracker AST(AA);
1338
1339 DenseMap<Value *, MemoryAccess *> PtrToAcc;
Johannes Doerfert13771732014-10-01 12:40:46 +00001340 DenseSet<Value *> HasWriteAccess;
Johannes Doerfertb164c792014-09-18 11:17:17 +00001341 for (ScopStmt *Stmt : *this) {
Johannes Doerfertf1ee2622014-10-06 17:43:00 +00001342
1343 // Skip statements with an empty domain as they will never be executed.
1344 isl_set *StmtDomain = Stmt->getDomain();
1345 bool StmtDomainEmpty = isl_set_is_empty(StmtDomain);
1346 isl_set_free(StmtDomain);
1347 if (StmtDomainEmpty)
1348 continue;
1349
Johannes Doerfertb164c792014-09-18 11:17:17 +00001350 for (MemoryAccess *MA : *Stmt) {
1351 if (MA->isScalar())
1352 continue;
Johannes Doerfert13771732014-10-01 12:40:46 +00001353 if (!MA->isRead())
1354 HasWriteAccess.insert(MA->getBaseAddr());
Johannes Doerfertb164c792014-09-18 11:17:17 +00001355 Instruction *Acc = MA->getAccessInstruction();
1356 PtrToAcc[getPointerOperand(*Acc)] = MA;
1357 AST.add(Acc);
1358 }
1359 }
1360
1361 SmallVector<AliasGroupTy, 4> AliasGroups;
1362 for (AliasSet &AS : AST) {
Johannes Doerfert74f68692014-10-08 02:23:48 +00001363 if (AS.isMustAlias() || AS.isForwardingAliasSet())
Johannes Doerfertb164c792014-09-18 11:17:17 +00001364 continue;
1365 AliasGroupTy AG;
1366 for (auto PR : AS)
1367 AG.push_back(PtrToAcc[PR.getValue()]);
1368 assert(AG.size() > 1 &&
1369 "Alias groups should contain at least two accesses");
1370 AliasGroups.push_back(std::move(AG));
1371 }
1372
Johannes Doerferteeab05a2014-10-01 12:42:37 +00001373 // Split the alias groups based on their domain.
1374 for (unsigned u = 0; u < AliasGroups.size(); u++) {
1375 AliasGroupTy NewAG;
1376 AliasGroupTy &AG = AliasGroups[u];
1377 AliasGroupTy::iterator AGI = AG.begin();
1378 isl_set *AGDomain = getAccessDomain(*AGI);
1379 while (AGI != AG.end()) {
1380 MemoryAccess *MA = *AGI;
1381 isl_set *MADomain = getAccessDomain(MA);
1382 if (isl_set_is_disjoint(AGDomain, MADomain)) {
1383 NewAG.push_back(MA);
1384 AGI = AG.erase(AGI);
1385 isl_set_free(MADomain);
1386 } else {
1387 AGDomain = isl_set_union(AGDomain, MADomain);
1388 AGI++;
1389 }
1390 }
1391 if (NewAG.size() > 1)
1392 AliasGroups.push_back(std::move(NewAG));
1393 isl_set_free(AGDomain);
1394 }
1395
Johannes Doerfert13771732014-10-01 12:40:46 +00001396 DenseMap<const Value *, SmallPtrSet<MemoryAccess *, 8>> ReadOnlyPairs;
1397 SmallPtrSet<const Value *, 4> NonReadOnlyBaseValues;
1398 for (AliasGroupTy &AG : AliasGroups) {
1399 NonReadOnlyBaseValues.clear();
1400 ReadOnlyPairs.clear();
1401
Johannes Doerferteeab05a2014-10-01 12:42:37 +00001402 if (AG.size() < 2) {
1403 AG.clear();
1404 continue;
1405 }
1406
Johannes Doerfert13771732014-10-01 12:40:46 +00001407 for (auto II = AG.begin(); II != AG.end();) {
1408 Value *BaseAddr = (*II)->getBaseAddr();
1409 if (HasWriteAccess.count(BaseAddr)) {
1410 NonReadOnlyBaseValues.insert(BaseAddr);
1411 II++;
1412 } else {
1413 ReadOnlyPairs[BaseAddr].insert(*II);
1414 II = AG.erase(II);
1415 }
1416 }
1417
1418 // If we don't have read only pointers check if there are at least two
1419 // non read only pointers, otherwise clear the alias group.
1420 if (ReadOnlyPairs.empty()) {
1421 if (NonReadOnlyBaseValues.size() <= 1)
1422 AG.clear();
1423 continue;
1424 }
1425
1426 // If we don't have non read only pointers clear the alias group.
1427 if (NonReadOnlyBaseValues.empty()) {
1428 AG.clear();
1429 continue;
1430 }
1431
1432 // If we have both read only and non read only base pointers we combine
1433 // the non read only ones with exactly one read only one at a time into a
1434 // new alias group and clear the old alias group in the end.
1435 for (const auto &ReadOnlyPair : ReadOnlyPairs) {
1436 AliasGroupTy AGNonReadOnly = AG;
1437 for (MemoryAccess *MA : ReadOnlyPair.second)
1438 AGNonReadOnly.push_back(MA);
1439 AliasGroups.push_back(std::move(AGNonReadOnly));
1440 }
1441 AG.clear();
Johannes Doerfertb164c792014-09-18 11:17:17 +00001442 }
1443
Johannes Doerfert9143d672014-09-27 11:02:39 +00001444 bool Valid = true;
Johannes Doerfertb164c792014-09-18 11:17:17 +00001445 for (AliasGroupTy &AG : AliasGroups) {
Johannes Doerfert13771732014-10-01 12:40:46 +00001446 if (AG.empty())
1447 continue;
1448
Johannes Doerfertb164c792014-09-18 11:17:17 +00001449 MinMaxVectorTy *MinMaxAccesses = new MinMaxVectorTy();
1450 MinMaxAccesses->reserve(AG.size());
1451
1452 isl_union_map *Accesses = isl_union_map_empty(getParamSpace());
1453 for (MemoryAccess *MA : AG)
1454 Accesses = isl_union_map_add_map(Accesses, MA->getAccessRelation());
1455 Accesses = isl_union_map_intersect_domain(Accesses, getDomains());
1456
1457 isl_union_set *Locations = isl_union_map_range(Accesses);
1458 Locations = isl_union_set_intersect_params(Locations, getAssumedContext());
1459 Locations = isl_union_set_coalesce(Locations);
1460 Locations = isl_union_set_detect_equalities(Locations);
Johannes Doerfert9143d672014-09-27 11:02:39 +00001461 Valid = (0 == isl_union_set_foreach_set(Locations, buildMinMaxAccess,
1462 MinMaxAccesses));
Johannes Doerfertb164c792014-09-18 11:17:17 +00001463 isl_union_set_free(Locations);
Johannes Doerfertb164c792014-09-18 11:17:17 +00001464 MinMaxAliasGroups.push_back(MinMaxAccesses);
Johannes Doerfert9143d672014-09-27 11:02:39 +00001465
1466 if (!Valid)
1467 break;
Johannes Doerfertb164c792014-09-18 11:17:17 +00001468 }
Johannes Doerfert9143d672014-09-27 11:02:39 +00001469
1470 return Valid;
Johannes Doerfertb164c792014-09-18 11:17:17 +00001471}
1472
Johannes Doerferte3da05a2014-11-01 00:12:13 +00001473static unsigned getMaxLoopDepthInRegion(const Region &R, LoopInfo &LI) {
1474 unsigned MinLD = INT_MAX, MaxLD = 0;
1475 for (BasicBlock *BB : R.blocks()) {
1476 if (Loop *L = LI.getLoopFor(BB)) {
1477 unsigned LD = L->getLoopDepth();
1478 MinLD = std::min(MinLD, LD);
1479 MaxLD = std::max(MaxLD, LD);
1480 }
1481 }
1482
1483 // Handle the case that there is no loop in the SCoP first.
1484 if (MaxLD == 0)
1485 return 1;
1486
1487 assert(MinLD >= 1 && "Minimal loop depth should be at least one");
1488 assert(MaxLD >= MinLD &&
1489 "Maximal loop depth was smaller than mininaml loop depth?");
1490 return MaxLD - MinLD + 1;
1491}
1492
Tobias Grosser0e27e242011-10-06 00:03:48 +00001493Scop::Scop(TempScop &tempScop, LoopInfo &LI, ScalarEvolution &ScalarEvolution,
1494 isl_ctx *Context)
Tobias Grosserabfbe632013-02-05 12:09:06 +00001495 : SE(&ScalarEvolution), R(tempScop.getMaxRegion()),
Johannes Doerferte3da05a2014-11-01 00:12:13 +00001496 MaxLoopDepth(getMaxLoopDepthInRegion(tempScop.getMaxRegion(), LI)) {
Tobias Grosser9a38ab82011-11-08 15:41:03 +00001497 IslCtx = Context;
Tobias Grosser6be480c2011-11-08 15:41:13 +00001498 buildContext();
Tobias Grosser75805372011-04-29 06:27:02 +00001499
Tobias Grosserabfbe632013-02-05 12:09:06 +00001500 SmallVector<Loop *, 8> NestLoops;
Tobias Grosser75805372011-04-29 06:27:02 +00001501 SmallVector<unsigned, 8> Scatter;
1502
1503 Scatter.assign(MaxLoopDepth + 1, 0);
1504
1505 // Build the iteration domain, access functions and scattering functions
1506 // traversing the region tree.
1507 buildScop(tempScop, getRegion(), NestLoops, Scatter, LI);
Tobias Grosser75805372011-04-29 06:27:02 +00001508
Tobias Grosser8cae72f2011-11-08 15:41:08 +00001509 realignParams();
Tobias Grosser18daaca2012-05-22 10:47:27 +00001510 addParameterBounds();
Tobias Grosser5e6813d2014-07-02 17:47:48 +00001511 simplifyAssumedContext();
Tobias Grosser8cae72f2011-11-08 15:41:08 +00001512
Tobias Grosser75805372011-04-29 06:27:02 +00001513 assert(NestLoops.empty() && "NestLoops not empty at top level!");
1514}
1515
1516Scop::~Scop() {
1517 isl_set_free(Context);
Tobias Grossere86109f2013-10-29 21:05:49 +00001518 isl_set_free(AssumedContext);
Tobias Grosser75805372011-04-29 06:27:02 +00001519
1520 // Free the statements;
Tobias Grosser083d3d32014-06-28 08:59:45 +00001521 for (ScopStmt *Stmt : *this)
1522 delete Stmt;
Johannes Doerfertb164c792014-09-18 11:17:17 +00001523
Johannes Doerfert1a28a892014-10-05 11:32:18 +00001524 // Free the ScopArrayInfo objects.
1525 for (auto &ScopArrayInfoPair : ScopArrayInfoMap)
1526 delete ScopArrayInfoPair.second;
1527
Johannes Doerfertb164c792014-09-18 11:17:17 +00001528 // Free the alias groups
1529 for (MinMaxVectorTy *MinMaxAccesses : MinMaxAliasGroups) {
1530 for (MinMaxAccessTy &MMA : *MinMaxAccesses) {
1531 isl_pw_multi_aff_free(MMA.first);
1532 isl_pw_multi_aff_free(MMA.second);
1533 }
1534 delete MinMaxAccesses;
1535 }
Tobias Grosser75805372011-04-29 06:27:02 +00001536}
1537
Johannes Doerfert80ef1102014-11-07 08:31:31 +00001538const ScopArrayInfo *
1539Scop::getOrCreateScopArrayInfo(Value *BasePtr, Type *AccessType,
1540 const SmallVector<const SCEV *, 4> &Sizes) {
Johannes Doerfert1a28a892014-10-05 11:32:18 +00001541 const ScopArrayInfo *&SAI = ScopArrayInfoMap[BasePtr];
Johannes Doerfert80ef1102014-11-07 08:31:31 +00001542 if (!SAI)
1543 SAI = new ScopArrayInfo(BasePtr, AccessType, getIslCtx(), Sizes);
Johannes Doerfert1a28a892014-10-05 11:32:18 +00001544 return SAI;
1545}
1546
1547const ScopArrayInfo *Scop::getScopArrayInfo(Value *BasePtr) {
1548 const SCEV *PtrSCEV = SE->getSCEV(BasePtr);
1549 const SCEVUnknown *PtrBaseSCEV =
1550 cast<SCEVUnknown>(SE->getPointerBase(PtrSCEV));
1551 const ScopArrayInfo *SAI = ScopArrayInfoMap[PtrBaseSCEV->getValue()];
1552 assert(SAI && "No ScopArrayInfo available for this base pointer");
1553 return SAI;
1554}
1555
Tobias Grosser74394f02013-01-14 22:40:23 +00001556std::string Scop::getContextStr() const { return stringFromIslObj(Context); }
Tobias Grosser5e6813d2014-07-02 17:47:48 +00001557std::string Scop::getAssumedContextStr() const {
1558 return stringFromIslObj(AssumedContext);
1559}
Tobias Grosser75805372011-04-29 06:27:02 +00001560
1561std::string Scop::getNameStr() const {
1562 std::string ExitName, EntryName;
1563 raw_string_ostream ExitStr(ExitName);
1564 raw_string_ostream EntryStr(EntryName);
1565
Tobias Grosserf240b482014-01-09 10:42:15 +00001566 R.getEntry()->printAsOperand(EntryStr, false);
Tobias Grosser75805372011-04-29 06:27:02 +00001567 EntryStr.str();
1568
1569 if (R.getExit()) {
Tobias Grosserf240b482014-01-09 10:42:15 +00001570 R.getExit()->printAsOperand(ExitStr, false);
Tobias Grosser75805372011-04-29 06:27:02 +00001571 ExitStr.str();
1572 } else
1573 ExitName = "FunctionExit";
1574
1575 return EntryName + "---" + ExitName;
1576}
1577
Tobias Grosser74394f02013-01-14 22:40:23 +00001578__isl_give isl_set *Scop::getContext() const { return isl_set_copy(Context); }
Tobias Grosser37487052011-10-06 00:03:42 +00001579__isl_give isl_space *Scop::getParamSpace() const {
1580 return isl_set_get_space(this->Context);
1581}
1582
Tobias Grossere86109f2013-10-29 21:05:49 +00001583__isl_give isl_set *Scop::getAssumedContext() const {
1584 return isl_set_copy(AssumedContext);
1585}
1586
Tobias Grosser5e6813d2014-07-02 17:47:48 +00001587void Scop::addAssumption(__isl_take isl_set *Set) {
1588 AssumedContext = isl_set_intersect(AssumedContext, Set);
Tobias Grosser7b50bee2014-11-25 10:51:12 +00001589 AssumedContext = isl_set_coalesce(AssumedContext);
Tobias Grosser5e6813d2014-07-02 17:47:48 +00001590}
1591
Tobias Grosser75805372011-04-29 06:27:02 +00001592void Scop::printContext(raw_ostream &OS) const {
1593 OS << "Context:\n";
1594
1595 if (!Context) {
1596 OS.indent(4) << "n/a\n\n";
1597 return;
1598 }
1599
1600 OS.indent(4) << getContextStr() << "\n";
Tobias Grosser60b54f12011-11-08 15:41:28 +00001601
Tobias Grosser5e6813d2014-07-02 17:47:48 +00001602 OS.indent(4) << "Assumed Context:\n";
1603 if (!AssumedContext) {
1604 OS.indent(4) << "n/a\n\n";
1605 return;
1606 }
1607
1608 OS.indent(4) << getAssumedContextStr() << "\n";
1609
Tobias Grosser083d3d32014-06-28 08:59:45 +00001610 for (const SCEV *Parameter : Parameters) {
Tobias Grosser60b54f12011-11-08 15:41:28 +00001611 int Dim = ParameterIds.find(Parameter)->second;
Tobias Grosser60b54f12011-11-08 15:41:28 +00001612 OS.indent(4) << "p" << Dim << ": " << *Parameter << "\n";
1613 }
Tobias Grosser75805372011-04-29 06:27:02 +00001614}
1615
Johannes Doerfertb164c792014-09-18 11:17:17 +00001616void Scop::printAliasAssumptions(raw_ostream &OS) const {
1617 OS.indent(4) << "Alias Groups (" << MinMaxAliasGroups.size() << "):\n";
1618 if (MinMaxAliasGroups.empty()) {
1619 OS.indent(8) << "n/a\n";
1620 return;
1621 }
1622 for (MinMaxVectorTy *MinMaxAccesses : MinMaxAliasGroups) {
1623 OS.indent(8) << "[[";
1624 for (MinMaxAccessTy &MinMacAccess : *MinMaxAccesses)
1625 OS << " <" << MinMacAccess.first << ", " << MinMacAccess.second << ">";
1626 OS << " ]]\n";
1627 }
1628}
1629
Tobias Grosser75805372011-04-29 06:27:02 +00001630void Scop::printStatements(raw_ostream &OS) const {
1631 OS << "Statements {\n";
1632
Tobias Grosser083d3d32014-06-28 08:59:45 +00001633 for (ScopStmt *Stmt : *this)
1634 OS.indent(4) << *Stmt;
Tobias Grosser75805372011-04-29 06:27:02 +00001635
1636 OS.indent(4) << "}\n";
1637}
1638
Tobias Grosser75805372011-04-29 06:27:02 +00001639void Scop::print(raw_ostream &OS) const {
Tobias Grosser4eb7ddb2014-03-18 18:51:11 +00001640 OS.indent(4) << "Function: " << getRegion().getEntry()->getParent()->getName()
1641 << "\n";
Tobias Grosser483fdd42014-03-18 18:05:38 +00001642 OS.indent(4) << "Region: " << getNameStr() << "\n";
Tobias Grosser75805372011-04-29 06:27:02 +00001643 printContext(OS.indent(4));
Johannes Doerfertb164c792014-09-18 11:17:17 +00001644 printAliasAssumptions(OS);
Tobias Grosser75805372011-04-29 06:27:02 +00001645 printStatements(OS.indent(4));
1646}
1647
1648void Scop::dump() const { print(dbgs()); }
1649
Tobias Grosser9a38ab82011-11-08 15:41:03 +00001650isl_ctx *Scop::getIslCtx() const { return IslCtx; }
Tobias Grosser75805372011-04-29 06:27:02 +00001651
Tobias Grosser5f9a7622012-02-14 14:02:40 +00001652__isl_give isl_union_set *Scop::getDomains() {
Tobias Grosserbc4ef902014-06-28 08:59:38 +00001653 isl_union_set *Domain = isl_union_set_empty(getParamSpace());
Tobias Grosser5f9a7622012-02-14 14:02:40 +00001654
Tobias Grosserbc4ef902014-06-28 08:59:38 +00001655 for (ScopStmt *Stmt : *this)
1656 Domain = isl_union_set_add_set(Domain, Stmt->getDomain());
Tobias Grosser5f9a7622012-02-14 14:02:40 +00001657
1658 return Domain;
1659}
1660
Tobias Grosser780ce0f2014-07-11 07:12:10 +00001661__isl_give isl_union_map *Scop::getMustWrites() {
1662 isl_union_map *Write = isl_union_map_empty(this->getParamSpace());
1663
1664 for (ScopStmt *Stmt : *this) {
1665 for (MemoryAccess *MA : *Stmt) {
1666 if (!MA->isMustWrite())
1667 continue;
1668
1669 isl_set *Domain = Stmt->getDomain();
1670 isl_map *AccessDomain = MA->getAccessRelation();
1671 AccessDomain = isl_map_intersect_domain(AccessDomain, Domain);
1672 Write = isl_union_map_add_map(Write, AccessDomain);
1673 }
1674 }
1675 return isl_union_map_coalesce(Write);
1676}
1677
1678__isl_give isl_union_map *Scop::getMayWrites() {
1679 isl_union_map *Write = isl_union_map_empty(this->getParamSpace());
1680
1681 for (ScopStmt *Stmt : *this) {
1682 for (MemoryAccess *MA : *Stmt) {
1683 if (!MA->isMayWrite())
1684 continue;
1685
1686 isl_set *Domain = Stmt->getDomain();
1687 isl_map *AccessDomain = MA->getAccessRelation();
1688 AccessDomain = isl_map_intersect_domain(AccessDomain, Domain);
1689 Write = isl_union_map_add_map(Write, AccessDomain);
1690 }
1691 }
1692 return isl_union_map_coalesce(Write);
1693}
1694
Tobias Grosser37eb4222014-02-20 21:43:54 +00001695__isl_give isl_union_map *Scop::getWrites() {
1696 isl_union_map *Write = isl_union_map_empty(this->getParamSpace());
1697
Tobias Grosserbc4ef902014-06-28 08:59:38 +00001698 for (ScopStmt *Stmt : *this) {
Johannes Doerfertf6752892014-06-13 18:01:45 +00001699 for (MemoryAccess *MA : *Stmt) {
1700 if (!MA->isWrite())
Tobias Grosser37eb4222014-02-20 21:43:54 +00001701 continue;
1702
1703 isl_set *Domain = Stmt->getDomain();
Johannes Doerfertf6752892014-06-13 18:01:45 +00001704 isl_map *AccessDomain = MA->getAccessRelation();
Tobias Grosser37eb4222014-02-20 21:43:54 +00001705 AccessDomain = isl_map_intersect_domain(AccessDomain, Domain);
1706 Write = isl_union_map_add_map(Write, AccessDomain);
1707 }
1708 }
1709 return isl_union_map_coalesce(Write);
1710}
1711
1712__isl_give isl_union_map *Scop::getReads() {
Tobias Grosserbc4ef902014-06-28 08:59:38 +00001713 isl_union_map *Read = isl_union_map_empty(getParamSpace());
Tobias Grosser37eb4222014-02-20 21:43:54 +00001714
Tobias Grosserbc4ef902014-06-28 08:59:38 +00001715 for (ScopStmt *Stmt : *this) {
Johannes Doerfertf6752892014-06-13 18:01:45 +00001716 for (MemoryAccess *MA : *Stmt) {
1717 if (!MA->isRead())
Tobias Grosser37eb4222014-02-20 21:43:54 +00001718 continue;
1719
1720 isl_set *Domain = Stmt->getDomain();
Johannes Doerfertf6752892014-06-13 18:01:45 +00001721 isl_map *AccessDomain = MA->getAccessRelation();
Tobias Grosser37eb4222014-02-20 21:43:54 +00001722
1723 AccessDomain = isl_map_intersect_domain(AccessDomain, Domain);
1724 Read = isl_union_map_add_map(Read, AccessDomain);
1725 }
1726 }
1727 return isl_union_map_coalesce(Read);
1728}
1729
1730__isl_give isl_union_map *Scop::getSchedule() {
Tobias Grosserbc4ef902014-06-28 08:59:38 +00001731 isl_union_map *Schedule = isl_union_map_empty(getParamSpace());
Tobias Grosser37eb4222014-02-20 21:43:54 +00001732
Tobias Grosserbc4ef902014-06-28 08:59:38 +00001733 for (ScopStmt *Stmt : *this)
Tobias Grosser37eb4222014-02-20 21:43:54 +00001734 Schedule = isl_union_map_add_map(Schedule, Stmt->getScattering());
Tobias Grosserbc4ef902014-06-28 08:59:38 +00001735
Tobias Grosser37eb4222014-02-20 21:43:54 +00001736 return isl_union_map_coalesce(Schedule);
1737}
1738
1739bool Scop::restrictDomains(__isl_take isl_union_set *Domain) {
1740 bool Changed = false;
Tobias Grosserbc4ef902014-06-28 08:59:38 +00001741 for (ScopStmt *Stmt : *this) {
Tobias Grosser37eb4222014-02-20 21:43:54 +00001742 isl_union_set *StmtDomain = isl_union_set_from_set(Stmt->getDomain());
Tobias Grosser37eb4222014-02-20 21:43:54 +00001743 isl_union_set *NewStmtDomain = isl_union_set_intersect(
1744 isl_union_set_copy(StmtDomain), isl_union_set_copy(Domain));
1745
1746 if (isl_union_set_is_subset(StmtDomain, NewStmtDomain)) {
1747 isl_union_set_free(StmtDomain);
1748 isl_union_set_free(NewStmtDomain);
1749 continue;
1750 }
1751
1752 Changed = true;
1753
1754 isl_union_set_free(StmtDomain);
1755 NewStmtDomain = isl_union_set_coalesce(NewStmtDomain);
1756
1757 if (isl_union_set_is_empty(NewStmtDomain)) {
1758 Stmt->restrictDomain(isl_set_empty(Stmt->getDomainSpace()));
1759 isl_union_set_free(NewStmtDomain);
1760 } else
1761 Stmt->restrictDomain(isl_set_from_union_set(NewStmtDomain));
1762 }
1763 isl_union_set_free(Domain);
1764 return Changed;
1765}
1766
Tobias Grosser75805372011-04-29 06:27:02 +00001767ScalarEvolution *Scop::getSE() const { return SE; }
1768
1769bool Scop::isTrivialBB(BasicBlock *BB, TempScop &tempScop) {
1770 if (tempScop.getAccessFunctions(BB))
1771 return false;
1772
1773 return true;
1774}
1775
Tobias Grosser74394f02013-01-14 22:40:23 +00001776void Scop::buildScop(TempScop &tempScop, const Region &CurRegion,
1777 SmallVectorImpl<Loop *> &NestLoops,
1778 SmallVectorImpl<unsigned> &Scatter, LoopInfo &LI) {
Tobias Grosser75805372011-04-29 06:27:02 +00001779 Loop *L = castToLoop(CurRegion, LI);
1780
1781 if (L)
1782 NestLoops.push_back(L);
1783
1784 unsigned loopDepth = NestLoops.size();
1785 assert(Scatter.size() > loopDepth && "Scatter not big enough!");
1786
1787 for (Region::const_element_iterator I = CurRegion.element_begin(),
Tobias Grosserabfbe632013-02-05 12:09:06 +00001788 E = CurRegion.element_end();
1789 I != E; ++I)
Tobias Grosser75805372011-04-29 06:27:02 +00001790 if (I->isSubRegion())
1791 buildScop(tempScop, *(I->getNodeAs<Region>()), NestLoops, Scatter, LI);
1792 else {
1793 BasicBlock *BB = I->getNodeAs<BasicBlock>();
1794
1795 if (isTrivialBB(BB, tempScop))
1796 continue;
1797
Johannes Doerfert7c494212014-10-31 23:13:39 +00001798 ScopStmt *Stmt =
1799 new ScopStmt(*this, tempScop, CurRegion, *BB, NestLoops, Scatter);
1800
1801 // Insert all statements into the statement map and the statement vector.
1802 StmtMap[BB] = Stmt;
1803 Stmts.push_back(Stmt);
Tobias Grosser75805372011-04-29 06:27:02 +00001804
1805 // Increasing the Scattering function is OK for the moment, because
1806 // we are using a depth first iterator and the program is well structured.
1807 ++Scatter[loopDepth];
1808 }
1809
1810 if (!L)
1811 return;
1812
1813 // Exiting a loop region.
1814 Scatter[loopDepth] = 0;
1815 NestLoops.pop_back();
Tobias Grosser74394f02013-01-14 22:40:23 +00001816 ++Scatter[loopDepth - 1];
Tobias Grosser75805372011-04-29 06:27:02 +00001817}
1818
Johannes Doerfert7c494212014-10-31 23:13:39 +00001819ScopStmt *Scop::getStmtForBasicBlock(BasicBlock *BB) const {
1820 const auto &StmtMapIt = StmtMap.find(BB);
1821 if (StmtMapIt == StmtMap.end())
1822 return nullptr;
1823 return StmtMapIt->second;
1824}
1825
Tobias Grosser75805372011-04-29 06:27:02 +00001826//===----------------------------------------------------------------------===//
Tobias Grosserb76f38532011-08-20 11:11:25 +00001827ScopInfo::ScopInfo() : RegionPass(ID), scop(0) {
1828 ctx = isl_ctx_alloc();
Tobias Grosser4a8e3562011-12-07 07:42:51 +00001829 isl_options_set_on_error(ctx, ISL_ON_ERROR_ABORT);
Tobias Grosserb76f38532011-08-20 11:11:25 +00001830}
1831
1832ScopInfo::~ScopInfo() {
1833 clear();
1834 isl_ctx_free(ctx);
1835}
1836
Tobias Grosser75805372011-04-29 06:27:02 +00001837void ScopInfo::getAnalysisUsage(AnalysisUsage &AU) const {
1838 AU.addRequired<LoopInfo>();
Matt Arsenault8ca36812014-07-19 18:40:17 +00001839 AU.addRequired<RegionInfoPass>();
Tobias Grosser75805372011-04-29 06:27:02 +00001840 AU.addRequired<ScalarEvolution>();
1841 AU.addRequired<TempScopInfo>();
Johannes Doerfertb164c792014-09-18 11:17:17 +00001842 AU.addRequired<AliasAnalysis>();
Tobias Grosser75805372011-04-29 06:27:02 +00001843 AU.setPreservesAll();
1844}
1845
1846bool ScopInfo::runOnRegion(Region *R, RGPassManager &RGM) {
1847 LoopInfo &LI = getAnalysis<LoopInfo>();
Johannes Doerfertb164c792014-09-18 11:17:17 +00001848 AliasAnalysis &AA = getAnalysis<AliasAnalysis>();
Tobias Grosser75805372011-04-29 06:27:02 +00001849 ScalarEvolution &SE = getAnalysis<ScalarEvolution>();
1850
1851 TempScop *tempScop = getAnalysis<TempScopInfo>().getTempScop(R);
1852
1853 // This region is no Scop.
1854 if (!tempScop) {
Tobias Grosserc98a8fc2014-11-14 11:12:31 +00001855 scop = nullptr;
Tobias Grosser75805372011-04-29 06:27:02 +00001856 return false;
1857 }
1858
Tobias Grosserb76f38532011-08-20 11:11:25 +00001859 scop = new Scop(*tempScop, LI, SE, ctx);
Tobias Grosser75805372011-04-29 06:27:02 +00001860
Johannes Doerfert21aa3dc2014-11-01 01:30:11 +00001861 if (!PollyUseRuntimeAliasChecks) {
1862 // Statistics.
1863 ++ScopFound;
1864 if (scop->getMaxLoopDepth() > 0)
1865 ++RichScopFound;
Johannes Doerfert9143d672014-09-27 11:02:39 +00001866 return false;
Johannes Doerfert21aa3dc2014-11-01 01:30:11 +00001867 }
Johannes Doerfertb164c792014-09-18 11:17:17 +00001868
Johannes Doerfert9143d672014-09-27 11:02:39 +00001869 // If a problem occurs while building the alias groups we need to delete
1870 // this SCoP and pretend it wasn't valid in the first place.
Johannes Doerfert21aa3dc2014-11-01 01:30:11 +00001871 if (scop->buildAliasGroups(AA)) {
1872 // Statistics.
1873 ++ScopFound;
1874 if (scop->getMaxLoopDepth() > 0)
1875 ++RichScopFound;
Johannes Doerfert9143d672014-09-27 11:02:39 +00001876 return false;
Johannes Doerfert21aa3dc2014-11-01 01:30:11 +00001877 }
Johannes Doerfert9143d672014-09-27 11:02:39 +00001878
1879 DEBUG(dbgs()
1880 << "\n\nNOTE: Run time checks for " << scop->getNameStr()
1881 << " could not be created as the number of parameters involved is too "
1882 "high. The SCoP will be "
1883 "dismissed.\nUse:\n\t--polly-rtc-max-parameters=X\nto adjust the "
1884 "maximal number of parameters but be advised that the compile time "
1885 "might increase exponentially.\n\n");
1886
1887 delete scop;
1888 scop = nullptr;
Tobias Grosser75805372011-04-29 06:27:02 +00001889 return false;
1890}
1891
1892char ScopInfo::ID = 0;
1893
Tobias Grosser4d96c8d2013-03-23 01:05:07 +00001894Pass *polly::createScopInfoPass() { return new ScopInfo(); }
1895
Tobias Grosser73600b82011-10-08 00:30:40 +00001896INITIALIZE_PASS_BEGIN(ScopInfo, "polly-scops",
1897 "Polly - Create polyhedral description of Scops", false,
Tobias Grosser4d96c8d2013-03-23 01:05:07 +00001898 false);
Johannes Doerfertb164c792014-09-18 11:17:17 +00001899INITIALIZE_AG_DEPENDENCY(AliasAnalysis);
Tobias Grosser4d96c8d2013-03-23 01:05:07 +00001900INITIALIZE_PASS_DEPENDENCY(LoopInfo);
Matt Arsenault8ca36812014-07-19 18:40:17 +00001901INITIALIZE_PASS_DEPENDENCY(RegionInfoPass);
Tobias Grosser4d96c8d2013-03-23 01:05:07 +00001902INITIALIZE_PASS_DEPENDENCY(ScalarEvolution);
1903INITIALIZE_PASS_DEPENDENCY(TempScopInfo);
Tobias Grosser73600b82011-10-08 00:30:40 +00001904INITIALIZE_PASS_END(ScopInfo, "polly-scops",
1905 "Polly - Create polyhedral description of Scops", false,
1906 false)