blob: c8d9fe095fbc4a7b71ec6cdde487b8de73b23b89 [file] [log] [blame]
Tobias Grosser30aa24c2011-05-14 19:02:06 +00001//===- Schedule.cpp - Calculate an optimized schedule ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass the isl to calculate a schedule that is optimized for parallelism
11// and tileablility. The algorithm used in isl is an optimized version of the
12// algorithm described in following paper:
13//
14// U. Bondhugula, A. Hartono, J. Ramanujam, and P. Sadayappan.
15// A Practical Automatic Polyhedral Parallelizer and Locality Optimizer.
16// In Proceedings of the 2008 ACM SIGPLAN Conference On Programming Language
17// Design and Implementation, PLDI ’08, pages 101–113. ACM, 2008.
18//===----------------------------------------------------------------------===//
19
Tobias Grosser967239c2011-10-23 20:59:44 +000020#include "polly/ScheduleOptimizer.h"
Tobias Grosser2493e922011-12-07 07:42:57 +000021#include "isl/aff.h"
Tobias Grosserde68cc92011-06-30 20:01:02 +000022#include "isl/band.h"
Tobias Grosser8ad6bc32012-01-31 13:26:29 +000023#include "isl/constraint.h"
24#include "isl/map.h"
Tobias Grosser42152ff2012-01-30 19:38:47 +000025#include "isl/options.h"
Tobias Grosser8ad6bc32012-01-31 13:26:29 +000026#include "isl/schedule.h"
27#include "isl/space.h"
Tobias Grosser83628182013-05-07 08:11:54 +000028#include "polly/CodeGen/CodeGeneration.h"
29#include "polly/Dependences.h"
30#include "polly/LinkAllPasses.h"
31#include "polly/Options.h"
32#include "polly/ScopInfo.h"
Tobias Grosser01aea582014-10-22 23:16:28 +000033#include "polly/Support/GICHelper.h"
Tobias Grosser30aa24c2011-05-14 19:02:06 +000034#include "llvm/Support/Debug.h"
35
36using namespace llvm;
37using namespace polly;
38
Chandler Carruth95fef942014-04-22 03:30:19 +000039#define DEBUG_TYPE "polly-opt-isl"
40
Tobias Grosser58032cb2013-06-23 01:29:29 +000041namespace polly {
42bool DisablePollyTiling;
43}
Tobias Grossere602a072013-05-07 07:30:56 +000044static cl::opt<bool, true>
Tobias Grosser483a90d2014-07-09 10:50:10 +000045 DisableTiling("polly-no-tiling",
46 cl::desc("Disable tiling in the scheduler"),
47 cl::location(polly::DisablePollyTiling), cl::init(false),
48 cl::ZeroOrMore, cl::cat(PollyCategory));
Tobias Grosser353a2682011-10-23 20:59:26 +000049
Tobias Grossera26db472012-01-30 19:38:43 +000050static cl::opt<std::string>
Tobias Grosser483a90d2014-07-09 10:50:10 +000051 OptimizeDeps("polly-opt-optimize-only",
52 cl::desc("Only a certain kind of dependences (all/raw)"),
53 cl::Hidden, cl::init("all"), cl::ZeroOrMore,
54 cl::cat(PollyCategory));
Tobias Grosser1deda292012-02-14 14:02:48 +000055
56static cl::opt<std::string>
Tobias Grosser483a90d2014-07-09 10:50:10 +000057 SimplifyDeps("polly-opt-simplify-deps",
58 cl::desc("Dependences should be simplified (yes/no)"),
59 cl::Hidden, cl::init("yes"), cl::ZeroOrMore,
60 cl::cat(PollyCategory));
Tobias Grossera26db472012-01-30 19:38:43 +000061
Tobias Grosser483a90d2014-07-09 10:50:10 +000062static cl::opt<int> MaxConstantTerm(
63 "polly-opt-max-constant-term",
64 cl::desc("The maximal constant term allowed (-1 is unlimited)"), cl::Hidden,
65 cl::init(20), cl::ZeroOrMore, cl::cat(PollyCategory));
Tobias Grosser992e60c2012-02-20 08:41:15 +000066
Tobias Grosser483a90d2014-07-09 10:50:10 +000067static cl::opt<int> MaxCoefficient(
68 "polly-opt-max-coefficient",
69 cl::desc("The maximal coefficient allowed (-1 is unlimited)"), cl::Hidden,
70 cl::init(20), cl::ZeroOrMore, cl::cat(PollyCategory));
71
72static cl::opt<std::string> FusionStrategy(
73 "polly-opt-fusion", cl::desc("The fusion strategy to choose (min/max)"),
74 cl::Hidden, cl::init("min"), cl::ZeroOrMore, cl::cat(PollyCategory));
Tobias Grosser92f54802012-02-20 08:41:47 +000075
Tobias Grossere602a072013-05-07 07:30:56 +000076static cl::opt<std::string>
Tobias Grosser483a90d2014-07-09 10:50:10 +000077 MaximizeBandDepth("polly-opt-maximize-bands",
78 cl::desc("Maximize the band depth (yes/no)"), cl::Hidden,
79 cl::init("yes"), cl::ZeroOrMore, cl::cat(PollyCategory));
Tobias Grosserb3ad85b2012-01-30 19:38:50 +000080
Tobias Grosser483a90d2014-07-09 10:50:10 +000081static cl::opt<int> DefaultTileSize(
82 "polly-default-tile-size",
83 cl::desc("The default tile size (if not enough were provided by"
84 " --polly-tile-sizes)"),
85 cl::Hidden, cl::init(32), cl::ZeroOrMore, cl::cat(PollyCategory));
Johannes Doerfertc3958b22014-05-28 17:21:02 +000086
87static cl::list<int> TileSizes("polly-tile-sizes",
88 cl::desc("A tile size"
89 " for each loop dimension, filled with"
90 " --polly-default-tile-size"),
91 cl::Hidden, cl::ZeroOrMore, cl::CommaSeparated,
92 cl::cat(PollyCategory));
Tobias Grosser30aa24c2011-05-14 19:02:06 +000093namespace {
94
Tobias Grosser4d96c8d2013-03-23 01:05:07 +000095class IslScheduleOptimizer : public ScopPass {
Tobias Grosser4d96c8d2013-03-23 01:05:07 +000096public:
97 static char ID;
Tobias Grosser5a56cbf2014-04-16 07:33:47 +000098 explicit IslScheduleOptimizer() : ScopPass(ID) { LastSchedule = nullptr; }
Tobias Grosser28781422012-10-16 07:29:19 +000099
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000100 ~IslScheduleOptimizer() { isl_schedule_free(LastSchedule); }
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000101
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000102 virtual bool runOnScop(Scop &S);
103 void printScop(llvm::raw_ostream &OS) const;
104 void getAnalysisUsage(AnalysisUsage &AU) const;
Tobias Grosser460e9a42012-04-25 13:22:43 +0000105
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000106private:
107 isl_schedule *LastSchedule;
Tobias Grosser28781422012-10-16 07:29:19 +0000108
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000109 static void extendScattering(Scop &S, unsigned NewDimensions);
Tobias Grosser460e9a42012-04-25 13:22:43 +0000110
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000111 /// @brief Create a map that describes a n-dimensonal tiling.
112 ///
113 /// getTileMap creates a map from a n-dimensional scattering space into an
114 /// 2*n-dimensional scattering space. The map describes a rectangular
115 /// tiling.
116 ///
117 /// Example:
Johannes Doerfertc3958b22014-05-28 17:21:02 +0000118 /// scheduleDimensions = 2, parameterDimensions = 1, TileSizes = <32, 64>
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000119 ///
120 /// tileMap := [p0] -> {[s0, s1] -> [t0, t1, s0, s1]:
121 /// t0 % 32 = 0 and t0 <= s0 < t0 + 32 and
Johannes Doerfertc3958b22014-05-28 17:21:02 +0000122 /// t1 % 64 = 0 and t1 <= s1 < t1 + 64}
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000123 ///
124 /// Before tiling:
125 ///
126 /// for (i = 0; i < N; i++)
127 /// for (j = 0; j < M; j++)
128 /// S(i,j)
129 ///
130 /// After tiling:
131 ///
132 /// for (t_i = 0; t_i < N; i+=32)
Johannes Doerfertc3958b22014-05-28 17:21:02 +0000133 /// for (t_j = 0; t_j < M; j+=64)
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000134 /// for (i = t_i; i < min(t_i + 32, N); i++) | Unknown that N % 32 = 0
Johannes Doerfertc3958b22014-05-28 17:21:02 +0000135 /// for (j = t_j; j < t_j + 64; j++) | Known that M % 64 = 0
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000136 /// S(i,j)
137 ///
138 static isl_basic_map *getTileMap(isl_ctx *ctx, int scheduleDimensions,
Johannes Doerfertc3958b22014-05-28 17:21:02 +0000139 isl_space *SpaceModel);
Tobias Grosser460e9a42012-04-25 13:22:43 +0000140
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000141 /// @brief Get the schedule for this band.
142 ///
143 /// Polly applies transformations like tiling on top of the isl calculated
144 /// value. This can influence the number of scheduling dimension. The
145 /// number of schedule dimensions is returned in the parameter 'Dimension'.
146 static isl_union_map *getScheduleForBand(isl_band *Band, int *Dimensions);
Tobias Grosser460e9a42012-04-25 13:22:43 +0000147
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000148 /// @brief Create a map that pre-vectorizes one scheduling dimension.
149 ///
150 /// getPrevectorMap creates a map that maps each input dimension to the same
151 /// output dimension, except for the dimension DimToVectorize.
152 /// DimToVectorize is strip mined by 'VectorWidth' and the newly created
153 /// point loop of DimToVectorize is moved to the innermost level.
154 ///
155 /// Example (DimToVectorize=0, ScheduleDimensions=2, VectorWidth=4):
156 ///
157 /// | Before transformation
158 /// |
159 /// | A[i,j] -> [i,j]
160 /// |
161 /// | for (i = 0; i < 128; i++)
162 /// | for (j = 0; j < 128; j++)
163 /// | A(i,j);
164 ///
165 /// Prevector map:
166 /// [i,j] -> [it,j,ip] : it % 4 = 0 and it <= ip <= it + 3 and i = ip
167 ///
168 /// | After transformation:
169 /// |
170 /// | A[i,j] -> [it,j,ip] : it % 4 = 0 and it <= ip <= it + 3 and i = ip
171 /// |
172 /// | for (it = 0; it < 128; it+=4)
173 /// | for (j = 0; j < 128; j++)
174 /// | for (ip = max(0,it); ip < min(128, it + 3); ip++)
175 /// | A(ip,j);
176 ///
177 /// The goal of this transformation is to create a trivially vectorizable
178 /// loop. This means a parallel loop at the innermost level that has a
179 /// constant number of iterations corresponding to the target vector width.
180 ///
181 /// This transformation creates a loop at the innermost level. The loop has
182 /// a constant number of iterations, if the number of loop iterations at
183 /// DimToVectorize can be divided by VectorWidth. The default VectorWidth is
184 /// currently constant and not yet target specific. This function does not
185 /// reason about parallelism.
186 static isl_map *getPrevectorMap(isl_ctx *ctx, int DimToVectorize,
187 int ScheduleDimensions, int VectorWidth = 4);
Tobias Grosser460e9a42012-04-25 13:22:43 +0000188
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000189 /// @brief Get the scheduling map for a list of bands.
190 ///
191 /// Walk recursively the forest of bands to combine the schedules of the
192 /// individual bands to the overall schedule. In case tiling is requested,
193 /// the individual bands are tiled.
194 static isl_union_map *getScheduleForBandList(isl_band_list *BandList);
Tobias Grosser460e9a42012-04-25 13:22:43 +0000195
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000196 static isl_union_map *getScheduleMap(isl_schedule *Schedule);
Tobias Grosser28781422012-10-16 07:29:19 +0000197
Tobias Grosser20532b82014-04-11 17:56:49 +0000198 using llvm::Pass::doFinalization;
199
200 virtual bool doFinalization() {
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000201 isl_schedule_free(LastSchedule);
Tobias Grosser5a56cbf2014-04-16 07:33:47 +0000202 LastSchedule = nullptr;
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000203 return true;
204 }
205};
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000206}
207
Tobias Grosser73600b82011-10-08 00:30:40 +0000208char IslScheduleOptimizer::ID = 0;
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000209
Tobias Grosser460e9a42012-04-25 13:22:43 +0000210void IslScheduleOptimizer::extendScattering(Scop &S, unsigned NewDimensions) {
Tobias Grosser083d3d32014-06-28 08:59:45 +0000211 for (ScopStmt *Stmt : S) {
Tobias Grossercf3942d2011-10-06 00:04:05 +0000212 unsigned OldDimensions = Stmt->getNumScattering();
213 isl_space *Space;
Tobias Grosser29666112012-05-22 10:47:31 +0000214 isl_map *Map, *New;
Tobias Grossercf3942d2011-10-06 00:04:05 +0000215
216 Space = isl_space_alloc(Stmt->getIslCtx(), 0, OldDimensions, NewDimensions);
Tobias Grosser29666112012-05-22 10:47:31 +0000217 Map = isl_map_universe(Space);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000218
Tobias Grosser29666112012-05-22 10:47:31 +0000219 for (unsigned i = 0; i < OldDimensions; i++)
220 Map = isl_map_equate(Map, isl_dim_in, i, isl_dim_out, i);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000221
Tobias Grosser29666112012-05-22 10:47:31 +0000222 for (unsigned i = OldDimensions; i < NewDimensions; i++)
223 Map = isl_map_fix_si(Map, isl_dim_out, i, 0);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000224
Tobias Grosser29666112012-05-22 10:47:31 +0000225 Map = isl_map_align_params(Map, S.getParamSpace());
226 New = isl_map_apply_range(Stmt->getScattering(), Map);
227 Stmt->setScattering(New);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000228 }
229}
230
Tobias Grossere602a072013-05-07 07:30:56 +0000231isl_basic_map *IslScheduleOptimizer::getTileMap(isl_ctx *ctx,
232 int scheduleDimensions,
Johannes Doerfertc3958b22014-05-28 17:21:02 +0000233 isl_space *SpaceModel) {
Tobias Grosserde68cc92011-06-30 20:01:02 +0000234 // We construct
235 //
236 // tileMap := [p0] -> {[s0, s1] -> [t0, t1, p0, p1, a0, a1]:
Johannes Doerfertc3958b22014-05-28 17:21:02 +0000237 // s0 = a0 * 32 and s0 = p0 and t0 <= p0 < t0 + 64 and
238 // s1 = a1 * 64 and s1 = p1 and t1 <= p1 < t1 + 64}
Tobias Grosserde68cc92011-06-30 20:01:02 +0000239 //
240 // and project out the auxilary dimensions a0 and a1.
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000241 isl_space *Space =
242 isl_space_alloc(ctx, 0, scheduleDimensions, scheduleDimensions * 3);
Tobias Grosserf5338802011-10-06 00:03:35 +0000243 isl_basic_map *tileMap = isl_basic_map_universe(isl_space_copy(Space));
244
245 isl_local_space *LocalSpace = isl_local_space_from_space(Space);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000246
Tobias Grosserde68cc92011-06-30 20:01:02 +0000247 for (int x = 0; x < scheduleDimensions; x++) {
248 int sX = x;
249 int tX = x;
250 int pX = scheduleDimensions + x;
251 int aX = 2 * scheduleDimensions + x;
Johannes Doerfertc3958b22014-05-28 17:21:02 +0000252 int tileSize = (int)TileSizes.size() > x ? TileSizes[x] : DefaultTileSize;
253 assert(tileSize > 0 && "Invalid tile size");
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000254
Tobias Grosserde68cc92011-06-30 20:01:02 +0000255 isl_constraint *c;
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000256
Tobias Grosserde68cc92011-06-30 20:01:02 +0000257 // sX = aX * tileSize;
Tobias Grosserf5338802011-10-06 00:03:35 +0000258 c = isl_equality_alloc(isl_local_space_copy(LocalSpace));
Tobias Grosserde68cc92011-06-30 20:01:02 +0000259 isl_constraint_set_coefficient_si(c, isl_dim_out, sX, 1);
260 isl_constraint_set_coefficient_si(c, isl_dim_out, aX, -tileSize);
261 tileMap = isl_basic_map_add_constraint(tileMap, c);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000262
Tobias Grosserde68cc92011-06-30 20:01:02 +0000263 // pX = sX;
Tobias Grosserf5338802011-10-06 00:03:35 +0000264 c = isl_equality_alloc(isl_local_space_copy(LocalSpace));
Tobias Grosserde68cc92011-06-30 20:01:02 +0000265 isl_constraint_set_coefficient_si(c, isl_dim_out, pX, 1);
266 isl_constraint_set_coefficient_si(c, isl_dim_in, sX, -1);
267 tileMap = isl_basic_map_add_constraint(tileMap, c);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000268
Tobias Grosserde68cc92011-06-30 20:01:02 +0000269 // tX <= pX
Tobias Grosserf5338802011-10-06 00:03:35 +0000270 c = isl_inequality_alloc(isl_local_space_copy(LocalSpace));
Tobias Grosserde68cc92011-06-30 20:01:02 +0000271 isl_constraint_set_coefficient_si(c, isl_dim_out, pX, 1);
272 isl_constraint_set_coefficient_si(c, isl_dim_out, tX, -1);
273 tileMap = isl_basic_map_add_constraint(tileMap, c);
274
275 // pX <= tX + (tileSize - 1)
Tobias Grosserf5338802011-10-06 00:03:35 +0000276 c = isl_inequality_alloc(isl_local_space_copy(LocalSpace));
Tobias Grosserde68cc92011-06-30 20:01:02 +0000277 isl_constraint_set_coefficient_si(c, isl_dim_out, tX, 1);
278 isl_constraint_set_coefficient_si(c, isl_dim_out, pX, -1);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000279 isl_constraint_set_constant_si(c, tileSize - 1);
Tobias Grosserde68cc92011-06-30 20:01:02 +0000280 tileMap = isl_basic_map_add_constraint(tileMap, c);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000281 }
282
Tobias Grosserde68cc92011-06-30 20:01:02 +0000283 // Project out auxilary dimensions.
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000284 //
Tobias Grosserde68cc92011-06-30 20:01:02 +0000285 // The auxilary dimensions are transformed into existentially quantified ones.
286 // This reduces the number of visible scattering dimensions and allows Cloog
287 // to produces better code.
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000288 tileMap = isl_basic_map_project_out(
289 tileMap, isl_dim_out, 2 * scheduleDimensions, scheduleDimensions);
Tobias Grosserf5338802011-10-06 00:03:35 +0000290 isl_local_space_free(LocalSpace);
Tobias Grosserde68cc92011-06-30 20:01:02 +0000291 return tileMap;
292}
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000293
Tobias Grossere602a072013-05-07 07:30:56 +0000294isl_union_map *IslScheduleOptimizer::getScheduleForBand(isl_band *Band,
295 int *Dimensions) {
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000296 isl_union_map *PartialSchedule;
Tobias Grosserde68cc92011-06-30 20:01:02 +0000297 isl_ctx *ctx;
Tobias Grosserf5338802011-10-06 00:03:35 +0000298 isl_space *Space;
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000299 isl_basic_map *TileMap;
300 isl_union_map *TileUMap;
Tobias Grosserde68cc92011-06-30 20:01:02 +0000301
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000302 PartialSchedule = isl_band_get_partial_schedule(Band);
Tobias Grosserb6033392011-12-08 13:02:58 +0000303 *Dimensions = isl_band_n_member(Band);
Tobias Grosserde68cc92011-06-30 20:01:02 +0000304
Tobias Grosser79b30202011-11-17 12:56:00 +0000305 if (DisableTiling)
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000306 return PartialSchedule;
Tobias Grosser353a2682011-10-23 20:59:26 +0000307
Tobias Grosserb6033392011-12-08 13:02:58 +0000308 // It does not make any sense to tile a band with just one dimension.
309 if (*Dimensions == 1)
310 return PartialSchedule;
311
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000312 ctx = isl_union_map_get_ctx(PartialSchedule);
313 Space = isl_union_map_get_space(PartialSchedule);
Tobias Grosserde68cc92011-06-30 20:01:02 +0000314
Tobias Grosserb6033392011-12-08 13:02:58 +0000315 TileMap = getTileMap(ctx, *Dimensions, Space);
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000316 TileUMap = isl_union_map_from_map(isl_map_from_basic_map(TileMap));
317 TileUMap = isl_union_map_align_params(TileUMap, Space);
Tobias Grosserb6033392011-12-08 13:02:58 +0000318 *Dimensions = 2 * *Dimensions;
319
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000320 return isl_union_map_apply_range(PartialSchedule, TileUMap);
Tobias Grosserde68cc92011-06-30 20:01:02 +0000321}
322
Tobias Grossere602a072013-05-07 07:30:56 +0000323isl_map *IslScheduleOptimizer::getPrevectorMap(isl_ctx *ctx, int DimToVectorize,
324 int ScheduleDimensions,
325 int VectorWidth) {
Tobias Grosser2493e922011-12-07 07:42:57 +0000326 isl_space *Space;
327 isl_local_space *LocalSpace, *LocalSpaceRange;
328 isl_set *Modulo;
329 isl_map *TilingMap;
Tobias Grosserc6699b72011-06-30 20:29:13 +0000330 isl_constraint *c;
Tobias Grosser2493e922011-12-07 07:42:57 +0000331 isl_aff *Aff;
332 int PointDimension; /* ip */
333 int TileDimension; /* it */
Tobias Grosseredab1352013-06-21 06:41:31 +0000334 isl_val *VectorWidthMP;
Tobias Grosserc6699b72011-06-30 20:29:13 +0000335
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000336 assert(0 <= DimToVectorize && DimToVectorize < ScheduleDimensions);
Tobias Grosserf5338802011-10-06 00:03:35 +0000337
Tobias Grosser2493e922011-12-07 07:42:57 +0000338 Space = isl_space_alloc(ctx, 0, ScheduleDimensions, ScheduleDimensions + 1);
339 TilingMap = isl_map_universe(isl_space_copy(Space));
340 LocalSpace = isl_local_space_from_space(Space);
341 PointDimension = ScheduleDimensions;
342 TileDimension = DimToVectorize;
343
344 // Create an identity map for everything except DimToVectorize and map
345 // DimToVectorize to the point loop at the innermost dimension.
346 for (int i = 0; i < ScheduleDimensions; i++) {
Tobias Grosserf5338802011-10-06 00:03:35 +0000347 c = isl_equality_alloc(isl_local_space_copy(LocalSpace));
Johannes Doerfert495dd052014-08-20 17:15:34 +0000348 c = isl_constraint_set_coefficient_si(c, isl_dim_in, i, -1);
Tobias Grosser2493e922011-12-07 07:42:57 +0000349
350 if (i == DimToVectorize)
Johannes Doerfert495dd052014-08-20 17:15:34 +0000351 c = isl_constraint_set_coefficient_si(c, isl_dim_out, PointDimension, 1);
Tobias Grosser2493e922011-12-07 07:42:57 +0000352 else
Johannes Doerfert495dd052014-08-20 17:15:34 +0000353 c = isl_constraint_set_coefficient_si(c, isl_dim_out, i, 1);
Tobias Grosser2493e922011-12-07 07:42:57 +0000354
355 TilingMap = isl_map_add_constraint(TilingMap, c);
Tobias Grosserc6699b72011-06-30 20:29:13 +0000356 }
357
Tobias Grosser2493e922011-12-07 07:42:57 +0000358 // it % 'VectorWidth' = 0
359 LocalSpaceRange = isl_local_space_range(isl_local_space_copy(LocalSpace));
360 Aff = isl_aff_zero_on_domain(LocalSpaceRange);
361 Aff = isl_aff_set_constant_si(Aff, VectorWidth);
362 Aff = isl_aff_set_coefficient_si(Aff, isl_dim_in, TileDimension, 1);
Tobias Grosseredab1352013-06-21 06:41:31 +0000363 VectorWidthMP = isl_val_int_from_si(ctx, VectorWidth);
364 Aff = isl_aff_mod_val(Aff, VectorWidthMP);
Tobias Grosser2493e922011-12-07 07:42:57 +0000365 Modulo = isl_pw_aff_zero_set(isl_pw_aff_from_aff(Aff));
366 TilingMap = isl_map_intersect_range(TilingMap, Modulo);
Tobias Grosserc6699b72011-06-30 20:29:13 +0000367
Tobias Grosser2493e922011-12-07 07:42:57 +0000368 // it <= ip
Tobias Grosserf5338802011-10-06 00:03:35 +0000369 c = isl_inequality_alloc(isl_local_space_copy(LocalSpace));
Tobias Grosser2493e922011-12-07 07:42:57 +0000370 isl_constraint_set_coefficient_si(c, isl_dim_out, TileDimension, -1);
371 isl_constraint_set_coefficient_si(c, isl_dim_out, PointDimension, 1);
372 TilingMap = isl_map_add_constraint(TilingMap, c);
Tobias Grosserc6699b72011-06-30 20:29:13 +0000373
Tobias Grosser2493e922011-12-07 07:42:57 +0000374 // ip <= it + ('VectorWidth' - 1)
Tobias Grosserf5338802011-10-06 00:03:35 +0000375 c = isl_inequality_alloc(LocalSpace);
Tobias Grosser2493e922011-12-07 07:42:57 +0000376 isl_constraint_set_coefficient_si(c, isl_dim_out, TileDimension, 1);
377 isl_constraint_set_coefficient_si(c, isl_dim_out, PointDimension, -1);
378 isl_constraint_set_constant_si(c, VectorWidth - 1);
379 TilingMap = isl_map_add_constraint(TilingMap, c);
Tobias Grosserc6699b72011-06-30 20:29:13 +0000380
Tobias Grosser2493e922011-12-07 07:42:57 +0000381 return TilingMap;
Tobias Grosserc6699b72011-06-30 20:29:13 +0000382}
383
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000384isl_union_map *
385IslScheduleOptimizer::getScheduleForBandList(isl_band_list *BandList) {
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000386 int NumBands;
387 isl_union_map *Schedule;
388 isl_ctx *ctx;
Tobias Grosserde68cc92011-06-30 20:01:02 +0000389
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000390 ctx = isl_band_list_get_ctx(BandList);
391 NumBands = isl_band_list_n_band(BandList);
Tobias Grosser62872012011-11-17 12:56:04 +0000392 Schedule = isl_union_map_empty(isl_space_params_alloc(ctx, 0));
Tobias Grosserde68cc92011-06-30 20:01:02 +0000393
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000394 for (int i = 0; i < NumBands; i++) {
395 isl_band *Band;
396 isl_union_map *PartialSchedule;
397 int ScheduleDimensions;
398 isl_space *Space;
Tobias Grosser44f19ac2011-07-05 22:15:53 +0000399
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000400 Band = isl_band_list_get_band(BandList, i);
Tobias Grosserb6033392011-12-08 13:02:58 +0000401 PartialSchedule = getScheduleForBand(Band, &ScheduleDimensions);
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000402 Space = isl_union_map_get_space(PartialSchedule);
Tobias Grosserde68cc92011-06-30 20:01:02 +0000403
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000404 if (isl_band_has_children(Band)) {
405 isl_band_list *Children;
406 isl_union_map *SuffixSchedule;
407
408 Children = isl_band_get_children(Band);
409 SuffixSchedule = getScheduleForBandList(Children);
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000410 PartialSchedule =
411 isl_union_map_flat_range_product(PartialSchedule, SuffixSchedule);
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000412 isl_band_list_free(Children);
Hongbin Zheng68794212012-05-06 10:22:19 +0000413 } else if (PollyVectorizerChoice != VECTORIZER_NONE) {
Tobias Grosser4ba60fe2014-03-11 06:27:36 +0000414 // In case we are at the innermost band, we try to prepare for
415 // vectorization. This means, we look for the innermost parallel loop
416 // and strip mine this loop to the innermost level using a strip-mine
417 // factor corresponding to the number of vector iterations.
418 int NumDims = isl_band_n_member(Band);
419 for (int j = NumDims - 1; j >= 0; j--) {
Tobias Grossera38c9242014-01-26 19:36:28 +0000420 if (isl_band_member_is_coincident(Band, j)) {
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000421 isl_map *TileMap;
422 isl_union_map *TileUMap;
Tobias Grosserc6699b72011-06-30 20:29:13 +0000423
Tobias Grosser4ba60fe2014-03-11 06:27:36 +0000424 TileMap = getPrevectorMap(ctx, ScheduleDimensions - NumDims + j,
Tobias Grosser216ea582012-04-16 11:06:06 +0000425 ScheduleDimensions);
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000426 TileUMap = isl_union_map_from_map(TileMap);
427 TileUMap =
428 isl_union_map_align_params(TileUMap, isl_space_copy(Space));
429 PartialSchedule =
430 isl_union_map_apply_range(PartialSchedule, TileUMap);
431 break;
432 }
Tobias Grosser7c5ba832011-06-30 20:29:20 +0000433 }
Tobias Grosserde68cc92011-06-30 20:01:02 +0000434 }
435
Tobias Grosser62872012011-11-17 12:56:04 +0000436 Schedule = isl_union_map_union(Schedule, PartialSchedule);
Tobias Grosserde68cc92011-06-30 20:01:02 +0000437
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000438 isl_band_free(Band);
Tobias Grosserf5338802011-10-06 00:03:35 +0000439 isl_space_free(Space);
Tobias Grosserde68cc92011-06-30 20:01:02 +0000440 }
441
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000442 return Schedule;
Tobias Grosserde68cc92011-06-30 20:01:02 +0000443}
444
Tobias Grosser460e9a42012-04-25 13:22:43 +0000445isl_union_map *IslScheduleOptimizer::getScheduleMap(isl_schedule *Schedule) {
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000446 isl_band_list *BandList = isl_schedule_get_band_forest(Schedule);
447 isl_union_map *ScheduleMap = getScheduleForBandList(BandList);
448 isl_band_list_free(BandList);
449 return ScheduleMap;
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000450}
451
Tobias Grosser73600b82011-10-08 00:30:40 +0000452bool IslScheduleOptimizer::runOnScop(Scop &S) {
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000453 Dependences *D = &getAnalysis<Dependences>();
454
Tobias Grosser38c36ea2014-02-23 15:15:44 +0000455 if (!D->hasValidDependences())
456 return false;
457
Tobias Grosser28781422012-10-16 07:29:19 +0000458 isl_schedule_free(LastSchedule);
Tobias Grosser5a56cbf2014-04-16 07:33:47 +0000459 LastSchedule = nullptr;
Tobias Grosser28781422012-10-16 07:29:19 +0000460
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000461 // Build input data.
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000462 int ValidityKinds =
463 Dependences::TYPE_RAW | Dependences::TYPE_WAR | Dependences::TYPE_WAW;
Tobias Grosser1deda292012-02-14 14:02:48 +0000464 int ProximityKinds;
465
466 if (OptimizeDeps == "all")
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000467 ProximityKinds =
468 Dependences::TYPE_RAW | Dependences::TYPE_WAR | Dependences::TYPE_WAW;
Tobias Grosser1deda292012-02-14 14:02:48 +0000469 else if (OptimizeDeps == "raw")
Tobias Grosser1e03ad72012-02-15 09:58:42 +0000470 ProximityKinds = Dependences::TYPE_RAW;
Tobias Grosser1deda292012-02-14 14:02:48 +0000471 else {
472 errs() << "Do not know how to optimize for '" << OptimizeDeps << "'"
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000473 << " Falling back to optimizing all dependences.\n";
474 ProximityKinds =
475 Dependences::TYPE_RAW | Dependences::TYPE_WAR | Dependences::TYPE_WAW;
Tobias Grosser1deda292012-02-14 14:02:48 +0000476 }
477
Tobias Grosser5f9a7622012-02-14 14:02:40 +0000478 isl_union_set *Domain = S.getDomains();
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000479
Tobias Grosser98610ee2012-02-13 23:31:39 +0000480 if (!Domain)
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000481 return false;
482
Tobias Grosser8a507022012-03-16 11:51:41 +0000483 isl_union_map *Validity = D->getDependences(ValidityKinds);
484 isl_union_map *Proximity = D->getDependences(ProximityKinds);
485
Tobias Grossera26db472012-01-30 19:38:43 +0000486 // Simplify the dependences by removing the constraints introduced by the
487 // domains. This can speed up the scheduling time significantly, as large
488 // constant coefficients will be removed from the dependences. The
489 // introduction of some additional dependences reduces the possible
490 // transformations, but in most cases, such transformation do not seem to be
491 // interesting anyway. In some cases this option may stop the scheduler to
492 // find any schedule.
493 if (SimplifyDeps == "yes") {
Tobias Grosser00383a72012-02-14 14:02:44 +0000494 Validity = isl_union_map_gist_domain(Validity, isl_union_set_copy(Domain));
495 Validity = isl_union_map_gist_range(Validity, isl_union_set_copy(Domain));
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000496 Proximity =
497 isl_union_map_gist_domain(Proximity, isl_union_set_copy(Domain));
Tobias Grosser00383a72012-02-14 14:02:44 +0000498 Proximity = isl_union_map_gist_range(Proximity, isl_union_set_copy(Domain));
Tobias Grossera26db472012-01-30 19:38:43 +0000499 } else if (SimplifyDeps != "no") {
500 errs() << "warning: Option -polly-opt-simplify-deps should either be 'yes' "
501 "or 'no'. Falling back to default: 'yes'\n";
502 }
503
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000504 DEBUG(dbgs() << "\n\nCompute schedule from: ");
Tobias Grosser01aea582014-10-22 23:16:28 +0000505 DEBUG(dbgs() << "Domain := " << stringFromIslObj(Domain) << ";\n");
506 DEBUG(dbgs() << "Proximity := " << stringFromIslObj(Proximity) << ";\n");
507 DEBUG(dbgs() << "Validity := " << stringFromIslObj(Validity) << ";\n");
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000508
Tobias Grosserb3ad85b2012-01-30 19:38:50 +0000509 int IslFusionStrategy;
510
511 if (FusionStrategy == "max") {
512 IslFusionStrategy = ISL_SCHEDULE_FUSE_MAX;
513 } else if (FusionStrategy == "min") {
514 IslFusionStrategy = ISL_SCHEDULE_FUSE_MIN;
515 } else {
516 errs() << "warning: Unknown fusion strategy. Falling back to maximal "
517 "fusion.\n";
518 IslFusionStrategy = ISL_SCHEDULE_FUSE_MAX;
519 }
520
Tobias Grosser95e860c2012-01-30 19:38:54 +0000521 int IslMaximizeBands;
522
Tobias Grossera4ea90b2012-01-30 22:43:56 +0000523 if (MaximizeBandDepth == "yes") {
Tobias Grosser95e860c2012-01-30 19:38:54 +0000524 IslMaximizeBands = 1;
Tobias Grossera4ea90b2012-01-30 22:43:56 +0000525 } else if (MaximizeBandDepth == "no") {
Tobias Grosser95e860c2012-01-30 19:38:54 +0000526 IslMaximizeBands = 0;
527 } else {
528 errs() << "warning: Option -polly-opt-maximize-bands should either be 'yes'"
529 " or 'no'. Falling back to default: 'yes'\n";
530 IslMaximizeBands = 1;
531 }
532
Tobias Grosserb3ad85b2012-01-30 19:38:50 +0000533 isl_options_set_schedule_fuse(S.getIslCtx(), IslFusionStrategy);
Tobias Grosser95e860c2012-01-30 19:38:54 +0000534 isl_options_set_schedule_maximize_band_depth(S.getIslCtx(), IslMaximizeBands);
Tobias Grosser992e60c2012-02-20 08:41:15 +0000535 isl_options_set_schedule_max_constant_term(S.getIslCtx(), MaxConstantTerm);
Tobias Grosser92f54802012-02-20 08:41:47 +0000536 isl_options_set_schedule_max_coefficient(S.getIslCtx(), MaxCoefficient);
Tobias Grosser42152ff2012-01-30 19:38:47 +0000537
538 isl_options_set_on_error(S.getIslCtx(), ISL_ON_ERROR_CONTINUE);
Tobias Grossera38c9242014-01-26 19:36:28 +0000539
540 isl_schedule_constraints *ScheduleConstraints;
541 ScheduleConstraints = isl_schedule_constraints_on_domain(Domain);
542 ScheduleConstraints =
543 isl_schedule_constraints_set_proximity(ScheduleConstraints, Proximity);
544 ScheduleConstraints = isl_schedule_constraints_set_validity(
545 ScheduleConstraints, isl_union_map_copy(Validity));
546 ScheduleConstraints =
547 isl_schedule_constraints_set_coincidence(ScheduleConstraints, Validity);
Tobias Grosser00383a72012-02-14 14:02:44 +0000548 isl_schedule *Schedule;
Tobias Grossera38c9242014-01-26 19:36:28 +0000549 Schedule = isl_schedule_constraints_compute_schedule(ScheduleConstraints);
Tobias Grosser42152ff2012-01-30 19:38:47 +0000550 isl_options_set_on_error(S.getIslCtx(), ISL_ON_ERROR_ABORT);
551
552 // In cases the scheduler is not able to optimize the code, we just do not
553 // touch the schedule.
Tobias Grosser98610ee2012-02-13 23:31:39 +0000554 if (!Schedule)
Tobias Grosser42152ff2012-01-30 19:38:47 +0000555 return false;
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000556
Tobias Grosser01aea582014-10-22 23:16:28 +0000557 DEBUG(dbgs() << "Schedule := " << stringFromIslObj(Schedule) << ";\n");
Tobias Grosser4d63b9d2012-02-20 08:41:21 +0000558
Tobias Grosser98610ee2012-02-13 23:31:39 +0000559 isl_union_map *ScheduleMap = getScheduleMap(Schedule);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000560
Tobias Grosser083d3d32014-06-28 08:59:45 +0000561 for (ScopStmt *Stmt : S) {
Tobias Grosser249c4b12013-04-11 05:55:13 +0000562 isl_map *StmtSchedule;
Tobias Grosser98610ee2012-02-13 23:31:39 +0000563 isl_set *Domain = Stmt->getDomain();
564 isl_union_map *StmtBand;
565 StmtBand = isl_union_map_intersect_domain(isl_union_map_copy(ScheduleMap),
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000566 isl_union_set_from_set(Domain));
Tobias Grosser249c4b12013-04-11 05:55:13 +0000567 if (isl_union_map_is_empty(StmtBand)) {
Tobias Grosser34f06132014-02-21 20:51:36 +0000568 StmtSchedule = isl_map_from_domain(isl_set_empty(Stmt->getDomainSpace()));
Tobias Grosser249c4b12013-04-11 05:55:13 +0000569 isl_union_map_free(StmtBand);
570 } else {
571 assert(isl_union_map_n_map(StmtBand) == 1);
572 StmtSchedule = isl_map_from_union_map(StmtBand);
Tobias Grosserf242b802013-04-10 22:48:08 +0000573 }
574
Tobias Grosser98610ee2012-02-13 23:31:39 +0000575 Stmt->setScattering(StmtSchedule);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000576 }
577
Tobias Grosser1ae9a602011-11-17 12:56:03 +0000578 isl_union_map_free(ScheduleMap);
Tobias Grosser28781422012-10-16 07:29:19 +0000579 LastSchedule = Schedule;
Tobias Grosserde68cc92011-06-30 20:01:02 +0000580
Tobias Grosser98610ee2012-02-13 23:31:39 +0000581 unsigned MaxScatDims = 0;
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000582
Tobias Grosser083d3d32014-06-28 08:59:45 +0000583 for (ScopStmt *Stmt : S)
584 MaxScatDims = std::max(Stmt->getNumScattering(), MaxScatDims);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000585
Tobias Grosser98610ee2012-02-13 23:31:39 +0000586 extendScattering(S, MaxScatDims);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000587 return false;
588}
589
Tobias Grosser73600b82011-10-08 00:30:40 +0000590void IslScheduleOptimizer::printScop(raw_ostream &OS) const {
Tobias Grosser28781422012-10-16 07:29:19 +0000591 isl_printer *p;
592 char *ScheduleStr;
593
594 OS << "Calculated schedule:\n";
595
596 if (!LastSchedule) {
597 OS << "n/a\n";
598 return;
599 }
600
601 p = isl_printer_to_str(isl_schedule_get_ctx(LastSchedule));
602 p = isl_printer_print_schedule(p, LastSchedule);
603 ScheduleStr = isl_printer_get_str(p);
604 isl_printer_free(p);
605
606 OS << ScheduleStr << "\n";
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000607}
608
Tobias Grosser73600b82011-10-08 00:30:40 +0000609void IslScheduleOptimizer::getAnalysisUsage(AnalysisUsage &AU) const {
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000610 ScopPass::getAnalysisUsage(AU);
611 AU.addRequired<Dependences>();
612}
613
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000614Pass *polly::createIslScheduleOptimizerPass() {
Tobias Grosser73600b82011-10-08 00:30:40 +0000615 return new IslScheduleOptimizer();
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000616}
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000617
618INITIALIZE_PASS_BEGIN(IslScheduleOptimizer, "polly-opt-isl",
619 "Polly - Optimize schedule of SCoP", false, false);
620INITIALIZE_PASS_DEPENDENCY(Dependences);
621INITIALIZE_PASS_DEPENDENCY(ScopInfo);
622INITIALIZE_PASS_END(IslScheduleOptimizer, "polly-opt-isl",
623 "Polly - Optimize schedule of SCoP", false, false)