blob: 0296af4e9095c71f51f706ae56f1c26b2ed65c42 [file] [log] [blame]
Tobias Grosser30aa24c2011-05-14 19:02:06 +00001//===- Schedule.cpp - Calculate an optimized schedule ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
Tobias Grosser234a4822015-08-15 09:34:33 +000010// This pass generates an entirey new schedule tree from the data dependences
11// and iteration domains. The new schedule tree is computed in two steps:
Tobias Grosser30aa24c2011-05-14 19:02:06 +000012//
Tobias Grosser234a4822015-08-15 09:34:33 +000013// 1) The isl scheduling optimizer is run
14//
15// The isl scheduling optimizer creates a new schedule tree that maximizes
16// parallelism and tileability and minimizes data-dependence distances. The
17// algorithm used is a modified version of the ``Pluto'' algorithm:
18//
19// U. Bondhugula, A. Hartono, J. Ramanujam, and P. Sadayappan.
20// A Practical Automatic Polyhedral Parallelizer and Locality Optimizer.
21// In Proceedings of the 2008 ACM SIGPLAN Conference On Programming Language
22// Design and Implementation, PLDI ’08, pages 101–113. ACM, 2008.
23//
24// 2) A set of post-scheduling transformations is applied on the schedule tree.
25//
26// These optimizations include:
27//
28// - Tiling of the innermost tilable bands
29// - Prevectorization - The coice of a possible outer loop that is strip-mined
30// to the innermost level to enable inner-loop
31// vectorization.
32// - Some optimizations for spatial locality are also planned.
33//
34// For a detailed description of the schedule tree itself please see section 6
35// of:
36//
37// Polyhedral AST generation is more than scanning polyhedra
38// Tobias Grosser, Sven Verdoolaege, Albert Cohen
39// ACM Transations on Programming Languages and Systems (TOPLAS),
40// 37(4), July 2015
41// http://www.grosser.es/#pub-polyhedral-AST-generation
42//
43// This publication also contains a detailed discussion of the different options
44// for polyhedral loop unrolling, full/partial tile separation and other uses
45// of the schedule tree.
46//
Tobias Grosser30aa24c2011-05-14 19:02:06 +000047//===----------------------------------------------------------------------===//
48
Tobias Grosser967239c2011-10-23 20:59:44 +000049#include "polly/ScheduleOptimizer.h"
Tobias Grosserba0d0922015-05-09 09:13:42 +000050#include "polly/CodeGen/CodeGeneration.h"
51#include "polly/DependenceInfo.h"
52#include "polly/LinkAllPasses.h"
53#include "polly/Options.h"
54#include "polly/ScopInfo.h"
55#include "polly/Support/GICHelper.h"
56#include "llvm/Support/Debug.h"
Tobias Grosser2493e922011-12-07 07:42:57 +000057#include "isl/aff.h"
Tobias Grosserde68cc92011-06-30 20:01:02 +000058#include "isl/band.h"
Tobias Grosser8ad6bc32012-01-31 13:26:29 +000059#include "isl/constraint.h"
60#include "isl/map.h"
Tobias Grosser42152ff2012-01-30 19:38:47 +000061#include "isl/options.h"
Tobias Grosser97d87452015-05-30 06:46:59 +000062#include "isl/printer.h"
Tobias Grosser8ad6bc32012-01-31 13:26:29 +000063#include "isl/schedule.h"
Tobias Grosserbbb4cec2015-03-22 12:06:39 +000064#include "isl/schedule_node.h"
Tobias Grosser8ad6bc32012-01-31 13:26:29 +000065#include "isl/space.h"
Tobias Grossercd524dc2015-05-09 09:36:38 +000066#include "isl/union_map.h"
67#include "isl/union_set.h"
Tobias Grosser30aa24c2011-05-14 19:02:06 +000068
69using namespace llvm;
70using namespace polly;
71
Chandler Carruth95fef942014-04-22 03:30:19 +000072#define DEBUG_TYPE "polly-opt-isl"
73
Tobias Grossera26db472012-01-30 19:38:43 +000074static cl::opt<std::string>
Tobias Grosser483a90d2014-07-09 10:50:10 +000075 OptimizeDeps("polly-opt-optimize-only",
76 cl::desc("Only a certain kind of dependences (all/raw)"),
77 cl::Hidden, cl::init("all"), cl::ZeroOrMore,
78 cl::cat(PollyCategory));
Tobias Grosser1deda292012-02-14 14:02:48 +000079
80static cl::opt<std::string>
Tobias Grosser483a90d2014-07-09 10:50:10 +000081 SimplifyDeps("polly-opt-simplify-deps",
82 cl::desc("Dependences should be simplified (yes/no)"),
83 cl::Hidden, cl::init("yes"), cl::ZeroOrMore,
84 cl::cat(PollyCategory));
Tobias Grossera26db472012-01-30 19:38:43 +000085
Tobias Grosser483a90d2014-07-09 10:50:10 +000086static cl::opt<int> MaxConstantTerm(
87 "polly-opt-max-constant-term",
88 cl::desc("The maximal constant term allowed (-1 is unlimited)"), cl::Hidden,
89 cl::init(20), cl::ZeroOrMore, cl::cat(PollyCategory));
Tobias Grosser992e60c2012-02-20 08:41:15 +000090
Tobias Grosser483a90d2014-07-09 10:50:10 +000091static cl::opt<int> MaxCoefficient(
92 "polly-opt-max-coefficient",
93 cl::desc("The maximal coefficient allowed (-1 is unlimited)"), cl::Hidden,
94 cl::init(20), cl::ZeroOrMore, cl::cat(PollyCategory));
95
96static cl::opt<std::string> FusionStrategy(
97 "polly-opt-fusion", cl::desc("The fusion strategy to choose (min/max)"),
98 cl::Hidden, cl::init("min"), cl::ZeroOrMore, cl::cat(PollyCategory));
Tobias Grosser92f54802012-02-20 08:41:47 +000099
Tobias Grossere602a072013-05-07 07:30:56 +0000100static cl::opt<std::string>
Tobias Grosser483a90d2014-07-09 10:50:10 +0000101 MaximizeBandDepth("polly-opt-maximize-bands",
102 cl::desc("Maximize the band depth (yes/no)"), cl::Hidden,
103 cl::init("yes"), cl::ZeroOrMore, cl::cat(PollyCategory));
Tobias Grosserb3ad85b2012-01-30 19:38:50 +0000104
Tobias Grosser07c1c2f2015-08-19 08:46:11 +0000105static cl::opt<int> PrevectorWidth(
106 "polly-prevect-width",
107 cl::desc(
108 "The number of loop iterations to strip-mine for pre-vectorization"),
109 cl::Hidden, cl::init(4), cl::ZeroOrMore, cl::cat(PollyCategory));
110
Tobias Grosser04832712015-08-20 13:45:02 +0000111static cl::opt<bool> FirstLevelTiling("polly-tiling",
112 cl::desc("Enable loop tiling"),
113 cl::init(true), cl::ZeroOrMore,
114 cl::cat(PollyCategory));
115
116static cl::opt<int> FirstLevelDefaultTileSize(
Tobias Grosser483a90d2014-07-09 10:50:10 +0000117 "polly-default-tile-size",
118 cl::desc("The default tile size (if not enough were provided by"
119 " --polly-tile-sizes)"),
120 cl::Hidden, cl::init(32), cl::ZeroOrMore, cl::cat(PollyCategory));
Johannes Doerfertc3958b22014-05-28 17:21:02 +0000121
Tobias Grosser04832712015-08-20 13:45:02 +0000122static cl::list<int> FirstLevelTileSizes(
123 "polly-tile-sizes", cl::desc("A tile size for each loop dimension, filled "
124 "with --polly-default-tile-size"),
125 cl::Hidden, cl::ZeroOrMore, cl::CommaSeparated, cl::cat(PollyCategory));
126
127static cl::opt<bool>
128 SecondLevelTiling("polly-2nd-level-tiling",
129 cl::desc("Enable a 2nd level loop of loop tiling"),
130 cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
131
132static cl::opt<int> SecondLevelDefaultTileSize(
133 "polly-2nd-level-default-tile-size",
134 cl::desc("The default 2nd-level tile size (if not enough were provided by"
135 " --polly-2nd-level-tile-sizes)"),
136 cl::Hidden, cl::init(16), cl::ZeroOrMore, cl::cat(PollyCategory));
137
138static cl::list<int>
139 SecondLevelTileSizes("polly-2nd-level-tile-sizes",
140 cl::desc("A tile size for each loop dimension, filled "
141 "with --polly-default-tile-size"),
142 cl::Hidden, cl::ZeroOrMore, cl::CommaSeparated,
143 cl::cat(PollyCategory));
144
Tobias Grosser42e24892015-08-20 13:45:05 +0000145static cl::opt<bool> RegisterTiling("polly-register-tiling",
146 cl::desc("Enable register tiling"),
147 cl::init(false), cl::ZeroOrMore,
148 cl::cat(PollyCategory));
149
150static cl::opt<int> RegisterDefaultTileSize(
151 "polly-register-tiling-default-tile-size",
152 cl::desc("The default register tile size (if not enough were provided by"
153 " --polly-register-tile-sizes)"),
154 cl::Hidden, cl::init(2), cl::ZeroOrMore, cl::cat(PollyCategory));
155
156static cl::list<int>
157 RegisterTileSizes("polly-register-tile-sizes",
158 cl::desc("A tile size for each loop dimension, filled "
159 "with --polly-register-tile-size"),
160 cl::Hidden, cl::ZeroOrMore, cl::CommaSeparated,
161 cl::cat(PollyCategory));
162
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000163namespace {
164
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000165class IslScheduleOptimizer : public ScopPass {
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000166public:
167 static char ID;
Tobias Grosser5a56cbf2014-04-16 07:33:47 +0000168 explicit IslScheduleOptimizer() : ScopPass(ID) { LastSchedule = nullptr; }
Tobias Grosser28781422012-10-16 07:29:19 +0000169
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000170 ~IslScheduleOptimizer() { isl_schedule_free(LastSchedule); }
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000171
Johannes Doerfert909a3bf2015-03-01 18:42:08 +0000172 bool runOnScop(Scop &S) override;
173 void printScop(raw_ostream &OS, Scop &S) const override;
174 void getAnalysisUsage(AnalysisUsage &AU) const override;
Tobias Grosser460e9a42012-04-25 13:22:43 +0000175
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000176private:
177 isl_schedule *LastSchedule;
Tobias Grosser28781422012-10-16 07:29:19 +0000178
Johannes Doerfert7ceb0402015-02-11 17:25:09 +0000179 /// @brief Decide if the @p NewSchedule is profitable for @p S.
180 ///
181 /// @param S The SCoP we optimize.
182 /// @param NewSchedule The new schedule we computed.
183 ///
184 /// @return True, if we believe @p NewSchedule is an improvement for @p S.
185 bool isProfitableSchedule(Scop &S, __isl_keep isl_union_map *NewSchedule);
186
Tobias Grosser9bdea572015-08-20 12:22:37 +0000187 /// @brief Tile a schedule node.
188 ///
189 /// @param Node The node to tile.
Tobias Grosser1ac884d2015-08-23 09:11:00 +0000190 /// @param Identifier An name that identifies this kind of tiling and
191 /// that is used to mark the tiled loops in the
192 /// generated AST.
Tobias Grosser9bdea572015-08-20 12:22:37 +0000193 /// @param TileSizes A vector of tile sizes that should be used for
194 /// tiling.
195 /// @param DefaultTileSize A default tile size that is used for dimensions
196 /// that are not covered by the TileSizes vector.
197 static __isl_give isl_schedule_node *
Tobias Grosser1ac884d2015-08-23 09:11:00 +0000198 tileNode(__isl_take isl_schedule_node *Node, const char *Identifier,
199 ArrayRef<int> TileSizes, int DefaultTileSize);
Tobias Grosser9bdea572015-08-20 12:22:37 +0000200
Tobias Grosser862b9b52015-08-20 12:32:45 +0000201 /// @brief Check if this node is a band node we want to tile.
202 ///
203 /// We look for innermost band nodes where individual dimensions are marked as
204 /// permutable.
205 ///
206 /// @param Node The node to check.
207 static bool isTileableBandNode(__isl_keep isl_schedule_node *Node);
208
Tobias Grosserb241d922015-07-28 18:03:36 +0000209 /// @brief Pre-vectorizes one scheduling dimension of a schedule band.
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000210 ///
Tobias Grosserb241d922015-07-28 18:03:36 +0000211 /// prevectSchedBand splits out the dimension DimToVectorize, tiles it and
212 /// sinks the resulting point loop.
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000213 ///
Tobias Grosserb241d922015-07-28 18:03:36 +0000214 /// Example (DimToVectorize=0, VectorWidth=4):
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000215 ///
Tobias Grosserb241d922015-07-28 18:03:36 +0000216 /// | Before transformation:
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000217 /// |
218 /// | A[i,j] -> [i,j]
219 /// |
220 /// | for (i = 0; i < 128; i++)
221 /// | for (j = 0; j < 128; j++)
222 /// | A(i,j);
223 ///
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000224 /// | After transformation:
225 /// |
Tobias Grosserb241d922015-07-28 18:03:36 +0000226 /// | for (it = 0; it < 32; it+=1)
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000227 /// | for (j = 0; j < 128; j++)
Tobias Grosserb241d922015-07-28 18:03:36 +0000228 /// | for (ip = 0; ip <= 3; ip++)
229 /// | A(4 * it + ip,j);
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000230 ///
231 /// The goal of this transformation is to create a trivially vectorizable
232 /// loop. This means a parallel loop at the innermost level that has a
233 /// constant number of iterations corresponding to the target vector width.
234 ///
235 /// This transformation creates a loop at the innermost level. The loop has
236 /// a constant number of iterations, if the number of loop iterations at
237 /// DimToVectorize can be divided by VectorWidth. The default VectorWidth is
238 /// currently constant and not yet target specific. This function does not
239 /// reason about parallelism.
Tobias Grosserb241d922015-07-28 18:03:36 +0000240 static __isl_give isl_schedule_node *
241 prevectSchedBand(__isl_take isl_schedule_node *Node, unsigned DimToVectorize,
Tobias Grosser07c1c2f2015-08-19 08:46:11 +0000242 int VectorWidth);
Tobias Grosser460e9a42012-04-25 13:22:43 +0000243
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000244 /// @brief Apply additional optimizations on the bands in the schedule tree.
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000245 ///
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000246 /// We are looking for an innermost band node and apply the following
247 /// transformations:
248 ///
249 /// - Tile the band
250 /// - if the band is tileable
251 /// - if the band has more than one loop dimension
252 ///
Tobias Grosser3b10c942015-07-25 12:28:56 +0000253 /// - Prevectorize the schedule of the band (or the point loop in case of
Tobias Grosserb241d922015-07-28 18:03:36 +0000254 /// tiling).
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000255 /// - if vectorization is enabled
256 ///
257 /// @param Node The schedule node to (possibly) optimize.
258 /// @param User A pointer to forward some use information (currently unused).
259 static isl_schedule_node *optimizeBand(isl_schedule_node *Node, void *User);
Tobias Grosser460e9a42012-04-25 13:22:43 +0000260
Tobias Grosser808cd692015-07-14 09:33:13 +0000261 /// @brief Apply post-scheduling transformations.
262 ///
263 /// This function applies a set of additional local transformations on the
264 /// schedule tree as it computed by the isl scheduler. Local transformations
265 /// applied include:
266 ///
267 /// - Tiling
268 /// - Prevectorization
269 ///
270 /// @param Schedule The schedule object post-transformations will be applied
271 /// on.
272 /// @returns The transformed schedule.
273 static __isl_give isl_schedule *
274 addPostTransforms(__isl_take isl_schedule *Schedule);
Tobias Grosser28781422012-10-16 07:29:19 +0000275
Tobias Grosser20532b82014-04-11 17:56:49 +0000276 using llvm::Pass::doFinalization;
277
Johannes Doerfert909a3bf2015-03-01 18:42:08 +0000278 virtual bool doFinalization() override {
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000279 isl_schedule_free(LastSchedule);
Tobias Grosser5a56cbf2014-04-16 07:33:47 +0000280 LastSchedule = nullptr;
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000281 return true;
282 }
283};
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000284}
285
Tobias Grosser73600b82011-10-08 00:30:40 +0000286char IslScheduleOptimizer::ID = 0;
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000287
Tobias Grosserb241d922015-07-28 18:03:36 +0000288__isl_give isl_schedule_node *
289IslScheduleOptimizer::prevectSchedBand(__isl_take isl_schedule_node *Node,
290 unsigned DimToVectorize,
291 int VectorWidth) {
292 assert(isl_schedule_node_get_type(Node) == isl_schedule_node_band);
Tobias Grosserc6699b72011-06-30 20:29:13 +0000293
Tobias Grosserb241d922015-07-28 18:03:36 +0000294 auto Space = isl_schedule_node_band_get_space(Node);
295 auto ScheduleDimensions = isl_space_dim(Space, isl_dim_set);
296 isl_space_free(Space);
297 assert(DimToVectorize < ScheduleDimensions);
Tobias Grosserf5338802011-10-06 00:03:35 +0000298
Tobias Grosserb241d922015-07-28 18:03:36 +0000299 if (DimToVectorize > 0) {
300 Node = isl_schedule_node_band_split(Node, DimToVectorize);
301 Node = isl_schedule_node_child(Node, 0);
302 }
303 if (DimToVectorize < ScheduleDimensions - 1)
304 Node = isl_schedule_node_band_split(Node, 1);
305 Space = isl_schedule_node_band_get_space(Node);
306 auto Sizes = isl_multi_val_zero(Space);
307 auto Ctx = isl_schedule_node_get_ctx(Node);
308 Sizes =
309 isl_multi_val_set_val(Sizes, 0, isl_val_int_from_si(Ctx, VectorWidth));
310 Node = isl_schedule_node_band_tile(Node, Sizes);
311 Node = isl_schedule_node_child(Node, 0);
Tobias Grosser42e24892015-08-20 13:45:05 +0000312 // Make sure the "trivially vectorizable loop" is not unrolled. Otherwise,
313 // we will have troubles to match it in the backend.
314 Node = isl_schedule_node_band_set_ast_build_options(
Tobias Grosserfc490a92015-08-20 19:08:16 +0000315 Node, isl_union_set_read_from_str(Ctx, "{ unroll[x]: 1 = 0 }"));
316 Node = isl_schedule_node_band_sink(Node);
Tobias Grosserb241d922015-07-28 18:03:36 +0000317 Node = isl_schedule_node_child(Node, 0);
318 return Node;
Tobias Grosserc6699b72011-06-30 20:29:13 +0000319}
320
Tobias Grosserd891b542015-08-20 12:16:23 +0000321__isl_give isl_schedule_node *
Tobias Grosser9bdea572015-08-20 12:22:37 +0000322IslScheduleOptimizer::tileNode(__isl_take isl_schedule_node *Node,
Tobias Grosser1ac884d2015-08-23 09:11:00 +0000323 const char *Identifier, ArrayRef<int> TileSizes,
324 int DefaultTileSize) {
Tobias Grosser9bdea572015-08-20 12:22:37 +0000325 auto Ctx = isl_schedule_node_get_ctx(Node);
326 auto Space = isl_schedule_node_band_get_space(Node);
327 auto Dims = isl_space_dim(Space, isl_dim_set);
328 auto Sizes = isl_multi_val_zero(Space);
Tobias Grosser1ac884d2015-08-23 09:11:00 +0000329 std::string IdentifierString(Identifier);
Tobias Grosser9bdea572015-08-20 12:22:37 +0000330 for (unsigned i = 0; i < Dims; i++) {
331 auto tileSize = i < TileSizes.size() ? TileSizes[i] : DefaultTileSize;
332 Sizes = isl_multi_val_set_val(Sizes, i, isl_val_int_from_si(Ctx, tileSize));
333 }
Tobias Grosser1ac884d2015-08-23 09:11:00 +0000334 auto TileLoopMarkerStr = IdentifierString + " - Tiles";
335 isl_id *TileLoopMarker =
336 isl_id_alloc(Ctx, TileLoopMarkerStr.c_str(), nullptr);
337 Node = isl_schedule_node_insert_mark(Node, TileLoopMarker);
338 Node = isl_schedule_node_child(Node, 0);
Tobias Grosser9bdea572015-08-20 12:22:37 +0000339 Node = isl_schedule_node_band_tile(Node, Sizes);
Tobias Grosser1ac884d2015-08-23 09:11:00 +0000340 Node = isl_schedule_node_child(Node, 0);
341 auto PointLoopMarkerStr = IdentifierString + " - Points";
342 isl_id *PointLoopMarker =
343 isl_id_alloc(Ctx, PointLoopMarkerStr.c_str(), nullptr);
344 Node = isl_schedule_node_insert_mark(Node, PointLoopMarker);
345 Node = isl_schedule_node_child(Node, 0);
346 return Node;
Tobias Grosser9bdea572015-08-20 12:22:37 +0000347}
348
Tobias Grosser862b9b52015-08-20 12:32:45 +0000349bool IslScheduleOptimizer::isTileableBandNode(
350 __isl_keep isl_schedule_node *Node) {
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000351 if (isl_schedule_node_get_type(Node) != isl_schedule_node_band)
Tobias Grosser862b9b52015-08-20 12:32:45 +0000352 return false;
Tobias Grosserde68cc92011-06-30 20:01:02 +0000353
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000354 if (isl_schedule_node_n_children(Node) != 1)
Tobias Grosser862b9b52015-08-20 12:32:45 +0000355 return false;
Tobias Grosserde68cc92011-06-30 20:01:02 +0000356
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000357 if (!isl_schedule_node_band_get_permutable(Node))
Tobias Grosser862b9b52015-08-20 12:32:45 +0000358 return false;
Tobias Grosser44f19ac2011-07-05 22:15:53 +0000359
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000360 auto Space = isl_schedule_node_band_get_space(Node);
361 auto Dims = isl_space_dim(Space, isl_dim_set);
Tobias Grosser9bdea572015-08-20 12:22:37 +0000362 isl_space_free(Space);
Tobias Grosserde68cc92011-06-30 20:01:02 +0000363
Tobias Grosser9bdea572015-08-20 12:22:37 +0000364 if (Dims <= 1)
Tobias Grosser862b9b52015-08-20 12:32:45 +0000365 return false;
Tobias Grosserde68cc92011-06-30 20:01:02 +0000366
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000367 auto Child = isl_schedule_node_get_child(Node, 0);
368 auto Type = isl_schedule_node_get_type(Child);
369 isl_schedule_node_free(Child);
370
Tobias Grosser9bdea572015-08-20 12:22:37 +0000371 if (Type != isl_schedule_node_leaf)
Tobias Grosser862b9b52015-08-20 12:32:45 +0000372 return false;
373
374 return true;
375}
376
377__isl_give isl_schedule_node *
378IslScheduleOptimizer::optimizeBand(__isl_take isl_schedule_node *Node,
379 void *User) {
380 if (!isTileableBandNode(Node))
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000381 return Node;
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000382
Tobias Grosser04832712015-08-20 13:45:02 +0000383 if (FirstLevelTiling)
Tobias Grosser1ac884d2015-08-23 09:11:00 +0000384 Node = tileNode(Node, "1st level tiling", FirstLevelTileSizes,
385 FirstLevelDefaultTileSize);
Tobias Grosser04832712015-08-20 13:45:02 +0000386
387 if (SecondLevelTiling)
Tobias Grosser1ac884d2015-08-23 09:11:00 +0000388 Node = tileNode(Node, "2nd level tiling", SecondLevelTileSizes,
389 SecondLevelDefaultTileSize);
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000390
Tobias Grosser42e24892015-08-20 13:45:05 +0000391 if (RegisterTiling) {
392 auto *Ctx = isl_schedule_node_get_ctx(Node);
Tobias Grosser1ac884d2015-08-23 09:11:00 +0000393 Node = tileNode(Node, "Register tiling", RegisterTileSizes,
394 RegisterDefaultTileSize);
Tobias Grosser42e24892015-08-20 13:45:05 +0000395 Node = isl_schedule_node_band_set_ast_build_options(
396 Node, isl_union_set_read_from_str(Ctx, "{unroll[x]}"));
397 }
398
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000399 if (PollyVectorizerChoice == VECTORIZER_NONE)
Tobias Grosserf10f4632015-08-19 08:03:37 +0000400 return Node;
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000401
Tobias Grosser862b9b52015-08-20 12:32:45 +0000402 auto Space = isl_schedule_node_band_get_space(Node);
403 auto Dims = isl_space_dim(Space, isl_dim_set);
404 isl_space_free(Space);
405
Tobias Grosserb241d922015-07-28 18:03:36 +0000406 for (int i = Dims - 1; i >= 0; i--)
Tobias Grosserf10f4632015-08-19 08:03:37 +0000407 if (isl_schedule_node_band_member_get_coincident(Node, i)) {
Tobias Grosser07c1c2f2015-08-19 08:46:11 +0000408 Node = IslScheduleOptimizer::prevectSchedBand(Node, i, PrevectorWidth);
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000409 break;
410 }
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000411
Tobias Grosserf10f4632015-08-19 08:03:37 +0000412 return Node;
Tobias Grosserde68cc92011-06-30 20:01:02 +0000413}
414
Tobias Grosser808cd692015-07-14 09:33:13 +0000415__isl_give isl_schedule *
416IslScheduleOptimizer::addPostTransforms(__isl_take isl_schedule *Schedule) {
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000417 isl_schedule_node *Root = isl_schedule_get_root(Schedule);
Tobias Grosser808cd692015-07-14 09:33:13 +0000418 isl_schedule_free(Schedule);
Tobias Grosserb2f39922015-05-28 13:32:11 +0000419 Root = isl_schedule_node_map_descendant_bottom_up(
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000420 Root, IslScheduleOptimizer::optimizeBand, NULL);
Tobias Grosser808cd692015-07-14 09:33:13 +0000421 auto S = isl_schedule_node_get_schedule(Root);
Tobias Grosserbbb4cec2015-03-22 12:06:39 +0000422 isl_schedule_node_free(Root);
Tobias Grosser808cd692015-07-14 09:33:13 +0000423 return S;
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000424}
425
Johannes Doerfert7ceb0402015-02-11 17:25:09 +0000426bool IslScheduleOptimizer::isProfitableSchedule(
427 Scop &S, __isl_keep isl_union_map *NewSchedule) {
428 // To understand if the schedule has been optimized we check if the schedule
429 // has changed at all.
430 // TODO: We can improve this by tracking if any necessarily beneficial
431 // transformations have been performed. This can e.g. be tiling, loop
432 // interchange, or ...) We can track this either at the place where the
433 // transformation has been performed or, in case of automatic ILP based
434 // optimizations, by comparing (yet to be defined) performance metrics
435 // before/after the scheduling optimizer
436 // (e.g., #stride-one accesses)
437 isl_union_map *OldSchedule = S.getSchedule();
438 bool changed = !isl_union_map_is_equal(OldSchedule, NewSchedule);
439 isl_union_map_free(OldSchedule);
440 return changed;
441}
442
Tobias Grosser73600b82011-10-08 00:30:40 +0000443bool IslScheduleOptimizer::runOnScop(Scop &S) {
Johannes Doerfert6f7921f2015-02-14 12:02:24 +0000444
445 // Skip empty SCoPs but still allow code generation as it will delete the
446 // loops present but not needed.
447 if (S.getSize() == 0) {
448 S.markAsOptimized();
449 return false;
450 }
451
Johannes Doerfert7e6424b2015-03-05 00:43:48 +0000452 const Dependences &D = getAnalysis<DependenceInfo>().getDependences();
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000453
Johannes Doerfert7e6424b2015-03-05 00:43:48 +0000454 if (!D.hasValidDependences())
Tobias Grosser38c36ea2014-02-23 15:15:44 +0000455 return false;
456
Tobias Grosser28781422012-10-16 07:29:19 +0000457 isl_schedule_free(LastSchedule);
Tobias Grosser5a56cbf2014-04-16 07:33:47 +0000458 LastSchedule = nullptr;
Tobias Grosser28781422012-10-16 07:29:19 +0000459
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000460 // Build input data.
Johannes Doerfert7e6424b2015-03-05 00:43:48 +0000461 int ValidityKinds =
462 Dependences::TYPE_RAW | Dependences::TYPE_WAR | Dependences::TYPE_WAW;
Tobias Grosser1deda292012-02-14 14:02:48 +0000463 int ProximityKinds;
464
465 if (OptimizeDeps == "all")
Johannes Doerfert7e6424b2015-03-05 00:43:48 +0000466 ProximityKinds =
467 Dependences::TYPE_RAW | Dependences::TYPE_WAR | Dependences::TYPE_WAW;
Tobias Grosser1deda292012-02-14 14:02:48 +0000468 else if (OptimizeDeps == "raw")
Johannes Doerfert7e6424b2015-03-05 00:43:48 +0000469 ProximityKinds = Dependences::TYPE_RAW;
Tobias Grosser1deda292012-02-14 14:02:48 +0000470 else {
471 errs() << "Do not know how to optimize for '" << OptimizeDeps << "'"
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000472 << " Falling back to optimizing all dependences.\n";
Johannes Doerfert7e6424b2015-03-05 00:43:48 +0000473 ProximityKinds =
474 Dependences::TYPE_RAW | Dependences::TYPE_WAR | Dependences::TYPE_WAW;
Tobias Grosser1deda292012-02-14 14:02:48 +0000475 }
476
Tobias Grosser5f9a7622012-02-14 14:02:40 +0000477 isl_union_set *Domain = S.getDomains();
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000478
Tobias Grosser98610ee2012-02-13 23:31:39 +0000479 if (!Domain)
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000480 return false;
481
Johannes Doerfert7e6424b2015-03-05 00:43:48 +0000482 isl_union_map *Validity = D.getDependences(ValidityKinds);
483 isl_union_map *Proximity = D.getDependences(ProximityKinds);
Tobias Grosser8a507022012-03-16 11:51:41 +0000484
Tobias Grossera26db472012-01-30 19:38:43 +0000485 // Simplify the dependences by removing the constraints introduced by the
486 // domains. This can speed up the scheduling time significantly, as large
487 // constant coefficients will be removed from the dependences. The
488 // introduction of some additional dependences reduces the possible
489 // transformations, but in most cases, such transformation do not seem to be
490 // interesting anyway. In some cases this option may stop the scheduler to
491 // find any schedule.
492 if (SimplifyDeps == "yes") {
Tobias Grosser00383a72012-02-14 14:02:44 +0000493 Validity = isl_union_map_gist_domain(Validity, isl_union_set_copy(Domain));
494 Validity = isl_union_map_gist_range(Validity, isl_union_set_copy(Domain));
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000495 Proximity =
496 isl_union_map_gist_domain(Proximity, isl_union_set_copy(Domain));
Tobias Grosser00383a72012-02-14 14:02:44 +0000497 Proximity = isl_union_map_gist_range(Proximity, isl_union_set_copy(Domain));
Tobias Grossera26db472012-01-30 19:38:43 +0000498 } else if (SimplifyDeps != "no") {
499 errs() << "warning: Option -polly-opt-simplify-deps should either be 'yes' "
500 "or 'no'. Falling back to default: 'yes'\n";
501 }
502
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000503 DEBUG(dbgs() << "\n\nCompute schedule from: ");
Tobias Grosser01aea582014-10-22 23:16:28 +0000504 DEBUG(dbgs() << "Domain := " << stringFromIslObj(Domain) << ";\n");
505 DEBUG(dbgs() << "Proximity := " << stringFromIslObj(Proximity) << ";\n");
506 DEBUG(dbgs() << "Validity := " << stringFromIslObj(Validity) << ";\n");
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000507
Michael Krusec59f22c2015-06-18 16:45:40 +0000508 unsigned IslSerializeSCCs;
Tobias Grosserb3ad85b2012-01-30 19:38:50 +0000509
510 if (FusionStrategy == "max") {
Michael Krusec59f22c2015-06-18 16:45:40 +0000511 IslSerializeSCCs = 0;
Tobias Grosserb3ad85b2012-01-30 19:38:50 +0000512 } else if (FusionStrategy == "min") {
Michael Krusec59f22c2015-06-18 16:45:40 +0000513 IslSerializeSCCs = 1;
Tobias Grosserb3ad85b2012-01-30 19:38:50 +0000514 } else {
515 errs() << "warning: Unknown fusion strategy. Falling back to maximal "
516 "fusion.\n";
Michael Krusec59f22c2015-06-18 16:45:40 +0000517 IslSerializeSCCs = 0;
Tobias Grosserb3ad85b2012-01-30 19:38:50 +0000518 }
519
Tobias Grosser95e860c2012-01-30 19:38:54 +0000520 int IslMaximizeBands;
521
Tobias Grossera4ea90b2012-01-30 22:43:56 +0000522 if (MaximizeBandDepth == "yes") {
Tobias Grosser95e860c2012-01-30 19:38:54 +0000523 IslMaximizeBands = 1;
Tobias Grossera4ea90b2012-01-30 22:43:56 +0000524 } else if (MaximizeBandDepth == "no") {
Tobias Grosser95e860c2012-01-30 19:38:54 +0000525 IslMaximizeBands = 0;
526 } else {
527 errs() << "warning: Option -polly-opt-maximize-bands should either be 'yes'"
528 " or 'no'. Falling back to default: 'yes'\n";
529 IslMaximizeBands = 1;
530 }
531
Michael Krusec59f22c2015-06-18 16:45:40 +0000532 isl_options_set_schedule_serialize_sccs(S.getIslCtx(), IslSerializeSCCs);
Tobias Grosser95e860c2012-01-30 19:38:54 +0000533 isl_options_set_schedule_maximize_band_depth(S.getIslCtx(), IslMaximizeBands);
Tobias Grosser992e60c2012-02-20 08:41:15 +0000534 isl_options_set_schedule_max_constant_term(S.getIslCtx(), MaxConstantTerm);
Tobias Grosser92f54802012-02-20 08:41:47 +0000535 isl_options_set_schedule_max_coefficient(S.getIslCtx(), MaxCoefficient);
Tobias Grosser4f6bcef2015-03-31 07:52:36 +0000536 isl_options_set_tile_scale_tile_loops(S.getIslCtx(), 0);
Tobias Grosser42152ff2012-01-30 19:38:47 +0000537
538 isl_options_set_on_error(S.getIslCtx(), ISL_ON_ERROR_CONTINUE);
Tobias Grossera38c9242014-01-26 19:36:28 +0000539
540 isl_schedule_constraints *ScheduleConstraints;
541 ScheduleConstraints = isl_schedule_constraints_on_domain(Domain);
542 ScheduleConstraints =
543 isl_schedule_constraints_set_proximity(ScheduleConstraints, Proximity);
544 ScheduleConstraints = isl_schedule_constraints_set_validity(
545 ScheduleConstraints, isl_union_map_copy(Validity));
546 ScheduleConstraints =
547 isl_schedule_constraints_set_coincidence(ScheduleConstraints, Validity);
Tobias Grosser00383a72012-02-14 14:02:44 +0000548 isl_schedule *Schedule;
Tobias Grossera38c9242014-01-26 19:36:28 +0000549 Schedule = isl_schedule_constraints_compute_schedule(ScheduleConstraints);
Tobias Grosser42152ff2012-01-30 19:38:47 +0000550 isl_options_set_on_error(S.getIslCtx(), ISL_ON_ERROR_ABORT);
551
552 // In cases the scheduler is not able to optimize the code, we just do not
553 // touch the schedule.
Tobias Grosser98610ee2012-02-13 23:31:39 +0000554 if (!Schedule)
Tobias Grosser42152ff2012-01-30 19:38:47 +0000555 return false;
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000556
Tobias Grosser97d87452015-05-30 06:46:59 +0000557 DEBUG({
558 auto *P = isl_printer_to_str(S.getIslCtx());
559 P = isl_printer_set_yaml_style(P, ISL_YAML_STYLE_BLOCK);
560 P = isl_printer_print_schedule(P, Schedule);
561 dbgs() << "NewScheduleTree: \n" << isl_printer_get_str(P) << "\n";
562 isl_printer_free(P);
563 });
Tobias Grosser4d63b9d2012-02-20 08:41:21 +0000564
Tobias Grosser808cd692015-07-14 09:33:13 +0000565 isl_schedule *NewSchedule = addPostTransforms(Schedule);
566 isl_union_map *NewScheduleMap = isl_schedule_get_map(NewSchedule);
Johannes Doerfert7ceb0402015-02-11 17:25:09 +0000567
Tobias Grosser808cd692015-07-14 09:33:13 +0000568 if (!isProfitableSchedule(S, NewScheduleMap)) {
569 isl_union_map_free(NewScheduleMap);
570 isl_schedule_free(NewSchedule);
Johannes Doerfert7ceb0402015-02-11 17:25:09 +0000571 return false;
572 }
573
Tobias Grosser808cd692015-07-14 09:33:13 +0000574 S.setScheduleTree(NewSchedule);
Johannes Doerfert7ceb0402015-02-11 17:25:09 +0000575 S.markAsOptimized();
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000576
Tobias Grosser808cd692015-07-14 09:33:13 +0000577 isl_union_map_free(NewScheduleMap);
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000578 return false;
579}
580
Johannes Doerfert3fe584d2015-03-01 18:40:25 +0000581void IslScheduleOptimizer::printScop(raw_ostream &OS, Scop &) const {
Tobias Grosser28781422012-10-16 07:29:19 +0000582 isl_printer *p;
583 char *ScheduleStr;
584
585 OS << "Calculated schedule:\n";
586
587 if (!LastSchedule) {
588 OS << "n/a\n";
589 return;
590 }
591
592 p = isl_printer_to_str(isl_schedule_get_ctx(LastSchedule));
593 p = isl_printer_print_schedule(p, LastSchedule);
594 ScheduleStr = isl_printer_get_str(p);
595 isl_printer_free(p);
596
597 OS << ScheduleStr << "\n";
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000598}
599
Tobias Grosser73600b82011-10-08 00:30:40 +0000600void IslScheduleOptimizer::getAnalysisUsage(AnalysisUsage &AU) const {
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000601 ScopPass::getAnalysisUsage(AU);
Johannes Doerfertf6557f92015-03-04 22:43:40 +0000602 AU.addRequired<DependenceInfo>();
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000603}
604
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000605Pass *polly::createIslScheduleOptimizerPass() {
Tobias Grosser73600b82011-10-08 00:30:40 +0000606 return new IslScheduleOptimizer();
Tobias Grosser30aa24c2011-05-14 19:02:06 +0000607}
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000608
609INITIALIZE_PASS_BEGIN(IslScheduleOptimizer, "polly-opt-isl",
610 "Polly - Optimize schedule of SCoP", false, false);
Johannes Doerfertf6557f92015-03-04 22:43:40 +0000611INITIALIZE_PASS_DEPENDENCY(DependenceInfo);
Tobias Grosser4d96c8d2013-03-23 01:05:07 +0000612INITIALIZE_PASS_DEPENDENCY(ScopInfo);
613INITIALIZE_PASS_END(IslScheduleOptimizer, "polly-opt-isl",
614 "Polly - Optimize schedule of SCoP", false, false)