blob: bb1991511b350e7be2571b246ec42901d7252c7b [file] [log] [blame]
Sebastian Pop082cea82012-05-07 16:20:07 +00001//===------ IslCodeGeneration.cpp - Code generate the Scops using ISL. ----===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// The IslCodeGeneration pass takes a Scop created by ScopInfo and translates it
11// back to LLVM-IR using the ISL code generator.
12//
13// The Scop describes the high level memory behaviour of a control flow region.
14// Transformation passes can update the schedule (execution order) of statements
15// in the Scop. ISL is used to generate an abstract syntax tree that reflects
16// the updated execution order. This clast is used to create new LLVM-IR that is
17// computationally equivalent to the original control flow region, but executes
18// its code in the new execution order defined by the changed scattering.
19//
20//===----------------------------------------------------------------------===//
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000021#include "polly/Config/config.h"
Johannes Doerferta63b2572014-08-03 01:51:59 +000022#include "polly/CodeGen/IslExprBuilder.h"
Tobias Grosser83628182013-05-07 08:11:54 +000023#include "polly/CodeGen/BlockGenerators.h"
24#include "polly/CodeGen/CodeGeneration.h"
25#include "polly/CodeGen/IslAst.h"
26#include "polly/CodeGen/LoopGenerators.h"
27#include "polly/CodeGen/Utils.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000028#include "polly/Dependences.h"
29#include "polly/LinkAllPasses.h"
30#include "polly/ScopInfo.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000031#include "polly/Support/GICHelper.h"
Tobias Grosser0ee50f62013-04-10 06:55:31 +000032#include "polly/Support/ScopHelper.h"
Tobias Grosser83628182013-05-07 08:11:54 +000033#include "polly/TempScopInfo.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000034#include "llvm/Analysis/LoopInfo.h"
Matt Arsenault8ca36812014-07-19 18:40:17 +000035#include "llvm/Analysis/PostDominators.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000036#include "llvm/Analysis/ScalarEvolutionExpander.h"
Tobias Grosser83628182013-05-07 08:11:54 +000037#include "llvm/IR/Module.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000038#include "llvm/Support/CommandLine.h"
39#include "llvm/Support/Debug.h"
Chandler Carruth535d52c2013-01-02 11:47:44 +000040#include "llvm/IR/DataLayout.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000041#include "llvm/Transforms/Utils/BasicBlockUtils.h"
42
43#include "isl/union_map.h"
44#include "isl/list.h"
45#include "isl/ast.h"
46#include "isl/ast_build.h"
47#include "isl/set.h"
48#include "isl/map.h"
49#include "isl/aff.h"
50
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000051using namespace polly;
52using namespace llvm;
53
Chandler Carruth95fef942014-04-22 03:30:19 +000054#define DEBUG_TYPE "polly-codegen-isl"
55
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000056class IslNodeBuilder {
57public:
Johannes Doerfert51d1c742014-10-02 15:32:17 +000058 IslNodeBuilder(PollyIRBuilder &Builder, ScopAnnotator &Annotator, Pass *P,
Johannes Doerfert2ef3f4f2014-08-07 17:14:54 +000059 LoopInfo &LI, ScalarEvolution &SE, DominatorTree &DT)
Johannes Doerfert2ef33e92014-10-05 11:33:59 +000060 : Builder(Builder), Annotator(Annotator),
61 Rewriter(new SCEVExpander(SE, "polly")),
62 ExprBuilder(Builder, IDToValue, *Rewriter), P(P), LI(LI), SE(SE),
63 DT(DT) {}
64
65 ~IslNodeBuilder() { delete Rewriter; }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000066
67 void addParameters(__isl_take isl_set *Context);
68 void create(__isl_take isl_ast_node *Node);
Tobias Grosser54ee0ba2013-11-17 03:18:25 +000069 IslExprBuilder &getExprBuilder() { return ExprBuilder; }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000070
71private:
Tobias Grosser5103ba72014-03-04 14:58:49 +000072 PollyIRBuilder &Builder;
Johannes Doerfert51d1c742014-10-02 15:32:17 +000073 ScopAnnotator &Annotator;
Johannes Doerfert2ef33e92014-10-05 11:33:59 +000074
75 /// @brief A SCEVExpander to create llvm values from SCEVs.
76 SCEVExpander *Rewriter;
77
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000078 IslExprBuilder ExprBuilder;
79 Pass *P;
Johannes Doerfert2ef3f4f2014-08-07 17:14:54 +000080 LoopInfo &LI;
81 ScalarEvolution &SE;
82 DominatorTree &DT;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000083
84 // This maps an isl_id* to the Value* it has in the generated program. For now
85 // on, the only isl_ids that are stored here are the newly calculated loop
86 // ivs.
Tobias Grosser566ad582014-08-31 16:21:12 +000087 IslExprBuilder::IDToValueTy IDToValue;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000088
Tobias Grosserec7d67e2014-11-06 00:27:01 +000089 /// Generate code for a given SCEV*
90 ///
91 /// This function generates code for a given SCEV expression. It generated
92 /// code is emmitted at the end of the basic block our Builder currently
93 /// points to and the resulting value is returned.
94 ///
95 /// @param Expr The expression to code generate.
96 Value *generateSCEV(const SCEV *Expr);
97
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000098 // Extract the upper bound of this loop
99 //
100 // The isl code generation can generate arbitrary expressions to check if the
101 // upper bound of a loop is reached, but it provides an option to enforce
102 // 'atomic' upper bounds. An 'atomic upper bound is always of the form
103 // iv <= expr, where expr is an (arbitrary) expression not containing iv.
104 //
105 // This function extracts 'atomic' upper bounds. Polly, in general, requires
106 // atomic upper bounds for the following reasons:
107 //
108 // 1. An atomic upper bound is loop invariant
109 //
110 // It must not be calculated at each loop iteration and can often even be
111 // hoisted out further by the loop invariant code motion.
112 //
113 // 2. OpenMP needs a loop invarient upper bound to calculate the number
114 // of loop iterations.
115 //
116 // 3. With the existing code, upper bounds have been easier to implement.
Tobias Grossere602a072013-05-07 07:30:56 +0000117 __isl_give isl_ast_expr *getUpperBound(__isl_keep isl_ast_node *For,
118 CmpInst::Predicate &Predicate);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000119
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000120 unsigned getNumberOfIterations(__isl_keep isl_ast_node *For);
121
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000122 void createFor(__isl_take isl_ast_node *For);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000123 void createForVector(__isl_take isl_ast_node *For, int VectorWidth);
124 void createForSequential(__isl_take isl_ast_node *For);
Tobias Grosserce67a042014-07-02 16:26:47 +0000125
126 /// Generate LLVM-IR that computes the values of the original induction
127 /// variables in function of the newly generated loop induction variables.
128 ///
129 /// Example:
130 ///
131 /// // Original
132 /// for i
133 /// for j
134 /// S(i)
135 ///
136 /// Schedule: [i,j] -> [i+j, j]
137 ///
138 /// // New
139 /// for c0
140 /// for c1
141 /// S(c0 - c1, c1)
142 ///
143 /// Assuming the original code consists of two loops which are
144 /// transformed according to a schedule [i,j] -> [c0=i+j,c1=j]. The resulting
145 /// ast models the original statement as a call expression where each argument
146 /// is an expression that computes the old induction variables from the new
147 /// ones, ordered such that the first argument computes the value of induction
148 /// variable that was outermost in the original code.
149 ///
150 /// @param Expr The call expression that represents the statement.
151 /// @param Stmt The statement that is called.
152 /// @param VMap The value map into which the mapping from the old induction
153 /// variable to the new one is inserted. This mapping is used
154 /// for the classical code generation (not scev-based) and
155 /// gives an explicit mapping from an original, materialized
156 /// induction variable. It consequently can only be expressed
157 /// if there was an explicit induction variable.
158 /// @param LTS The loop to SCEV map in which the mapping from the original
159 /// loop to a SCEV representing the new loop iv is added. This
160 /// mapping does not require an explicit induction variable.
161 /// Instead, we think in terms of an implicit induction variable
162 /// that counts the number of times a loop is executed. For each
163 /// original loop this count, expressed in function of the new
164 /// induction variables, is added to the LTS map.
165 void createSubstitutions(__isl_take isl_ast_expr *Expr, ScopStmt *Stmt,
Sebastian Pop9d10fff2013-02-15 20:55:59 +0000166 ValueMapT &VMap, LoopToScevMapT &LTS);
Tobias Grosserce67a042014-07-02 16:26:47 +0000167 void createSubstitutionsVector(__isl_take isl_ast_expr *Expr, ScopStmt *Stmt,
168 VectorValueMapT &VMap,
Tobias Grossere602a072013-05-07 07:30:56 +0000169 std::vector<LoopToScevMapT> &VLTS,
170 std::vector<Value *> &IVS,
171 __isl_take isl_id *IteratorID);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000172 void createIf(__isl_take isl_ast_node *If);
Tobias Grossere602a072013-05-07 07:30:56 +0000173 void createUserVector(__isl_take isl_ast_node *User,
174 std::vector<Value *> &IVS,
175 __isl_take isl_id *IteratorID,
176 __isl_take isl_union_map *Schedule);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000177 void createUser(__isl_take isl_ast_node *User);
178 void createBlock(__isl_take isl_ast_node *Block);
179};
180
Tobias Grossere602a072013-05-07 07:30:56 +0000181__isl_give isl_ast_expr *
182IslNodeBuilder::getUpperBound(__isl_keep isl_ast_node *For,
183 ICmpInst::Predicate &Predicate) {
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000184 isl_id *UBID, *IteratorID;
185 isl_ast_expr *Cond, *Iterator, *UB, *Arg0;
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000186 isl_ast_op_type Type;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000187
188 Cond = isl_ast_node_for_get_cond(For);
189 Iterator = isl_ast_node_for_get_iterator(For);
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000190 Type = isl_ast_expr_get_op_type(Cond);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000191
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000192 assert(isl_ast_expr_get_type(Cond) == isl_ast_expr_op &&
193 "conditional expression is not an atomic upper bound");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000194
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000195 switch (Type) {
Tobias Grosserc14582f2013-02-05 18:01:29 +0000196 case isl_ast_op_le:
197 Predicate = ICmpInst::ICMP_SLE;
198 break;
199 case isl_ast_op_lt:
200 Predicate = ICmpInst::ICMP_SLT;
201 break;
202 default:
203 llvm_unreachable("Unexpected comparision type in loop conditon");
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000204 }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000205
206 Arg0 = isl_ast_expr_get_op_arg(Cond, 0);
207
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000208 assert(isl_ast_expr_get_type(Arg0) == isl_ast_expr_id &&
209 "conditional expression is not an atomic upper bound");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000210
211 UBID = isl_ast_expr_get_id(Arg0);
212
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000213 assert(isl_ast_expr_get_type(Iterator) == isl_ast_expr_id &&
214 "Could not get the iterator");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000215
216 IteratorID = isl_ast_expr_get_id(Iterator);
217
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000218 assert(UBID == IteratorID &&
219 "conditional expression is not an atomic upper bound");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000220
221 UB = isl_ast_expr_get_op_arg(Cond, 1);
222
223 isl_ast_expr_free(Cond);
224 isl_ast_expr_free(Iterator);
225 isl_ast_expr_free(Arg0);
226 isl_id_free(IteratorID);
227 isl_id_free(UBID);
228
229 return UB;
230}
231
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000232unsigned IslNodeBuilder::getNumberOfIterations(__isl_keep isl_ast_node *For) {
Johannes Doerfert94d90822014-07-23 20:26:25 +0000233 isl_union_map *Schedule = IslAstInfo::getSchedule(For);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000234 isl_set *LoopDomain = isl_set_from_union_set(isl_union_map_range(Schedule));
Sebastian Pop2aa5c242012-12-18 08:56:51 +0000235 int NumberOfIterations = polly::getNumberOfIterations(LoopDomain);
236 if (NumberOfIterations == -1)
237 return -1;
238 return NumberOfIterations + 1;
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000239}
240
Tobias Grossere602a072013-05-07 07:30:56 +0000241void IslNodeBuilder::createUserVector(__isl_take isl_ast_node *User,
242 std::vector<Value *> &IVS,
243 __isl_take isl_id *IteratorID,
244 __isl_take isl_union_map *Schedule) {
Tobias Grosserce67a042014-07-02 16:26:47 +0000245 isl_ast_expr *Expr = isl_ast_node_user_get_expr(User);
246 isl_ast_expr *StmtExpr = isl_ast_expr_get_op_arg(Expr, 0);
247 isl_id *Id = isl_ast_expr_get_id(StmtExpr);
248 isl_ast_expr_free(StmtExpr);
Tobias Grosserc14582f2013-02-05 18:01:29 +0000249 ScopStmt *Stmt = (ScopStmt *)isl_id_get_user(Id);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000250 VectorValueMapT VectorMap(IVS.size());
Sebastian Pop9d10fff2013-02-15 20:55:59 +0000251 std::vector<LoopToScevMapT> VLTS(IVS.size());
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000252
253 isl_union_set *Domain = isl_union_set_from_set(Stmt->getDomain());
254 Schedule = isl_union_map_intersect_domain(Schedule, Domain);
255 isl_map *S = isl_map_from_union_map(Schedule);
256
Tobias Grosserce67a042014-07-02 16:26:47 +0000257 createSubstitutionsVector(Expr, Stmt, VectorMap, VLTS, IVS, IteratorID);
Johannes Doerfert731685e2014-10-08 17:25:30 +0000258 VectorBlockGenerator::generate(Builder, *Stmt, VectorMap, VLTS, S, P, LI, SE,
259 IslAstInfo::getBuild(User), &ExprBuilder);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000260
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000261 isl_map_free(S);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000262 isl_id_free(Id);
263 isl_ast_node_free(User);
264}
265
Tobias Grossere602a072013-05-07 07:30:56 +0000266void IslNodeBuilder::createForVector(__isl_take isl_ast_node *For,
267 int VectorWidth) {
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000268 isl_ast_node *Body = isl_ast_node_for_get_body(For);
269 isl_ast_expr *Init = isl_ast_node_for_get_init(For);
270 isl_ast_expr *Inc = isl_ast_node_for_get_inc(For);
271 isl_ast_expr *Iterator = isl_ast_node_for_get_iterator(For);
272 isl_id *IteratorID = isl_ast_expr_get_id(Iterator);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000273
274 Value *ValueLB = ExprBuilder.create(Init);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000275 Value *ValueInc = ExprBuilder.create(Inc);
276
277 Type *MaxType = ExprBuilder.getType(Iterator);
278 MaxType = ExprBuilder.getWidestType(MaxType, ValueLB->getType());
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000279 MaxType = ExprBuilder.getWidestType(MaxType, ValueInc->getType());
280
281 if (MaxType != ValueLB->getType())
282 ValueLB = Builder.CreateSExt(ValueLB, MaxType);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000283 if (MaxType != ValueInc->getType())
284 ValueInc = Builder.CreateSExt(ValueInc, MaxType);
285
Tobias Grosserc14582f2013-02-05 18:01:29 +0000286 std::vector<Value *> IVS(VectorWidth);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000287 IVS[0] = ValueLB;
288
289 for (int i = 1; i < VectorWidth; i++)
Tobias Grosserc14582f2013-02-05 18:01:29 +0000290 IVS[i] = Builder.CreateAdd(IVS[i - 1], ValueInc, "p_vector_iv");
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000291
Johannes Doerfert94d90822014-07-23 20:26:25 +0000292 isl_union_map *Schedule = IslAstInfo::getSchedule(For);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000293 assert(Schedule && "For statement annotation does not contain its schedule");
294
295 IDToValue[IteratorID] = ValueLB;
296
297 switch (isl_ast_node_get_type(Body)) {
298 case isl_ast_node_user:
299 createUserVector(Body, IVS, isl_id_copy(IteratorID),
300 isl_union_map_copy(Schedule));
301 break;
302 case isl_ast_node_block: {
303 isl_ast_node_list *List = isl_ast_node_block_get_children(Body);
304
305 for (int i = 0; i < isl_ast_node_list_n_ast_node(List); ++i)
306 createUserVector(isl_ast_node_list_get_ast_node(List, i), IVS,
Tobias Grosser1bb59b02012-12-29 23:47:38 +0000307 isl_id_copy(IteratorID), isl_union_map_copy(Schedule));
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000308
309 isl_ast_node_free(Body);
310 isl_ast_node_list_free(List);
311 break;
312 }
313 default:
314 isl_ast_node_dump(Body);
315 llvm_unreachable("Unhandled isl_ast_node in vectorizer");
316 }
317
318 IDToValue.erase(IteratorID);
319 isl_id_free(IteratorID);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000320 isl_union_map_free(Schedule);
321
322 isl_ast_node_free(For);
323 isl_ast_expr_free(Iterator);
324}
325
326void IslNodeBuilder::createForSequential(__isl_take isl_ast_node *For) {
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000327 isl_ast_node *Body;
328 isl_ast_expr *Init, *Inc, *Iterator, *UB;
329 isl_id *IteratorID;
330 Value *ValueLB, *ValueUB, *ValueInc;
331 Type *MaxType;
Tobias Grosser5db6ffd2013-05-16 06:40:06 +0000332 BasicBlock *ExitBlock;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000333 Value *IV;
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000334 CmpInst::Predicate Predicate;
Tobias Grosser37c9b8e2014-03-04 14:59:00 +0000335 bool Parallel;
336
Johannes Doerfertc7b719f2014-10-01 20:10:44 +0000337 Parallel =
338 IslAstInfo::isParallel(For) && !IslAstInfo::isReductionParallel(For);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000339
340 Body = isl_ast_node_for_get_body(For);
341
342 // isl_ast_node_for_is_degenerate(For)
343 //
344 // TODO: For degenerated loops we could generate a plain assignment.
345 // However, for now we just reuse the logic for normal loops, which will
346 // create a loop with a single iteration.
347
348 Init = isl_ast_node_for_get_init(For);
349 Inc = isl_ast_node_for_get_inc(For);
350 Iterator = isl_ast_node_for_get_iterator(For);
351 IteratorID = isl_ast_expr_get_id(Iterator);
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000352 UB = getUpperBound(For, Predicate);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000353
354 ValueLB = ExprBuilder.create(Init);
355 ValueUB = ExprBuilder.create(UB);
356 ValueInc = ExprBuilder.create(Inc);
357
358 MaxType = ExprBuilder.getType(Iterator);
359 MaxType = ExprBuilder.getWidestType(MaxType, ValueLB->getType());
360 MaxType = ExprBuilder.getWidestType(MaxType, ValueUB->getType());
361 MaxType = ExprBuilder.getWidestType(MaxType, ValueInc->getType());
362
363 if (MaxType != ValueLB->getType())
364 ValueLB = Builder.CreateSExt(ValueLB, MaxType);
365 if (MaxType != ValueUB->getType())
366 ValueUB = Builder.CreateSExt(ValueUB, MaxType);
367 if (MaxType != ValueInc->getType())
368 ValueInc = Builder.CreateSExt(ValueInc, MaxType);
369
Johannes Doerfertdd5c1442014-09-10 17:33:32 +0000370 // If we can show that LB <Predicate> UB holds at least once, we can
371 // omit the GuardBB in front of the loop.
372 bool UseGuardBB =
373 !SE.isKnownPredicate(Predicate, SE.getSCEV(ValueLB), SE.getSCEV(ValueUB));
Johannes Doerfert2ef3f4f2014-08-07 17:14:54 +0000374 IV = createLoop(ValueLB, ValueUB, ValueInc, Builder, P, LI, DT, ExitBlock,
Johannes Doerfertdd5c1442014-09-10 17:33:32 +0000375 Predicate, &Annotator, Parallel, UseGuardBB);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000376 IDToValue[IteratorID] = IV;
377
378 create(Body);
379
Johannes Doerfertc7b719f2014-10-01 20:10:44 +0000380 Annotator.popLoop(Parallel);
Tobias Grosser37c9b8e2014-03-04 14:59:00 +0000381
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000382 IDToValue.erase(IteratorID);
383
Tobias Grosser5db6ffd2013-05-16 06:40:06 +0000384 Builder.SetInsertPoint(ExitBlock->begin());
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000385
386 isl_ast_node_free(For);
387 isl_ast_expr_free(Iterator);
388 isl_id_free(IteratorID);
389}
390
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000391void IslNodeBuilder::createFor(__isl_take isl_ast_node *For) {
392 bool Vector = PollyVectorizerChoice != VECTORIZER_NONE;
393
Johannes Doerferted67f8b2014-08-01 08:14:28 +0000394 if (Vector && IslAstInfo::isInnermostParallel(For) &&
395 !IslAstInfo::isReductionParallel(For)) {
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000396 int VectorWidth = getNumberOfIterations(For);
397 if (1 < VectorWidth && VectorWidth <= 16) {
398 createForVector(For, VectorWidth);
399 return;
400 }
401 }
402 createForSequential(For);
403}
404
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000405void IslNodeBuilder::createIf(__isl_take isl_ast_node *If) {
406 isl_ast_expr *Cond = isl_ast_node_if_get_cond(If);
407
408 Function *F = Builder.GetInsertBlock()->getParent();
409 LLVMContext &Context = F->getContext();
410
Tobias Grosserc14582f2013-02-05 18:01:29 +0000411 BasicBlock *CondBB =
412 SplitBlock(Builder.GetInsertBlock(), Builder.GetInsertPoint(), P);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000413 CondBB->setName("polly.cond");
414 BasicBlock *MergeBB = SplitBlock(CondBB, CondBB->begin(), P);
415 MergeBB->setName("polly.merge");
416 BasicBlock *ThenBB = BasicBlock::Create(Context, "polly.then", F);
417 BasicBlock *ElseBB = BasicBlock::Create(Context, "polly.else", F);
418
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000419 DT.addNewBlock(ThenBB, CondBB);
420 DT.addNewBlock(ElseBB, CondBB);
421 DT.changeImmediateDominator(MergeBB, CondBB);
422
Tobias Grosser3081b0f2013-05-16 06:40:24 +0000423 Loop *L = LI.getLoopFor(CondBB);
424 if (L) {
425 L->addBasicBlockToLoop(ThenBB, LI.getBase());
426 L->addBasicBlockToLoop(ElseBB, LI.getBase());
427 }
428
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000429 CondBB->getTerminator()->eraseFromParent();
430
431 Builder.SetInsertPoint(CondBB);
432 Value *Predicate = ExprBuilder.create(Cond);
433 Builder.CreateCondBr(Predicate, ThenBB, ElseBB);
434 Builder.SetInsertPoint(ThenBB);
435 Builder.CreateBr(MergeBB);
436 Builder.SetInsertPoint(ElseBB);
437 Builder.CreateBr(MergeBB);
438 Builder.SetInsertPoint(ThenBB->begin());
439
440 create(isl_ast_node_if_get_then(If));
441
442 Builder.SetInsertPoint(ElseBB->begin());
443
444 if (isl_ast_node_if_has_else(If))
445 create(isl_ast_node_if_get_else(If));
446
447 Builder.SetInsertPoint(MergeBB->begin());
448
449 isl_ast_node_free(If);
450}
451
Tobias Grosserce67a042014-07-02 16:26:47 +0000452void IslNodeBuilder::createSubstitutions(isl_ast_expr *Expr, ScopStmt *Stmt,
453 ValueMapT &VMap, LoopToScevMapT &LTS) {
454 assert(isl_ast_expr_get_type(Expr) == isl_ast_expr_op &&
455 "Expression of type 'op' expected");
456 assert(isl_ast_expr_get_op_type(Expr) == isl_ast_op_call &&
457 "Opertation of type 'call' expected");
458 for (int i = 0; i < isl_ast_expr_get_op_n_arg(Expr) - 1; ++i) {
459 isl_ast_expr *SubExpr;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000460 Value *V;
461
Tobias Grosserce67a042014-07-02 16:26:47 +0000462 SubExpr = isl_ast_expr_get_op_arg(Expr, i + 1);
463 V = ExprBuilder.create(SubExpr);
Sebastian Pope039bb12013-03-18 19:09:49 +0000464 ScalarEvolution *SE = Stmt->getParent()->getSE();
465 LTS[Stmt->getLoopForDimension(i)] = SE->getUnknown(V);
466
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000467 // CreateIntCast can introduce trunc expressions. This is correct, as the
468 // result will always fit into the type of the original induction variable
469 // (because we calculate a value of the original induction variable).
Sebastian Pope039bb12013-03-18 19:09:49 +0000470 const Value *OldIV = Stmt->getInductionVariableForDimension(i);
471 if (OldIV) {
472 V = Builder.CreateIntCast(V, OldIV->getType(), true);
473 VMap[OldIV] = V;
474 }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000475 }
476
Tobias Grosserce67a042014-07-02 16:26:47 +0000477 isl_ast_expr_free(Expr);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000478}
479
Tobias Grosserc14582f2013-02-05 18:01:29 +0000480void IslNodeBuilder::createSubstitutionsVector(
Tobias Grosserce67a042014-07-02 16:26:47 +0000481 __isl_take isl_ast_expr *Expr, ScopStmt *Stmt, VectorValueMapT &VMap,
482 std::vector<LoopToScevMapT> &VLTS, std::vector<Value *> &IVS,
483 __isl_take isl_id *IteratorID) {
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000484 int i = 0;
485
486 Value *OldValue = IDToValue[IteratorID];
Tobias Grosser91f5b262014-06-04 08:06:40 +0000487 for (Value *IV : IVS) {
488 IDToValue[IteratorID] = IV;
Tobias Grosserce67a042014-07-02 16:26:47 +0000489 createSubstitutions(isl_ast_expr_copy(Expr), Stmt, VMap[i], VLTS[i]);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000490 i++;
491 }
492
493 IDToValue[IteratorID] = OldValue;
494 isl_id_free(IteratorID);
Tobias Grosserce67a042014-07-02 16:26:47 +0000495 isl_ast_expr_free(Expr);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000496}
497
498void IslNodeBuilder::createUser(__isl_take isl_ast_node *User) {
499 ValueMapT VMap;
Sebastian Pop9d10fff2013-02-15 20:55:59 +0000500 LoopToScevMapT LTS;
Tobias Grosserce67a042014-07-02 16:26:47 +0000501 isl_id *Id;
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000502 ScopStmt *Stmt;
503
Tobias Grosserce67a042014-07-02 16:26:47 +0000504 isl_ast_expr *Expr = isl_ast_node_user_get_expr(User);
505 isl_ast_expr *StmtExpr = isl_ast_expr_get_op_arg(Expr, 0);
506 Id = isl_ast_expr_get_id(StmtExpr);
507 isl_ast_expr_free(StmtExpr);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000508
Tobias Grosserc14582f2013-02-05 18:01:29 +0000509 Stmt = (ScopStmt *)isl_id_get_user(Id);
Johannes Doerferta63b2572014-08-03 01:51:59 +0000510
Tobias Grosserce67a042014-07-02 16:26:47 +0000511 createSubstitutions(Expr, Stmt, VMap, LTS);
Johannes Doerfert2ef3f4f2014-08-07 17:14:54 +0000512 BlockGenerator::generate(Builder, *Stmt, VMap, LTS, P, LI, SE,
Johannes Doerferta63b2572014-08-03 01:51:59 +0000513 IslAstInfo::getBuild(User), &ExprBuilder);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000514
515 isl_ast_node_free(User);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000516 isl_id_free(Id);
517}
518
519void IslNodeBuilder::createBlock(__isl_take isl_ast_node *Block) {
520 isl_ast_node_list *List = isl_ast_node_block_get_children(Block);
521
522 for (int i = 0; i < isl_ast_node_list_n_ast_node(List); ++i)
523 create(isl_ast_node_list_get_ast_node(List, i));
524
525 isl_ast_node_free(Block);
526 isl_ast_node_list_free(List);
527}
528
529void IslNodeBuilder::create(__isl_take isl_ast_node *Node) {
530 switch (isl_ast_node_get_type(Node)) {
531 case isl_ast_node_error:
532 llvm_unreachable("code generation error");
533 case isl_ast_node_for:
534 createFor(Node);
535 return;
536 case isl_ast_node_if:
537 createIf(Node);
538 return;
539 case isl_ast_node_user:
540 createUser(Node);
541 return;
542 case isl_ast_node_block:
543 createBlock(Node);
544 return;
545 }
546
547 llvm_unreachable("Unknown isl_ast_node type");
548}
549
550void IslNodeBuilder::addParameters(__isl_take isl_set *Context) {
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000551
552 for (unsigned i = 0; i < isl_set_dim(Context, isl_dim_param); ++i) {
553 isl_id *Id;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000554
555 Id = isl_set_get_dim_id(Context, isl_dim_param, i);
Tobias Grosserec7d67e2014-11-06 00:27:01 +0000556 IDToValue[Id] = generateSCEV((const SCEV *)isl_id_get_user(Id));
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000557
558 isl_id_free(Id);
559 }
560
561 isl_set_free(Context);
562}
563
Tobias Grosserec7d67e2014-11-06 00:27:01 +0000564Value *IslNodeBuilder::generateSCEV(const SCEV *Expr) {
565 Instruction *InsertLocation = --(Builder.GetInsertBlock()->end());
566 return Rewriter->expandCodeFor(Expr, cast<IntegerType>(Expr->getType()),
567 InsertLocation);
568}
569
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000570namespace {
571class IslCodeGeneration : public ScopPass {
Tobias Grosser1bb59b02012-12-29 23:47:38 +0000572public:
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000573 static char ID;
574
575 IslCodeGeneration() : ScopPass(ID) {}
576
Johannes Doerfert38262242014-09-10 14:50:23 +0000577 /// @name The analysis passes we need to generate code.
578 ///
579 ///{
580 LoopInfo *LI;
581 IslAstInfo *AI;
582 DominatorTree *DT;
583 ScalarEvolution *SE;
584 ///}
585
586 /// @brief The loop annotator to generate llvm.loop metadata.
Johannes Doerfert51d1c742014-10-02 15:32:17 +0000587 ScopAnnotator Annotator;
Johannes Doerfert38262242014-09-10 14:50:23 +0000588
589 /// @brief Build the runtime condition.
590 ///
591 /// Build the condition that evaluates at run-time to true iff all
592 /// assumptions taken for the SCoP hold, and to false otherwise.
593 ///
594 /// @return A value evaluating to true/false if execution is save/unsafe.
595 Value *buildRTC(PollyIRBuilder &Builder, IslExprBuilder &ExprBuilder) {
596 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
597 Value *RTC = ExprBuilder.create(AI->getRunCondition());
Johannes Doerfertb164c792014-09-18 11:17:17 +0000598 if (!RTC->getType()->isIntegerTy(1))
599 RTC = Builder.CreateIsNotNull(RTC);
600 return RTC;
Johannes Doerfert38262242014-09-10 14:50:23 +0000601 }
602
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000603 bool runOnScop(Scop &S) {
Johannes Doerfert38262242014-09-10 14:50:23 +0000604 LI = &getAnalysis<LoopInfo>();
605 AI = &getAnalysis<IslAstInfo>();
606 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
607 SE = &getAnalysis<ScalarEvolution>();
Tobias Grosser0ee50f62013-04-10 06:55:31 +0000608
Tobias Grossere602a072013-05-07 07:30:56 +0000609 assert(!S.getRegion().isTopLevelRegion() &&
610 "Top level regions are not supported");
Tobias Grosser0ee50f62013-04-10 06:55:31 +0000611
Johannes Doerfertecdf2632014-10-02 15:31:24 +0000612 // Build the alias scopes for annotations first.
613 if (PollyAnnotateAliasScopes)
614 Annotator.buildAliasScopes(S);
615
Johannes Doerfert38262242014-09-10 14:50:23 +0000616 BasicBlock *EnteringBB = simplifyRegion(&S, this);
617 PollyIRBuilder Builder = createPollyIRBuilder(EnteringBB, Annotator);
Johannes Doerfert9744c4a2014-08-12 18:35:54 +0000618
Johannes Doerfert38262242014-09-10 14:50:23 +0000619 IslNodeBuilder NodeBuilder(Builder, Annotator, this, *LI, *SE, *DT);
Tobias Grosser28735942014-08-16 09:09:15 +0000620 NodeBuilder.addParameters(S.getContext());
Johannes Doerfert38262242014-09-10 14:50:23 +0000621
622 Value *RTC = buildRTC(Builder, NodeBuilder.getExprBuilder());
623 BasicBlock *StartBlock = executeScopConditionally(S, this, RTC);
Tobias Grosser28735942014-08-16 09:09:15 +0000624 Builder.SetInsertPoint(StartBlock->begin());
625
Johannes Doerfert38262242014-09-10 14:50:23 +0000626 NodeBuilder.create(AI->getAst());
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000627 return true;
628 }
629
Tobias Grosserc14582f2013-02-05 18:01:29 +0000630 virtual void printScop(raw_ostream &OS) const {}
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000631
632 virtual void getAnalysisUsage(AnalysisUsage &AU) const {
Tobias Grosser42aff302014-01-13 22:29:56 +0000633 AU.addRequired<DominatorTreeWrapperPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000634 AU.addRequired<IslAstInfo>();
Matt Arsenault8ca36812014-07-19 18:40:17 +0000635 AU.addRequired<RegionInfoPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000636 AU.addRequired<ScalarEvolution>();
637 AU.addRequired<ScopDetection>();
638 AU.addRequired<ScopInfo>();
Tobias Grosserecfe21b2013-03-20 18:03:18 +0000639 AU.addRequired<LoopInfo>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000640
641 AU.addPreserved<Dependences>();
642
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000643 AU.addPreserved<LoopInfo>();
Tobias Grosser42aff302014-01-13 22:29:56 +0000644 AU.addPreserved<DominatorTreeWrapperPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000645 AU.addPreserved<IslAstInfo>();
646 AU.addPreserved<ScopDetection>();
647 AU.addPreserved<ScalarEvolution>();
648
649 // FIXME: We do not yet add regions for the newly generated code to the
650 // region tree.
Matt Arsenault8ca36812014-07-19 18:40:17 +0000651 AU.addPreserved<RegionInfoPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000652 AU.addPreserved<TempScopInfo>();
653 AU.addPreserved<ScopInfo>();
654 AU.addPreservedID(IndependentBlocksID);
655 }
656};
657}
658
659char IslCodeGeneration::ID = 1;
660
Tobias Grosser7242ad92013-02-22 08:07:06 +0000661Pass *polly::createIslCodeGenerationPass() { return new IslCodeGeneration(); }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000662
Tobias Grosser7242ad92013-02-22 08:07:06 +0000663INITIALIZE_PASS_BEGIN(IslCodeGeneration, "polly-codegen-isl",
664 "Polly - Create LLVM-IR from SCoPs", false, false);
665INITIALIZE_PASS_DEPENDENCY(Dependences);
Tobias Grosser42aff302014-01-13 22:29:56 +0000666INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass);
Tobias Grosser7242ad92013-02-22 08:07:06 +0000667INITIALIZE_PASS_DEPENDENCY(LoopInfo);
Matt Arsenault8ca36812014-07-19 18:40:17 +0000668INITIALIZE_PASS_DEPENDENCY(RegionInfoPass);
Tobias Grosser7242ad92013-02-22 08:07:06 +0000669INITIALIZE_PASS_DEPENDENCY(ScalarEvolution);
670INITIALIZE_PASS_DEPENDENCY(ScopDetection);
671INITIALIZE_PASS_END(IslCodeGeneration, "polly-codegen-isl",
672 "Polly - Create LLVM-IR from SCoPs", false, false)