blob: 0d9a798de04702bd6ef63bf1f78c46b7ba96ddeb [file] [log] [blame]
Sebastian Pop082cea82012-05-07 16:20:07 +00001//===------ IslCodeGeneration.cpp - Code generate the Scops using ISL. ----===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// The IslCodeGeneration pass takes a Scop created by ScopInfo and translates it
11// back to LLVM-IR using the ISL code generator.
12//
13// The Scop describes the high level memory behaviour of a control flow region.
14// Transformation passes can update the schedule (execution order) of statements
15// in the Scop. ISL is used to generate an abstract syntax tree that reflects
16// the updated execution order. This clast is used to create new LLVM-IR that is
17// computationally equivalent to the original control flow region, but executes
18// its code in the new execution order defined by the changed scattering.
19//
20//===----------------------------------------------------------------------===//
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000021#include "polly/Config/config.h"
Johannes Doerferta63b2572014-08-03 01:51:59 +000022#include "polly/CodeGen/IslExprBuilder.h"
Tobias Grosser83628182013-05-07 08:11:54 +000023#include "polly/CodeGen/BlockGenerators.h"
24#include "polly/CodeGen/CodeGeneration.h"
25#include "polly/CodeGen/IslAst.h"
Johannes Doerfert48cf6ec2014-07-29 20:50:09 +000026#include "polly/CodeGen/IslExprBuilder.h"
Tobias Grosser83628182013-05-07 08:11:54 +000027#include "polly/CodeGen/LoopGenerators.h"
28#include "polly/CodeGen/Utils.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000029#include "polly/Dependences.h"
30#include "polly/LinkAllPasses.h"
31#include "polly/ScopInfo.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000032#include "polly/Support/GICHelper.h"
Tobias Grosser0ee50f62013-04-10 06:55:31 +000033#include "polly/Support/ScopHelper.h"
Tobias Grosser83628182013-05-07 08:11:54 +000034#include "polly/TempScopInfo.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000035#include "llvm/Analysis/LoopInfo.h"
Matt Arsenault8ca36812014-07-19 18:40:17 +000036#include "llvm/Analysis/PostDominators.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000037#include "llvm/Analysis/ScalarEvolutionExpander.h"
Tobias Grosser83628182013-05-07 08:11:54 +000038#include "llvm/IR/Module.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000039#include "llvm/Support/CommandLine.h"
40#include "llvm/Support/Debug.h"
Chandler Carruth535d52c2013-01-02 11:47:44 +000041#include "llvm/IR/DataLayout.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000042#include "llvm/Transforms/Utils/BasicBlockUtils.h"
43
44#include "isl/union_map.h"
45#include "isl/list.h"
46#include "isl/ast.h"
47#include "isl/ast_build.h"
48#include "isl/set.h"
49#include "isl/map.h"
50#include "isl/aff.h"
51
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000052using namespace polly;
53using namespace llvm;
54
Chandler Carruth95fef942014-04-22 03:30:19 +000055#define DEBUG_TYPE "polly-codegen-isl"
56
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000057class IslNodeBuilder {
58public:
Johannes Doerfert2ef3f4f2014-08-07 17:14:54 +000059 IslNodeBuilder(PollyIRBuilder &Builder, LoopAnnotator &Annotator, Pass *P,
60 LoopInfo &LI, ScalarEvolution &SE, DominatorTree &DT)
Tobias Grosser34c87872014-04-22 16:39:41 +000061 : Builder(Builder), Annotator(Annotator), ExprBuilder(Builder, IDToValue),
Johannes Doerfert2ef3f4f2014-08-07 17:14:54 +000062 P(P), LI(LI), SE(SE), DT(DT) {}
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000063
Johannes Doerferta63b2572014-08-03 01:51:59 +000064 /// @brief Add the mappings from array id's to array llvm::Value's.
65 void addMemoryAccesses(Scop &S);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000066 void addParameters(__isl_take isl_set *Context);
67 void create(__isl_take isl_ast_node *Node);
Tobias Grosser54ee0ba2013-11-17 03:18:25 +000068 IslExprBuilder &getExprBuilder() { return ExprBuilder; }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000069
70private:
Tobias Grosser5103ba72014-03-04 14:58:49 +000071 PollyIRBuilder &Builder;
Tobias Grosser37c9b8e2014-03-04 14:59:00 +000072 LoopAnnotator &Annotator;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000073 IslExprBuilder ExprBuilder;
74 Pass *P;
Johannes Doerfert2ef3f4f2014-08-07 17:14:54 +000075 LoopInfo &LI;
76 ScalarEvolution &SE;
77 DominatorTree &DT;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000078
79 // This maps an isl_id* to the Value* it has in the generated program. For now
80 // on, the only isl_ids that are stored here are the newly calculated loop
81 // ivs.
Tobias Grosser566ad582014-08-31 16:21:12 +000082 IslExprBuilder::IDToValueTy IDToValue;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000083
84 // Extract the upper bound of this loop
85 //
86 // The isl code generation can generate arbitrary expressions to check if the
87 // upper bound of a loop is reached, but it provides an option to enforce
88 // 'atomic' upper bounds. An 'atomic upper bound is always of the form
89 // iv <= expr, where expr is an (arbitrary) expression not containing iv.
90 //
91 // This function extracts 'atomic' upper bounds. Polly, in general, requires
92 // atomic upper bounds for the following reasons:
93 //
94 // 1. An atomic upper bound is loop invariant
95 //
96 // It must not be calculated at each loop iteration and can often even be
97 // hoisted out further by the loop invariant code motion.
98 //
99 // 2. OpenMP needs a loop invarient upper bound to calculate the number
100 // of loop iterations.
101 //
102 // 3. With the existing code, upper bounds have been easier to implement.
Tobias Grossere602a072013-05-07 07:30:56 +0000103 __isl_give isl_ast_expr *getUpperBound(__isl_keep isl_ast_node *For,
104 CmpInst::Predicate &Predicate);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000105
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000106 unsigned getNumberOfIterations(__isl_keep isl_ast_node *For);
107
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000108 void createFor(__isl_take isl_ast_node *For);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000109 void createForVector(__isl_take isl_ast_node *For, int VectorWidth);
110 void createForSequential(__isl_take isl_ast_node *For);
Tobias Grosserce67a042014-07-02 16:26:47 +0000111
112 /// Generate LLVM-IR that computes the values of the original induction
113 /// variables in function of the newly generated loop induction variables.
114 ///
115 /// Example:
116 ///
117 /// // Original
118 /// for i
119 /// for j
120 /// S(i)
121 ///
122 /// Schedule: [i,j] -> [i+j, j]
123 ///
124 /// // New
125 /// for c0
126 /// for c1
127 /// S(c0 - c1, c1)
128 ///
129 /// Assuming the original code consists of two loops which are
130 /// transformed according to a schedule [i,j] -> [c0=i+j,c1=j]. The resulting
131 /// ast models the original statement as a call expression where each argument
132 /// is an expression that computes the old induction variables from the new
133 /// ones, ordered such that the first argument computes the value of induction
134 /// variable that was outermost in the original code.
135 ///
136 /// @param Expr The call expression that represents the statement.
137 /// @param Stmt The statement that is called.
138 /// @param VMap The value map into which the mapping from the old induction
139 /// variable to the new one is inserted. This mapping is used
140 /// for the classical code generation (not scev-based) and
141 /// gives an explicit mapping from an original, materialized
142 /// induction variable. It consequently can only be expressed
143 /// if there was an explicit induction variable.
144 /// @param LTS The loop to SCEV map in which the mapping from the original
145 /// loop to a SCEV representing the new loop iv is added. This
146 /// mapping does not require an explicit induction variable.
147 /// Instead, we think in terms of an implicit induction variable
148 /// that counts the number of times a loop is executed. For each
149 /// original loop this count, expressed in function of the new
150 /// induction variables, is added to the LTS map.
151 void createSubstitutions(__isl_take isl_ast_expr *Expr, ScopStmt *Stmt,
Sebastian Pop9d10fff2013-02-15 20:55:59 +0000152 ValueMapT &VMap, LoopToScevMapT &LTS);
Tobias Grosserce67a042014-07-02 16:26:47 +0000153 void createSubstitutionsVector(__isl_take isl_ast_expr *Expr, ScopStmt *Stmt,
154 VectorValueMapT &VMap,
Tobias Grossere602a072013-05-07 07:30:56 +0000155 std::vector<LoopToScevMapT> &VLTS,
156 std::vector<Value *> &IVS,
157 __isl_take isl_id *IteratorID);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000158 void createIf(__isl_take isl_ast_node *If);
Tobias Grossere602a072013-05-07 07:30:56 +0000159 void createUserVector(__isl_take isl_ast_node *User,
160 std::vector<Value *> &IVS,
161 __isl_take isl_id *IteratorID,
162 __isl_take isl_union_map *Schedule);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000163 void createUser(__isl_take isl_ast_node *User);
164 void createBlock(__isl_take isl_ast_node *Block);
165};
166
Tobias Grossere602a072013-05-07 07:30:56 +0000167__isl_give isl_ast_expr *
168IslNodeBuilder::getUpperBound(__isl_keep isl_ast_node *For,
169 ICmpInst::Predicate &Predicate) {
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000170 isl_id *UBID, *IteratorID;
171 isl_ast_expr *Cond, *Iterator, *UB, *Arg0;
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000172 isl_ast_op_type Type;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000173
174 Cond = isl_ast_node_for_get_cond(For);
175 Iterator = isl_ast_node_for_get_iterator(For);
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000176 Type = isl_ast_expr_get_op_type(Cond);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000177
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000178 assert(isl_ast_expr_get_type(Cond) == isl_ast_expr_op &&
179 "conditional expression is not an atomic upper bound");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000180
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000181 switch (Type) {
Tobias Grosserc14582f2013-02-05 18:01:29 +0000182 case isl_ast_op_le:
183 Predicate = ICmpInst::ICMP_SLE;
184 break;
185 case isl_ast_op_lt:
186 Predicate = ICmpInst::ICMP_SLT;
187 break;
188 default:
189 llvm_unreachable("Unexpected comparision type in loop conditon");
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000190 }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000191
192 Arg0 = isl_ast_expr_get_op_arg(Cond, 0);
193
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000194 assert(isl_ast_expr_get_type(Arg0) == isl_ast_expr_id &&
195 "conditional expression is not an atomic upper bound");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000196
197 UBID = isl_ast_expr_get_id(Arg0);
198
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000199 assert(isl_ast_expr_get_type(Iterator) == isl_ast_expr_id &&
200 "Could not get the iterator");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000201
202 IteratorID = isl_ast_expr_get_id(Iterator);
203
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000204 assert(UBID == IteratorID &&
205 "conditional expression is not an atomic upper bound");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000206
207 UB = isl_ast_expr_get_op_arg(Cond, 1);
208
209 isl_ast_expr_free(Cond);
210 isl_ast_expr_free(Iterator);
211 isl_ast_expr_free(Arg0);
212 isl_id_free(IteratorID);
213 isl_id_free(UBID);
214
215 return UB;
216}
217
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000218unsigned IslNodeBuilder::getNumberOfIterations(__isl_keep isl_ast_node *For) {
Johannes Doerfert94d90822014-07-23 20:26:25 +0000219 isl_union_map *Schedule = IslAstInfo::getSchedule(For);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000220 isl_set *LoopDomain = isl_set_from_union_set(isl_union_map_range(Schedule));
Sebastian Pop2aa5c242012-12-18 08:56:51 +0000221 int NumberOfIterations = polly::getNumberOfIterations(LoopDomain);
222 if (NumberOfIterations == -1)
223 return -1;
224 return NumberOfIterations + 1;
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000225}
226
Tobias Grossere602a072013-05-07 07:30:56 +0000227void IslNodeBuilder::createUserVector(__isl_take isl_ast_node *User,
228 std::vector<Value *> &IVS,
229 __isl_take isl_id *IteratorID,
230 __isl_take isl_union_map *Schedule) {
Tobias Grosserce67a042014-07-02 16:26:47 +0000231 isl_ast_expr *Expr = isl_ast_node_user_get_expr(User);
232 isl_ast_expr *StmtExpr = isl_ast_expr_get_op_arg(Expr, 0);
233 isl_id *Id = isl_ast_expr_get_id(StmtExpr);
234 isl_ast_expr_free(StmtExpr);
Tobias Grosserc14582f2013-02-05 18:01:29 +0000235 ScopStmt *Stmt = (ScopStmt *)isl_id_get_user(Id);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000236 VectorValueMapT VectorMap(IVS.size());
Sebastian Pop9d10fff2013-02-15 20:55:59 +0000237 std::vector<LoopToScevMapT> VLTS(IVS.size());
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000238
239 isl_union_set *Domain = isl_union_set_from_set(Stmt->getDomain());
240 Schedule = isl_union_map_intersect_domain(Schedule, Domain);
241 isl_map *S = isl_map_from_union_map(Schedule);
242
Tobias Grosserce67a042014-07-02 16:26:47 +0000243 createSubstitutionsVector(Expr, Stmt, VectorMap, VLTS, IVS, IteratorID);
Johannes Doerfert2ef3f4f2014-08-07 17:14:54 +0000244 VectorBlockGenerator::generate(Builder, *Stmt, VectorMap, VLTS, S, P, LI, SE);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000245
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000246 isl_map_free(S);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000247 isl_id_free(Id);
248 isl_ast_node_free(User);
249}
250
Tobias Grossere602a072013-05-07 07:30:56 +0000251void IslNodeBuilder::createForVector(__isl_take isl_ast_node *For,
252 int VectorWidth) {
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000253 isl_ast_node *Body = isl_ast_node_for_get_body(For);
254 isl_ast_expr *Init = isl_ast_node_for_get_init(For);
255 isl_ast_expr *Inc = isl_ast_node_for_get_inc(For);
256 isl_ast_expr *Iterator = isl_ast_node_for_get_iterator(For);
257 isl_id *IteratorID = isl_ast_expr_get_id(Iterator);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000258
259 Value *ValueLB = ExprBuilder.create(Init);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000260 Value *ValueInc = ExprBuilder.create(Inc);
261
262 Type *MaxType = ExprBuilder.getType(Iterator);
263 MaxType = ExprBuilder.getWidestType(MaxType, ValueLB->getType());
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000264 MaxType = ExprBuilder.getWidestType(MaxType, ValueInc->getType());
265
266 if (MaxType != ValueLB->getType())
267 ValueLB = Builder.CreateSExt(ValueLB, MaxType);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000268 if (MaxType != ValueInc->getType())
269 ValueInc = Builder.CreateSExt(ValueInc, MaxType);
270
Tobias Grosserc14582f2013-02-05 18:01:29 +0000271 std::vector<Value *> IVS(VectorWidth);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000272 IVS[0] = ValueLB;
273
274 for (int i = 1; i < VectorWidth; i++)
Tobias Grosserc14582f2013-02-05 18:01:29 +0000275 IVS[i] = Builder.CreateAdd(IVS[i - 1], ValueInc, "p_vector_iv");
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000276
Johannes Doerfert94d90822014-07-23 20:26:25 +0000277 isl_union_map *Schedule = IslAstInfo::getSchedule(For);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000278 assert(Schedule && "For statement annotation does not contain its schedule");
279
280 IDToValue[IteratorID] = ValueLB;
281
282 switch (isl_ast_node_get_type(Body)) {
283 case isl_ast_node_user:
284 createUserVector(Body, IVS, isl_id_copy(IteratorID),
285 isl_union_map_copy(Schedule));
286 break;
287 case isl_ast_node_block: {
288 isl_ast_node_list *List = isl_ast_node_block_get_children(Body);
289
290 for (int i = 0; i < isl_ast_node_list_n_ast_node(List); ++i)
291 createUserVector(isl_ast_node_list_get_ast_node(List, i), IVS,
Tobias Grosser1bb59b02012-12-29 23:47:38 +0000292 isl_id_copy(IteratorID), isl_union_map_copy(Schedule));
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000293
294 isl_ast_node_free(Body);
295 isl_ast_node_list_free(List);
296 break;
297 }
298 default:
299 isl_ast_node_dump(Body);
300 llvm_unreachable("Unhandled isl_ast_node in vectorizer");
301 }
302
303 IDToValue.erase(IteratorID);
304 isl_id_free(IteratorID);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000305 isl_union_map_free(Schedule);
306
307 isl_ast_node_free(For);
308 isl_ast_expr_free(Iterator);
309}
310
311void IslNodeBuilder::createForSequential(__isl_take isl_ast_node *For) {
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000312 isl_ast_node *Body;
313 isl_ast_expr *Init, *Inc, *Iterator, *UB;
314 isl_id *IteratorID;
315 Value *ValueLB, *ValueUB, *ValueInc;
316 Type *MaxType;
Tobias Grosser5db6ffd2013-05-16 06:40:06 +0000317 BasicBlock *ExitBlock;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000318 Value *IV;
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000319 CmpInst::Predicate Predicate;
Tobias Grosser37c9b8e2014-03-04 14:59:00 +0000320 bool Parallel;
321
Johannes Doerferted67f8b2014-08-01 08:14:28 +0000322 Parallel = IslAstInfo::isInnermostParallel(For) &&
323 !IslAstInfo::isReductionParallel(For);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000324
325 Body = isl_ast_node_for_get_body(For);
326
327 // isl_ast_node_for_is_degenerate(For)
328 //
329 // TODO: For degenerated loops we could generate a plain assignment.
330 // However, for now we just reuse the logic for normal loops, which will
331 // create a loop with a single iteration.
332
333 Init = isl_ast_node_for_get_init(For);
334 Inc = isl_ast_node_for_get_inc(For);
335 Iterator = isl_ast_node_for_get_iterator(For);
336 IteratorID = isl_ast_expr_get_id(Iterator);
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000337 UB = getUpperBound(For, Predicate);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000338
339 ValueLB = ExprBuilder.create(Init);
340 ValueUB = ExprBuilder.create(UB);
341 ValueInc = ExprBuilder.create(Inc);
342
343 MaxType = ExprBuilder.getType(Iterator);
344 MaxType = ExprBuilder.getWidestType(MaxType, ValueLB->getType());
345 MaxType = ExprBuilder.getWidestType(MaxType, ValueUB->getType());
346 MaxType = ExprBuilder.getWidestType(MaxType, ValueInc->getType());
347
348 if (MaxType != ValueLB->getType())
349 ValueLB = Builder.CreateSExt(ValueLB, MaxType);
350 if (MaxType != ValueUB->getType())
351 ValueUB = Builder.CreateSExt(ValueUB, MaxType);
352 if (MaxType != ValueInc->getType())
353 ValueInc = Builder.CreateSExt(ValueInc, MaxType);
354
Johannes Doerfertdd5c1442014-09-10 17:33:32 +0000355 // If we can show that LB <Predicate> UB holds at least once, we can
356 // omit the GuardBB in front of the loop.
357 bool UseGuardBB =
358 !SE.isKnownPredicate(Predicate, SE.getSCEV(ValueLB), SE.getSCEV(ValueUB));
Johannes Doerfert2ef3f4f2014-08-07 17:14:54 +0000359 IV = createLoop(ValueLB, ValueUB, ValueInc, Builder, P, LI, DT, ExitBlock,
Johannes Doerfertdd5c1442014-09-10 17:33:32 +0000360 Predicate, &Annotator, Parallel, UseGuardBB);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000361 IDToValue[IteratorID] = IV;
362
363 create(Body);
364
Tobias Grosser37c9b8e2014-03-04 14:59:00 +0000365 Annotator.End();
366
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000367 IDToValue.erase(IteratorID);
368
Tobias Grosser5db6ffd2013-05-16 06:40:06 +0000369 Builder.SetInsertPoint(ExitBlock->begin());
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000370
371 isl_ast_node_free(For);
372 isl_ast_expr_free(Iterator);
373 isl_id_free(IteratorID);
374}
375
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000376void IslNodeBuilder::createFor(__isl_take isl_ast_node *For) {
377 bool Vector = PollyVectorizerChoice != VECTORIZER_NONE;
378
Johannes Doerferted67f8b2014-08-01 08:14:28 +0000379 if (Vector && IslAstInfo::isInnermostParallel(For) &&
380 !IslAstInfo::isReductionParallel(For)) {
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000381 int VectorWidth = getNumberOfIterations(For);
382 if (1 < VectorWidth && VectorWidth <= 16) {
383 createForVector(For, VectorWidth);
384 return;
385 }
386 }
387 createForSequential(For);
388}
389
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000390void IslNodeBuilder::createIf(__isl_take isl_ast_node *If) {
391 isl_ast_expr *Cond = isl_ast_node_if_get_cond(If);
392
393 Function *F = Builder.GetInsertBlock()->getParent();
394 LLVMContext &Context = F->getContext();
395
Tobias Grosserc14582f2013-02-05 18:01:29 +0000396 BasicBlock *CondBB =
397 SplitBlock(Builder.GetInsertBlock(), Builder.GetInsertPoint(), P);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000398 CondBB->setName("polly.cond");
399 BasicBlock *MergeBB = SplitBlock(CondBB, CondBB->begin(), P);
400 MergeBB->setName("polly.merge");
401 BasicBlock *ThenBB = BasicBlock::Create(Context, "polly.then", F);
402 BasicBlock *ElseBB = BasicBlock::Create(Context, "polly.else", F);
403
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000404 DT.addNewBlock(ThenBB, CondBB);
405 DT.addNewBlock(ElseBB, CondBB);
406 DT.changeImmediateDominator(MergeBB, CondBB);
407
Tobias Grosser3081b0f2013-05-16 06:40:24 +0000408 Loop *L = LI.getLoopFor(CondBB);
409 if (L) {
410 L->addBasicBlockToLoop(ThenBB, LI.getBase());
411 L->addBasicBlockToLoop(ElseBB, LI.getBase());
412 }
413
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000414 CondBB->getTerminator()->eraseFromParent();
415
416 Builder.SetInsertPoint(CondBB);
417 Value *Predicate = ExprBuilder.create(Cond);
418 Builder.CreateCondBr(Predicate, ThenBB, ElseBB);
419 Builder.SetInsertPoint(ThenBB);
420 Builder.CreateBr(MergeBB);
421 Builder.SetInsertPoint(ElseBB);
422 Builder.CreateBr(MergeBB);
423 Builder.SetInsertPoint(ThenBB->begin());
424
425 create(isl_ast_node_if_get_then(If));
426
427 Builder.SetInsertPoint(ElseBB->begin());
428
429 if (isl_ast_node_if_has_else(If))
430 create(isl_ast_node_if_get_else(If));
431
432 Builder.SetInsertPoint(MergeBB->begin());
433
434 isl_ast_node_free(If);
435}
436
Tobias Grosserce67a042014-07-02 16:26:47 +0000437void IslNodeBuilder::createSubstitutions(isl_ast_expr *Expr, ScopStmt *Stmt,
438 ValueMapT &VMap, LoopToScevMapT &LTS) {
439 assert(isl_ast_expr_get_type(Expr) == isl_ast_expr_op &&
440 "Expression of type 'op' expected");
441 assert(isl_ast_expr_get_op_type(Expr) == isl_ast_op_call &&
442 "Opertation of type 'call' expected");
443 for (int i = 0; i < isl_ast_expr_get_op_n_arg(Expr) - 1; ++i) {
444 isl_ast_expr *SubExpr;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000445 Value *V;
446
Tobias Grosserce67a042014-07-02 16:26:47 +0000447 SubExpr = isl_ast_expr_get_op_arg(Expr, i + 1);
448 V = ExprBuilder.create(SubExpr);
Sebastian Pope039bb12013-03-18 19:09:49 +0000449 ScalarEvolution *SE = Stmt->getParent()->getSE();
450 LTS[Stmt->getLoopForDimension(i)] = SE->getUnknown(V);
451
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000452 // CreateIntCast can introduce trunc expressions. This is correct, as the
453 // result will always fit into the type of the original induction variable
454 // (because we calculate a value of the original induction variable).
Sebastian Pope039bb12013-03-18 19:09:49 +0000455 const Value *OldIV = Stmt->getInductionVariableForDimension(i);
456 if (OldIV) {
457 V = Builder.CreateIntCast(V, OldIV->getType(), true);
458 VMap[OldIV] = V;
459 }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000460 }
461
Tobias Grosserce67a042014-07-02 16:26:47 +0000462 isl_ast_expr_free(Expr);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000463}
464
Tobias Grosserc14582f2013-02-05 18:01:29 +0000465void IslNodeBuilder::createSubstitutionsVector(
Tobias Grosserce67a042014-07-02 16:26:47 +0000466 __isl_take isl_ast_expr *Expr, ScopStmt *Stmt, VectorValueMapT &VMap,
467 std::vector<LoopToScevMapT> &VLTS, std::vector<Value *> &IVS,
468 __isl_take isl_id *IteratorID) {
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000469 int i = 0;
470
471 Value *OldValue = IDToValue[IteratorID];
Tobias Grosser91f5b262014-06-04 08:06:40 +0000472 for (Value *IV : IVS) {
473 IDToValue[IteratorID] = IV;
Tobias Grosserce67a042014-07-02 16:26:47 +0000474 createSubstitutions(isl_ast_expr_copy(Expr), Stmt, VMap[i], VLTS[i]);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000475 i++;
476 }
477
478 IDToValue[IteratorID] = OldValue;
479 isl_id_free(IteratorID);
Tobias Grosserce67a042014-07-02 16:26:47 +0000480 isl_ast_expr_free(Expr);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000481}
482
483void IslNodeBuilder::createUser(__isl_take isl_ast_node *User) {
484 ValueMapT VMap;
Sebastian Pop9d10fff2013-02-15 20:55:59 +0000485 LoopToScevMapT LTS;
Tobias Grosserce67a042014-07-02 16:26:47 +0000486 isl_id *Id;
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000487 ScopStmt *Stmt;
488
Tobias Grosserce67a042014-07-02 16:26:47 +0000489 isl_ast_expr *Expr = isl_ast_node_user_get_expr(User);
490 isl_ast_expr *StmtExpr = isl_ast_expr_get_op_arg(Expr, 0);
491 Id = isl_ast_expr_get_id(StmtExpr);
492 isl_ast_expr_free(StmtExpr);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000493
Tobias Grosserc14582f2013-02-05 18:01:29 +0000494 Stmt = (ScopStmt *)isl_id_get_user(Id);
Johannes Doerferta63b2572014-08-03 01:51:59 +0000495
Tobias Grosserce67a042014-07-02 16:26:47 +0000496 createSubstitutions(Expr, Stmt, VMap, LTS);
Johannes Doerfert2ef3f4f2014-08-07 17:14:54 +0000497 BlockGenerator::generate(Builder, *Stmt, VMap, LTS, P, LI, SE,
Johannes Doerferta63b2572014-08-03 01:51:59 +0000498 IslAstInfo::getBuild(User), &ExprBuilder);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000499
500 isl_ast_node_free(User);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000501 isl_id_free(Id);
502}
503
504void IslNodeBuilder::createBlock(__isl_take isl_ast_node *Block) {
505 isl_ast_node_list *List = isl_ast_node_block_get_children(Block);
506
507 for (int i = 0; i < isl_ast_node_list_n_ast_node(List); ++i)
508 create(isl_ast_node_list_get_ast_node(List, i));
509
510 isl_ast_node_free(Block);
511 isl_ast_node_list_free(List);
512}
513
514void IslNodeBuilder::create(__isl_take isl_ast_node *Node) {
515 switch (isl_ast_node_get_type(Node)) {
516 case isl_ast_node_error:
517 llvm_unreachable("code generation error");
518 case isl_ast_node_for:
519 createFor(Node);
520 return;
521 case isl_ast_node_if:
522 createIf(Node);
523 return;
524 case isl_ast_node_user:
525 createUser(Node);
526 return;
527 case isl_ast_node_block:
528 createBlock(Node);
529 return;
530 }
531
532 llvm_unreachable("Unknown isl_ast_node type");
533}
534
535void IslNodeBuilder::addParameters(__isl_take isl_set *Context) {
Johannes Doerfert2ef3f4f2014-08-07 17:14:54 +0000536 SCEVExpander Rewriter(SE, "polly");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000537
538 for (unsigned i = 0; i < isl_set_dim(Context, isl_dim_param); ++i) {
539 isl_id *Id;
540 const SCEV *Scev;
541 IntegerType *T;
542 Instruction *InsertLocation;
543
544 Id = isl_set_get_dim_id(Context, isl_dim_param, i);
Tobias Grosserc14582f2013-02-05 18:01:29 +0000545 Scev = (const SCEV *)isl_id_get_user(Id);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000546 T = dyn_cast<IntegerType>(Scev->getType());
547 InsertLocation = --(Builder.GetInsertBlock()->end());
548 Value *V = Rewriter.expandCodeFor(Scev, T, InsertLocation);
549 IDToValue[Id] = V;
550
551 isl_id_free(Id);
552 }
553
554 isl_set_free(Context);
555}
556
Johannes Doerferta63b2572014-08-03 01:51:59 +0000557void IslNodeBuilder::addMemoryAccesses(Scop &S) {
558 for (ScopStmt *Stmt : S)
559 for (MemoryAccess *MA : *Stmt) {
560 isl_id *Id = MA->getArrayId();
561 IDToValue[Id] = MA->getBaseAddr();
562 isl_id_free(Id);
563 }
564}
565
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000566namespace {
567class IslCodeGeneration : public ScopPass {
Tobias Grosser1bb59b02012-12-29 23:47:38 +0000568public:
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000569 static char ID;
570
571 IslCodeGeneration() : ScopPass(ID) {}
572
Johannes Doerfert38262242014-09-10 14:50:23 +0000573 /// @name The analysis passes we need to generate code.
574 ///
575 ///{
576 LoopInfo *LI;
577 IslAstInfo *AI;
578 DominatorTree *DT;
579 ScalarEvolution *SE;
580 ///}
581
582 /// @brief The loop annotator to generate llvm.loop metadata.
583 LoopAnnotator Annotator;
584
585 /// @brief Build the runtime condition.
586 ///
587 /// Build the condition that evaluates at run-time to true iff all
588 /// assumptions taken for the SCoP hold, and to false otherwise.
589 ///
590 /// @return A value evaluating to true/false if execution is save/unsafe.
591 Value *buildRTC(PollyIRBuilder &Builder, IslExprBuilder &ExprBuilder) {
592 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
593 Value *RTC = ExprBuilder.create(AI->getRunCondition());
594 return Builder.CreateIsNotNull(RTC);
595 }
596
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000597 bool runOnScop(Scop &S) {
Johannes Doerfert38262242014-09-10 14:50:23 +0000598 LI = &getAnalysis<LoopInfo>();
599 AI = &getAnalysis<IslAstInfo>();
600 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
601 SE = &getAnalysis<ScalarEvolution>();
Tobias Grosser0ee50f62013-04-10 06:55:31 +0000602
Tobias Grossere602a072013-05-07 07:30:56 +0000603 assert(!S.getRegion().isTopLevelRegion() &&
604 "Top level regions are not supported");
Tobias Grosser0ee50f62013-04-10 06:55:31 +0000605
Johannes Doerfert38262242014-09-10 14:50:23 +0000606 BasicBlock *EnteringBB = simplifyRegion(&S, this);
607 PollyIRBuilder Builder = createPollyIRBuilder(EnteringBB, Annotator);
Johannes Doerfert9744c4a2014-08-12 18:35:54 +0000608
Johannes Doerfert38262242014-09-10 14:50:23 +0000609 IslNodeBuilder NodeBuilder(Builder, Annotator, this, *LI, *SE, *DT);
Tobias Grosser28735942014-08-16 09:09:15 +0000610 NodeBuilder.addMemoryAccesses(S);
611 NodeBuilder.addParameters(S.getContext());
Johannes Doerfert38262242014-09-10 14:50:23 +0000612
613 Value *RTC = buildRTC(Builder, NodeBuilder.getExprBuilder());
614 BasicBlock *StartBlock = executeScopConditionally(S, this, RTC);
Tobias Grosser28735942014-08-16 09:09:15 +0000615 Builder.SetInsertPoint(StartBlock->begin());
616
Johannes Doerfert38262242014-09-10 14:50:23 +0000617 NodeBuilder.create(AI->getAst());
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000618 return true;
619 }
620
Tobias Grosserc14582f2013-02-05 18:01:29 +0000621 virtual void printScop(raw_ostream &OS) const {}
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000622
623 virtual void getAnalysisUsage(AnalysisUsage &AU) const {
Tobias Grosser42aff302014-01-13 22:29:56 +0000624 AU.addRequired<DominatorTreeWrapperPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000625 AU.addRequired<IslAstInfo>();
Matt Arsenault8ca36812014-07-19 18:40:17 +0000626 AU.addRequired<RegionInfoPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000627 AU.addRequired<ScalarEvolution>();
628 AU.addRequired<ScopDetection>();
629 AU.addRequired<ScopInfo>();
Tobias Grosserecfe21b2013-03-20 18:03:18 +0000630 AU.addRequired<LoopInfo>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000631
632 AU.addPreserved<Dependences>();
633
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000634 AU.addPreserved<LoopInfo>();
Tobias Grosser42aff302014-01-13 22:29:56 +0000635 AU.addPreserved<DominatorTreeWrapperPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000636 AU.addPreserved<IslAstInfo>();
637 AU.addPreserved<ScopDetection>();
638 AU.addPreserved<ScalarEvolution>();
639
640 // FIXME: We do not yet add regions for the newly generated code to the
641 // region tree.
Matt Arsenault8ca36812014-07-19 18:40:17 +0000642 AU.addPreserved<RegionInfoPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000643 AU.addPreserved<TempScopInfo>();
644 AU.addPreserved<ScopInfo>();
645 AU.addPreservedID(IndependentBlocksID);
646 }
647};
648}
649
650char IslCodeGeneration::ID = 1;
651
Tobias Grosser7242ad92013-02-22 08:07:06 +0000652Pass *polly::createIslCodeGenerationPass() { return new IslCodeGeneration(); }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000653
Tobias Grosser7242ad92013-02-22 08:07:06 +0000654INITIALIZE_PASS_BEGIN(IslCodeGeneration, "polly-codegen-isl",
655 "Polly - Create LLVM-IR from SCoPs", false, false);
656INITIALIZE_PASS_DEPENDENCY(Dependences);
Tobias Grosser42aff302014-01-13 22:29:56 +0000657INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass);
Tobias Grosser7242ad92013-02-22 08:07:06 +0000658INITIALIZE_PASS_DEPENDENCY(LoopInfo);
Matt Arsenault8ca36812014-07-19 18:40:17 +0000659INITIALIZE_PASS_DEPENDENCY(RegionInfoPass);
Tobias Grosser7242ad92013-02-22 08:07:06 +0000660INITIALIZE_PASS_DEPENDENCY(ScalarEvolution);
661INITIALIZE_PASS_DEPENDENCY(ScopDetection);
662INITIALIZE_PASS_END(IslCodeGeneration, "polly-codegen-isl",
663 "Polly - Create LLVM-IR from SCoPs", false, false)