blob: 90b73214bbe7874103904b5f8a84d8f3f31c8b23 [file] [log] [blame]
Sebastian Pop082cea82012-05-07 16:20:07 +00001//===------ IslCodeGeneration.cpp - Code generate the Scops using ISL. ----===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// The IslCodeGeneration pass takes a Scop created by ScopInfo and translates it
11// back to LLVM-IR using the ISL code generator.
12//
13// The Scop describes the high level memory behaviour of a control flow region.
14// Transformation passes can update the schedule (execution order) of statements
15// in the Scop. ISL is used to generate an abstract syntax tree that reflects
16// the updated execution order. This clast is used to create new LLVM-IR that is
17// computationally equivalent to the original control flow region, but executes
18// its code in the new execution order defined by the changed scattering.
19//
20//===----------------------------------------------------------------------===//
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000021#include "polly/Config/config.h"
Tobias Grosser83628182013-05-07 08:11:54 +000022#include "polly/CodeGen/BlockGenerators.h"
23#include "polly/CodeGen/CodeGeneration.h"
24#include "polly/CodeGen/IslAst.h"
Johannes Doerfert48cf6ec2014-07-29 20:50:09 +000025#include "polly/CodeGen/IslExprBuilder.h"
Tobias Grosser83628182013-05-07 08:11:54 +000026#include "polly/CodeGen/LoopGenerators.h"
27#include "polly/CodeGen/Utils.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000028#include "polly/Dependences.h"
29#include "polly/LinkAllPasses.h"
30#include "polly/ScopInfo.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000031#include "polly/Support/GICHelper.h"
Tobias Grosser0ee50f62013-04-10 06:55:31 +000032#include "polly/Support/ScopHelper.h"
Tobias Grosser83628182013-05-07 08:11:54 +000033#include "polly/TempScopInfo.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000034#include "llvm/Analysis/LoopInfo.h"
Matt Arsenault8ca36812014-07-19 18:40:17 +000035#include "llvm/Analysis/PostDominators.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000036#include "llvm/Analysis/ScalarEvolutionExpander.h"
Tobias Grosser83628182013-05-07 08:11:54 +000037#include "llvm/IR/Module.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000038#include "llvm/Support/CommandLine.h"
39#include "llvm/Support/Debug.h"
Chandler Carruth535d52c2013-01-02 11:47:44 +000040#include "llvm/IR/DataLayout.h"
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000041#include "llvm/Transforms/Utils/BasicBlockUtils.h"
42
43#include "isl/union_map.h"
44#include "isl/list.h"
45#include "isl/ast.h"
46#include "isl/ast_build.h"
47#include "isl/set.h"
48#include "isl/map.h"
49#include "isl/aff.h"
50
51#include <map>
52
53using namespace polly;
54using namespace llvm;
55
Chandler Carruth95fef942014-04-22 03:30:19 +000056#define DEBUG_TYPE "polly-codegen-isl"
57
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000058class IslNodeBuilder {
59public:
Tobias Grosser37c9b8e2014-03-04 14:59:00 +000060 IslNodeBuilder(PollyIRBuilder &Builder, LoopAnnotator &Annotator, Pass *P)
Tobias Grosser34c87872014-04-22 16:39:41 +000061 : Builder(Builder), Annotator(Annotator), ExprBuilder(Builder, IDToValue),
62 P(P) {}
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000063
64 void addParameters(__isl_take isl_set *Context);
65 void create(__isl_take isl_ast_node *Node);
Tobias Grosser54ee0ba2013-11-17 03:18:25 +000066 IslExprBuilder &getExprBuilder() { return ExprBuilder; }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000067
68private:
Tobias Grosser5103ba72014-03-04 14:58:49 +000069 PollyIRBuilder &Builder;
Tobias Grosser37c9b8e2014-03-04 14:59:00 +000070 LoopAnnotator &Annotator;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000071 IslExprBuilder ExprBuilder;
72 Pass *P;
73
74 // This maps an isl_id* to the Value* it has in the generated program. For now
75 // on, the only isl_ids that are stored here are the newly calculated loop
76 // ivs.
Tobias Grosserc14582f2013-02-05 18:01:29 +000077 std::map<isl_id *, Value *> IDToValue;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +000078
79 // Extract the upper bound of this loop
80 //
81 // The isl code generation can generate arbitrary expressions to check if the
82 // upper bound of a loop is reached, but it provides an option to enforce
83 // 'atomic' upper bounds. An 'atomic upper bound is always of the form
84 // iv <= expr, where expr is an (arbitrary) expression not containing iv.
85 //
86 // This function extracts 'atomic' upper bounds. Polly, in general, requires
87 // atomic upper bounds for the following reasons:
88 //
89 // 1. An atomic upper bound is loop invariant
90 //
91 // It must not be calculated at each loop iteration and can often even be
92 // hoisted out further by the loop invariant code motion.
93 //
94 // 2. OpenMP needs a loop invarient upper bound to calculate the number
95 // of loop iterations.
96 //
97 // 3. With the existing code, upper bounds have been easier to implement.
Tobias Grossere602a072013-05-07 07:30:56 +000098 __isl_give isl_ast_expr *getUpperBound(__isl_keep isl_ast_node *For,
99 CmpInst::Predicate &Predicate);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000100
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000101 unsigned getNumberOfIterations(__isl_keep isl_ast_node *For);
102
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000103 void createFor(__isl_take isl_ast_node *For);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000104 void createForVector(__isl_take isl_ast_node *For, int VectorWidth);
105 void createForSequential(__isl_take isl_ast_node *For);
Tobias Grosserce67a042014-07-02 16:26:47 +0000106
107 /// Generate LLVM-IR that computes the values of the original induction
108 /// variables in function of the newly generated loop induction variables.
109 ///
110 /// Example:
111 ///
112 /// // Original
113 /// for i
114 /// for j
115 /// S(i)
116 ///
117 /// Schedule: [i,j] -> [i+j, j]
118 ///
119 /// // New
120 /// for c0
121 /// for c1
122 /// S(c0 - c1, c1)
123 ///
124 /// Assuming the original code consists of two loops which are
125 /// transformed according to a schedule [i,j] -> [c0=i+j,c1=j]. The resulting
126 /// ast models the original statement as a call expression where each argument
127 /// is an expression that computes the old induction variables from the new
128 /// ones, ordered such that the first argument computes the value of induction
129 /// variable that was outermost in the original code.
130 ///
131 /// @param Expr The call expression that represents the statement.
132 /// @param Stmt The statement that is called.
133 /// @param VMap The value map into which the mapping from the old induction
134 /// variable to the new one is inserted. This mapping is used
135 /// for the classical code generation (not scev-based) and
136 /// gives an explicit mapping from an original, materialized
137 /// induction variable. It consequently can only be expressed
138 /// if there was an explicit induction variable.
139 /// @param LTS The loop to SCEV map in which the mapping from the original
140 /// loop to a SCEV representing the new loop iv is added. This
141 /// mapping does not require an explicit induction variable.
142 /// Instead, we think in terms of an implicit induction variable
143 /// that counts the number of times a loop is executed. For each
144 /// original loop this count, expressed in function of the new
145 /// induction variables, is added to the LTS map.
146 void createSubstitutions(__isl_take isl_ast_expr *Expr, ScopStmt *Stmt,
Sebastian Pop9d10fff2013-02-15 20:55:59 +0000147 ValueMapT &VMap, LoopToScevMapT &LTS);
Tobias Grosserce67a042014-07-02 16:26:47 +0000148 void createSubstitutionsVector(__isl_take isl_ast_expr *Expr, ScopStmt *Stmt,
149 VectorValueMapT &VMap,
Tobias Grossere602a072013-05-07 07:30:56 +0000150 std::vector<LoopToScevMapT> &VLTS,
151 std::vector<Value *> &IVS,
152 __isl_take isl_id *IteratorID);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000153 void createIf(__isl_take isl_ast_node *If);
Tobias Grossere602a072013-05-07 07:30:56 +0000154 void createUserVector(__isl_take isl_ast_node *User,
155 std::vector<Value *> &IVS,
156 __isl_take isl_id *IteratorID,
157 __isl_take isl_union_map *Schedule);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000158 void createUser(__isl_take isl_ast_node *User);
159 void createBlock(__isl_take isl_ast_node *Block);
160};
161
Tobias Grossere602a072013-05-07 07:30:56 +0000162__isl_give isl_ast_expr *
163IslNodeBuilder::getUpperBound(__isl_keep isl_ast_node *For,
164 ICmpInst::Predicate &Predicate) {
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000165 isl_id *UBID, *IteratorID;
166 isl_ast_expr *Cond, *Iterator, *UB, *Arg0;
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000167 isl_ast_op_type Type;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000168
169 Cond = isl_ast_node_for_get_cond(For);
170 Iterator = isl_ast_node_for_get_iterator(For);
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000171 Type = isl_ast_expr_get_op_type(Cond);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000172
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000173 assert(isl_ast_expr_get_type(Cond) == isl_ast_expr_op &&
174 "conditional expression is not an atomic upper bound");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000175
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000176 switch (Type) {
Tobias Grosserc14582f2013-02-05 18:01:29 +0000177 case isl_ast_op_le:
178 Predicate = ICmpInst::ICMP_SLE;
179 break;
180 case isl_ast_op_lt:
181 Predicate = ICmpInst::ICMP_SLT;
182 break;
183 default:
184 llvm_unreachable("Unexpected comparision type in loop conditon");
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000185 }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000186
187 Arg0 = isl_ast_expr_get_op_arg(Cond, 0);
188
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000189 assert(isl_ast_expr_get_type(Arg0) == isl_ast_expr_id &&
190 "conditional expression is not an atomic upper bound");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000191
192 UBID = isl_ast_expr_get_id(Arg0);
193
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000194 assert(isl_ast_expr_get_type(Iterator) == isl_ast_expr_id &&
195 "Could not get the iterator");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000196
197 IteratorID = isl_ast_expr_get_id(Iterator);
198
Tobias Grosserae2d83e2012-12-29 23:57:18 +0000199 assert(UBID == IteratorID &&
200 "conditional expression is not an atomic upper bound");
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000201
202 UB = isl_ast_expr_get_op_arg(Cond, 1);
203
204 isl_ast_expr_free(Cond);
205 isl_ast_expr_free(Iterator);
206 isl_ast_expr_free(Arg0);
207 isl_id_free(IteratorID);
208 isl_id_free(UBID);
209
210 return UB;
211}
212
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000213unsigned IslNodeBuilder::getNumberOfIterations(__isl_keep isl_ast_node *For) {
Johannes Doerfert94d90822014-07-23 20:26:25 +0000214 isl_union_map *Schedule = IslAstInfo::getSchedule(For);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000215 isl_set *LoopDomain = isl_set_from_union_set(isl_union_map_range(Schedule));
Sebastian Pop2aa5c242012-12-18 08:56:51 +0000216 int NumberOfIterations = polly::getNumberOfIterations(LoopDomain);
217 if (NumberOfIterations == -1)
218 return -1;
219 return NumberOfIterations + 1;
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000220}
221
Tobias Grossere602a072013-05-07 07:30:56 +0000222void IslNodeBuilder::createUserVector(__isl_take isl_ast_node *User,
223 std::vector<Value *> &IVS,
224 __isl_take isl_id *IteratorID,
225 __isl_take isl_union_map *Schedule) {
Tobias Grosserce67a042014-07-02 16:26:47 +0000226 isl_ast_expr *Expr = isl_ast_node_user_get_expr(User);
227 isl_ast_expr *StmtExpr = isl_ast_expr_get_op_arg(Expr, 0);
228 isl_id *Id = isl_ast_expr_get_id(StmtExpr);
229 isl_ast_expr_free(StmtExpr);
Tobias Grosserc14582f2013-02-05 18:01:29 +0000230 ScopStmt *Stmt = (ScopStmt *)isl_id_get_user(Id);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000231 VectorValueMapT VectorMap(IVS.size());
Sebastian Pop9d10fff2013-02-15 20:55:59 +0000232 std::vector<LoopToScevMapT> VLTS(IVS.size());
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000233
234 isl_union_set *Domain = isl_union_set_from_set(Stmt->getDomain());
235 Schedule = isl_union_map_intersect_domain(Schedule, Domain);
236 isl_map *S = isl_map_from_union_map(Schedule);
237
Tobias Grosserce67a042014-07-02 16:26:47 +0000238 createSubstitutionsVector(Expr, Stmt, VectorMap, VLTS, IVS, IteratorID);
Sebastian Pop9d10fff2013-02-15 20:55:59 +0000239 VectorBlockGenerator::generate(Builder, *Stmt, VectorMap, VLTS, S, P);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000240
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000241 isl_map_free(S);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000242 isl_id_free(Id);
243 isl_ast_node_free(User);
244}
245
Tobias Grossere602a072013-05-07 07:30:56 +0000246void IslNodeBuilder::createForVector(__isl_take isl_ast_node *For,
247 int VectorWidth) {
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000248 isl_ast_node *Body = isl_ast_node_for_get_body(For);
249 isl_ast_expr *Init = isl_ast_node_for_get_init(For);
250 isl_ast_expr *Inc = isl_ast_node_for_get_inc(For);
251 isl_ast_expr *Iterator = isl_ast_node_for_get_iterator(For);
252 isl_id *IteratorID = isl_ast_expr_get_id(Iterator);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000253
254 Value *ValueLB = ExprBuilder.create(Init);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000255 Value *ValueInc = ExprBuilder.create(Inc);
256
257 Type *MaxType = ExprBuilder.getType(Iterator);
258 MaxType = ExprBuilder.getWidestType(MaxType, ValueLB->getType());
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000259 MaxType = ExprBuilder.getWidestType(MaxType, ValueInc->getType());
260
261 if (MaxType != ValueLB->getType())
262 ValueLB = Builder.CreateSExt(ValueLB, MaxType);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000263 if (MaxType != ValueInc->getType())
264 ValueInc = Builder.CreateSExt(ValueInc, MaxType);
265
Tobias Grosserc14582f2013-02-05 18:01:29 +0000266 std::vector<Value *> IVS(VectorWidth);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000267 IVS[0] = ValueLB;
268
269 for (int i = 1; i < VectorWidth; i++)
Tobias Grosserc14582f2013-02-05 18:01:29 +0000270 IVS[i] = Builder.CreateAdd(IVS[i - 1], ValueInc, "p_vector_iv");
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000271
Johannes Doerfert94d90822014-07-23 20:26:25 +0000272 isl_union_map *Schedule = IslAstInfo::getSchedule(For);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000273 assert(Schedule && "For statement annotation does not contain its schedule");
274
275 IDToValue[IteratorID] = ValueLB;
276
277 switch (isl_ast_node_get_type(Body)) {
278 case isl_ast_node_user:
279 createUserVector(Body, IVS, isl_id_copy(IteratorID),
280 isl_union_map_copy(Schedule));
281 break;
282 case isl_ast_node_block: {
283 isl_ast_node_list *List = isl_ast_node_block_get_children(Body);
284
285 for (int i = 0; i < isl_ast_node_list_n_ast_node(List); ++i)
286 createUserVector(isl_ast_node_list_get_ast_node(List, i), IVS,
Tobias Grosser1bb59b02012-12-29 23:47:38 +0000287 isl_id_copy(IteratorID), isl_union_map_copy(Schedule));
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000288
289 isl_ast_node_free(Body);
290 isl_ast_node_list_free(List);
291 break;
292 }
293 default:
294 isl_ast_node_dump(Body);
295 llvm_unreachable("Unhandled isl_ast_node in vectorizer");
296 }
297
298 IDToValue.erase(IteratorID);
299 isl_id_free(IteratorID);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000300 isl_union_map_free(Schedule);
301
302 isl_ast_node_free(For);
303 isl_ast_expr_free(Iterator);
304}
305
306void IslNodeBuilder::createForSequential(__isl_take isl_ast_node *For) {
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000307 isl_ast_node *Body;
308 isl_ast_expr *Init, *Inc, *Iterator, *UB;
309 isl_id *IteratorID;
310 Value *ValueLB, *ValueUB, *ValueInc;
311 Type *MaxType;
Tobias Grosser5db6ffd2013-05-16 06:40:06 +0000312 BasicBlock *ExitBlock;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000313 Value *IV;
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000314 CmpInst::Predicate Predicate;
Tobias Grosser37c9b8e2014-03-04 14:59:00 +0000315 bool Parallel;
316
Johannes Doerferted67f8b2014-08-01 08:14:28 +0000317 Parallel = IslAstInfo::isInnermostParallel(For) &&
318 !IslAstInfo::isReductionParallel(For);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000319
320 Body = isl_ast_node_for_get_body(For);
321
322 // isl_ast_node_for_is_degenerate(For)
323 //
324 // TODO: For degenerated loops we could generate a plain assignment.
325 // However, for now we just reuse the logic for normal loops, which will
326 // create a loop with a single iteration.
327
328 Init = isl_ast_node_for_get_init(For);
329 Inc = isl_ast_node_for_get_inc(For);
330 Iterator = isl_ast_node_for_get_iterator(For);
331 IteratorID = isl_ast_expr_get_id(Iterator);
Tobias Grosserc967d8e2012-10-16 07:29:13 +0000332 UB = getUpperBound(For, Predicate);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000333
334 ValueLB = ExprBuilder.create(Init);
335 ValueUB = ExprBuilder.create(UB);
336 ValueInc = ExprBuilder.create(Inc);
337
338 MaxType = ExprBuilder.getType(Iterator);
339 MaxType = ExprBuilder.getWidestType(MaxType, ValueLB->getType());
340 MaxType = ExprBuilder.getWidestType(MaxType, ValueUB->getType());
341 MaxType = ExprBuilder.getWidestType(MaxType, ValueInc->getType());
342
343 if (MaxType != ValueLB->getType())
344 ValueLB = Builder.CreateSExt(ValueLB, MaxType);
345 if (MaxType != ValueUB->getType())
346 ValueUB = Builder.CreateSExt(ValueUB, MaxType);
347 if (MaxType != ValueInc->getType())
348 ValueInc = Builder.CreateSExt(ValueInc, MaxType);
349
Tobias Grosser37c9b8e2014-03-04 14:59:00 +0000350 IV = createLoop(ValueLB, ValueUB, ValueInc, Builder, P, ExitBlock, Predicate,
351 &Annotator, Parallel);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000352 IDToValue[IteratorID] = IV;
353
354 create(Body);
355
Tobias Grosser37c9b8e2014-03-04 14:59:00 +0000356 Annotator.End();
357
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000358 IDToValue.erase(IteratorID);
359
Tobias Grosser5db6ffd2013-05-16 06:40:06 +0000360 Builder.SetInsertPoint(ExitBlock->begin());
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000361
362 isl_ast_node_free(For);
363 isl_ast_expr_free(Iterator);
364 isl_id_free(IteratorID);
365}
366
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000367void IslNodeBuilder::createFor(__isl_take isl_ast_node *For) {
368 bool Vector = PollyVectorizerChoice != VECTORIZER_NONE;
369
Johannes Doerferted67f8b2014-08-01 08:14:28 +0000370 if (Vector && IslAstInfo::isInnermostParallel(For) &&
371 !IslAstInfo::isReductionParallel(For)) {
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000372 int VectorWidth = getNumberOfIterations(For);
373 if (1 < VectorWidth && VectorWidth <= 16) {
374 createForVector(For, VectorWidth);
375 return;
376 }
377 }
378 createForSequential(For);
379}
380
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000381void IslNodeBuilder::createIf(__isl_take isl_ast_node *If) {
382 isl_ast_expr *Cond = isl_ast_node_if_get_cond(If);
383
384 Function *F = Builder.GetInsertBlock()->getParent();
385 LLVMContext &Context = F->getContext();
386
Tobias Grosserc14582f2013-02-05 18:01:29 +0000387 BasicBlock *CondBB =
388 SplitBlock(Builder.GetInsertBlock(), Builder.GetInsertPoint(), P);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000389 CondBB->setName("polly.cond");
390 BasicBlock *MergeBB = SplitBlock(CondBB, CondBB->begin(), P);
391 MergeBB->setName("polly.merge");
392 BasicBlock *ThenBB = BasicBlock::Create(Context, "polly.then", F);
393 BasicBlock *ElseBB = BasicBlock::Create(Context, "polly.else", F);
394
Tobias Grosser42aff302014-01-13 22:29:56 +0000395 DominatorTree &DT = P->getAnalysis<DominatorTreeWrapperPass>().getDomTree();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000396 DT.addNewBlock(ThenBB, CondBB);
397 DT.addNewBlock(ElseBB, CondBB);
398 DT.changeImmediateDominator(MergeBB, CondBB);
399
Tobias Grosser3081b0f2013-05-16 06:40:24 +0000400 LoopInfo &LI = P->getAnalysis<LoopInfo>();
401 Loop *L = LI.getLoopFor(CondBB);
402 if (L) {
403 L->addBasicBlockToLoop(ThenBB, LI.getBase());
404 L->addBasicBlockToLoop(ElseBB, LI.getBase());
405 }
406
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000407 CondBB->getTerminator()->eraseFromParent();
408
409 Builder.SetInsertPoint(CondBB);
410 Value *Predicate = ExprBuilder.create(Cond);
411 Builder.CreateCondBr(Predicate, ThenBB, ElseBB);
412 Builder.SetInsertPoint(ThenBB);
413 Builder.CreateBr(MergeBB);
414 Builder.SetInsertPoint(ElseBB);
415 Builder.CreateBr(MergeBB);
416 Builder.SetInsertPoint(ThenBB->begin());
417
418 create(isl_ast_node_if_get_then(If));
419
420 Builder.SetInsertPoint(ElseBB->begin());
421
422 if (isl_ast_node_if_has_else(If))
423 create(isl_ast_node_if_get_else(If));
424
425 Builder.SetInsertPoint(MergeBB->begin());
426
427 isl_ast_node_free(If);
428}
429
Tobias Grosserce67a042014-07-02 16:26:47 +0000430void IslNodeBuilder::createSubstitutions(isl_ast_expr *Expr, ScopStmt *Stmt,
431 ValueMapT &VMap, LoopToScevMapT &LTS) {
432 assert(isl_ast_expr_get_type(Expr) == isl_ast_expr_op &&
433 "Expression of type 'op' expected");
434 assert(isl_ast_expr_get_op_type(Expr) == isl_ast_op_call &&
435 "Opertation of type 'call' expected");
436 for (int i = 0; i < isl_ast_expr_get_op_n_arg(Expr) - 1; ++i) {
437 isl_ast_expr *SubExpr;
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000438 Value *V;
439
Tobias Grosserce67a042014-07-02 16:26:47 +0000440 SubExpr = isl_ast_expr_get_op_arg(Expr, i + 1);
441 V = ExprBuilder.create(SubExpr);
Sebastian Pope039bb12013-03-18 19:09:49 +0000442 ScalarEvolution *SE = Stmt->getParent()->getSE();
443 LTS[Stmt->getLoopForDimension(i)] = SE->getUnknown(V);
444
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000445 // CreateIntCast can introduce trunc expressions. This is correct, as the
446 // result will always fit into the type of the original induction variable
447 // (because we calculate a value of the original induction variable).
Sebastian Pope039bb12013-03-18 19:09:49 +0000448 const Value *OldIV = Stmt->getInductionVariableForDimension(i);
449 if (OldIV) {
450 V = Builder.CreateIntCast(V, OldIV->getType(), true);
451 VMap[OldIV] = V;
452 }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000453 }
454
Tobias Grosserce67a042014-07-02 16:26:47 +0000455 isl_ast_expr_free(Expr);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000456}
457
Tobias Grosserc14582f2013-02-05 18:01:29 +0000458void IslNodeBuilder::createSubstitutionsVector(
Tobias Grosserce67a042014-07-02 16:26:47 +0000459 __isl_take isl_ast_expr *Expr, ScopStmt *Stmt, VectorValueMapT &VMap,
460 std::vector<LoopToScevMapT> &VLTS, std::vector<Value *> &IVS,
461 __isl_take isl_id *IteratorID) {
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000462 int i = 0;
463
464 Value *OldValue = IDToValue[IteratorID];
Tobias Grosser91f5b262014-06-04 08:06:40 +0000465 for (Value *IV : IVS) {
466 IDToValue[IteratorID] = IV;
Tobias Grosserce67a042014-07-02 16:26:47 +0000467 createSubstitutions(isl_ast_expr_copy(Expr), Stmt, VMap[i], VLTS[i]);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000468 i++;
469 }
470
471 IDToValue[IteratorID] = OldValue;
472 isl_id_free(IteratorID);
Tobias Grosserce67a042014-07-02 16:26:47 +0000473 isl_ast_expr_free(Expr);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000474}
475
476void IslNodeBuilder::createUser(__isl_take isl_ast_node *User) {
477 ValueMapT VMap;
Sebastian Pop9d10fff2013-02-15 20:55:59 +0000478 LoopToScevMapT LTS;
Tobias Grosserce67a042014-07-02 16:26:47 +0000479 isl_id *Id;
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000480 ScopStmt *Stmt;
481
Tobias Grosserce67a042014-07-02 16:26:47 +0000482 isl_ast_expr *Expr = isl_ast_node_user_get_expr(User);
483 isl_ast_expr *StmtExpr = isl_ast_expr_get_op_arg(Expr, 0);
484 Id = isl_ast_expr_get_id(StmtExpr);
485 isl_ast_expr_free(StmtExpr);
Sebastian Pop04c4ce32012-12-18 07:46:13 +0000486
Tobias Grosserc14582f2013-02-05 18:01:29 +0000487 Stmt = (ScopStmt *)isl_id_get_user(Id);
Tobias Grosserce67a042014-07-02 16:26:47 +0000488 createSubstitutions(Expr, Stmt, VMap, LTS);
Sebastian Pop9d10fff2013-02-15 20:55:59 +0000489 BlockGenerator::generate(Builder, *Stmt, VMap, LTS, P);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000490
491 isl_ast_node_free(User);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000492 isl_id_free(Id);
493}
494
495void IslNodeBuilder::createBlock(__isl_take isl_ast_node *Block) {
496 isl_ast_node_list *List = isl_ast_node_block_get_children(Block);
497
498 for (int i = 0; i < isl_ast_node_list_n_ast_node(List); ++i)
499 create(isl_ast_node_list_get_ast_node(List, i));
500
501 isl_ast_node_free(Block);
502 isl_ast_node_list_free(List);
503}
504
505void IslNodeBuilder::create(__isl_take isl_ast_node *Node) {
506 switch (isl_ast_node_get_type(Node)) {
507 case isl_ast_node_error:
508 llvm_unreachable("code generation error");
509 case isl_ast_node_for:
510 createFor(Node);
511 return;
512 case isl_ast_node_if:
513 createIf(Node);
514 return;
515 case isl_ast_node_user:
516 createUser(Node);
517 return;
518 case isl_ast_node_block:
519 createBlock(Node);
520 return;
521 }
522
523 llvm_unreachable("Unknown isl_ast_node type");
524}
525
526void IslNodeBuilder::addParameters(__isl_take isl_set *Context) {
527 SCEVExpander Rewriter(P->getAnalysis<ScalarEvolution>(), "polly");
528
529 for (unsigned i = 0; i < isl_set_dim(Context, isl_dim_param); ++i) {
530 isl_id *Id;
531 const SCEV *Scev;
532 IntegerType *T;
533 Instruction *InsertLocation;
534
535 Id = isl_set_get_dim_id(Context, isl_dim_param, i);
Tobias Grosserc14582f2013-02-05 18:01:29 +0000536 Scev = (const SCEV *)isl_id_get_user(Id);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000537 T = dyn_cast<IntegerType>(Scev->getType());
538 InsertLocation = --(Builder.GetInsertBlock()->end());
539 Value *V = Rewriter.expandCodeFor(Scev, T, InsertLocation);
540 IDToValue[Id] = V;
541
542 isl_id_free(Id);
543 }
544
545 isl_set_free(Context);
546}
547
548namespace {
549class IslCodeGeneration : public ScopPass {
Tobias Grosser1bb59b02012-12-29 23:47:38 +0000550public:
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000551 static char ID;
552
553 IslCodeGeneration() : ScopPass(ID) {}
554
555 bool runOnScop(Scop &S) {
556 IslAstInfo &AstInfo = getAnalysis<IslAstInfo>();
Tobias Grosser0ee50f62013-04-10 06:55:31 +0000557
Tobias Grossere602a072013-05-07 07:30:56 +0000558 assert(!S.getRegion().isTopLevelRegion() &&
559 "Top level regions are not supported");
Tobias Grosser0ee50f62013-04-10 06:55:31 +0000560
Tobias Grosser8edce4e2013-04-16 08:04:42 +0000561 simplifyRegion(&S, this);
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000562
563 BasicBlock *StartBlock = executeScopConditionally(S, this);
564 isl_ast_node *Ast = AstInfo.getAst();
Tobias Grosser37c9b8e2014-03-04 14:59:00 +0000565 LoopAnnotator Annotator;
566 PollyIRBuilder Builder(StartBlock->getContext(), llvm::ConstantFolder(),
567 polly::IRInserter(Annotator));
568 Builder.SetInsertPoint(StartBlock->begin());
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000569
Tobias Grosser37c9b8e2014-03-04 14:59:00 +0000570 IslNodeBuilder NodeBuilder(Builder, Annotator, this);
Tobias Grosser54ee0ba2013-11-17 03:18:25 +0000571
Tobias Grosser5e6813d2014-07-02 17:47:48 +0000572 Builder.SetInsertPoint(StartBlock->getSinglePredecessor()->begin());
573 NodeBuilder.addParameters(S.getContext());
Tobias Grosser54ee0ba2013-11-17 03:18:25 +0000574 // Build condition that evaluates at run-time if all assumptions taken
575 // for the scop hold. If we detect some assumptions do not hold, the
576 // original code is executed.
577 Value *V = NodeBuilder.getExprBuilder().create(AstInfo.getRunCondition());
578 Value *Zero = ConstantInt::get(V->getType(), 0);
579 V = Builder.CreateICmp(CmpInst::ICMP_NE, Zero, V);
580 BasicBlock *PrevBB = StartBlock->getUniquePredecessor();
581 BranchInst *Branch = dyn_cast<BranchInst>(PrevBB->getTerminator());
582 Branch->setCondition(V);
Tobias Grosser5e6813d2014-07-02 17:47:48 +0000583 Builder.SetInsertPoint(StartBlock->begin());
Tobias Grosser54ee0ba2013-11-17 03:18:25 +0000584
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000585 NodeBuilder.create(Ast);
586 return true;
587 }
588
Tobias Grosserc14582f2013-02-05 18:01:29 +0000589 virtual void printScop(raw_ostream &OS) const {}
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000590
591 virtual void getAnalysisUsage(AnalysisUsage &AU) const {
Tobias Grosser42aff302014-01-13 22:29:56 +0000592 AU.addRequired<DominatorTreeWrapperPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000593 AU.addRequired<IslAstInfo>();
Matt Arsenault8ca36812014-07-19 18:40:17 +0000594 AU.addRequired<RegionInfoPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000595 AU.addRequired<ScalarEvolution>();
596 AU.addRequired<ScopDetection>();
597 AU.addRequired<ScopInfo>();
Tobias Grosserecfe21b2013-03-20 18:03:18 +0000598 AU.addRequired<LoopInfo>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000599
600 AU.addPreserved<Dependences>();
601
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000602 AU.addPreserved<LoopInfo>();
Tobias Grosser42aff302014-01-13 22:29:56 +0000603 AU.addPreserved<DominatorTreeWrapperPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000604 AU.addPreserved<IslAstInfo>();
605 AU.addPreserved<ScopDetection>();
606 AU.addPreserved<ScalarEvolution>();
607
608 // FIXME: We do not yet add regions for the newly generated code to the
609 // region tree.
Matt Arsenault8ca36812014-07-19 18:40:17 +0000610 AU.addPreserved<RegionInfoPass>();
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000611 AU.addPreserved<TempScopInfo>();
612 AU.addPreserved<ScopInfo>();
613 AU.addPreservedID(IndependentBlocksID);
614 }
615};
616}
617
618char IslCodeGeneration::ID = 1;
619
Tobias Grosser7242ad92013-02-22 08:07:06 +0000620Pass *polly::createIslCodeGenerationPass() { return new IslCodeGeneration(); }
Tobias Grosser8a5bc6e2012-10-02 19:50:43 +0000621
Tobias Grosser7242ad92013-02-22 08:07:06 +0000622INITIALIZE_PASS_BEGIN(IslCodeGeneration, "polly-codegen-isl",
623 "Polly - Create LLVM-IR from SCoPs", false, false);
624INITIALIZE_PASS_DEPENDENCY(Dependences);
Tobias Grosser42aff302014-01-13 22:29:56 +0000625INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass);
Tobias Grosser7242ad92013-02-22 08:07:06 +0000626INITIALIZE_PASS_DEPENDENCY(LoopInfo);
Matt Arsenault8ca36812014-07-19 18:40:17 +0000627INITIALIZE_PASS_DEPENDENCY(RegionInfoPass);
Tobias Grosser7242ad92013-02-22 08:07:06 +0000628INITIALIZE_PASS_DEPENDENCY(ScalarEvolution);
629INITIALIZE_PASS_DEPENDENCY(ScopDetection);
630INITIALIZE_PASS_END(IslCodeGeneration, "polly-codegen-isl",
631 "Polly - Create LLVM-IR from SCoPs", false, false)