blob: b044f195077259fb8a7bfdbb17f1cef658346137 [file] [log] [blame]
Tobias Grosser75805372011-04-29 06:27:02 +00001//===----- ScopDetection.cpp - Detect Scops --------------------*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Detect the maximal Scops of a function.
11//
12// A static control part (Scop) is a subgraph of the control flow graph (CFG)
13// that only has statically known control flow and can therefore be described
14// within the polyhedral model.
15//
16// Every Scop fullfills these restrictions:
17//
18// * It is a single entry single exit region
19//
20// * Only affine linear bounds in the loops
21//
22// Every natural loop in a Scop must have a number of loop iterations that can
23// be described as an affine linear function in surrounding loop iterators or
24// parameters. (A parameter is a scalar that does not change its value during
25// execution of the Scop).
26//
27// * Only comparisons of affine linear expressions in conditions
28//
29// * All loops and conditions perfectly nested
30//
31// The control flow needs to be structured such that it could be written using
32// just 'for' and 'if' statements, without the need for any 'goto', 'break' or
33// 'continue'.
34//
35// * Side effect free functions call
36//
37// Only function calls and intrinsics that do not have side effects are allowed
38// (readnone).
39//
40// The Scop detection finds the largest Scops by checking if the largest
41// region is a Scop. If this is not the case, its canonical subregions are
42// checked until a region is a Scop. It is now tried to extend this Scop by
43// creating a larger non canonical region.
44//
45//===----------------------------------------------------------------------===//
46
Tobias Grosser5624d3c2015-12-21 12:38:56 +000047#include "polly/ScopDetection.h"
Tobias Grosserba0d0922015-05-09 09:13:42 +000048#include "polly/CodeGen/CodeGeneration.h"
Tobias Grosser75805372011-04-29 06:27:02 +000049#include "polly/LinkAllPasses.h"
Tobias Grosser637bd632013-05-07 07:31:10 +000050#include "polly/Options.h"
Tobias Grosserba0d0922015-05-09 09:13:42 +000051#include "polly/ScopDetectionDiagnostic.h"
Tobias Grosser120db6b2011-11-07 12:58:54 +000052#include "polly/Support/SCEVValidator.h"
Tobias Grossera63b7ce2015-05-03 05:21:36 +000053#include "polly/Support/ScopLocation.h"
Tobias Grosser75805372011-04-29 06:27:02 +000054#include "llvm/ADT/Statistic.h"
55#include "llvm/Analysis/AliasAnalysis.h"
Tobias Grosser6e9f25a2011-11-09 22:35:00 +000056#include "llvm/Analysis/LoopInfo.h"
Matt Arsenault8ca36812014-07-19 18:40:17 +000057#include "llvm/Analysis/PostDominators.h"
Tobias Grosser75805372011-04-29 06:27:02 +000058#include "llvm/Analysis/RegionIterator.h"
Tobias Grosser6e9f25a2011-11-09 22:35:00 +000059#include "llvm/Analysis/ScalarEvolution.h"
Tobias Grosserb8710b52011-11-10 12:44:50 +000060#include "llvm/Analysis/ScalarEvolutionExpressions.h"
Chandler Carruth6b96c242014-03-06 00:47:27 +000061#include "llvm/IR/DebugInfo.h"
Tobias Grosser8519f892013-12-18 10:49:53 +000062#include "llvm/IR/DiagnosticInfo.h"
63#include "llvm/IR/DiagnosticPrinter.h"
Tobias Grosserba0d0922015-05-09 09:13:42 +000064#include "llvm/IR/IntrinsicInst.h"
Chandler Carruth6b96c242014-03-06 00:47:27 +000065#include "llvm/IR/LLVMContext.h"
Tobias Grosser75805372011-04-29 06:27:02 +000066#include "llvm/Support/Debug.h"
Tobias Grosser60b54f12011-11-08 15:41:28 +000067#include <set>
Tobias Grosser1c3a6d72016-01-22 09:44:37 +000068#include <stack>
Tobias Grosser60b54f12011-11-08 15:41:28 +000069
Tobias Grosser75805372011-04-29 06:27:02 +000070using namespace llvm;
71using namespace polly;
72
Chandler Carruth95fef942014-04-22 03:30:19 +000073#define DEBUG_TYPE "polly-detect"
74
Tobias Grosserc1a269b2015-12-21 21:00:43 +000075// This option is set to a very high value, as analyzing such loops increases
76// compile time on several cases. For experiments that enable this option,
77// a value of around 40 has been working to avoid run-time regressions with
78// Polly while still exposing interesting optimization opportunities.
79static cl::opt<int> ProfitabilityMinPerLoopInstructions(
80 "polly-detect-profitability-min-per-loop-insts",
81 cl::desc("The minimal number of per-loop instructions before a single loop "
82 "region is considered profitable"),
83 cl::Hidden, cl::ValueRequired, cl::init(100000000), cl::cat(PollyCategory));
84
Tobias Grosser575aca82015-10-06 16:10:29 +000085bool polly::PollyProcessUnprofitable;
86static cl::opt<bool, true> XPollyProcessUnprofitable(
87 "polly-process-unprofitable",
88 cl::desc(
89 "Process scops that are unlikely to benefit from Polly optimizations."),
90 cl::location(PollyProcessUnprofitable), cl::init(false), cl::ZeroOrMore,
91 cl::cat(PollyCategory));
Tobias Grosserd1e33e72015-02-19 05:31:07 +000092
Tobias Grosser483a90d2014-07-09 10:50:10 +000093static cl::opt<std::string> OnlyFunction(
94 "polly-only-func",
95 cl::desc("Only run on functions that contain a certain string"),
96 cl::value_desc("string"), cl::ValueRequired, cl::init(""),
97 cl::cat(PollyCategory));
Tobias Grosser2ff87232011-10-23 11:17:06 +000098
Tobias Grosser483a90d2014-07-09 10:50:10 +000099static cl::opt<std::string> OnlyRegion(
100 "polly-only-region",
101 cl::desc("Only run on certain regions (The provided identifier must "
102 "appear in the name of the region's entry block"),
103 cl::value_desc("identifier"), cl::ValueRequired, cl::init(""),
104 cl::cat(PollyCategory));
Tobias Grosser4449e522014-01-27 14:24:53 +0000105
Tobias Grosser60cd9322011-11-10 12:47:26 +0000106static cl::opt<bool>
Tobias Grosser483a90d2014-07-09 10:50:10 +0000107 IgnoreAliasing("polly-ignore-aliasing",
108 cl::desc("Ignore possible aliasing of the array bases"),
109 cl::Hidden, cl::init(false), cl::ZeroOrMore,
110 cl::cat(PollyCategory));
Tobias Grosser2ff87232011-10-23 11:17:06 +0000111
Johannes Doerfertb164c792014-09-18 11:17:17 +0000112bool polly::PollyUseRuntimeAliasChecks;
113static cl::opt<bool, true> XPollyUseRuntimeAliasChecks(
114 "polly-use-runtime-alias-checks",
115 cl::desc("Use runtime alias checks to resolve possible aliasing."),
116 cl::location(PollyUseRuntimeAliasChecks), cl::Hidden, cl::ZeroOrMore,
117 cl::init(true), cl::cat(PollyCategory));
118
Tobias Grosser637bd632013-05-07 07:31:10 +0000119static cl::opt<bool>
Tobias Grosser483a90d2014-07-09 10:50:10 +0000120 ReportLevel("polly-report",
121 cl::desc("Print information about the activities of Polly"),
122 cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
Tobias Grosser531891e2012-11-01 16:45:20 +0000123
Tobias Grosser8ebdc2d2016-02-07 08:48:57 +0000124static cl::opt<bool> AllowDifferentTypes(
125 "polly-allow-differing-element-types",
126 cl::desc("Allow different element types for array accesses"), cl::Hidden,
Tobias Grossera2ee0032016-02-16 14:37:24 +0000127 cl::init(true), cl::ZeroOrMore, cl::cat(PollyCategory));
Tobias Grosser8ebdc2d2016-02-07 08:48:57 +0000128
Tobias Grosser531891e2012-11-01 16:45:20 +0000129static cl::opt<bool>
Tobias Grosser483a90d2014-07-09 10:50:10 +0000130 AllowNonAffine("polly-allow-nonaffine",
131 cl::desc("Allow non affine access functions in arrays"),
132 cl::Hidden, cl::init(false), cl::ZeroOrMore,
133 cl::cat(PollyCategory));
Tobias Grossera1879642011-12-20 10:43:14 +0000134
Johannes Doerfertba65c162015-02-24 11:45:21 +0000135static cl::opt<bool> AllowNonAffineSubRegions(
136 "polly-allow-nonaffine-branches",
137 cl::desc("Allow non affine conditions for branches"), cl::Hidden,
Johannes Doerferta36842f2015-02-26 11:09:24 +0000138 cl::init(true), cl::ZeroOrMore, cl::cat(PollyCategory));
Johannes Doerfertba65c162015-02-24 11:45:21 +0000139
Johannes Doerfertf3e98f42015-04-12 22:52:20 +0000140static cl::opt<bool>
141 AllowNonAffineSubLoops("polly-allow-nonaffine-loops",
142 cl::desc("Allow non affine conditions for loops"),
143 cl::Hidden, cl::init(false), cl::ZeroOrMore,
144 cl::cat(PollyCategory));
145
Tobias Grosserbfbc3692015-01-09 00:01:33 +0000146static cl::opt<bool> AllowUnsigned("polly-allow-unsigned",
147 cl::desc("Allow unsigned expressions"),
148 cl::Hidden, cl::init(false), cl::ZeroOrMore,
149 cl::cat(PollyCategory));
150
Tobias Grosserc7d3fc52013-07-25 03:02:29 +0000151static cl::opt<bool, true>
Tobias Grosser483a90d2014-07-09 10:50:10 +0000152 TrackFailures("polly-detect-track-failures",
153 cl::desc("Track failure strings in detecting scop regions"),
154 cl::location(PollyTrackFailures), cl::Hidden, cl::ZeroOrMore,
Andreas Simbuerger3efe40b2014-08-17 10:09:03 +0000155 cl::init(true), cl::cat(PollyCategory));
Tobias Grosserc7d3fc52013-07-25 03:02:29 +0000156
Andreas Simbuerger04472402014-05-24 09:25:10 +0000157static cl::opt<bool> KeepGoing("polly-detect-keep-going",
158 cl::desc("Do not fail on the first error."),
159 cl::Hidden, cl::ZeroOrMore, cl::init(false),
160 cl::cat(PollyCategory));
161
Sebastian Pop18016682014-04-08 21:20:44 +0000162static cl::opt<bool, true>
Tobias Grosser483a90d2014-07-09 10:50:10 +0000163 PollyDelinearizeX("polly-delinearize",
164 cl::desc("Delinearize array access functions"),
165 cl::location(PollyDelinearize), cl::Hidden,
Tobias Grosser6973cb62015-03-08 15:21:18 +0000166 cl::ZeroOrMore, cl::init(true), cl::cat(PollyCategory));
Sebastian Pop18016682014-04-08 21:20:44 +0000167
Tobias Grossera1689932014-02-18 18:49:49 +0000168static cl::opt<bool>
Tobias Grosser483a90d2014-07-09 10:50:10 +0000169 VerifyScops("polly-detect-verify",
170 cl::desc("Verify the detected SCoPs after each transformation"),
171 cl::Hidden, cl::init(false), cl::ZeroOrMore,
172 cl::cat(PollyCategory));
Tobias Grossera1689932014-02-18 18:49:49 +0000173
Johannes Doerferte526de52015-09-21 19:10:11 +0000174/// @brief The minimal trip count under which loops are considered unprofitable.
175static const unsigned MIN_LOOP_TRIP_COUNT = 8;
176
Tobias Grosserc7d3fc52013-07-25 03:02:29 +0000177bool polly::PollyTrackFailures = false;
Sebastian Pop18016682014-04-08 21:20:44 +0000178bool polly::PollyDelinearize = false;
Johannes Doerfert43e1ead2014-07-15 21:06:48 +0000179StringRef polly::PollySkipFnAttr = "polly.skip.fn";
Tobias Grosserc7d3fc52013-07-25 03:02:29 +0000180
Tobias Grosser75805372011-04-29 06:27:02 +0000181//===----------------------------------------------------------------------===//
182// Statistics.
183
184STATISTIC(ValidRegion, "Number of regions that a valid part of Scop");
185
Tobias Grosser8519f892013-12-18 10:49:53 +0000186class DiagnosticScopFound : public DiagnosticInfo {
187private:
188 static int PluginDiagnosticKind;
189
190 Function &F;
191 std::string FileName;
192 unsigned EntryLine, ExitLine;
193
194public:
Tobias Grosser1b12f462013-12-18 11:14:36 +0000195 DiagnosticScopFound(Function &F, std::string FileName, unsigned EntryLine,
196 unsigned ExitLine)
Tobias Grosser8519f892013-12-18 10:49:53 +0000197 : DiagnosticInfo(PluginDiagnosticKind, DS_Note), F(F), FileName(FileName),
Tobias Grosser1b12f462013-12-18 11:14:36 +0000198 EntryLine(EntryLine), ExitLine(ExitLine) {}
Tobias Grosser8519f892013-12-18 10:49:53 +0000199
200 virtual void print(DiagnosticPrinter &DP) const;
201
202 static bool classof(const DiagnosticInfo *DI) {
203 return DI->getKind() == PluginDiagnosticKind;
204 }
205};
206
207int DiagnosticScopFound::PluginDiagnosticKind = 10;
208
Tobias Grosser8519f892013-12-18 10:49:53 +0000209void DiagnosticScopFound::print(DiagnosticPrinter &DP) const {
Tobias Grosser1b12f462013-12-18 11:14:36 +0000210 DP << "Polly detected an optimizable loop region (scop) in function '" << F
211 << "'\n";
Tobias Grosser8519f892013-12-18 10:49:53 +0000212
213 if (FileName.empty()) {
214 DP << "Scop location is unknown. Compile with debug info "
215 "(-g) to get more precise information. ";
Tobias Grosser1b12f462013-12-18 11:14:36 +0000216 return;
Tobias Grosser8519f892013-12-18 10:49:53 +0000217 }
218
219 DP << FileName << ":" << EntryLine << ": Start of scop\n";
220 DP << FileName << ":" << ExitLine << ": End of scop";
221}
222
Tobias Grosser75805372011-04-29 06:27:02 +0000223//===----------------------------------------------------------------------===//
224// ScopDetection.
Andreas Simbuerger01a37a02014-04-02 11:54:01 +0000225
Johannes Doerfertb164c792014-09-18 11:17:17 +0000226ScopDetection::ScopDetection() : FunctionPass(ID) {
227 if (!PollyUseRuntimeAliasChecks)
228 return;
229
Johannes Doerfert928229f2014-09-29 17:06:29 +0000230 // Disable runtime alias checks if we ignore aliasing all together.
231 if (IgnoreAliasing) {
232 PollyUseRuntimeAliasChecks = false;
233 return;
234 }
235
Johannes Doerfertb164c792014-09-18 11:17:17 +0000236 if (AllowNonAffine) {
237 DEBUG(errs() << "WARNING: We disable runtime alias checks as non affine "
238 "accesses are enabled.\n");
239 PollyUseRuntimeAliasChecks = false;
240 }
Johannes Doerfertb164c792014-09-18 11:17:17 +0000241}
242
Andreas Simbuerger01a37a02014-04-02 11:54:01 +0000243template <class RR, typename... Args>
244inline bool ScopDetection::invalid(DetectionContext &Context, bool Assert,
245 Args &&... Arguments) const {
246
247 if (!Context.Verifying) {
Andreas Simbuerger4870e092014-05-24 09:25:01 +0000248 RejectLog &Log = Context.Log;
249 std::shared_ptr<RR> RejectReason = std::make_shared<RR>(Arguments...);
Andreas Simbuerger01a37a02014-04-02 11:54:01 +0000250
Andreas Simbuerger8a00c9b2014-05-24 09:25:06 +0000251 if (PollyTrackFailures)
Andreas Simbuerger4870e092014-05-24 09:25:01 +0000252 Log.report(RejectReason);
Andreas Simbuerger4870e092014-05-24 09:25:01 +0000253
254 DEBUG(dbgs() << RejectReason->getMessage());
Andreas Simbuerger01a37a02014-04-02 11:54:01 +0000255 DEBUG(dbgs() << "\n");
256 } else {
257 assert(!Assert && "Verification of detected scop failed");
258 }
259
260 return false;
261}
262
Tobias Grossera1689932014-02-18 18:49:49 +0000263bool ScopDetection::isMaxRegionInScop(const Region &R, bool Verify) const {
264 if (!ValidRegions.count(&R))
265 return false;
266
Johannes Doerfert3f1c2852015-02-19 18:11:50 +0000267 if (Verify) {
Tobias Grosser907090c2015-10-25 10:55:35 +0000268 DetectionContextMap.erase(&R);
269 const auto &It = DetectionContextMap.insert(
270 std::make_pair(&R, DetectionContext(const_cast<Region &>(R), *AA,
271 false /*verifying*/)));
272 DetectionContext &Context = It.first->second;
Johannes Doerfert3f1c2852015-02-19 18:11:50 +0000273 return isValidRegion(Context);
274 }
Tobias Grossera1689932014-02-18 18:49:49 +0000275
276 return true;
Tobias Grosser75805372011-04-29 06:27:02 +0000277}
278
Tobias Grosser4f129a62011-10-08 00:30:55 +0000279std::string ScopDetection::regionIsInvalidBecause(const Region *R) const {
Andreas Simbuerger8a00c9b2014-05-24 09:25:06 +0000280 if (!RejectLogs.count(R))
Tobias Grosser4f129a62011-10-08 00:30:55 +0000281 return "";
282
Andreas Simbuerger8a00c9b2014-05-24 09:25:06 +0000283 // Get the first error we found. Even in keep-going mode, this is the first
284 // reason that caused the candidate to be rejected.
285 RejectLog Errors = RejectLogs.at(R);
Andreas Simbuerger83ed8612014-06-12 07:25:08 +0000286
287 // This can happen when we marked a region invalid, but didn't track
288 // an error for it.
289 if (Errors.size() == 0)
290 return "";
291
292 RejectReasonPtr RR = *Errors.begin();
293 return RR->getMessage();
Tobias Grosser4f129a62011-10-08 00:30:55 +0000294}
295
Johannes Doerfertf3e98f42015-04-12 22:52:20 +0000296bool ScopDetection::addOverApproximatedRegion(Region *AR,
297 DetectionContext &Context) const {
298
299 // If we already know about Ar we can exit.
300 if (!Context.NonAffineSubRegionSet.insert(AR))
301 return true;
302
303 // All loops in the region have to be overapproximated too if there
304 // are accesses that depend on the iteration count.
305 for (BasicBlock *BB : AR->blocks()) {
Johannes Doerfertba65c162015-02-24 11:45:21 +0000306 Loop *L = LI->getLoopFor(BB);
Johannes Doerfertf3e98f42015-04-12 22:52:20 +0000307 if (AR->contains(L))
308 Context.BoxedLoopsSet.insert(L);
Johannes Doerfertba65c162015-02-24 11:45:21 +0000309 }
Johannes Doerfertf3e98f42015-04-12 22:52:20 +0000310
311 return (AllowNonAffineSubLoops || Context.BoxedLoopsSet.empty());
Johannes Doerfertba65c162015-02-24 11:45:21 +0000312}
313
Johannes Doerfert09e36972015-10-07 20:17:36 +0000314bool ScopDetection::onlyValidRequiredInvariantLoads(
315 InvariantLoadsSetTy &RequiredILS, DetectionContext &Context) const {
316 Region &CurRegion = Context.CurRegion;
317
318 for (LoadInst *Load : RequiredILS)
319 if (!isHoistableLoad(Load, CurRegion, *LI, *SE))
320 return false;
321
322 Context.RequiredILS.insert(RequiredILS.begin(), RequiredILS.end());
323
324 return true;
325}
326
327bool ScopDetection::isAffine(const SCEV *S, DetectionContext &Context,
328 Value *BaseAddress) const {
329
330 InvariantLoadsSetTy AccessILS;
331 if (!isAffineExpr(&Context.CurRegion, S, *SE, BaseAddress, &AccessILS))
332 return false;
333
334 if (!onlyValidRequiredInvariantLoads(AccessILS, Context))
335 return false;
336
337 return true;
338}
339
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000340bool ScopDetection::isValidSwitch(BasicBlock &BB, SwitchInst *SI,
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000341 Value *Condition, bool IsLoopBranch,
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000342 DetectionContext &Context) const {
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000343 Loop *L = LI->getLoopFor(&BB);
344 const SCEV *ConditionSCEV = SE->getSCEVAtScope(Condition, L);
Tobias Grosser75805372011-04-29 06:27:02 +0000345
Johannes Doerfert09e36972015-10-07 20:17:36 +0000346 if (isAffine(ConditionSCEV, Context))
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000347 return true;
Tobias Grosser75805372011-04-29 06:27:02 +0000348
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000349 if (!IsLoopBranch && AllowNonAffineSubRegions &&
350 addOverApproximatedRegion(RI->getRegionFor(&BB), Context))
351 return true;
352
353 if (IsLoopBranch)
354 return false;
355
356 return invalid<ReportNonAffBranch>(Context, /*Assert=*/true, &BB,
357 ConditionSCEV, ConditionSCEV, SI);
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000358}
359
360bool ScopDetection::isValidBranch(BasicBlock &BB, BranchInst *BI,
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000361 Value *Condition, bool IsLoopBranch,
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000362 DetectionContext &Context) const {
Johannes Doerfert9b1f9c82015-10-11 13:21:03 +0000363
364 if (BinaryOperator *BinOp = dyn_cast<BinaryOperator>(Condition)) {
365 auto Opcode = BinOp->getOpcode();
366 if (Opcode == Instruction::And || Opcode == Instruction::Or) {
367 Value *Op0 = BinOp->getOperand(0);
368 Value *Op1 = BinOp->getOperand(1);
369 return isValidBranch(BB, BI, Op0, IsLoopBranch, Context) &&
370 isValidBranch(BB, BI, Op1, IsLoopBranch, Context);
371 }
372 }
373
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000374 // Non constant conditions of branches need to be ICmpInst.
375 if (!isa<ICmpInst>(Condition)) {
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000376 if (!IsLoopBranch && AllowNonAffineSubRegions &&
377 addOverApproximatedRegion(RI->getRegionFor(&BB), Context))
378 return true;
379 return invalid<ReportInvalidCond>(Context, /*Assert=*/true, BI, &BB);
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000380 }
Tobias Grosser75805372011-04-29 06:27:02 +0000381
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000382 ICmpInst *ICmp = cast<ICmpInst>(Condition);
383 // Unsigned comparisons are not allowed. They trigger overflow problems
384 // in the code generation.
385 //
386 // TODO: This is not sufficient and just hides bugs. However it does pretty
387 // well.
388 if (ICmp->isUnsigned() && !AllowUnsigned)
389 return invalid<ReportUnsignedCond>(Context, /*Assert=*/true, BI, &BB);
Tobias Grosser75805372011-04-29 06:27:02 +0000390
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000391 // Are both operands of the ICmp affine?
392 if (isa<UndefValue>(ICmp->getOperand(0)) ||
393 isa<UndefValue>(ICmp->getOperand(1)))
394 return invalid<ReportUndefOperand>(Context, /*Assert=*/true, &BB, ICmp);
Tobias Grosser75805372011-04-29 06:27:02 +0000395
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000396 // TODO: FIXME: IslExprBuilder is not capable of producing valid code
397 // for arbitrary pointer expressions at the moment. Until
398 // this is fixed we disallow pointer expressions completely.
399 if (ICmp->getOperand(0)->getType()->isPointerTy())
400 return false;
Johannes Doerfert7ca8dc22015-09-09 14:19:04 +0000401
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000402 Loop *L = LI->getLoopFor(ICmp->getParent());
403 const SCEV *LHS = SE->getSCEVAtScope(ICmp->getOperand(0), L);
404 const SCEV *RHS = SE->getSCEVAtScope(ICmp->getOperand(1), L);
Tobias Grosser75805372011-04-29 06:27:02 +0000405
Johannes Doerfert09e36972015-10-07 20:17:36 +0000406 if (isAffine(LHS, Context) && isAffine(RHS, Context))
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000407 return true;
Tobias Grosser75805372011-04-29 06:27:02 +0000408
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000409 if (!IsLoopBranch && AllowNonAffineSubRegions &&
410 addOverApproximatedRegion(RI->getRegionFor(&BB), Context))
411 return true;
412
413 if (IsLoopBranch)
414 return false;
415
416 return invalid<ReportNonAffBranch>(Context, /*Assert=*/true, &BB, LHS, RHS,
417 ICmp);
Tobias Grosser75805372011-04-29 06:27:02 +0000418}
419
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000420bool ScopDetection::isValidCFG(BasicBlock &BB, bool IsLoopBranch,
Tobias Grosserb76cd3c2015-11-11 08:42:20 +0000421 bool AllowUnreachable,
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000422 DetectionContext &Context) const {
423 Region &CurRegion = Context.CurRegion;
424
425 TerminatorInst *TI = BB.getTerminator();
426
Tobias Grosserb76cd3c2015-11-11 08:42:20 +0000427 if (AllowUnreachable && isa<UnreachableInst>(TI))
428 return true;
429
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000430 // Return instructions are only valid if the region is the top level region.
431 if (isa<ReturnInst>(TI) && !CurRegion.getExit() && TI->getNumOperands() == 0)
432 return true;
433
434 Value *Condition = getConditionFromTerminator(TI);
435
436 if (!Condition)
437 return invalid<ReportInvalidTerminator>(Context, /*Assert=*/true, &BB);
438
439 // UndefValue is not allowed as condition.
440 if (isa<UndefValue>(Condition))
441 return invalid<ReportUndefCond>(Context, /*Assert=*/true, TI, &BB);
442
Johannes Doerfert9c28bfa2015-10-18 22:56:42 +0000443 // Constant integer conditions are always affine.
444 if (isa<ConstantInt>(Condition))
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000445 return true;
446
447 if (BranchInst *BI = dyn_cast<BranchInst>(TI))
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000448 return isValidBranch(BB, BI, Condition, IsLoopBranch, Context);
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000449
450 SwitchInst *SI = dyn_cast<SwitchInst>(TI);
451 assert(SI && "Terminator was neither branch nor switch");
452
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000453 return isValidSwitch(BB, SI, Condition, IsLoopBranch, Context);
Johannes Doerfert9a132f32015-09-28 09:33:22 +0000454}
455
Tobias Grosser75805372011-04-29 06:27:02 +0000456bool ScopDetection::isValidCallInst(CallInst &CI) {
Johannes Doerfert3f500fa2015-01-25 18:07:30 +0000457 if (CI.doesNotReturn())
Tobias Grosser75805372011-04-29 06:27:02 +0000458 return false;
459
460 if (CI.doesNotAccessMemory())
461 return true;
462
463 Function *CalledFunction = CI.getCalledFunction();
464
465 // Indirect calls are not supported.
466 if (CalledFunction == 0)
467 return false;
468
Tobias Grosser9c0ffe32015-08-30 16:57:15 +0000469 if (isIgnoredIntrinsic(&CI))
470 return true;
Johannes Doerfert3f500fa2015-01-25 18:07:30 +0000471
Tobias Grosser75805372011-04-29 06:27:02 +0000472 return false;
473}
474
Tobias Grosser458fb782014-01-28 12:58:58 +0000475bool ScopDetection::isInvariant(const Value &Val, const Region &Reg) const {
476 // A reference to function argument or constant value is invariant.
477 if (isa<Argument>(Val) || isa<Constant>(Val))
478 return true;
479
480 const Instruction *I = dyn_cast<Instruction>(&Val);
481 if (!I)
482 return false;
483
484 if (!Reg.contains(I))
485 return true;
486
487 if (I->mayHaveSideEffects())
488 return false;
489
490 // When Val is a Phi node, it is likely not invariant. We do not check whether
491 // Phi nodes are actually invariant, we assume that Phi nodes are usually not
492 // invariant. Recursively checking the operators of Phi nodes would lead to
493 // infinite recursion.
494 if (isa<PHINode>(*I))
495 return false;
496
Tobias Grosser26108892014-04-02 20:18:19 +0000497 for (const Use &Operand : I->operands())
Tobias Grosser1d191902014-03-03 13:13:55 +0000498 if (!isInvariant(*Operand, Reg))
Tobias Grosser458fb782014-01-28 12:58:58 +0000499 return false;
Tobias Grosser458fb782014-01-28 12:58:58 +0000500
Tobias Grosser458fb782014-01-28 12:58:58 +0000501 return true;
502}
503
Tobias Grosser2f8e43d2015-11-24 17:06:38 +0000504/// @brief Remove smax of smax(0, size) expressions from a SCEV expression and
505/// register the '...' components.
506///
507/// Array access expressions as they are generated by gfortran contain smax(0,
508/// size) expressions that confuse the 'normal' delinearization algorithm.
509/// However, if we extract such expressions before the normal delinearization
510/// takes place they can actually help to identify array size expressions in
511/// fortran accesses. For the subsequently following delinearization the smax(0,
512/// size) component can be replaced by just 'size'. This is correct as we will
513/// always add and verify the assumption that for all subscript expressions
514/// 'exp' the inequality 0 <= exp < size holds. Hence, we will also verify
515/// that 0 <= size, which means smax(0, size) == size.
516struct SCEVRemoveMax : public SCEVVisitor<SCEVRemoveMax, const SCEV *> {
517public:
518 static const SCEV *remove(ScalarEvolution &SE, const SCEV *Expr,
519 std::vector<const SCEV *> *Terms = nullptr) {
520
521 SCEVRemoveMax D(SE, Terms);
522 return D.visit(Expr);
523 }
524
525 SCEVRemoveMax(ScalarEvolution &SE, std::vector<const SCEV *> *Terms)
526 : SE(SE), Terms(Terms) {}
527
528 const SCEV *visitTruncateExpr(const SCEVTruncateExpr *Expr) { return Expr; }
529
530 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
531 return Expr;
532 }
533
534 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
535 return SE.getSignExtendExpr(visit(Expr->getOperand()), Expr->getType());
536 }
537
538 const SCEV *visitUDivExpr(const SCEVUDivExpr *Expr) { return Expr; }
539
540 const SCEV *visitSMaxExpr(const SCEVSMaxExpr *Expr) {
Michael Kruse8fc28962015-12-20 14:42:32 +0000541 if ((Expr->getNumOperands() == 2) && Expr->getOperand(0)->isZero()) {
Tobias Grosser2f8e43d2015-11-24 17:06:38 +0000542 auto Res = visit(Expr->getOperand(1));
543 if (Terms)
544 (*Terms).push_back(Res);
545 return Res;
546 }
547
548 return Expr;
549 }
550
Roman Gareev8aa43752015-12-17 20:37:17 +0000551 const SCEV *visitUMaxExpr(const SCEVUMaxExpr *Expr) { return Expr; }
Tobias Grosser2f8e43d2015-11-24 17:06:38 +0000552
553 const SCEV *visitUnknown(const SCEVUnknown *Expr) { return Expr; }
554
555 const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) {
556 return Expr;
557 }
558
559 const SCEV *visitConstant(const SCEVConstant *Expr) { return Expr; }
560
561 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
562 SmallVector<const SCEV *, 5> NewOps;
563 for (const SCEV *Op : Expr->operands())
564 NewOps.push_back(visit(Op));
565
566 return SE.getAddRecExpr(NewOps, Expr->getLoop(), Expr->getNoWrapFlags());
567 }
568
569 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
570 SmallVector<const SCEV *, 5> NewOps;
571 for (const SCEV *Op : Expr->operands())
572 NewOps.push_back(visit(Op));
573
574 return SE.getAddExpr(NewOps);
575 }
576
577 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
578 SmallVector<const SCEV *, 5> NewOps;
579 for (const SCEV *Op : Expr->operands())
580 NewOps.push_back(visit(Op));
581
582 return SE.getMulExpr(NewOps);
583 }
584
585private:
586 ScalarEvolution &SE;
587 std::vector<const SCEV *> *Terms;
588};
589
Tobias Grosserd68ba422015-11-24 05:00:36 +0000590SmallVector<const SCEV *, 4>
591ScopDetection::getDelinearizationTerms(DetectionContext &Context,
592 const SCEVUnknown *BasePointer) const {
593 SmallVector<const SCEV *, 4> Terms;
594 for (const auto &Pair : Context.Accesses[BasePointer]) {
Tobias Grosser2f8e43d2015-11-24 17:06:38 +0000595 std::vector<const SCEV *> MaxTerms;
596 SCEVRemoveMax::remove(*SE, Pair.second, &MaxTerms);
597 if (MaxTerms.size() > 0) {
598 Terms.insert(Terms.begin(), MaxTerms.begin(), MaxTerms.end());
599 continue;
600 }
Tobias Grosserd68ba422015-11-24 05:00:36 +0000601 // In case the outermost expression is a plain add, we check if any of its
602 // terms has the form 4 * %inst * %param * %param ..., aka a term that
603 // contains a product between a parameter and an instruction that is
604 // inside the scop. Such instructions, if allowed at all, are instructions
605 // SCEV can not represent, but Polly is still looking through. As a
606 // result, these instructions can depend on induction variables and are
607 // most likely no array sizes. However, terms that are multiplied with
608 // them are likely candidates for array sizes.
609 if (auto *AF = dyn_cast<SCEVAddExpr>(Pair.second)) {
610 for (auto Op : AF->operands()) {
611 if (auto *AF2 = dyn_cast<SCEVAddRecExpr>(Op))
612 SE->collectParametricTerms(AF2, Terms);
613 if (auto *AF2 = dyn_cast<SCEVMulExpr>(Op)) {
614 SmallVector<const SCEV *, 0> Operands;
Johannes Doerfertfb79a962015-02-23 14:18:28 +0000615
Tobias Grosserd68ba422015-11-24 05:00:36 +0000616 for (auto *MulOp : AF2->operands()) {
617 if (auto *Const = dyn_cast<SCEVConstant>(MulOp))
618 Operands.push_back(Const);
619 if (auto *Unknown = dyn_cast<SCEVUnknown>(MulOp)) {
620 if (auto *Inst = dyn_cast<Instruction>(Unknown->getValue())) {
621 if (!Context.CurRegion.contains(Inst))
Tobias Grosser1b13dde2015-06-29 14:44:22 +0000622 Operands.push_back(MulOp);
Tobias Grosserd68ba422015-11-24 05:00:36 +0000623
624 } else {
625 Operands.push_back(MulOp);
Tobias Grosser1b13dde2015-06-29 14:44:22 +0000626 }
627 }
Tobias Grosser1b13dde2015-06-29 14:44:22 +0000628 }
Tobias Grosserd68ba422015-11-24 05:00:36 +0000629 if (Operands.size())
630 Terms.push_back(SE->getMulExpr(Operands));
Tobias Grosser1b13dde2015-06-29 14:44:22 +0000631 }
632 }
Tobias Grosser230acc42014-09-13 14:47:55 +0000633 }
Tobias Grosserd68ba422015-11-24 05:00:36 +0000634 if (Terms.empty())
635 SE->collectParametricTerms(Pair.second, Terms);
636 }
637 return Terms;
638}
Sebastian Pope8863b82014-05-12 19:02:02 +0000639
Tobias Grosserd68ba422015-11-24 05:00:36 +0000640bool ScopDetection::hasValidArraySizes(DetectionContext &Context,
641 SmallVectorImpl<const SCEV *> &Sizes,
642 const SCEVUnknown *BasePointer) const {
643 Value *BaseValue = BasePointer->getValue();
644 Region &CurRegion = Context.CurRegion;
645 for (const SCEV *DelinearizedSize : Sizes) {
646 if (!isAffine(DelinearizedSize, Context, nullptr)) {
647 Sizes.clear();
648 break;
Tobias Grosser5528dcd2015-10-25 08:40:38 +0000649 }
Tobias Grosserd68ba422015-11-24 05:00:36 +0000650 if (auto *Unknown = dyn_cast<SCEVUnknown>(DelinearizedSize)) {
651 auto *V = dyn_cast<Value>(Unknown->getValue());
652 if (auto *Load = dyn_cast<LoadInst>(V)) {
653 if (Context.CurRegion.contains(Load) &&
654 isHoistableLoad(Load, CurRegion, *LI, *SE))
655 Context.RequiredILS.insert(Load);
Tobias Grosser230acc42014-09-13 14:47:55 +0000656 continue;
Tobias Grosser230acc42014-09-13 14:47:55 +0000657 }
Sebastian Pop46e1ecd2014-05-09 22:45:15 +0000658 }
Tobias Grosserd68ba422015-11-24 05:00:36 +0000659 if (hasScalarDepsInsideRegion(DelinearizedSize, &CurRegion))
Tobias Grosserbfaf1ae2015-12-21 09:09:39 +0000660 return invalid<ReportNonAffineAccess>(
Tobias Grosserd68ba422015-11-24 05:00:36 +0000661 Context, /*Assert=*/true, DelinearizedSize,
662 Context.Accesses[BasePointer].front().first, BaseValue);
663 }
Tobias Grosser230acc42014-09-13 14:47:55 +0000664
Tobias Grosserd68ba422015-11-24 05:00:36 +0000665 // No array shape derived.
666 if (Sizes.empty()) {
667 if (AllowNonAffine)
668 return true;
669
Tobias Grosser230acc42014-09-13 14:47:55 +0000670 for (const auto &Pair : Context.Accesses[BasePointer]) {
671 const Instruction *Insn = Pair.first;
Tobias Grosserd68ba422015-11-24 05:00:36 +0000672 const SCEV *AF = Pair.second;
Tobias Grosser230acc42014-09-13 14:47:55 +0000673
Tobias Grosserd68ba422015-11-24 05:00:36 +0000674 if (!isAffine(AF, Context, BaseValue)) {
675 invalid<ReportNonAffineAccess>(Context, /*Assert=*/true, AF, Insn,
676 BaseValue);
677 if (!KeepGoing)
Tobias Grosser230acc42014-09-13 14:47:55 +0000678 return false;
679 }
680 }
Tobias Grosserd68ba422015-11-24 05:00:36 +0000681 return false;
Sebastian Pop46e1ecd2014-05-09 22:45:15 +0000682 }
Sebastian Pope8863b82014-05-12 19:02:02 +0000683 return true;
Sebastian Pop46e1ecd2014-05-09 22:45:15 +0000684}
685
Tobias Grosserd68ba422015-11-24 05:00:36 +0000686// We first store the resulting memory accesses in TempMemoryAccesses. Only
687// if the access functions for all memory accesses have been successfully
688// delinearized we continue. Otherwise, we either report a failure or, if
689// non-affine accesses are allowed, we drop the information. In case the
690// information is dropped the memory accesses need to be overapproximated
691// when translated to a polyhedral representation.
692bool ScopDetection::computeAccessFunctions(
693 DetectionContext &Context, const SCEVUnknown *BasePointer,
694 std::shared_ptr<ArrayShape> Shape) const {
695 Value *BaseValue = BasePointer->getValue();
696 bool BasePtrHasNonAffine = false;
697 MapInsnToMemAcc TempMemoryAccesses;
698 for (const auto &Pair : Context.Accesses[BasePointer]) {
699 const Instruction *Insn = Pair.first;
700 auto *AF = Pair.second;
Tobias Grosser2f8e43d2015-11-24 17:06:38 +0000701 AF = SCEVRemoveMax::remove(*SE, AF);
Tobias Grosserd68ba422015-11-24 05:00:36 +0000702 bool IsNonAffine = false;
703 TempMemoryAccesses.insert(std::make_pair(Insn, MemAcc(Insn, Shape)));
704 MemAcc *Acc = &TempMemoryAccesses.find(Insn)->second;
705
706 if (!AF) {
707 if (isAffine(Pair.second, Context, BaseValue))
708 Acc->DelinearizedSubscripts.push_back(Pair.second);
709 else
710 IsNonAffine = true;
711 } else {
712 SE->computeAccessFunctions(AF, Acc->DelinearizedSubscripts,
713 Shape->DelinearizedSizes);
714 if (Acc->DelinearizedSubscripts.size() == 0)
715 IsNonAffine = true;
716 for (const SCEV *S : Acc->DelinearizedSubscripts)
717 if (!isAffine(S, Context, BaseValue))
718 IsNonAffine = true;
719 }
720
721 // (Possibly) report non affine access
722 if (IsNonAffine) {
723 BasePtrHasNonAffine = true;
724 if (!AllowNonAffine)
725 invalid<ReportNonAffineAccess>(Context, /*Assert=*/true, Pair.second,
726 Insn, BaseValue);
727 if (!KeepGoing && !AllowNonAffine)
728 return false;
729 }
730 }
731
732 if (!BasePtrHasNonAffine)
Hongbin Zheng22623202016-02-15 00:20:58 +0000733 Context.InsnToMemAcc.insert(TempMemoryAccesses.begin(),
734 TempMemoryAccesses.end());
Tobias Grosserd68ba422015-11-24 05:00:36 +0000735
736 return true;
737}
738
739bool ScopDetection::hasBaseAffineAccesses(
740 DetectionContext &Context, const SCEVUnknown *BasePointer) const {
741 auto Shape = std::shared_ptr<ArrayShape>(new ArrayShape(BasePointer));
742
743 auto Terms = getDelinearizationTerms(Context, BasePointer);
744
745 SE->findArrayDimensions(Terms, Shape->DelinearizedSizes,
746 Context.ElementSize[BasePointer]);
747
748 if (!hasValidArraySizes(Context, Shape->DelinearizedSizes, BasePointer))
749 return false;
750
751 return computeAccessFunctions(Context, BasePointer, Shape);
752}
753
754bool ScopDetection::hasAffineMemoryAccesses(DetectionContext &Context) const {
755 for (const SCEVUnknown *BasePointer : Context.NonAffineAccesses)
756 if (!hasBaseAffineAccesses(Context, BasePointer)) {
757 if (KeepGoing)
758 continue;
759 else
760 return false;
761 }
762 return true;
763}
764
Michael Kruse70131d32016-01-27 17:09:17 +0000765bool ScopDetection::isValidMemoryAccess(MemAccInst Inst,
Tobias Grosser75805372011-04-29 06:27:02 +0000766 DetectionContext &Context) const {
Johannes Doerfertfb79a962015-02-23 14:18:28 +0000767 Region &CurRegion = Context.CurRegion;
768
Michael Kruse70131d32016-01-27 17:09:17 +0000769 Value *Ptr = Inst.getPointerOperand();
Sebastian Pop9f57c5b2013-04-10 04:05:18 +0000770 Loop *L = LI->getLoopFor(Inst.getParent());
771 const SCEV *AccessFunction = SE->getSCEVAtScope(Ptr, L);
Tobias Grosserb8710b52011-11-10 12:44:50 +0000772 const SCEVUnknown *BasePointer;
Tobias Grossere5e171e2011-11-10 12:45:03 +0000773 Value *BaseValue;
Tobias Grosser75805372011-04-29 06:27:02 +0000774
Tobias Grosserb8710b52011-11-10 12:44:50 +0000775 BasePointer = dyn_cast<SCEVUnknown>(SE->getPointerBase(AccessFunction));
776
Andreas Simbuerger01a37a02014-04-02 11:54:01 +0000777 if (!BasePointer)
Michael Kruse70131d32016-01-27 17:09:17 +0000778 return invalid<ReportNoBasePtr>(Context, /*Assert=*/true, Inst);
Tobias Grosserb8710b52011-11-10 12:44:50 +0000779
780 BaseValue = BasePointer->getValue();
781
Andreas Simbuerger01a37a02014-04-02 11:54:01 +0000782 if (isa<UndefValue>(BaseValue))
Michael Kruse70131d32016-01-27 17:09:17 +0000783 return invalid<ReportUndefBasePtr>(Context, /*Assert=*/true, Inst);
Tobias Grosserb8710b52011-11-10 12:44:50 +0000784
Tobias Grosser458fb782014-01-28 12:58:58 +0000785 // Check that the base address of the access is invariant in the current
786 // region.
Johannes Doerfertfb79a962015-02-23 14:18:28 +0000787 if (!isInvariant(*BaseValue, CurRegion))
Johannes Doerfert01978cf2015-10-18 12:28:00 +0000788 return invalid<ReportVariantBasePtr>(Context, /*Assert=*/true, BaseValue,
Michael Kruse70131d32016-01-27 17:09:17 +0000789 Inst);
Tobias Grosser458fb782014-01-28 12:58:58 +0000790
Tobias Grosserb8710b52011-11-10 12:44:50 +0000791 AccessFunction = SE->getMinusSCEV(AccessFunction, BasePointer);
792
Michael Kruse70131d32016-01-27 17:09:17 +0000793 const SCEV *Size = SE->getElementSize(Inst);
Tobias Grosser8ebdc2d2016-02-07 08:48:57 +0000794 if (Context.ElementSize[BasePointer]) {
795 if (!AllowDifferentTypes && Context.ElementSize[BasePointer] != Size)
796 return invalid<ReportDifferentArrayElementSize>(Context, /*Assert=*/true,
797 Inst, BaseValue);
798
Tobias Grosserd840fc72016-02-04 13:18:42 +0000799 Context.ElementSize[BasePointer] =
800 SE->getSMinExpr(Size, Context.ElementSize[BasePointer]);
Tobias Grosser8ebdc2d2016-02-07 08:48:57 +0000801 } else {
Tobias Grossere2c31212016-02-03 05:53:27 +0000802 Context.ElementSize[BasePointer] = Size;
Tobias Grosser8ebdc2d2016-02-07 08:48:57 +0000803 }
Tobias Grosserbcd4eff2014-09-13 14:47:40 +0000804
Johannes Doerfertf3e98f42015-04-12 22:52:20 +0000805 bool isVariantInNonAffineLoop = false;
806 SetVector<const Loop *> Loops;
807 findLoops(AccessFunction, Loops);
808 for (const Loop *L : Loops)
809 if (Context.BoxedLoopsSet.count(L))
810 isVariantInNonAffineLoop = true;
811
812 if (PollyDelinearize && !isVariantInNonAffineLoop) {
Michael Kruse70131d32016-01-27 17:09:17 +0000813 Context.Accesses[BasePointer].push_back({Inst, AccessFunction});
Sebastian Pop46e1ecd2014-05-09 22:45:15 +0000814
Johannes Doerfert09e36972015-10-07 20:17:36 +0000815 if (!isAffine(AccessFunction, Context, BaseValue))
Tobias Grosser230acc42014-09-13 14:47:55 +0000816 Context.NonAffineAccesses.insert(BasePointer);
817 } else if (!AllowNonAffine) {
Johannes Doerfertf3e98f42015-04-12 22:52:20 +0000818 if (isVariantInNonAffineLoop ||
Johannes Doerfert09e36972015-10-07 20:17:36 +0000819 !isAffine(AccessFunction, Context, BaseValue))
Sebastian Popcd3bb592014-04-10 16:08:11 +0000820 return invalid<ReportNonAffineAccess>(Context, /*Assert=*/true,
Michael Kruse70131d32016-01-27 17:09:17 +0000821 AccessFunction, Inst, BaseValue);
Sebastian Pop18016682014-04-08 21:20:44 +0000822 }
Tobias Grosser75805372011-04-29 06:27:02 +0000823
Johannes Doerfert01978cf2015-10-18 12:28:00 +0000824 // FIXME: Think about allowing IntToPtrInst
Andreas Simbuerger5569bf32014-06-26 10:06:40 +0000825 if (IntToPtrInst *Inst = dyn_cast<IntToPtrInst>(BaseValue))
826 return invalid<ReportIntToPtr>(Context, /*Assert=*/true, Inst);
Tobias Grosser75805372011-04-29 06:27:02 +0000827
Tobias Grosser1eedb672014-09-24 21:04:29 +0000828 if (IgnoreAliasing)
Sebastian Pop8c2d7532013-07-03 22:50:36 +0000829 return true;
Tobias Grosserfff5adc2011-11-10 13:21:43 +0000830
Sebastian Pop8c2d7532013-07-03 22:50:36 +0000831 // Check if the base pointer of the memory access does alias with
832 // any other pointer. This cannot be handled at the moment.
Benjamin Kramerae81abf2014-10-05 11:58:57 +0000833 AAMDNodes AATags;
834 Inst.getAAMetadata(AATags);
835 AliasSet &AS = Context.AST.getAliasSetForPointer(
Chandler Carruthafa4ea72015-06-17 08:29:32 +0000836 BaseValue, MemoryLocation::UnknownSize, AATags);
Tobias Grosser428b3e42013-02-04 15:46:25 +0000837
Tobias Grosser1eedb672014-09-24 21:04:29 +0000838 if (!AS.isMustAlias()) {
839 if (PollyUseRuntimeAliasChecks) {
840 bool CanBuildRunTimeCheck = true;
841 // The run-time alias check places code that involves the base pointer at
842 // the beginning of the SCoP. This breaks if the base pointer is defined
843 // inside the scop. Hence, we can only create a run-time check if we are
844 // sure the base pointer is not an instruction defined inside the scop.
Johannes Doerfert09e36972015-10-07 20:17:36 +0000845 // However, we can ignore loads that will be hoisted.
Tobias Grosser1eedb672014-09-24 21:04:29 +0000846 for (const auto &Ptr : AS) {
847 Instruction *Inst = dyn_cast<Instruction>(Ptr.getValue());
Johannes Doerfertfb79a962015-02-23 14:18:28 +0000848 if (Inst && CurRegion.contains(Inst)) {
Johannes Doerfert09e36972015-10-07 20:17:36 +0000849 auto *Load = dyn_cast<LoadInst>(Inst);
850 if (Load && isHoistableLoad(Load, CurRegion, *LI, *SE)) {
851 Context.RequiredILS.insert(Load);
852 continue;
853 }
854
Tobias Grosser1eedb672014-09-24 21:04:29 +0000855 CanBuildRunTimeCheck = false;
856 break;
857 }
858 }
859
860 if (CanBuildRunTimeCheck)
861 return true;
862 }
Michael Kruse70131d32016-01-27 17:09:17 +0000863 return invalid<ReportAlias>(Context, /*Assert=*/true, Inst, AS);
Tobias Grosser1eedb672014-09-24 21:04:29 +0000864 }
Tobias Grosser75805372011-04-29 06:27:02 +0000865
866 return true;
867}
868
Tobias Grosser75805372011-04-29 06:27:02 +0000869bool ScopDetection::isValidInstruction(Instruction &Inst,
870 DetectionContext &Context) const {
Tobias Grosserb12b0062015-11-11 12:44:18 +0000871 for (auto &Op : Inst.operands()) {
872 auto *OpInst = dyn_cast<Instruction>(&Op);
873
874 if (!OpInst)
875 continue;
876
877 if (isErrorBlock(*OpInst->getParent(), Context.CurRegion, *LI, *DT))
878 return false;
879 }
880
Tobias Grosser75805372011-04-29 06:27:02 +0000881 // We only check the call instruction but not invoke instruction.
882 if (CallInst *CI = dyn_cast<CallInst>(&Inst)) {
883 if (isValidCallInst(*CI))
884 return true;
885
Andreas Simbuerger01a37a02014-04-02 11:54:01 +0000886 return invalid<ReportFuncCall>(Context, /*Assert=*/true, &Inst);
Tobias Grosser75805372011-04-29 06:27:02 +0000887 }
888
889 if (!Inst.mayWriteToMemory() && !Inst.mayReadFromMemory()) {
Tobias Grosser5b1a7f22013-07-22 03:50:33 +0000890 if (!isa<AllocaInst>(Inst))
891 return true;
Tobias Grosser75805372011-04-29 06:27:02 +0000892
Andreas Simbuerger01a37a02014-04-02 11:54:01 +0000893 return invalid<ReportAlloca>(Context, /*Assert=*/true, &Inst);
Tobias Grosser75805372011-04-29 06:27:02 +0000894 }
895
896 // Check the access function.
Michael Kruse70131d32016-01-27 17:09:17 +0000897 if (auto MemInst = MemAccInst::dyn_cast(Inst)) {
898 Context.hasStores |= MemInst.isLoad();
899 Context.hasLoads |= MemInst.isStore();
900 if (!MemInst.isSimple())
901 return invalid<ReportNonSimpleMemoryAccess>(Context, /*Assert=*/true,
902 &Inst);
Tobias Grosserbf45e742015-10-25 13:48:40 +0000903
Michael Kruse70131d32016-01-27 17:09:17 +0000904 return isValidMemoryAccess(MemInst, Context);
Tobias Grosserd1e33e72015-02-19 05:31:07 +0000905 }
Tobias Grosser75805372011-04-29 06:27:02 +0000906
907 // We do not know this instruction, therefore we assume it is invalid.
Andreas Simbuerger01a37a02014-04-02 11:54:01 +0000908 return invalid<ReportUnknownInst>(Context, /*Assert=*/true, &Inst);
Tobias Grosser75805372011-04-29 06:27:02 +0000909}
910
Johannes Doerfertd020b772015-08-27 06:53:52 +0000911bool ScopDetection::canUseISLTripCount(Loop *L,
912 DetectionContext &Context) const {
Johannes Doerfert30ffb6f2015-10-04 14:53:18 +0000913 // Ensure the loop has valid exiting blocks as well as latches, otherwise we
914 // need to overapproximate it as a boxed loop.
915 SmallVector<BasicBlock *, 4> LoopControlBlocks;
916 L->getLoopLatches(LoopControlBlocks);
917 L->getExitingBlocks(LoopControlBlocks);
918 for (BasicBlock *ControlBB : LoopControlBlocks) {
Tobias Grosserb76cd3c2015-11-11 08:42:20 +0000919 if (!isValidCFG(*ControlBB, true, false, Context))
Johannes Doerfertd020b772015-08-27 06:53:52 +0000920 return false;
921 }
922
Johannes Doerfertd020b772015-08-27 06:53:52 +0000923 // We can use ISL to compute the trip count of L.
924 return true;
925}
926
Tobias Grosser75805372011-04-29 06:27:02 +0000927bool ScopDetection::isValidLoop(Loop *L, DetectionContext &Context) const {
Johannes Doerfertf61df692015-10-04 14:56:08 +0000928 if (canUseISLTripCount(L, Context))
Johannes Doerfertba65c162015-02-24 11:45:21 +0000929 return true;
Johannes Doerfertf3e98f42015-04-12 22:52:20 +0000930
Johannes Doerfertb68cffb2015-09-10 15:27:46 +0000931 if (AllowNonAffineSubLoops && AllowNonAffineSubRegions) {
Johannes Doerfertf3e98f42015-04-12 22:52:20 +0000932 Region *R = RI->getRegionFor(L->getHeader());
Johannes Doerfert757a32b2015-10-04 14:54:27 +0000933 while (R != &Context.CurRegion && !R->contains(L))
934 R = R->getParent();
935
936 if (addOverApproximatedRegion(R, Context))
937 return true;
Johannes Doerfertf3e98f42015-04-12 22:52:20 +0000938 }
Tobias Grosser75805372011-04-29 06:27:02 +0000939
Johannes Doerfertb68cffb2015-09-10 15:27:46 +0000940 const SCEV *LoopCount = SE->getBackedgeTakenCount(L);
Johannes Doerfertba65c162015-02-24 11:45:21 +0000941 return invalid<ReportLoopBound>(Context, /*Assert=*/true, L, LoopCount);
Tobias Grosser75805372011-04-29 06:27:02 +0000942}
943
Johannes Doerfert7175bdf2015-09-20 14:56:54 +0000944/// @brief Return the number of loops in @p L (incl. @p L) that have a trip
Johannes Doerferte526de52015-09-21 19:10:11 +0000945/// count that is not known to be less than MIN_LOOP_TRIP_COUNT.
Johannes Doerfertf61df692015-10-04 14:56:08 +0000946static int countBeneficialSubLoops(Loop *L, ScalarEvolution &SE) {
Johannes Doerfert7175bdf2015-09-20 14:56:54 +0000947 auto *TripCount = SE.getBackedgeTakenCount(L);
948
Johannes Doerfertf61df692015-10-04 14:56:08 +0000949 int count = 1;
Johannes Doerfert7175bdf2015-09-20 14:56:54 +0000950 if (auto *TripCountC = dyn_cast<SCEVConstant>(TripCount))
Johannes Doerfertf61df692015-10-04 14:56:08 +0000951 if (TripCountC->getType()->getScalarSizeInBits() <= 64)
952 if (TripCountC->getValue()->getZExtValue() < MIN_LOOP_TRIP_COUNT)
953 count -= 1;
Johannes Doerfert7175bdf2015-09-20 14:56:54 +0000954
955 for (auto &SubLoop : *L)
Johannes Doerfertf61df692015-10-04 14:56:08 +0000956 count += countBeneficialSubLoops(SubLoop, SE);
Johannes Doerfert7175bdf2015-09-20 14:56:54 +0000957
958 return count;
959}
960
Johannes Doerfertf61df692015-10-04 14:56:08 +0000961int ScopDetection::countBeneficialLoops(Region *R) const {
962 int LoopNum = 0;
Tobias Grossered21a1f2015-08-27 16:55:18 +0000963
Tobias Grosser050e0cb2015-08-31 12:08:11 +0000964 auto L = LI->getLoopFor(R->getEntry());
965 L = L ? R->outermostLoopInRegion(L) : nullptr;
966 L = L ? L->getParentLoop() : nullptr;
Tobias Grossered21a1f2015-08-27 16:55:18 +0000967
Tobias Grosser050e0cb2015-08-31 12:08:11 +0000968 auto SubLoops =
969 L ? L->getSubLoopsVector() : std::vector<Loop *>(LI->begin(), LI->end());
970
971 for (auto &SubLoop : SubLoops)
Johannes Doerfertf61df692015-10-04 14:56:08 +0000972 if (R->contains(SubLoop))
973 LoopNum += countBeneficialSubLoops(SubLoop, *SE);
Tobias Grosser050e0cb2015-08-31 12:08:11 +0000974
Johannes Doerfertf61df692015-10-04 14:56:08 +0000975 return LoopNum;
Tobias Grossered21a1f2015-08-27 16:55:18 +0000976}
977
Tobias Grosser75805372011-04-29 06:27:02 +0000978Region *ScopDetection::expandRegion(Region &R) {
Hongbin Zhenged986ab2012-04-07 15:14:28 +0000979 // Initial no valid region was found (greater than R)
Tobias Grosserd5d93ec2015-06-04 17:59:54 +0000980 std::unique_ptr<Region> LastValidRegion;
981 auto ExpandedRegion = std::unique_ptr<Region>(R.getExpandedRegion());
Tobias Grosser75805372011-04-29 06:27:02 +0000982
983 DEBUG(dbgs() << "\tExpanding " << R.getNameStr() << "\n");
984
Hongbin Zhenged986ab2012-04-07 15:14:28 +0000985 while (ExpandedRegion) {
Johannes Doerfertc06b7d62015-10-07 20:46:06 +0000986 const auto &It = DetectionContextMap.insert(std::make_pair(
987 ExpandedRegion.get(),
988 DetectionContext(*ExpandedRegion, *AA, false /*verifying*/)));
989 DetectionContext &Context = It.first->second;
Hongbin Zhenged986ab2012-04-07 15:14:28 +0000990 DEBUG(dbgs() << "\t\tTrying " << ExpandedRegion->getNameStr() << "\n");
Andreas Simbuergerb379edb2014-06-27 06:21:14 +0000991 // Only expand when we did not collect errors.
Tobias Grosser75805372011-04-29 06:27:02 +0000992
Johannes Doerfert717b8662015-09-08 21:44:27 +0000993 if (!Context.Log.hasErrors()) {
Hongbin Zhenged986ab2012-04-07 15:14:28 +0000994 // If the exit is valid check all blocks
995 // - if true, a valid region was found => store it + keep expanding
996 // - if false, .tbd. => stop (should this really end the loop?)
Johannes Doerferte46925f2015-10-01 10:59:14 +0000997 if (!allBlocksValid(Context) || Context.Log.hasErrors()) {
998 removeCachedResults(*ExpandedRegion);
Andreas Simbuergerb379edb2014-06-27 06:21:14 +0000999 break;
Johannes Doerferte46925f2015-10-01 10:59:14 +00001000 }
Andreas Simbuergerb379edb2014-06-27 06:21:14 +00001001
Tobias Grosserd7e58642013-04-10 06:55:45 +00001002 // Store this region, because it is the greatest valid (encountered so
1003 // far).
Johannes Doerferte46925f2015-10-01 10:59:14 +00001004 removeCachedResults(*LastValidRegion);
Tobias Grosserd5d93ec2015-06-04 17:59:54 +00001005 LastValidRegion = std::move(ExpandedRegion);
Hongbin Zhenged986ab2012-04-07 15:14:28 +00001006
1007 // Create and test the next greater region (if any)
Tobias Grosserd5d93ec2015-06-04 17:59:54 +00001008 ExpandedRegion =
1009 std::unique_ptr<Region>(LastValidRegion->getExpandedRegion());
Hongbin Zhenged986ab2012-04-07 15:14:28 +00001010
1011 } else {
1012 // Create and test the next greater region (if any)
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001013 removeCachedResults(*ExpandedRegion);
Tobias Grosserd5d93ec2015-06-04 17:59:54 +00001014 ExpandedRegion =
1015 std::unique_ptr<Region>(ExpandedRegion->getExpandedRegion());
Tobias Grosser75805372011-04-29 06:27:02 +00001016 }
Tobias Grosser75805372011-04-29 06:27:02 +00001017 }
1018
Tobias Grosser378a9f22013-11-16 19:34:11 +00001019 DEBUG({
1020 if (LastValidRegion)
1021 dbgs() << "\tto " << LastValidRegion->getNameStr() << "\n";
1022 else
1023 dbgs() << "\tExpanding " << R.getNameStr() << " failed\n";
1024 });
Tobias Grosser75805372011-04-29 06:27:02 +00001025
Tobias Grosserd5d93ec2015-06-04 17:59:54 +00001026 return LastValidRegion.release();
Tobias Grosser75805372011-04-29 06:27:02 +00001027}
Sebastian Pop2c9ec2e2013-06-03 16:35:37 +00001028static bool regionWithoutLoops(Region &R, LoopInfo *LI) {
Tobias Grosser26108892014-04-02 20:18:19 +00001029 for (const BasicBlock *BB : R.blocks())
Tobias Grosser1d191902014-03-03 13:13:55 +00001030 if (R.contains(LI->getLoopFor(BB)))
Sebastian Pop2c9ec2e2013-06-03 16:35:37 +00001031 return false;
1032
1033 return true;
1034}
Tobias Grosser75805372011-04-29 06:27:02 +00001035
Johannes Doerferte46925f2015-10-01 10:59:14 +00001036unsigned ScopDetection::removeCachedResultsRecursively(const Region &R) {
Tobias Grosser28a70c52014-01-29 19:05:30 +00001037 unsigned Count = 0;
David Blaikieb035f6d2014-04-15 18:45:27 +00001038 for (auto &SubRegion : R) {
Johannes Doerferte46925f2015-10-01 10:59:14 +00001039 if (ValidRegions.count(SubRegion.get())) {
1040 removeCachedResults(*SubRegion.get());
Tobias Grosser28a70c52014-01-29 19:05:30 +00001041 ++Count;
Johannes Doerferte46925f2015-10-01 10:59:14 +00001042 } else
1043 Count += removeCachedResultsRecursively(*SubRegion);
Tobias Grosser28a70c52014-01-29 19:05:30 +00001044 }
1045 return Count;
1046}
1047
Johannes Doerferte46925f2015-10-01 10:59:14 +00001048void ScopDetection::removeCachedResults(const Region &R) {
1049 ValidRegions.remove(&R);
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001050 DetectionContextMap.erase(&R);
Johannes Doerferte46925f2015-10-01 10:59:14 +00001051}
1052
Tobias Grosser75805372011-04-29 06:27:02 +00001053void ScopDetection::findScops(Region &R) {
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001054 const auto &It = DetectionContextMap.insert(
1055 std::make_pair(&R, DetectionContext(R, *AA, false /*verifying*/)));
1056 DetectionContext &Context = It.first->second;
Johannes Doerfert6a4d81c2015-03-08 15:11:50 +00001057
1058 bool RegionIsValid = false;
Tobias Grosser575aca82015-10-06 16:10:29 +00001059 if (!PollyProcessUnprofitable && regionWithoutLoops(R, LI)) {
Johannes Doerferte46925f2015-10-01 10:59:14 +00001060 removeCachedResults(R);
Johannes Doerfert6a4d81c2015-03-08 15:11:50 +00001061 invalid<ReportUnprofitable>(Context, /*Assert=*/true, &R);
Johannes Doerferte46925f2015-10-01 10:59:14 +00001062 } else
Johannes Doerfert6a4d81c2015-03-08 15:11:50 +00001063 RegionIsValid = isValidRegion(Context);
1064
Johannes Doerfert3f1c2852015-02-19 18:11:50 +00001065 bool HasErrors = !RegionIsValid || Context.Log.size() > 0;
Andreas Simbuerger04472402014-05-24 09:25:10 +00001066
Johannes Doerfert3f1c2852015-02-19 18:11:50 +00001067 if (PollyTrackFailures && HasErrors)
1068 RejectLogs.insert(std::make_pair(&R, Context.Log));
1069
Johannes Doerferte46925f2015-10-01 10:59:14 +00001070 if (HasErrors) {
1071 removeCachedResults(R);
1072 } else {
Tobias Grosser75805372011-04-29 06:27:02 +00001073 ++ValidRegion;
1074 ValidRegions.insert(&R);
1075 return;
1076 }
1077
David Blaikieb035f6d2014-04-15 18:45:27 +00001078 for (auto &SubRegion : R)
Tobias Grosser00dc3092014-03-02 12:02:46 +00001079 findScops(*SubRegion);
Tobias Grosser75805372011-04-29 06:27:02 +00001080
1081 // Try to expand regions.
1082 //
1083 // As the region tree normally only contains canonical regions, non canonical
1084 // regions that form a Scop are not found. Therefore, those non canonical
1085 // regions are checked by expanding the canonical ones.
1086
Tobias Grosser0d1eee32013-02-05 11:56:05 +00001087 std::vector<Region *> ToExpand;
Tobias Grosser75805372011-04-29 06:27:02 +00001088
David Blaikieb035f6d2014-04-15 18:45:27 +00001089 for (auto &SubRegion : R)
1090 ToExpand.push_back(SubRegion.get());
Tobias Grosser75805372011-04-29 06:27:02 +00001091
Tobias Grosser26108892014-04-02 20:18:19 +00001092 for (Region *CurrentRegion : ToExpand) {
Andreas Simbuergerb379edb2014-06-27 06:21:14 +00001093 // Skip regions that had errors.
1094 bool HadErrors = RejectLogs.hasErrors(CurrentRegion);
1095 if (HadErrors)
1096 continue;
1097
Tobias Grosser75805372011-04-29 06:27:02 +00001098 // Skip invalid regions. Regions may become invalid, if they are element of
1099 // an already expanded region.
David Peixotto8da2b932014-10-22 20:39:07 +00001100 if (!ValidRegions.count(CurrentRegion))
Tobias Grosser75805372011-04-29 06:27:02 +00001101 continue;
1102
1103 Region *ExpandedR = expandRegion(*CurrentRegion);
1104
1105 if (!ExpandedR)
1106 continue;
1107
1108 R.addSubRegion(ExpandedR, true);
1109 ValidRegions.insert(ExpandedR);
Johannes Doerferte46925f2015-10-01 10:59:14 +00001110 removeCachedResults(*CurrentRegion);
Tobias Grosser75805372011-04-29 06:27:02 +00001111
Tobias Grosser28a70c52014-01-29 19:05:30 +00001112 // Erase all (direct and indirect) children of ExpandedR from the valid
1113 // regions and update the number of valid regions.
Johannes Doerferte46925f2015-10-01 10:59:14 +00001114 ValidRegion -= removeCachedResultsRecursively(*ExpandedR);
Tobias Grosser75805372011-04-29 06:27:02 +00001115 }
1116}
1117
1118bool ScopDetection::allBlocksValid(DetectionContext &Context) const {
Johannes Doerfertfb79a962015-02-23 14:18:28 +00001119 Region &CurRegion = Context.CurRegion;
Tobias Grosser75805372011-04-29 06:27:02 +00001120
Johannes Doerfertfb79a962015-02-23 14:18:28 +00001121 for (const BasicBlock *BB : CurRegion.blocks()) {
Tobias Grosser1d191902014-03-03 13:13:55 +00001122 Loop *L = LI->getLoopFor(BB);
Andreas Simbuerger04472402014-05-24 09:25:10 +00001123 if (L && L->getHeader() == BB && (!isValidLoop(L, Context) && !KeepGoing))
Sebastian Popb88ea5e2013-06-11 22:20:32 +00001124 return false;
1125 }
1126
Johannes Doerfert90db75e2015-09-10 17:51:27 +00001127 for (BasicBlock *BB : CurRegion.blocks()) {
Tobias Grosserb76cd3c2015-11-11 08:42:20 +00001128 bool IsErrorBlock = isErrorBlock(*BB, CurRegion, *LI, *DT);
1129
1130 // Also check exception blocks (and possibly register them as non-affine
1131 // regions). Even though exception blocks are not modeled, we use them
1132 // to forward-propagate domain constraints during ScopInfo construction.
1133 if (!isValidCFG(*BB, false, IsErrorBlock, Context) && !KeepGoing)
1134 return false;
1135
1136 if (IsErrorBlock)
Johannes Doerfert90db75e2015-09-10 17:51:27 +00001137 continue;
1138
Tobias Grosser1d191902014-03-03 13:13:55 +00001139 for (BasicBlock::iterator I = BB->begin(), E = --BB->end(); I != E; ++I)
Andreas Simbuerger04472402014-05-24 09:25:10 +00001140 if (!isValidInstruction(*I, Context) && !KeepGoing)
Sebastian Pop8ca899c2013-06-14 20:20:43 +00001141 return false;
Johannes Doerfert90db75e2015-09-10 17:51:27 +00001142 }
Tobias Grosser75805372011-04-29 06:27:02 +00001143
Sebastian Pope8863b82014-05-12 19:02:02 +00001144 if (!hasAffineMemoryAccesses(Context))
Sebastian Pop46e1ecd2014-05-09 22:45:15 +00001145 return false;
1146
Tobias Grosser75805372011-04-29 06:27:02 +00001147 return true;
1148}
1149
Tobias Grosserc1a269b2015-12-21 21:00:43 +00001150bool ScopDetection::hasSufficientCompute(DetectionContext &Context,
1151 int NumLoops) const {
1152 int InstCount = 0;
1153
1154 for (auto *BB : Context.CurRegion.blocks())
1155 if (Context.CurRegion.contains(LI->getLoopFor(BB)))
Tobias Grosserc6424ae2015-12-22 17:38:59 +00001156 InstCount += BB->size();
Tobias Grosserc1a269b2015-12-21 21:00:43 +00001157
1158 InstCount = InstCount / NumLoops;
1159
1160 return InstCount >= ProfitabilityMinPerLoopInstructions;
1161}
1162
Tobias Grosser97fc5bb2015-12-21 12:14:48 +00001163bool ScopDetection::isProfitableRegion(DetectionContext &Context) const {
1164 Region &CurRegion = Context.CurRegion;
1165
1166 if (PollyProcessUnprofitable)
1167 return true;
1168
1169 // We can probably not do a lot on scops that only write or only read
1170 // data.
1171 if (!Context.hasStores || !Context.hasLoads)
1172 return invalid<ReportUnprofitable>(Context, /*Assert=*/true, &CurRegion);
1173
Tobias Grosser97fc5bb2015-12-21 12:14:48 +00001174 int NumLoops = countBeneficialLoops(&CurRegion);
1175 int NumAffineLoops = NumLoops - Context.BoxedLoopsSet.size();
Tobias Grosser97fc5bb2015-12-21 12:14:48 +00001176
Tobias Grosserc1a269b2015-12-21 21:00:43 +00001177 // Scops with at least two loops may allow either loop fusion or tiling and
1178 // are consequently interesting to look at.
1179 if (NumAffineLoops >= 2)
1180 return true;
1181
1182 // Scops that contain a loop with a non-trivial amount of computation per
1183 // loop-iteration are interesting as we may be able to parallelize such
1184 // loops. Individual loops that have only a small amount of computation
1185 // per-iteration are performance-wise very fragile as any change to the
1186 // loop induction variables may affect performance. To not cause spurious
1187 // performance regressions, we do not consider such loops.
1188 if (NumAffineLoops == 1 && hasSufficientCompute(Context, NumLoops))
1189 return true;
1190
1191 return invalid<ReportUnprofitable>(Context, /*Assert=*/true, &CurRegion);
Tobias Grosser97fc5bb2015-12-21 12:14:48 +00001192}
1193
Tobias Grosser75805372011-04-29 06:27:02 +00001194bool ScopDetection::isValidRegion(DetectionContext &Context) const {
Johannes Doerfertfb79a962015-02-23 14:18:28 +00001195 Region &CurRegion = Context.CurRegion;
Tobias Grosser75805372011-04-29 06:27:02 +00001196
Johannes Doerfertfb79a962015-02-23 14:18:28 +00001197 DEBUG(dbgs() << "Checking region: " << CurRegion.getNameStr() << "\n\t");
Tobias Grosser75805372011-04-29 06:27:02 +00001198
Johannes Doerfertfb79a962015-02-23 14:18:28 +00001199 if (CurRegion.isTopLevelRegion()) {
Tobias Grosserf084edd2014-10-22 23:00:03 +00001200 DEBUG(dbgs() << "Top level region is invalid\n");
Tobias Grosser75805372011-04-29 06:27:02 +00001201 return false;
1202 }
1203
Johannes Doerfertfb79a962015-02-23 14:18:28 +00001204 if (!CurRegion.getEntry()->getName().count(OnlyRegion)) {
Tobias Grosser4449e522014-01-27 14:24:53 +00001205 DEBUG({
1206 dbgs() << "Region entry does not match -polly-region-only";
1207 dbgs() << "\n";
1208 });
1209 return false;
1210 }
1211
Tobias Grosserd654c252012-04-10 18:12:19 +00001212 // SCoP cannot contain the entry block of the function, because we need
Tobias Grosser75805372011-04-29 06:27:02 +00001213 // to insert alloca instruction there when translate scalar to array.
Johannes Doerfertfb79a962015-02-23 14:18:28 +00001214 if (CurRegion.getEntry() ==
1215 &(CurRegion.getEntry()->getParent()->getEntryBlock()))
1216 return invalid<ReportEntry>(Context, /*Assert=*/true, CurRegion.getEntry());
Tobias Grosser75805372011-04-29 06:27:02 +00001217
Hongbin Zheng94868e62012-04-07 12:29:17 +00001218 if (!allBlocksValid(Context))
Tobias Grosser75805372011-04-29 06:27:02 +00001219 return false;
1220
Tobias Grosser1c3a6d72016-01-22 09:44:37 +00001221 DebugLoc DbgLoc;
1222 if (!isReducibleRegion(CurRegion, DbgLoc))
1223 return invalid<ReportIrreducibleRegion>(Context, /*Assert=*/true,
1224 &CurRegion, DbgLoc);
1225
Tobias Grosser97fc5bb2015-12-21 12:14:48 +00001226 if (!isProfitableRegion(Context))
1227 return false;
Johannes Doerfertf3e98f42015-04-12 22:52:20 +00001228
Tobias Grosser75805372011-04-29 06:27:02 +00001229 DEBUG(dbgs() << "OK\n");
1230 return true;
1231}
1232
Johannes Doerfert43e1ead2014-07-15 21:06:48 +00001233void ScopDetection::markFunctionAsInvalid(Function *F) const {
1234 F->addFnAttr(PollySkipFnAttr);
1235}
1236
Tobias Grosser75805372011-04-29 06:27:02 +00001237bool ScopDetection::isValidFunction(llvm::Function &F) {
Johannes Doerfert43e1ead2014-07-15 21:06:48 +00001238 return !F.hasFnAttribute(PollySkipFnAttr);
Tobias Grosser75805372011-04-29 06:27:02 +00001239}
1240
Tobias Grosserb2863ca2013-03-04 19:49:51 +00001241void ScopDetection::printLocations(llvm::Function &F) {
Tobias Grosser26108892014-04-02 20:18:19 +00001242 for (const Region *R : *this) {
Tobias Grosser531891e2012-11-01 16:45:20 +00001243 unsigned LineEntry, LineExit;
1244 std::string FileName;
1245
Tobias Grosser00dc3092014-03-02 12:02:46 +00001246 getDebugLocation(R, LineEntry, LineExit, FileName);
Tobias Grosser8519f892013-12-18 10:49:53 +00001247 DiagnosticScopFound Diagnostic(F, FileName, LineEntry, LineExit);
1248 F.getContext().diagnose(Diagnostic);
Tobias Grosser531891e2012-11-01 16:45:20 +00001249 }
1250}
1251
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001252void ScopDetection::emitMissedRemarksForValidRegions(const Function &F) {
Andreas Simbuerger5569bf32014-06-26 10:06:40 +00001253 for (const Region *R : ValidRegions) {
1254 const Region *Parent = R->getParent();
1255 if (Parent && !Parent->isTopLevelRegion() && RejectLogs.count(Parent))
1256 emitRejectionRemarks(F, RejectLogs.at(Parent));
1257 }
1258}
1259
1260void ScopDetection::emitMissedRemarksForLeaves(const Function &F,
1261 const Region *R) {
1262 for (const std::unique_ptr<Region> &Child : *R) {
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001263 bool IsValid = DetectionContextMap.count(Child.get());
Andreas Simbuerger5569bf32014-06-26 10:06:40 +00001264 if (IsValid)
1265 continue;
1266
1267 bool IsLeaf = Child->begin() == Child->end();
1268 if (!IsLeaf)
1269 emitMissedRemarksForLeaves(F, Child.get());
1270 else {
1271 if (RejectLogs.count(Child.get())) {
1272 emitRejectionRemarks(F, RejectLogs.at(Child.get()));
1273 }
1274 }
1275 }
1276}
1277
Tobias Grosser1c3a6d72016-01-22 09:44:37 +00001278bool ScopDetection::isReducibleRegion(Region &R, DebugLoc &DbgLoc) const {
1279 BasicBlock *REntry = R.getEntry();
1280 BasicBlock *RExit = R.getExit();
1281 // Map to match the color of a BasicBlock during the DFS walk.
1282 DenseMap<const BasicBlock *, Color> BBColorMap;
1283 // Stack keeping track of current BB and index of next child to be processed.
1284 std::stack<std::pair<BasicBlock *, unsigned>> DFSStack;
1285
1286 unsigned AdjacentBlockIndex = 0;
1287 BasicBlock *CurrBB, *SuccBB;
1288 CurrBB = REntry;
1289
1290 // Initialize the map for all BB with WHITE color.
1291 for (auto *BB : R.blocks())
1292 BBColorMap[BB] = ScopDetection::WHITE;
1293
1294 // Process the entry block of the Region.
1295 BBColorMap[CurrBB] = ScopDetection::GREY;
1296 DFSStack.push(std::make_pair(CurrBB, 0));
1297
1298 while (!DFSStack.empty()) {
1299 // Get next BB on stack to be processed.
1300 CurrBB = DFSStack.top().first;
1301 AdjacentBlockIndex = DFSStack.top().second;
1302 DFSStack.pop();
1303
1304 // Loop to iterate over the successors of current BB.
1305 const TerminatorInst *TInst = CurrBB->getTerminator();
1306 unsigned NSucc = TInst->getNumSuccessors();
1307 for (unsigned I = AdjacentBlockIndex; I < NSucc;
1308 ++I, ++AdjacentBlockIndex) {
1309 SuccBB = TInst->getSuccessor(I);
1310
1311 // Checks for region exit block and self-loops in BB.
1312 if (SuccBB == RExit || SuccBB == CurrBB)
1313 continue;
1314
1315 // WHITE indicates an unvisited BB in DFS walk.
1316 if (BBColorMap[SuccBB] == ScopDetection::WHITE) {
1317 // Push the current BB and the index of the next child to be visited.
1318 DFSStack.push(std::make_pair(CurrBB, I + 1));
1319 // Push the next BB to be processed.
1320 DFSStack.push(std::make_pair(SuccBB, 0));
1321 // First time the BB is being processed.
1322 BBColorMap[SuccBB] = ScopDetection::GREY;
1323 break;
1324 } else if (BBColorMap[SuccBB] == ScopDetection::GREY) {
1325 // GREY indicates a loop in the control flow.
1326 // If the destination dominates the source, it is a natural loop
1327 // else, an irreducible control flow in the region is detected.
1328 if (!DT->dominates(SuccBB, CurrBB)) {
1329 // Get debug info of instruction which causes irregular control flow.
1330 DbgLoc = TInst->getDebugLoc();
1331 return false;
1332 }
1333 }
1334 }
1335
1336 // If all children of current BB have been processed,
1337 // then mark that BB as fully processed.
1338 if (AdjacentBlockIndex == NSucc)
1339 BBColorMap[CurrBB] = ScopDetection::BLACK;
1340 }
1341
1342 return true;
1343}
1344
Tobias Grosser75805372011-04-29 06:27:02 +00001345bool ScopDetection::runOnFunction(llvm::Function &F) {
Chandler Carruthf5579872015-01-17 14:16:56 +00001346 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
Matt Arsenault8ca36812014-07-19 18:40:17 +00001347 RI = &getAnalysis<RegionInfoPass>().getRegionInfo();
Tobias Grosser575aca82015-10-06 16:10:29 +00001348 if (!PollyProcessUnprofitable && LI->empty())
Sebastian Pop8fe6d112013-05-30 17:47:32 +00001349 return false;
1350
Chandler Carruth66ef16b2015-09-09 22:13:56 +00001351 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
Tobias Grosserc5bcf242015-08-17 10:57:08 +00001352 SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
Johannes Doerfert08d90a32015-10-07 20:32:43 +00001353 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
Tobias Grosser75805372011-04-29 06:27:02 +00001354 Region *TopRegion = RI->getTopLevelRegion();
1355
Tobias Grosser2ff87232011-10-23 11:17:06 +00001356 releaseMemory();
1357
Tobias Grossera3ab27e2014-05-07 11:23:32 +00001358 if (OnlyFunction != "" && !F.getName().count(OnlyFunction))
Tobias Grosser2ff87232011-10-23 11:17:06 +00001359 return false;
1360
Tobias Grosser1bb59b02012-12-29 23:47:38 +00001361 if (!isValidFunction(F))
Tobias Grosser75805372011-04-29 06:27:02 +00001362 return false;
1363
1364 findScops(*TopRegion);
Tobias Grosser531891e2012-11-01 16:45:20 +00001365
Andreas Simbuerger5569bf32014-06-26 10:06:40 +00001366 // Only makes sense when we tracked errors.
1367 if (PollyTrackFailures) {
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001368 emitMissedRemarksForValidRegions(F);
Andreas Simbuerger5569bf32014-06-26 10:06:40 +00001369 emitMissedRemarksForLeaves(F, TopRegion);
1370 }
1371
Johannes Doerferta05214f2014-10-15 23:24:28 +00001372 if (ReportLevel)
Tobias Grosserb2863ca2013-03-04 19:49:51 +00001373 printLocations(F);
Tobias Grosser531891e2012-11-01 16:45:20 +00001374
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001375 assert(ValidRegions.size() == DetectionContextMap.size() &&
Johannes Doerferte46925f2015-10-01 10:59:14 +00001376 "Cached more results than valid regions");
Tobias Grosser75805372011-04-29 06:27:02 +00001377 return false;
1378}
1379
Johannes Doerfertba65c162015-02-24 11:45:21 +00001380bool ScopDetection::isNonAffineSubRegion(const Region *SubR,
1381 const Region *ScopR) const {
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001382 const DetectionContext *DC = getDetectionContext(ScopR);
1383 assert(DC && "ScopR is no valid region!");
1384 return DC->NonAffineSubRegionSet.count(SubR);
1385}
1386
1387const ScopDetection::DetectionContext *
1388ScopDetection::getDetectionContext(const Region *R) const {
1389 auto DCMIt = DetectionContextMap.find(R);
1390 if (DCMIt == DetectionContextMap.end())
1391 return nullptr;
1392 return &DCMIt->second;
Johannes Doerfertba65c162015-02-24 11:45:21 +00001393}
1394
Johannes Doerfertf3e98f42015-04-12 22:52:20 +00001395const ScopDetection::BoxedLoopsSetTy *
1396ScopDetection::getBoxedLoops(const Region *R) const {
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001397 const DetectionContext *DC = getDetectionContext(R);
1398 assert(DC && "ScopR is no valid region!");
1399 return &DC->BoxedLoopsSet;
Johannes Doerfertf3e98f42015-04-12 22:52:20 +00001400}
1401
Hongbin Zheng22623202016-02-15 00:20:58 +00001402const MapInsnToMemAcc *
1403ScopDetection::getInsnToMemAccMap(const Region *R) const {
1404 const DetectionContext *DC = getDetectionContext(R);
1405 assert(DC && "ScopR is no valid region!");
1406 return &DC->InsnToMemAcc;
1407}
1408
Johannes Doerfert09e36972015-10-07 20:17:36 +00001409const InvariantLoadsSetTy *
1410ScopDetection::getRequiredInvariantLoads(const Region *R) const {
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001411 const DetectionContext *DC = getDetectionContext(R);
1412 assert(DC && "ScopR is no valid region!");
1413 return &DC->RequiredILS;
Johannes Doerfert09e36972015-10-07 20:17:36 +00001414}
1415
Tobias Grosser75805372011-04-29 06:27:02 +00001416void polly::ScopDetection::verifyRegion(const Region &R) const {
1417 assert(isMaxRegionInScop(R) && "Expect R is a valid region.");
Johannes Doerfertf3e98f42015-04-12 22:52:20 +00001418
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001419 DetectionContext Context(const_cast<Region &>(R), *AA, true /*verifying*/);
Tobias Grosser75805372011-04-29 06:27:02 +00001420 isValidRegion(Context);
1421}
1422
1423void polly::ScopDetection::verifyAnalysis() const {
Tobias Grossera1689932014-02-18 18:49:49 +00001424 if (!VerifyScops)
1425 return;
1426
Tobias Grosser26108892014-04-02 20:18:19 +00001427 for (const Region *R : ValidRegions)
Tobias Grosser00dc3092014-03-02 12:02:46 +00001428 verifyRegion(*R);
Tobias Grosser75805372011-04-29 06:27:02 +00001429}
1430
1431void ScopDetection::getAnalysisUsage(AnalysisUsage &AU) const {
Chandler Carruthf5579872015-01-17 14:16:56 +00001432 AU.addRequired<LoopInfoWrapperPass>();
Tobias Grosserc5bcf242015-08-17 10:57:08 +00001433 AU.addRequired<ScalarEvolutionWrapperPass>();
Johannes Doerfert08d90a32015-10-07 20:32:43 +00001434 AU.addRequired<DominatorTreeWrapperPass>();
Tobias Grosser75805372011-04-29 06:27:02 +00001435 // We also need AA and RegionInfo when we are verifying analysis.
Chandler Carruth66ef16b2015-09-09 22:13:56 +00001436 AU.addRequiredTransitive<AAResultsWrapperPass>();
Matt Arsenault8ca36812014-07-19 18:40:17 +00001437 AU.addRequiredTransitive<RegionInfoPass>();
Tobias Grosser75805372011-04-29 06:27:02 +00001438 AU.setPreservesAll();
1439}
1440
Tobias Grosser4d96c8d2013-03-23 01:05:07 +00001441void ScopDetection::print(raw_ostream &OS, const Module *) const {
Tobias Grosser26108892014-04-02 20:18:19 +00001442 for (const Region *R : ValidRegions)
Tobias Grosser00dc3092014-03-02 12:02:46 +00001443 OS << "Valid Region for Scop: " << R->getNameStr() << '\n';
Tobias Grosser75805372011-04-29 06:27:02 +00001444
1445 OS << "\n";
1446}
1447
1448void ScopDetection::releaseMemory() {
Andreas Simbuerger4870e092014-05-24 09:25:01 +00001449 RejectLogs.clear();
Johannes Doerfert6206d7a2015-09-30 16:51:05 +00001450 ValidRegions.clear();
Johannes Doerfertc06b7d62015-10-07 20:46:06 +00001451 DetectionContextMap.clear();
Andreas Simbuerger4870e092014-05-24 09:25:01 +00001452
Hongbin Zheng94c5df12011-05-06 02:38:20 +00001453 // Do not clear the invalid function set.
Tobias Grosser75805372011-04-29 06:27:02 +00001454}
1455
1456char ScopDetection::ID = 0;
1457
Tobias Grosser4d96c8d2013-03-23 01:05:07 +00001458Pass *polly::createScopDetectionPass() { return new ScopDetection(); }
1459
Tobias Grosser73600b82011-10-08 00:30:40 +00001460INITIALIZE_PASS_BEGIN(ScopDetection, "polly-detect",
1461 "Polly - Detect static control parts (SCoPs)", false,
Tobias Grosser4d96c8d2013-03-23 01:05:07 +00001462 false);
Chandler Carruth66ef16b2015-09-09 22:13:56 +00001463INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass);
Chandler Carruthf5579872015-01-17 14:16:56 +00001464INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass);
Matt Arsenault8ca36812014-07-19 18:40:17 +00001465INITIALIZE_PASS_DEPENDENCY(RegionInfoPass);
Johannes Doerfert08d90a32015-10-07 20:32:43 +00001466INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass);
Tobias Grosserc5bcf242015-08-17 10:57:08 +00001467INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass);
Tobias Grosser73600b82011-10-08 00:30:40 +00001468INITIALIZE_PASS_END(ScopDetection, "polly-detect",
1469 "Polly - Detect static control parts (SCoPs)", false, false)