blob: eef94bf9012a2e3dca13690a8c1b3c51de846bb5 [file] [log] [blame]
Chandler Carruth1ff77242015-03-07 09:02:36 +00001//===- Parsing, selection, and construction of pass pipelines -------------===//
Chandler Carruth66445382014-01-11 08:16:35 +00002//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Chandler Carruth66445382014-01-11 08:16:35 +00006//
7//===----------------------------------------------------------------------===//
8/// \file
9///
Chandler Carruth1ff77242015-03-07 09:02:36 +000010/// This file provides the implementation of the PassBuilder based on our
11/// static pass registry as well as related functionality. It also provides
12/// helpers to aid in analyzing, debugging, and testing passes and pass
13/// pipelines.
Chandler Carruth66445382014-01-11 08:16:35 +000014///
15//===----------------------------------------------------------------------===//
16
Chandler Carruth1ff77242015-03-07 09:02:36 +000017#include "llvm/Passes/PassBuilder.h"
Chandler Carruth8b5a74192016-02-28 22:16:03 +000018#include "llvm/ADT/StringSwitch.h"
Chandler Carruth6f5770b102016-02-13 23:32:00 +000019#include "llvm/Analysis/AliasAnalysis.h"
Chandler Carruth4f846a52016-02-20 03:46:03 +000020#include "llvm/Analysis/AliasAnalysisEvaluator.h"
Daniel Jasperaec2fa32016-12-19 08:22:17 +000021#include "llvm/Analysis/AssumptionCache.h"
Chandler Carruthbece8d52016-02-13 23:46:24 +000022#include "llvm/Analysis/BasicAliasAnalysis.h"
Xinliang David Li28a93272016-05-05 21:13:27 +000023#include "llvm/Analysis/BlockFrequencyInfo.h"
Xinliang David Li6e5dd412016-05-05 02:59:57 +000024#include "llvm/Analysis/BranchProbabilityInfo.h"
Mehdi Amini27d23792016-09-16 16:56:30 +000025#include "llvm/Analysis/CFGPrinter.h"
George Burgess IVbfa401e2016-07-06 00:26:41 +000026#include "llvm/Analysis/CFLAndersAliasAnalysis.h"
27#include "llvm/Analysis/CFLSteensAliasAnalysis.h"
Chandler Carruth572e3402014-04-21 11:12:00 +000028#include "llvm/Analysis/CGSCCPassManager.h"
Chandler Carruth4c660f72016-03-10 11:24:11 +000029#include "llvm/Analysis/CallGraph.h"
Michael Kupersteinde16b442016-04-18 23:55:01 +000030#include "llvm/Analysis/DemandedBits.h"
Chandler Carruth49c22192016-05-12 22:19:39 +000031#include "llvm/Analysis/DependenceAnalysis.h"
Hongbin Zheng751337f2016-02-25 17:54:15 +000032#include "llvm/Analysis/DominanceFrontier.h"
Chandler Carruth45a9c202016-03-11 09:15:11 +000033#include "llvm/Analysis/GlobalsModRef.h"
Dehao Chen1a444522016-07-16 22:51:33 +000034#include "llvm/Analysis/IVUsers.h"
Chandler Carruthbf71a342014-02-06 04:37:03 +000035#include "llvm/Analysis/LazyCallGraph.h"
Sean Silva687019f2016-06-13 22:01:25 +000036#include "llvm/Analysis/LazyValueInfo.h"
Xinliang David Li8a021312016-07-02 21:18:40 +000037#include "llvm/Analysis/LoopAccessAnalysis.h"
Chandler Carruthaaf0b4c2015-01-20 10:58:50 +000038#include "llvm/Analysis/LoopInfo.h"
Chandler Carruth61440d22016-03-10 00:55:30 +000039#include "llvm/Analysis/MemoryDependenceAnalysis.h"
Daniel Berlin554dcd82017-04-11 20:06:36 +000040#include "llvm/Analysis/MemorySSA.h"
Teresa Johnsonf93b2462016-08-12 13:53:02 +000041#include "llvm/Analysis/ModuleSummaryAnalysis.h"
Adam Nemet0965da22017-10-09 23:19:02 +000042#include "llvm/Analysis/OptimizationRemarkEmitter.h"
John Brawnbdbbd832018-06-28 14:13:06 +000043#include "llvm/Analysis/PhiValues.h"
Hongbin Zheng3f978402016-02-25 17:54:07 +000044#include "llvm/Analysis/PostDominators.h"
Easwaran Raman019e0bf2016-06-03 22:54:26 +000045#include "llvm/Analysis/ProfileSummaryInfo.h"
Hongbin Zhengbc539772016-02-25 17:54:25 +000046#include "llvm/Analysis/RegionInfo.h"
Chandler Carruth2f1fd162015-08-17 02:08:17 +000047#include "llvm/Analysis/ScalarEvolution.h"
Chandler Carruth2b3d0442016-02-20 04:01:45 +000048#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
Chandler Carruthd6091a02016-02-20 04:03:06 +000049#include "llvm/Analysis/ScopedNoAliasAA.h"
Vitaly Buka4493fe12018-11-26 21:57:47 +000050#include "llvm/Analysis/StackSafetyAnalysis.h"
Chandler Carruth8ca43222015-01-15 11:39:46 +000051#include "llvm/Analysis/TargetLibraryInfo.h"
Chandler Carruthe0385522015-02-01 10:11:22 +000052#include "llvm/Analysis/TargetTransformInfo.h"
Chandler Carruthc1dc3842016-02-20 04:04:52 +000053#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
Michael Kuperstein82d5da52016-06-24 20:13:42 +000054#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
Wei Mi90d195a2016-07-08 03:32:49 +000055#include "llvm/CodeGen/UnreachableBlockElim.h"
Chandler Carruth64764b42015-01-14 10:19:28 +000056#include "llvm/IR/Dominators.h"
Chandler Carruth52eef882014-01-12 12:15:39 +000057#include "llvm/IR/IRPrintingPasses.h"
Chandler Carruth66445382014-01-11 08:16:35 +000058#include "llvm/IR/PassManager.h"
Fedor Sergeeva2ed4482019-03-31 10:15:39 +000059#include "llvm/IR/SafepointIRVerifier.h"
Chandler Carruth4d356312014-01-20 11:34:08 +000060#include "llvm/IR/Verifier.h"
Chandler Carruth52eef882014-01-12 12:15:39 +000061#include "llvm/Support/Debug.h"
Fedor Sergeevbd6b2132018-10-17 10:36:23 +000062#include "llvm/Support/FormatVariadic.h"
Chandler Carruth8b5a74192016-02-28 22:16:03 +000063#include "llvm/Support/Regex.h"
Chandler Carruthe0385522015-02-01 10:11:22 +000064#include "llvm/Target/TargetMachine.h"
Amjad Aboudf1f57a32018-01-25 12:06:32 +000065#include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
Chandler Carruth67fc52f2016-08-17 02:56:20 +000066#include "llvm/Transforms/IPO/AlwaysInliner.h"
Chandler Carruthaddcda42017-02-09 23:46:27 +000067#include "llvm/Transforms/IPO/ArgumentPromotion.h"
Johannes Doerfertaade7822019-06-05 03:02:24 +000068#include "llvm/Transforms/IPO/Attributor.h"
Matthew Simpsoncb585582017-10-25 13:40:08 +000069#include "llvm/Transforms/IPO/CalledValuePropagation.h"
Davide Italiano164b9bc2016-05-05 00:51:09 +000070#include "llvm/Transforms/IPO/ConstantMerge.h"
Davide Italiano92b933a2016-07-09 03:25:35 +000071#include "llvm/Transforms/IPO/CrossDSOCFI.h"
Sean Silvae3bb4572016-06-12 09:16:39 +000072#include "llvm/Transforms/IPO/DeadArgumentElimination.h"
Davide Italiano344e8382016-05-05 02:37:32 +000073#include "llvm/Transforms/IPO/ElimAvailExtern.h"
Chandler Carruthf49f1a872015-12-27 08:13:45 +000074#include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
Chandler Carruth9c4ed172016-02-18 11:03:11 +000075#include "llvm/Transforms/IPO/FunctionAttrs.h"
Teresa Johnson21241572016-07-18 21:22:24 +000076#include "llvm/Transforms/IPO/FunctionImport.h"
Davide Italiano66228c42016-05-03 19:39:15 +000077#include "llvm/Transforms/IPO/GlobalDCE.h"
Justin Bogner1a075012016-04-26 00:28:01 +000078#include "llvm/Transforms/IPO/GlobalOpt.h"
Davide Italiano2ae76dd2016-11-21 00:28:23 +000079#include "llvm/Transforms/IPO/GlobalSplit.h"
Aditya Kumar9e20ade2018-10-03 05:55:20 +000080#include "llvm/Transforms/IPO/HotColdSplitting.h"
Chandler Carruth3a040e62015-12-27 08:41:34 +000081#include "llvm/Transforms/IPO/InferFunctionAttrs.h"
Chandler Carruth1d963112016-12-20 03:15:32 +000082#include "llvm/Transforms/IPO/Inliner.h"
Justin Bogner4563a062016-04-26 20:15:52 +000083#include "llvm/Transforms/IPO/Internalize.h"
Davide Italianoe8ae0b52016-07-11 18:10:06 +000084#include "llvm/Transforms/IPO/LowerTypeTests.h"
Easwaran Raman1832bf62016-06-27 16:50:18 +000085#include "llvm/Transforms/IPO/PartialInlining.h"
Davide Italianof54f2f02016-05-05 21:05:36 +000086#include "llvm/Transforms/IPO/SCCP.h"
David Blaikie301627f2018-03-22 22:42:44 +000087#include "llvm/Transforms/IPO/SampleProfile.h"
Justin Bogner21e15372015-10-30 23:28:12 +000088#include "llvm/Transforms/IPO/StripDeadPrototypes.h"
Easwaran Ramanbdf20262018-01-09 19:39:35 +000089#include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
Davide Italianod737dd22016-06-14 21:44:19 +000090#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
Chandler Carruthf49f1a872015-12-27 08:13:45 +000091#include "llvm/Transforms/InstCombine/InstCombine.h"
Philip Pfaffeb39a97c2019-01-03 13:42:44 +000092#include "llvm/Transforms/Instrumentation.h"
Leonard Chan436fb2b2019-02-13 22:22:48 +000093#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
Chandler Carruth00a301d2017-11-14 01:30:04 +000094#include "llvm/Transforms/Instrumentation/BoundsChecking.h"
Vitaly Buka4493fe12018-11-26 21:57:47 +000095#include "llvm/Transforms/Instrumentation/CGProfile.h"
Hiroshi Yamauchi9775a622018-09-04 17:19:13 +000096#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
Chandler Carruth7c557f82018-06-29 23:36:03 +000097#include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
Leonard Chan0cdd3b12019-05-14 21:17:21 +000098#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
Manman Ren18295122019-02-28 20:13:38 +000099#include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
Chandler Carruth7c557f82018-06-29 23:36:03 +0000100#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
Philip Pfaffeb39a97c2019-01-03 13:42:44 +0000101#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
David Blaikie4fe1fe12018-03-23 22:11:06 +0000102#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
Philip Reamesf47a3132019-07-09 18:49:29 +0000103#include "llvm/Transforms/Instrumentation/PoisonChecking.h"
Leonard Chan5652f352019-07-11 22:35:40 +0000104#include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
Leonard Chan436fb2b2019-02-13 22:22:48 +0000105#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
Justin Bogner19b67992015-10-30 23:13:18 +0000106#include "llvm/Transforms/Scalar/ADCE.h"
Sean Silvaa4c2d152016-06-15 06:18:01 +0000107#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
Davide Italiano655a1452016-05-25 01:57:04 +0000108#include "llvm/Transforms/Scalar/BDCE.h"
Jun Bum Lim0c990072017-11-03 20:41:16 +0000109#include "llvm/Transforms/Scalar/CallSiteSplitting.h"
Adam Nemet3beef412016-07-18 16:29:17 +0000110#include "llvm/Transforms/Scalar/ConstantHoisting.h"
Sean Silvab025d372016-07-06 23:26:29 +0000111#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
Justin Bogner395c2122016-04-22 19:40:41 +0000112#include "llvm/Transforms/Scalar/DCE.h"
Justin Bogner594e07b2016-05-17 21:38:13 +0000113#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
Sanjay Patel6fd43912017-09-09 13:38:18 +0000114#include "llvm/Transforms/Scalar/DivRemPairs.h"
Chandler Carruthe8c686a2015-02-01 10:51:23 +0000115#include "llvm/Transforms/Scalar/EarlyCSE.h"
Michael Kuperstein83b753d2016-06-24 23:32:02 +0000116#include "llvm/Transforms/Scalar/Float2Int.h"
Easwaran Raman019e0bf2016-06-03 22:54:26 +0000117#include "llvm/Transforms/Scalar/GVN.h"
Davide Italiano89ab89d2016-06-14 00:49:23 +0000118#include "llvm/Transforms/Scalar/GuardWidening.h"
Chandler Carruth3bab7e12017-01-11 09:43:56 +0000119#include "llvm/Transforms/Scalar/IVUsersPrinter.h"
Sanjoy Das4d4339d2016-06-05 18:01:19 +0000120#include "llvm/Transforms/Scalar/IndVarSimplify.h"
Fedor Sergeev194a4072018-03-15 11:01:19 +0000121#include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
Chandler Carruth7c557f82018-06-29 23:36:03 +0000122#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
Sean Silva46590d52016-06-14 00:51:09 +0000123#include "llvm/Transforms/Scalar/JumpThreading.h"
Dehao Chenf400a092016-07-12 22:42:24 +0000124#include "llvm/Transforms/Scalar/LICM.h"
Chandler Carruth3bab7e12017-01-11 09:43:56 +0000125#include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
Teresa Johnson1eca6bc2016-08-13 04:11:27 +0000126#include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
Jun Bum Limc837af32016-07-14 18:28:29 +0000127#include "llvm/Transforms/Scalar/LoopDeletion.h"
Adam Nemetb2593f72016-07-18 16:29:27 +0000128#include "llvm/Transforms/Scalar/LoopDistribute.h"
Kit Barton3cdf8792019-04-17 18:53:27 +0000129#include "llvm/Transforms/Scalar/LoopFuse.h"
Dehao Chenb9f8e292016-07-12 18:45:51 +0000130#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
Chandler Carruthe6c30fd2018-05-25 01:32:36 +0000131#include "llvm/Transforms/Scalar/LoopInstSimplify.h"
Chandler Carruthbaabda92017-01-27 01:32:26 +0000132#include "llvm/Transforms/Scalar/LoopLoadElimination.h"
Chandler Carruth3bab7e12017-01-11 09:43:56 +0000133#include "llvm/Transforms/Scalar/LoopPassManager.h"
Artur Pilipenko8fb3d572017-01-25 16:00:44 +0000134#include "llvm/Transforms/Scalar/LoopPredication.h"
Justin Bognerd0d23412016-05-03 22:02:31 +0000135#include "llvm/Transforms/Scalar/LoopRotation.h"
Justin Bognerab6a5132016-05-03 21:47:32 +0000136#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
Chandler Carruthe9b18e32017-01-20 08:42:19 +0000137#include "llvm/Transforms/Scalar/LoopSink.h"
Dehao Chen6132ee82016-07-18 21:41:50 +0000138#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
David Green963401d2018-07-01 12:47:30 +0000139#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
Sean Silvae3c18a52016-07-19 23:54:23 +0000140#include "llvm/Transforms/Scalar/LoopUnrollPass.h"
Davide Italiano99223442016-05-13 22:52:35 +0000141#include "llvm/Transforms/Scalar/LowerAtomic.h"
Chandler Carruth43e590e2015-01-24 11:13:02 +0000142#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
Michael Kupersteine45d4d92016-07-28 22:08:41 +0000143#include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
Max Kazantsevf392bc82019-01-31 09:10:17 +0000144#include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
Max Kazantsevb9e65cb2018-12-07 14:39:46 +0000145#include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
Sean Silva6347df02016-06-14 02:44:55 +0000146#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
Clement Courbet43882b12019-05-23 12:35:26 +0000147#include "llvm/Transforms/Scalar/MergeICmps.h"
Leonard Chan5652f352019-07-11 22:35:40 +0000148#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
Wei Mi1cf58f82016-07-21 22:28:52 +0000149#include "llvm/Transforms/Scalar/NaryReassociate.h"
Davide Italianoe05e3302016-12-22 16:35:02 +0000150#include "llvm/Transforms/Scalar/NewGVN.h"
Davide Italiano1021c682016-05-25 23:38:53 +0000151#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
Justin Bognerc2bf63d2016-04-26 23:39:29 +0000152#include "llvm/Transforms/Scalar/Reassociate.h"
Fedor Sergeev4b86d792017-12-15 09:32:11 +0000153#include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
Davide Italiano98f7e0e2016-05-18 15:18:25 +0000154#include "llvm/Transforms/Scalar/SCCP.h"
Chandler Carruth29a18a42015-09-12 09:09:14 +0000155#include "llvm/Transforms/Scalar/SROA.h"
Mikael Holmenb6f76002018-11-21 14:00:17 +0000156#include "llvm/Transforms/Scalar/Scalarizer.h"
Chandler Carruth1353f9a2017-04-27 18:45:20 +0000157#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
Chandler Carruthf49f1a872015-12-27 08:13:45 +0000158#include "llvm/Transforms/Scalar/SimplifyCFG.h"
Justin Bognerb9394902016-04-22 19:54:10 +0000159#include "llvm/Transforms/Scalar/Sink.h"
Chandler Carruthc34f7892017-11-28 11:32:31 +0000160#include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
Michael Kupersteinc4061862016-08-01 21:48:33 +0000161#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
Sean Silva59fe82f2016-07-06 23:48:41 +0000162#include "llvm/Transforms/Scalar/TailRecursionElimination.h"
Michael Kruse72448522018-12-12 17:32:52 +0000163#include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
Xinliang David Li1eaecef2016-06-15 21:51:30 +0000164#include "llvm/Transforms/Utils/AddDiscriminators.h"
Wei Mie04d0ef2016-07-22 18:04:25 +0000165#include "llvm/Transforms/Utils/BreakCriticalEdges.h"
Teresa Johnson853b9622019-01-04 19:04:54 +0000166#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
Hans Wennborge1ecd612017-11-14 21:09:45 +0000167#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
Easwaran Ramane12c4872016-06-09 19:44:46 +0000168#include "llvm/Transforms/Utils/LCSSA.h"
Rong Xu1c0e9b92016-10-18 21:36:27 +0000169#include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
Davide Italianocd96cfd2016-07-09 03:03:01 +0000170#include "llvm/Transforms/Utils/LoopSimplify.h"
Michael Kuperstein31b83992016-08-12 17:28:27 +0000171#include "llvm/Transforms/Utils/LowerInvoke.h"
Davide Italianocccf4f02016-06-14 03:22:22 +0000172#include "llvm/Transforms/Utils/Mem2Reg.h"
Mehdi Amini27d23792016-09-16 16:56:30 +0000173#include "llvm/Transforms/Utils/NameAnonGlobals.h"
Michael Kuperstein39feb622016-07-25 20:52:00 +0000174#include "llvm/Transforms/Utils/SymbolRewriter.h"
Markus Lavin4dc4ebd2018-12-07 08:23:37 +0000175#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
Sean Silvadb90d4d2016-07-09 22:56:50 +0000176#include "llvm/Transforms/Vectorize/LoopVectorize.h"
Sean Silva0dacbd82016-07-09 03:11:29 +0000177#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
Davide Italianocccf4f02016-06-14 03:22:22 +0000178
Chandler Carruth66445382014-01-11 08:16:35 +0000179using namespace llvm;
180
Chandler Carruth719ffe12017-02-12 05:38:04 +0000181static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
182 cl::ReallyHidden, cl::init(4));
Xinliang David Li126157c2017-05-22 16:41:57 +0000183static cl::opt<bool>
184 RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
185 cl::Hidden, cl::ZeroOrMore,
186 cl::desc("Run Partial inlinining pass"));
Chandler Carruth719ffe12017-02-12 05:38:04 +0000187
Davide Italiano8a09b8e2017-05-22 23:41:40 +0000188static cl::opt<bool>
Davide Italiano8e7d11a2017-05-22 23:47:11 +0000189 RunNewGVN("enable-npm-newgvn", cl::init(false),
Davide Italiano8a09b8e2017-05-22 23:41:40 +0000190 cl::Hidden, cl::ZeroOrMore,
191 cl::desc("Run NewGVN instead of GVN"));
192
Chandler Carruthc246a4c2017-04-27 00:28:03 +0000193static cl::opt<bool> EnableGVNHoist(
Eric Christopherdcf1d972018-10-01 18:57:08 +0000194 "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
195 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
Chandler Carruthc246a4c2017-04-27 00:28:03 +0000196
Davide Italianobe1b6a92017-06-03 23:18:29 +0000197static cl::opt<bool> EnableGVNSink(
198 "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
199 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
200
David Green963401d2018-07-01 12:47:30 +0000201static cl::opt<bool> EnableUnrollAndJam(
202 "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden,
203 cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
204
Easwaran Ramanbdf20262018-01-09 19:39:35 +0000205static cl::opt<bool> EnableSyntheticCounts(
206 "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
207 cl::desc("Run synthetic function entry count generation "
208 "pass"));
209
Chandler Carruth8b3be4e2017-06-01 11:39:39 +0000210static Regex DefaultAliasRegex(
211 "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
Chandler Carruth8b5a74192016-02-28 22:16:03 +0000212
Eric Christophereff3b6f2019-04-18 06:17:40 +0000213// This option is used in simplifying testing SampleFDO optimizations for
214// profile loading.
Hiroshi Yamauchi9775a622018-09-04 17:19:13 +0000215static cl::opt<bool>
216 EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
217 cl::desc("Enable control height reduction optimization (CHR)"));
218
Alina Sbirlea0499a2f2019-04-19 16:11:59 +0000219PipelineTuningOptions::PipelineTuningOptions() {
220 LoopInterleaving = EnableLoopInterleaving;
221 LoopVectorization = EnableLoopVectorization;
Alina Sbirlea458c7332019-05-08 17:58:35 +0000222 SLPVectorization = RunSLPVectorization;
Alina Sbirleae4b27862019-05-23 19:35:40 +0000223 LoopUnrolling = true;
Alina Sbirlead82ddfa2019-05-23 21:52:59 +0000224 ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
Alina Sbirlea43709f72019-04-19 17:46:50 +0000225 LicmMssaOptCap = SetLicmMssaOptCap;
226 LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
Alina Sbirlea0499a2f2019-04-19 16:11:59 +0000227}
228
Aditya Kumar9e20ade2018-10-03 05:55:20 +0000229extern cl::opt<bool> EnableHotColdSplit;
Manman Ren18295122019-02-28 20:13:38 +0000230extern cl::opt<bool> EnableOrderFileInstrumentation;
Aditya Kumar9e20ade2018-10-03 05:55:20 +0000231
Wei Mi3bcccdf2019-01-17 20:48:34 +0000232extern cl::opt<bool> FlattenedProfileUsed;
233
Chandler Carruthe3f50642016-12-22 06:59:15 +0000234static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level) {
235 switch (Level) {
236 case PassBuilder::O0:
237 case PassBuilder::O1:
238 case PassBuilder::O2:
239 case PassBuilder::O3:
240 return false;
241
242 case PassBuilder::Os:
243 case PassBuilder::Oz:
244 return true;
245 }
246 llvm_unreachable("Invalid optimization level!");
247}
248
Chandler Carruth66445382014-01-11 08:16:35 +0000249namespace {
250
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000251/// No-op module pass which does nothing.
Chandler Carruth66445382014-01-11 08:16:35 +0000252struct NoOpModulePass {
Sean Silvafd03ac62016-08-09 00:28:38 +0000253 PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
Chandler Carruth164a2aa62016-06-17 00:11:01 +0000254 return PreservedAnalyses::all();
255 }
Chandler Carrutha13f27c2014-01-11 11:52:05 +0000256 static StringRef name() { return "NoOpModulePass"; }
Chandler Carruth66445382014-01-11 08:16:35 +0000257};
258
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000259/// No-op module analysis.
Chandler Carruth30a07302016-03-11 10:33:22 +0000260class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
261 friend AnalysisInfoMixin<NoOpModuleAnalysis>;
Chandler Carruthdab4eae2016-11-23 17:53:26 +0000262 static AnalysisKey Key;
Chandler Carruthb4faf132016-03-11 10:22:49 +0000263
264public:
Chandler Carruth0b576b32015-01-06 02:50:06 +0000265 struct Result {};
Sean Silvafd03ac62016-08-09 00:28:38 +0000266 Result run(Module &, ModuleAnalysisManager &) { return Result(); }
Chandler Carruth0b576b32015-01-06 02:50:06 +0000267 static StringRef name() { return "NoOpModuleAnalysis"; }
Chandler Carruth0b576b32015-01-06 02:50:06 +0000268};
269
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000270/// No-op CGSCC pass which does nothing.
Chandler Carruth572e3402014-04-21 11:12:00 +0000271struct NoOpCGSCCPass {
Chandler Carruth88823462016-08-24 09:37:14 +0000272 PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
273 LazyCallGraph &, CGSCCUpdateResult &UR) {
Chandler Carruth572e3402014-04-21 11:12:00 +0000274 return PreservedAnalyses::all();
275 }
276 static StringRef name() { return "NoOpCGSCCPass"; }
277};
278
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000279/// No-op CGSCC analysis.
Chandler Carruth30a07302016-03-11 10:33:22 +0000280class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
281 friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
Chandler Carruthdab4eae2016-11-23 17:53:26 +0000282 static AnalysisKey Key;
Chandler Carruthb4faf132016-03-11 10:22:49 +0000283
284public:
Chandler Carruth0b576b32015-01-06 02:50:06 +0000285 struct Result {};
Chandler Carruth88823462016-08-24 09:37:14 +0000286 Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
Chandler Carruth164a2aa62016-06-17 00:11:01 +0000287 return Result();
288 }
Chandler Carruth0b576b32015-01-06 02:50:06 +0000289 static StringRef name() { return "NoOpCGSCCAnalysis"; }
Chandler Carruth0b576b32015-01-06 02:50:06 +0000290};
291
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000292/// No-op function pass which does nothing.
Chandler Carruthd8330982014-01-12 09:34:22 +0000293struct NoOpFunctionPass {
Sean Silva36e0d012016-08-09 00:28:15 +0000294 PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
Chandler Carruth164a2aa62016-06-17 00:11:01 +0000295 return PreservedAnalyses::all();
296 }
Chandler Carruthd8330982014-01-12 09:34:22 +0000297 static StringRef name() { return "NoOpFunctionPass"; }
298};
299
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000300/// No-op function analysis.
Chandler Carruth30a07302016-03-11 10:33:22 +0000301class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
302 friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
Chandler Carruthdab4eae2016-11-23 17:53:26 +0000303 static AnalysisKey Key;
Chandler Carruthb4faf132016-03-11 10:22:49 +0000304
305public:
Chandler Carruth0b576b32015-01-06 02:50:06 +0000306 struct Result {};
Sean Silva36e0d012016-08-09 00:28:15 +0000307 Result run(Function &, FunctionAnalysisManager &) { return Result(); }
Chandler Carruth0b576b32015-01-06 02:50:06 +0000308 static StringRef name() { return "NoOpFunctionAnalysis"; }
Chandler Carruth0b576b32015-01-06 02:50:06 +0000309};
310
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000311/// No-op loop pass which does nothing.
Justin Bognereecc3c82016-02-25 07:23:08 +0000312struct NoOpLoopPass {
Chandler Carruth410eaeb2017-01-11 06:23:21 +0000313 PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
314 LoopStandardAnalysisResults &, LPMUpdater &) {
Chandler Carruth164a2aa62016-06-17 00:11:01 +0000315 return PreservedAnalyses::all();
316 }
Justin Bognereecc3c82016-02-25 07:23:08 +0000317 static StringRef name() { return "NoOpLoopPass"; }
318};
319
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000320/// No-op loop analysis.
Chandler Carruth30a07302016-03-11 10:33:22 +0000321class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
322 friend AnalysisInfoMixin<NoOpLoopAnalysis>;
Chandler Carruthdab4eae2016-11-23 17:53:26 +0000323 static AnalysisKey Key;
Chandler Carruthb4faf132016-03-11 10:22:49 +0000324
325public:
Justin Bognereecc3c82016-02-25 07:23:08 +0000326 struct Result {};
Chandler Carruth410eaeb2017-01-11 06:23:21 +0000327 Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
328 return Result();
329 }
Justin Bognereecc3c82016-02-25 07:23:08 +0000330 static StringRef name() { return "NoOpLoopAnalysis"; }
Justin Bognereecc3c82016-02-25 07:23:08 +0000331};
332
Chandler Carruthdab4eae2016-11-23 17:53:26 +0000333AnalysisKey NoOpModuleAnalysis::Key;
334AnalysisKey NoOpCGSCCAnalysis::Key;
335AnalysisKey NoOpFunctionAnalysis::Key;
336AnalysisKey NoOpLoopAnalysis::Key;
Chandler Carruthb4faf132016-03-11 10:22:49 +0000337
Chandler Carruth66445382014-01-11 08:16:35 +0000338} // End anonymous namespace.
339
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000340void PassBuilder::invokePeepholeEPCallbacks(
341 FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
342 for (auto &C : PeepholeEPCallbacks)
343 C(FPM, Level);
344}
345
Chandler Carruth1ff77242015-03-07 09:02:36 +0000346void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
Chandler Carruth74a8a222016-06-17 07:15:29 +0000347#define MODULE_ANALYSIS(NAME, CREATE_PASS) \
Chandler Carruthedf59962016-02-18 09:45:17 +0000348 MAM.registerPass([&] { return CREATE_PASS; });
Chandler Carruthb70f6732015-01-06 02:21:37 +0000349#include "PassRegistry.def"
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000350
351 for (auto &C : ModuleAnalysisRegistrationCallbacks)
352 C(MAM);
Chandler Carruthb70f6732015-01-06 02:21:37 +0000353}
354
Chandler Carruth1ff77242015-03-07 09:02:36 +0000355void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
Chandler Carruth74a8a222016-06-17 07:15:29 +0000356#define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
Chandler Carruthedf59962016-02-18 09:45:17 +0000357 CGAM.registerPass([&] { return CREATE_PASS; });
Chandler Carruthb70f6732015-01-06 02:21:37 +0000358#include "PassRegistry.def"
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000359
360 for (auto &C : CGSCCAnalysisRegistrationCallbacks)
361 C(CGAM);
Chandler Carruthb70f6732015-01-06 02:21:37 +0000362}
363
Chandler Carruth1ff77242015-03-07 09:02:36 +0000364void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
Chandler Carruth74a8a222016-06-17 07:15:29 +0000365#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
Chandler Carruthedf59962016-02-18 09:45:17 +0000366 FAM.registerPass([&] { return CREATE_PASS; });
Chandler Carruthb70f6732015-01-06 02:21:37 +0000367#include "PassRegistry.def"
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000368
369 for (auto &C : FunctionAnalysisRegistrationCallbacks)
370 C(FAM);
Chandler Carruthb70f6732015-01-06 02:21:37 +0000371}
372
Justin Bognereecc3c82016-02-25 07:23:08 +0000373void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
Chandler Carruth74a8a222016-06-17 07:15:29 +0000374#define LOOP_ANALYSIS(NAME, CREATE_PASS) \
Justin Bognereecc3c82016-02-25 07:23:08 +0000375 LAM.registerPass([&] { return CREATE_PASS; });
376#include "PassRegistry.def"
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000377
378 for (auto &C : LoopAnalysisRegistrationCallbacks)
379 C(LAM);
Justin Bognereecc3c82016-02-25 07:23:08 +0000380}
381
Chandler Carruthe3f50642016-12-22 06:59:15 +0000382FunctionPassManager
383PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
Dehao Chen95f00302017-07-30 04:55:39 +0000384 ThinLTOPhase Phase,
385 bool DebugLogging) {
Chandler Carruthe3f50642016-12-22 06:59:15 +0000386 assert(Level != O0 && "Must request optimizations!");
387 FunctionPassManager FPM(DebugLogging);
388
389 // Form SSA out of local memory accesses after breaking apart aggregates into
390 // scalars.
391 FPM.addPass(SROA());
392
393 // Catch trivial redundancies
Eric Christopherdfebd842019-04-19 22:18:53 +0000394 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
Chandler Carruthe3f50642016-12-22 06:59:15 +0000395
Davide Italianoc3688312017-06-01 23:08:14 +0000396 // Hoisting of scalars and load expressions.
397 if (EnableGVNHoist)
398 FPM.addPass(GVNHoistPass());
399
Davide Italianobe1b6a92017-06-03 23:18:29 +0000400 // Global value numbering based sinking.
401 if (EnableGVNSink) {
402 FPM.addPass(GVNSinkPass());
403 FPM.addPass(SimplifyCFGPass());
404 }
405
Chandler Carruthe3f50642016-12-22 06:59:15 +0000406 // Speculative execution if the target has divergent branches; otherwise nop.
407 FPM.addPass(SpeculativeExecutionPass());
408
409 // Optimize based on known information about branches, and cleanup afterward.
410 FPM.addPass(JumpThreadingPass());
411 FPM.addPass(CorrelatedValuePropagationPass());
412 FPM.addPass(SimplifyCFGPass());
Amjad Aboudf1f57a32018-01-25 12:06:32 +0000413 if (Level == O3)
414 FPM.addPass(AggressiveInstCombinePass());
Chandler Carruthe3f50642016-12-22 06:59:15 +0000415 FPM.addPass(InstCombinePass());
416
417 if (!isOptimizingForSize(Level))
418 FPM.addPass(LibCallsShrinkWrapPass());
419
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000420 invokePeepholeEPCallbacks(FPM, Level);
421
Rong Xue1f42452017-10-23 22:21:29 +0000422 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
423 // using the size value profile. Don't perform this when optimizing for size.
Rong Xudb29a3a2019-03-04 20:21:27 +0000424 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
Rong Xue1f42452017-10-23 22:21:29 +0000425 !isOptimizingForSize(Level))
426 FPM.addPass(PGOMemOPSizeOpt());
427
Chandler Carruthe3f50642016-12-22 06:59:15 +0000428 FPM.addPass(TailCallElimPass());
429 FPM.addPass(SimplifyCFGPass());
430
431 // Form canonically associated expression trees, and simplify the trees using
432 // basic mathematical properties. For example, this will form (nearly)
433 // minimal multiplication trees.
434 FPM.addPass(ReassociatePass());
435
436 // Add the primary loop simplification pipeline.
437 // FIXME: Currently this is split into two loop pass pipelines because we run
Chandler Carruth71fd2702018-05-30 02:46:45 +0000438 // some function passes in between them. These can and should be removed
439 // and/or replaced by scheduling the loop pass equivalents in the correct
440 // positions. But those equivalent passes aren't powerful enough yet.
441 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
442 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
443 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
444 // `LoopInstSimplify`.
Chandler Carruthe3f50642016-12-22 06:59:15 +0000445 LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
446
Chandler Carruth71fd2702018-05-30 02:46:45 +0000447 // Simplify the loop body. We do this initially to clean up after other loop
448 // passes run, either when iterating on a loop or on inner loops with
449 // implications on the outer loop.
450 LPM1.addPass(LoopInstSimplifyPass());
451 LPM1.addPass(LoopSimplifyCFGPass());
452
Chandler Carruthe3f50642016-12-22 06:59:15 +0000453 // Rotate Loop - disable header duplication at -Oz
454 LPM1.addPass(LoopRotatePass(Level != Oz));
Alina Sbirlea43709f72019-04-19 17:46:50 +0000455 LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
Chandler Carruth86248d52017-05-26 01:24:11 +0000456 LPM1.addPass(SimpleLoopUnswitchPass());
Chandler Carruth79b733b2017-01-26 23:21:17 +0000457 LPM2.addPass(IndVarSimplifyPass());
458 LPM2.addPass(LoopIdiomRecognizePass());
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000459
460 for (auto &C : LateLoopOptimizationsEPCallbacks)
461 C(LPM2, Level);
462
Chandler Carruth79b733b2017-01-26 23:21:17 +0000463 LPM2.addPass(LoopDeletionPass());
Dehao Chen95f00302017-07-30 04:55:39 +0000464 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
Dehao Chen3a9861422017-07-07 20:53:10 +0000465 // because it changes IR to makes profile annotation in back compile
466 // inaccurate.
Alina Sbirleae4b27862019-05-23 19:35:40 +0000467 if ((Phase != ThinLTOPhase::PreLink || !PGOOpt ||
468 PGOOpt->Action != PGOOptions::SampleUse) &&
469 PTO.LoopUnrolling)
Alina Sbirlead82ddfa2019-05-23 21:52:59 +0000470 LPM2.addPass(
471 LoopFullUnrollPass(Level, false, PTO.ForgetAllSCEVInLoopUnroll));
Chandler Carruth79b733b2017-01-26 23:21:17 +0000472
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000473 for (auto &C : LoopOptimizerEndEPCallbacks)
474 C(LPM2, Level);
475
Chandler Carruth0ede22e2017-02-09 23:54:57 +0000476 // We provide the opt remark emitter pass for LICM to use. We only need to do
477 // this once as it is immutable.
478 FPM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
Fedor Sergeev02e7f022017-12-29 08:16:06 +0000479 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1), DebugLogging));
Chandler Carruthe3f50642016-12-22 06:59:15 +0000480 FPM.addPass(SimplifyCFGPass());
481 FPM.addPass(InstCombinePass());
Fedor Sergeev02e7f022017-12-29 08:16:06 +0000482 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2), DebugLogging));
Chandler Carruthe3f50642016-12-22 06:59:15 +0000483
484 // Eliminate redundancies.
485 if (Level != O1) {
486 // These passes add substantial compile time so skip them at O1.
487 FPM.addPass(MergedLoadStoreMotionPass());
Davide Italiano8a09b8e2017-05-22 23:41:40 +0000488 if (RunNewGVN)
489 FPM.addPass(NewGVNPass());
490 else
491 FPM.addPass(GVN());
Chandler Carruthe3f50642016-12-22 06:59:15 +0000492 }
493
494 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
495 FPM.addPass(MemCpyOptPass());
496
497 // Sparse conditional constant propagation.
498 // FIXME: It isn't clear why we do this *after* loop passes rather than
499 // before...
500 FPM.addPass(SCCPPass());
501
502 // Delete dead bit computations (instcombine runs after to fold away the dead
503 // computations, and then ADCE will run later to exploit any new DCE
504 // opportunities that creates).
505 FPM.addPass(BDCEPass());
506
507 // Run instcombine after redundancy and dead bit elimination to exploit
508 // opportunities opened up by them.
509 FPM.addPass(InstCombinePass());
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000510 invokePeepholeEPCallbacks(FPM, Level);
Chandler Carruthe3f50642016-12-22 06:59:15 +0000511
512 // Re-consider control flow based optimizations after redundancy elimination,
513 // redo DCE, etc.
514 FPM.addPass(JumpThreadingPass());
515 FPM.addPass(CorrelatedValuePropagationPass());
516 FPM.addPass(DSEPass());
Alina Sbirlea43709f72019-04-19 17:46:50 +0000517 FPM.addPass(createFunctionToLoopPassAdaptor(
518 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
519 DebugLogging));
Chandler Carruthe3f50642016-12-22 06:59:15 +0000520
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000521 for (auto &C : ScalarOptimizerLateEPCallbacks)
522 C(FPM, Level);
523
Chandler Carruthe3f50642016-12-22 06:59:15 +0000524 // Finally, do an expensive DCE pass to catch all the dead code exposed by
525 // the simplifications and basic cleanup after all the simplifications.
526 FPM.addPass(ADCEPass());
527 FPM.addPass(SimplifyCFGPass());
528 FPM.addPass(InstCombinePass());
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000529 invokePeepholeEPCallbacks(FPM, Level);
Chandler Carruthe3f50642016-12-22 06:59:15 +0000530
Hiroshi Yamauchi9775a622018-09-04 17:19:13 +0000531 if (EnableCHR && Level == O3 && PGOOpt &&
Rong Xudb29a3a2019-03-04 20:21:27 +0000532 (PGOOpt->Action == PGOOptions::IRUse ||
533 PGOOpt->Action == PGOOptions::SampleUse))
Hiroshi Yamauchi9775a622018-09-04 17:19:13 +0000534 FPM.addPass(ControlHeightReductionPass());
535
Chandler Carruthe3f50642016-12-22 06:59:15 +0000536 return FPM;
537}
538
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000539void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
540 PassBuilder::OptimizationLevel Level,
Rong Xudb29a3a2019-03-04 20:21:27 +0000541 bool RunProfileGen, bool IsCS,
542 std::string ProfileFile,
Richard Smith6c676622018-10-10 23:13:47 +0000543 std::string ProfileRemappingFile) {
Davide Italiano513dfaa2017-02-13 15:26:22 +0000544 // Generally running simplification passes and the inliner with an high
545 // threshold results in smaller executables, but there may be cases where
546 // the size grows, so let's be conservative here and skip this simplification
Rong Xudb29a3a2019-03-04 20:21:27 +0000547 // at -Os/Oz. We will not do this inline for context sensistive PGO (when
548 // IsCS is true).
549 if (!isOptimizingForSize(Level) && !IsCS) {
Davide Italiano513dfaa2017-02-13 15:26:22 +0000550 InlineParams IP;
551
552 // In the old pass manager, this is a cl::opt. Should still this be one?
553 IP.DefaultThreshold = 75;
554
555 // FIXME: The hint threshold has the same value used by the regular inliner.
556 // This should probably be lowered after performance testing.
557 // FIXME: this comment is cargo culted from the old pass manager, revisit).
558 IP.HintThreshold = 325;
559
560 CGSCCPassManager CGPipeline(DebugLogging);
561
562 CGPipeline.addPass(InlinerPass(IP));
563
564 FunctionPassManager FPM;
565 FPM.addPass(SROA());
566 FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.
567 FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
568 FPM.addPass(InstCombinePass()); // Combine silly sequences.
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000569 invokePeepholeEPCallbacks(FPM, Level);
Davide Italiano513dfaa2017-02-13 15:26:22 +0000570
Davide Italiano513dfaa2017-02-13 15:26:22 +0000571 CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
572
573 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline)));
574 }
575
Chandler Carruthf4d62c42017-05-25 07:15:09 +0000576 // Delete anything that is now dead to make sure that we don't instrument
577 // dead code. Instrumentation can end up keeping dead code around and
578 // dramatically increase code size.
579 MPM.addPass(GlobalDCEPass());
580
Davide Italiano513dfaa2017-02-13 15:26:22 +0000581 if (RunProfileGen) {
Rong Xudb29a3a2019-03-04 20:21:27 +0000582 MPM.addPass(PGOInstrumentationGen(IsCS));
Davide Italiano513dfaa2017-02-13 15:26:22 +0000583
Xinliang David Lib67530e2017-06-25 00:26:43 +0000584 FunctionPassManager FPM;
Fedor Sergeev02e7f022017-12-29 08:16:06 +0000585 FPM.addPass(
586 createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging));
Xinliang David Lib67530e2017-06-25 00:26:43 +0000587 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
588
Davide Italiano513dfaa2017-02-13 15:26:22 +0000589 // Add the profile lowering pass.
590 InstrProfOptions Options;
Rong Xudb29a3a2019-03-04 20:21:27 +0000591 if (!ProfileFile.empty())
592 Options.InstrProfileOutput = ProfileFile;
Xinliang David Lib67530e2017-06-25 00:26:43 +0000593 Options.DoCounterPromotion = true;
Rong Xudb29a3a2019-03-04 20:21:27 +0000594 Options.UseBFIInPromotion = IsCS;
595 MPM.addPass(InstrProfiling(Options, IsCS));
Hiroshi Yamauchi09e539f2019-04-15 16:49:00 +0000596 } else if (!ProfileFile.empty()) {
Rong Xudb29a3a2019-03-04 20:21:27 +0000597 MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
Hiroshi Yamauchi09e539f2019-04-15 16:49:00 +0000598 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
599 // RequireAnalysisPass for PSI before subsequent non-module passes.
600 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
601 }
Davide Italiano513dfaa2017-02-13 15:26:22 +0000602}
603
Easwaran Raman8249fac2017-06-28 13:33:49 +0000604static InlineParams
605getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
606 auto O3 = PassBuilder::O3;
607 unsigned OptLevel = Level > O3 ? 2 : Level;
608 unsigned SizeLevel = Level > O3 ? Level - O3 : 0;
609 return getInlineParams(OptLevel, SizeLevel);
610}
611
Chandler Carruthe3f50642016-12-22 06:59:15 +0000612ModulePassManager
Chandler Carruth8b3be4e2017-06-01 11:39:39 +0000613PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
Dehao Chen95f00302017-07-30 04:55:39 +0000614 ThinLTOPhase Phase,
615 bool DebugLogging) {
Chandler Carruthe3f50642016-12-22 06:59:15 +0000616 ModulePassManager MPM(DebugLogging);
617
Rong Xudb29a3a2019-03-04 20:21:27 +0000618 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
Teresa Johnsonb0bf5302019-02-05 04:09:19 +0000619
620 // In ThinLTO mode, when flattened profile is used, all the available
621 // profile information will be annotated in PreLink phase so there is
622 // no need to load the profile again in PostLink.
623 bool LoadSampleProfile =
624 HasSampleProfile &&
625 !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink);
626
627 // During the ThinLTO backend phase we perform early indirect call promotion
628 // here, before globalopt. Otherwise imported available_externally functions
629 // look unreferenced and are removed. If we are going to load the sample
630 // profile then defer until later.
631 // TODO: See if we can move later and consolidate with the location where
632 // we perform ICP when we are loading a sample profile.
633 // TODO: We pass HasSampleProfile (whether there was a sample profile file
634 // passed to the compile) to the SamplePGO flag of ICP. This is used to
635 // determine whether the new direct calls are annotated with prof metadata.
636 // Ideally this should be determined from whether the IR is annotated with
637 // sample profile, and not whether the a sample profile was provided on the
638 // command line. E.g. for flattened profiles where we will not be reloading
639 // the sample profile in the ThinLTO backend, we ideally shouldn't have to
640 // provide the sample profile file.
641 if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile)
642 MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
643
Chandler Carruthe3f50642016-12-22 06:59:15 +0000644 // Do basic inference of function attributes from known properties of system
645 // libraries and other oracles.
646 MPM.addPass(InferFunctionAttrsPass());
647
648 // Create an early function pass manager to cleanup the output of the
649 // frontend.
Chandler Carruth8b5a74192016-02-28 22:16:03 +0000650 FunctionPassManager EarlyFPM(DebugLogging);
651 EarlyFPM.addPass(SimplifyCFGPass());
652 EarlyFPM.addPass(SROA());
653 EarlyFPM.addPass(EarlyCSEPass());
654 EarlyFPM.addPass(LowerExpectIntrinsicPass());
Jun Bum Lim0c990072017-11-03 20:41:16 +0000655 if (Level == O3)
656 EarlyFPM.addPass(CallSiteSplittingPass());
657
Dehao Chen08f88312017-08-07 20:23:20 +0000658 // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
659 // to convert bitcast to direct calls so that they can be inlined during the
660 // profile annotation prepration step.
661 // More details about SamplePGO design can be found in:
662 // https://research.google.com/pubs/pub45290.html
663 // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
Teresa Johnsonb0bf5302019-02-05 04:09:19 +0000664 if (LoadSampleProfile)
Dehao Chen08f88312017-08-07 20:23:20 +0000665 EarlyFPM.addPass(InstCombinePass());
Chandler Carruth8b5a74192016-02-28 22:16:03 +0000666 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
Chandler Carruthe3f50642016-12-22 06:59:15 +0000667
Teresa Johnsonb0bf5302019-02-05 04:09:19 +0000668 if (LoadSampleProfile) {
Dehao Chen08f88312017-08-07 20:23:20 +0000669 // Annotate sample profile right after early FPM to ensure freshness of
670 // the debug info.
Rong Xudb29a3a2019-03-04 20:21:27 +0000671 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
Teresa Johnsonb0bf5302019-02-05 04:09:19 +0000672 PGOOpt->ProfileRemappingFile,
673 Phase == ThinLTOPhase::PreLink));
Hiroshi Yamauchi09e539f2019-04-15 16:49:00 +0000674 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
675 // RequireAnalysisPass for PSI before subsequent non-module passes.
676 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
Dehao Chen08f88312017-08-07 20:23:20 +0000677 // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
678 // for the profile annotation to be accurate in the ThinLTO backend.
679 if (Phase != ThinLTOPhase::PreLink)
680 // We perform early indirect call promotion here, before globalopt.
681 // This is important for the ThinLTO backend phase because otherwise
682 // imported available_externally functions look unreferenced and are
683 // removed.
684 MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
Teresa Johnsonb0bf5302019-02-05 04:09:19 +0000685 true /* SamplePGO */));
Dehao Chen08f88312017-08-07 20:23:20 +0000686 }
687
Malcolm Parsons21e545d2018-01-24 10:33:39 +0000688 // Interprocedural constant propagation now that basic cleanup has occurred
Chandler Carruthe3f50642016-12-22 06:59:15 +0000689 // and prior to optimizing globals.
690 // FIXME: This position in the pipeline hasn't been carefully considered in
691 // years, it should be re-analyzed.
692 MPM.addPass(IPSCCPPass());
693
Matthew Simpsoncb585582017-10-25 13:40:08 +0000694 // Attach metadata to indirect call sites indicating the set of functions
695 // they may target at run-time. This should follow IPSCCP.
696 MPM.addPass(CalledValuePropagationPass());
697
Chandler Carruthe3f50642016-12-22 06:59:15 +0000698 // Optimize globals to try and fold them into constants.
699 MPM.addPass(GlobalOptPass());
700
701 // Promote any localized globals to SSA registers.
702 // FIXME: Should this instead by a run of SROA?
703 // FIXME: We should probably run instcombine and simplify-cfg afterward to
704 // delete control flows that are dead once globals have been folded to
705 // constants.
706 MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
707
708 // Remove any dead arguments exposed by cleanups and constand folding
709 // globals.
710 MPM.addPass(DeadArgumentEliminationPass());
711
712 // Create a small function pass pipeline to cleanup after all the global
713 // optimizations.
714 FunctionPassManager GlobalCleanupPM(DebugLogging);
715 GlobalCleanupPM.addPass(InstCombinePass());
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000716 invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
717
Chandler Carruthe3f50642016-12-22 06:59:15 +0000718 GlobalCleanupPM.addPass(SimplifyCFGPass());
719 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
720
Dehao Chen89d32262017-08-02 01:28:31 +0000721 // Add all the requested passes for instrumentation PGO, if requested.
722 if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
Rong Xudb29a3a2019-03-04 20:21:27 +0000723 (PGOOpt->Action == PGOOptions::IRInstr ||
724 PGOOpt->Action == PGOOptions::IRUse)) {
725 addPGOInstrPasses(MPM, DebugLogging, Level,
726 /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
727 /* IsCS */ false, PGOOpt->ProfileFile,
Richard Smith6c676622018-10-10 23:13:47 +0000728 PGOOpt->ProfileRemappingFile);
Dehao Chen89d32262017-08-02 01:28:31 +0000729 MPM.addPass(PGOIndirectCallPromotion(false, false));
Chandler Carruth8b3be4e2017-06-01 11:39:39 +0000730 }
Rong Xudb29a3a2019-03-04 20:21:27 +0000731 if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
732 PGOOpt->CSAction == PGOOptions::CSIRInstr)
733 MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
Davide Italiano513dfaa2017-02-13 15:26:22 +0000734
Easwaran Ramanbdf20262018-01-09 19:39:35 +0000735 // Synthesize function entry counts for non-PGO compilation.
736 if (EnableSyntheticCounts && !PGOOpt)
737 MPM.addPass(SyntheticCountsPropagation());
738
Chandler Carruthe87fc8c2017-02-12 05:34:04 +0000739 // Require the GlobalsAA analysis for the module so we can query it within
740 // the CGSCC pipeline.
741 MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
Chandler Carruthe3f50642016-12-22 06:59:15 +0000742
Easwaran Raman5e6f9bd2017-05-04 16:58:45 +0000743 // Require the ProfileSummaryAnalysis for the module so we can query it within
744 // the inliner pass.
745 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
746
Chandler Carruthe3f50642016-12-22 06:59:15 +0000747 // Now begin the main postorder CGSCC pipeline.
748 // FIXME: The current CGSCC pipeline has its origins in the legacy pass
749 // manager and trying to emulate its precise behavior. Much of this doesn't
750 // make a lot of sense and we should revisit the core CGSCC structure.
751 CGSCCPassManager MainCGPipeline(DebugLogging);
752
753 // Note: historically, the PruneEH pass was run first to deduce nounwind and
754 // generally clean up exception handling overhead. It isn't clear this is
755 // valuable as the inliner doesn't currently care whether it is inlining an
756 // invoke or a call.
757
758 // Run the inliner first. The theory is that we are walking bottom-up and so
759 // the callees have already been fully optimized, and we want to inline them
760 // into the callers so that our optimizations can reflect that.
Dehao Chen95f00302017-07-30 04:55:39 +0000761 // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
Dehao Chen3a9861422017-07-07 20:53:10 +0000762 // because it makes profile annotation in the backend inaccurate.
763 InlineParams IP = getInlineParamsFromOptLevel(Level);
Rong Xudb29a3a2019-03-04 20:21:27 +0000764 if (Phase == ThinLTOPhase::PreLink && PGOOpt &&
765 PGOOpt->Action == PGOOptions::SampleUse)
Dehao Chen3a9861422017-07-07 20:53:10 +0000766 IP.HotCallSiteThreshold = 0;
767 MainCGPipeline.addPass(InlinerPass(IP));
Chandler Carruthe3f50642016-12-22 06:59:15 +0000768
769 // Now deduce any function attributes based in the current code.
770 MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
771
Chandler Carruth0ede22e2017-02-09 23:54:57 +0000772 // When at O3 add argument promotion to the pass pipeline.
773 // FIXME: It isn't at all clear why this should be limited to O3.
774 if (Level == O3)
775 MainCGPipeline.addPass(ArgumentPromotionPass());
776
Chandler Carruthe3f50642016-12-22 06:59:15 +0000777 // Lastly, add the core function simplification pipeline nested inside the
778 // CGSCC walk.
779 MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
Dehao Chen95f00302017-07-30 04:55:39 +0000780 buildFunctionSimplificationPipeline(Level, Phase, DebugLogging)));
Chandler Carruthe3f50642016-12-22 06:59:15 +0000781
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000782 for (auto &C : CGSCCOptimizerLateEPCallbacks)
783 C(MainCGPipeline, Level);
784
Chandler Carruth719ffe12017-02-12 05:38:04 +0000785 // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
786 // to detect when we devirtualize indirect calls and iterate the SCC passes
787 // in that case to try and catch knock-on inlining or function attrs
788 // opportunities. Then we add it to the module pipeline by walking the SCCs
789 // in postorder (or bottom-up).
Chandler Carruthe3f50642016-12-22 06:59:15 +0000790 MPM.addPass(
Chandler Carruth719ffe12017-02-12 05:38:04 +0000791 createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
Chandler Carruth19913b22017-08-11 05:47:13 +0000792 std::move(MainCGPipeline), MaxDevirtIterations)));
Chandler Carruthe3f50642016-12-22 06:59:15 +0000793
Chandler Carruth8b3be4e2017-06-01 11:39:39 +0000794 return MPM;
795}
796
Vedant Kumar47a0c9b2019-02-15 18:46:44 +0000797ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
798 OptimizationLevel Level, bool DebugLogging, bool LTOPreLink) {
Chandler Carruth8b3be4e2017-06-01 11:39:39 +0000799 ModulePassManager MPM(DebugLogging);
800
801 // Optimize globals now that the module is fully simplified.
802 MPM.addPass(GlobalOptPass());
Davide Italianoe0707212017-10-05 18:36:01 +0000803 MPM.addPass(GlobalDCEPass());
Chandler Carruthe3f50642016-12-22 06:59:15 +0000804
Xinliang David Li126157c2017-05-22 16:41:57 +0000805 // Run partial inlining pass to partially inline functions that have
806 // large bodies.
807 if (RunPartialInlining)
808 MPM.addPass(PartialInlinerPass());
809
Chandler Carruth8b3be4e2017-06-01 11:39:39 +0000810 // Remove avail extern fns and globals definitions since we aren't compiling
811 // an object file for later LTO. For LTO we want to preserve these so they
812 // are eligible for inlining at link-time. Note if they are unreferenced they
813 // will be removed by GlobalDCE later, so this only impacts referenced
814 // available externally globals. Eventually they will be suppressed during
815 // codegen, but eliminating here enables more opportunity for GlobalDCE as it
816 // may make globals referenced by available external functions dead and saves
Leonard Chan97dc6222019-06-20 19:44:51 +0000817 // running remaining passes on the eliminated functions. These should be
818 // preserved during prelinking for link-time inlining decisions.
819 if (!LTOPreLink)
820 MPM.addPass(EliminateAvailableExternallyPass());
Chandler Carruthe3f50642016-12-22 06:59:15 +0000821
Manman Ren18295122019-02-28 20:13:38 +0000822 if (EnableOrderFileInstrumentation)
823 MPM.addPass(InstrOrderFilePass());
824
Chandler Carruthe3f50642016-12-22 06:59:15 +0000825 // Do RPO function attribute inference across the module to forward-propagate
826 // attributes where applicable.
827 // FIXME: Is this really an optimization rather than a canonicalization?
828 MPM.addPass(ReversePostOrderFunctionAttrsPass());
829
Rong Xudb29a3a2019-03-04 20:21:27 +0000830 // Do a post inline PGO instrumentation and use pass. This is a context
831 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
832 // cross-module inline has not been done yet. The context sensitive
833 // instrumentation is after all the inlines are done.
834 if (!LTOPreLink && PGOOpt) {
835 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
836 addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
837 /* IsCS */ true, PGOOpt->CSProfileGenFile,
838 PGOOpt->ProfileRemappingFile);
839 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
840 addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
841 /* IsCS */ true, PGOOpt->ProfileFile,
842 PGOOpt->ProfileRemappingFile);
843 }
844
Chandler Carruthe87fc8c2017-02-12 05:34:04 +0000845 // Re-require GloblasAA here prior to function passes. This is particularly
Chandler Carruthe3f50642016-12-22 06:59:15 +0000846 // useful as the above will have inlined, DCE'ed, and function-attr
847 // propagated everything. We should at this point have a reasonably minimal
848 // and richly annotated call graph. By computing aliasing and mod/ref
849 // information for all local globals here, the late loop passes and notably
850 // the vectorizer will be able to use them to help recognize vectorizable
851 // memory operations.
Chandler Carruthe87fc8c2017-02-12 05:34:04 +0000852 MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
Chandler Carruthe3f50642016-12-22 06:59:15 +0000853
854 FunctionPassManager OptimizePM(DebugLogging);
855 OptimizePM.addPass(Float2IntPass());
856 // FIXME: We need to run some loop optimizations to re-rotate loops after
857 // simplify-cfg and others undo their rotation.
858
859 // Optimize the loop execution. These passes operate on entire loop nests
860 // rather than on each loop in an inside-out manner, and so they are actually
861 // function passes.
Chandler Carrutha95ff382017-01-27 00:50:21 +0000862
Philip Pfaffe730f2f92017-07-10 10:57:55 +0000863 for (auto &C : VectorizerStartEPCallbacks)
864 C(OptimizePM, Level);
865
Chandler Carrutha95ff382017-01-27 00:50:21 +0000866 // First rotate loops that may have been un-rotated by prior passes.
Fedor Sergeev02e7f022017-12-29 08:16:06 +0000867 OptimizePM.addPass(
868 createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging));
Chandler Carrutha95ff382017-01-27 00:50:21 +0000869
870 // Distribute loops to allow partial vectorization. I.e. isolate dependences
871 // into separate loop that would otherwise inhibit vectorization. This is
872 // currently only performed for loops marked with the metadata
873 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
Chandler Carruthe3f50642016-12-22 06:59:15 +0000874 OptimizePM.addPass(LoopDistributePass());
Chandler Carrutha95ff382017-01-27 00:50:21 +0000875
876 // Now run the core loop vectorizer.
Alina Sbirlea0499a2f2019-04-19 16:11:59 +0000877 OptimizePM.addPass(LoopVectorizePass(
878 LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
Chandler Carrutha95ff382017-01-27 00:50:21 +0000879
Chandler Carruthbaabda92017-01-27 01:32:26 +0000880 // Eliminate loads by forwarding stores from the previous iteration to loads
881 // of the current iteration.
882 OptimizePM.addPass(LoopLoadEliminationPass());
Chandler Carrutha95ff382017-01-27 00:50:21 +0000883
884 // Cleanup after the loop optimization passes.
Chandler Carruthe3f50642016-12-22 06:59:15 +0000885 OptimizePM.addPass(InstCombinePass());
886
Chandler Carrutha95ff382017-01-27 00:50:21 +0000887 // Now that we've formed fast to execute loop structures, we do further
888 // optimizations. These are run afterward as they might block doing complex
889 // analyses and transforms such as what are needed for loop vectorization.
890
Sanjay Patel0ab0c1a2017-12-14 22:05:20 +0000891 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
892 // GVN, loop transforms, and others have already run, so it's now better to
893 // convert to more optimized IR using more aggressive simplify CFG options.
894 // The extra sinking transform can create larger basic blocks, so do this
895 // before SLP vectorization.
896 OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions().
897 forwardSwitchCondToPhi(true).
898 convertSwitchToLookupTable(true).
899 needCanonicalLoops(false).
900 sinkCommonInsts(true)));
901
Chandler Carruthe3f50642016-12-22 06:59:15 +0000902 // Optimize parallel scalar instruction chains into SIMD instructions.
Alina Sbirlea458c7332019-05-08 17:58:35 +0000903 if (PTO.SLPVectorization)
904 OptimizePM.addPass(SLPVectorizerPass());
Chandler Carruthe3f50642016-12-22 06:59:15 +0000905
Chandler Carruthe3f50642016-12-22 06:59:15 +0000906 OptimizePM.addPass(InstCombinePass());
907
908 // Unroll small loops to hide loop backedge latency and saturate any parallel
Chandler Carrutha95ff382017-01-27 00:50:21 +0000909 // execution resources of an out-of-order processor. We also then need to
910 // clean up redundancies and loop invariant code.
911 // FIXME: It would be really good to use a loop-integrated instruction
912 // combiner for cleanup here so that the unrolling and LICM can be pipelined
913 // across the loop nests.
David Green963401d2018-07-01 12:47:30 +0000914 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
915 if (EnableUnrollAndJam) {
916 OptimizePM.addPass(
917 createFunctionToLoopPassAdaptor(LoopUnrollAndJamPass(Level)));
918 }
Alina Sbirleae4b27862019-05-23 19:35:40 +0000919 if (PTO.LoopUnrolling)
Alina Sbirlead82ddfa2019-05-23 21:52:59 +0000920 OptimizePM.addPass(LoopUnrollPass(
921 LoopUnrollOptions(Level, false, PTO.ForgetAllSCEVInLoopUnroll)));
Michael Kruse72448522018-12-12 17:32:52 +0000922 OptimizePM.addPass(WarnMissedTransformationsPass());
Chandler Carrutha95ff382017-01-27 00:50:21 +0000923 OptimizePM.addPass(InstCombinePass());
Chandler Carruth0ede22e2017-02-09 23:54:57 +0000924 OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
Alina Sbirlea43709f72019-04-19 17:46:50 +0000925 OptimizePM.addPass(createFunctionToLoopPassAdaptor(
926 LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
927 DebugLogging));
Chandler Carruthe3f50642016-12-22 06:59:15 +0000928
929 // Now that we've vectorized and unrolled loops, we may have more refined
930 // alignment information, try to re-derive it here.
931 OptimizePM.addPass(AlignmentFromAssumptionsPass());
932
Vedant Kumar47a0c9b2019-02-15 18:46:44 +0000933 // Split out cold code. Splitting is done late to avoid hiding context from
934 // other optimizations and inadvertently regressing performance. The tradeoff
935 // is that this has a higher code size cost than splitting early.
936 if (EnableHotColdSplit && !LTOPreLink)
937 MPM.addPass(HotColdSplittingPass());
938
Chandler Carrutha95ff382017-01-27 00:50:21 +0000939 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
940 // canonicalization pass that enables other optimizations. As a result,
941 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
942 // result too early.
943 OptimizePM.addPass(LoopSinkPass());
944
945 // And finally clean up LCSSA form before generating code.
Chandler Carruth7c557f82018-06-29 23:36:03 +0000946 OptimizePM.addPass(InstSimplifyPass());
Chandler Carrutha95ff382017-01-27 00:50:21 +0000947
Sanjay Patel6fd43912017-09-09 13:38:18 +0000948 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
949 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
950 // flattening of blocks.
951 OptimizePM.addPass(DivRemPairsPass());
952
Filipe Cabecinhas92dc3482017-04-26 12:02:41 +0000953 // LoopSink (and other loop passes since the last simplifyCFG) might have
954 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
955 OptimizePM.addPass(SimplifyCFGPass());
956
Chandler Carruthc34f7892017-11-28 11:32:31 +0000957 // Optimize PHIs by speculating around them when profitable. Note that this
958 // pass needs to be run after any PRE or similar pass as it is essentially
Joerg Sonnenbergerec6ee792019-05-16 18:01:57 +0000959 // inserting redundancies into the program. This even includes SimplifyCFG.
Chandler Carruthc34f7892017-11-28 11:32:31 +0000960 OptimizePM.addPass(SpeculateAroundPHIsPass());
961
Philip Pfaffe2d4effb2018-11-12 11:17:07 +0000962 for (auto &C : OptimizerLastEPCallbacks)
963 C(OptimizePM, Level);
964
Chandler Carrutha95ff382017-01-27 00:50:21 +0000965 // Add the core optimizing pipeline.
Chandler Carruthe3f50642016-12-22 06:59:15 +0000966 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
967
Michael J. Spencer7bb27672018-07-16 00:28:24 +0000968 MPM.addPass(CGProfilePass());
969
Chandler Carruthe3f50642016-12-22 06:59:15 +0000970 // Now we need to do some global optimization transforms.
971 // FIXME: It would seem like these should come first in the optimization
972 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
973 // ordering here.
974 MPM.addPass(GlobalDCEPass());
975 MPM.addPass(ConstantMergePass());
976
977 return MPM;
Chandler Carruth8b5a74192016-02-28 22:16:03 +0000978}
979
Chandler Carruthe3f50642016-12-22 06:59:15 +0000980ModulePassManager
Chandler Carruth8b3be4e2017-06-01 11:39:39 +0000981PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
Vedant Kumar47a0c9b2019-02-15 18:46:44 +0000982 bool DebugLogging, bool LTOPreLink) {
Chandler Carruth8b3be4e2017-06-01 11:39:39 +0000983 assert(Level != O0 && "Must request optimizations for the default pipeline!");
984
985 ModulePassManager MPM(DebugLogging);
986
987 // Force any function attributes we want the rest of the pipeline to observe.
988 MPM.addPass(ForceFunctionAttrsPass());
989
David Blaikie0c64f5a2018-01-23 01:25:20 +0000990 // Apply module pipeline start EP callback.
991 for (auto &C : PipelineStartEPCallbacks)
992 C(MPM);
993
Dehao Chen08f88312017-08-07 20:23:20 +0000994 if (PGOOpt && PGOOpt->SamplePGOSupport)
Dehao Chence0842c2017-07-29 04:10:24 +0000995 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
996
Chandler Carruth8b3be4e2017-06-01 11:39:39 +0000997 // Add the core simplification pipeline.
Dehao Chen95f00302017-07-30 04:55:39 +0000998 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None,
999 DebugLogging));
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001000
1001 // Now add the optimization pipeline.
Vedant Kumar47a0c9b2019-02-15 18:46:44 +00001002 MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging, LTOPreLink));
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001003
1004 return MPM;
1005}
1006
1007ModulePassManager
1008PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1009 bool DebugLogging) {
1010 assert(Level != O0 && "Must request optimizations for the default pipeline!");
1011
1012 ModulePassManager MPM(DebugLogging);
1013
1014 // Force any function attributes we want the rest of the pipeline to observe.
1015 MPM.addPass(ForceFunctionAttrsPass());
1016
Dehao Chen08f88312017-08-07 20:23:20 +00001017 if (PGOOpt && PGOOpt->SamplePGOSupport)
Dehao Chence0842c2017-07-29 04:10:24 +00001018 MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1019
David Blaikie0c64f5a2018-01-23 01:25:20 +00001020 // Apply module pipeline start EP callback.
1021 for (auto &C : PipelineStartEPCallbacks)
1022 C(MPM);
1023
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001024 // If we are planning to perform ThinLTO later, we don't bloat the code with
1025 // unrolling/vectorization/... now. Just simplify the module as much as we
1026 // can.
Dehao Chen95f00302017-07-30 04:55:39 +00001027 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink,
1028 DebugLogging));
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001029
1030 // Run partial inlining pass to partially inline functions that have
1031 // large bodies.
1032 // FIXME: It isn't clear whether this is really the right place to run this
1033 // in ThinLTO. Because there is another canonicalization and simplification
1034 // phase that will run after the thin link, running this here ends up with
1035 // less information than will be available later and it may grow functions in
1036 // ways that aren't beneficial.
1037 if (RunPartialInlining)
1038 MPM.addPass(PartialInlinerPass());
1039
1040 // Reduce the size of the IR as much as possible.
1041 MPM.addPass(GlobalOptPass());
1042
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001043 return MPM;
1044}
1045
Teresa Johnson28023db2018-07-19 14:51:32 +00001046ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1047 OptimizationLevel Level, bool DebugLogging,
1048 const ModuleSummaryIndex *ImportSummary) {
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001049 ModulePassManager MPM(DebugLogging);
1050
Teresa Johnson28023db2018-07-19 14:51:32 +00001051 if (ImportSummary) {
1052 // These passes import type identifier resolutions for whole-program
1053 // devirtualization and CFI. They must run early because other passes may
1054 // disturb the specific instruction patterns that these passes look for,
1055 // creating dependencies on resolutions that may not appear in the summary.
1056 //
1057 // For example, GVN may transform the pattern assume(type.test) appearing in
1058 // two basic blocks into assume(phi(type.test, type.test)), which would
1059 // transform a dependency on a WPD resolution into a dependency on a type
1060 // identifier resolution for CFI.
1061 //
1062 // Also, WPD has access to more precise information than ICP and can
1063 // devirtualize more effectively, so it should operate on the IR first.
Teresa Johnson867bc392019-04-23 18:56:19 +00001064 //
1065 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1066 // metadata and intrinsics.
Teresa Johnson28023db2018-07-19 14:51:32 +00001067 MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1068 MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1069 }
1070
Teresa Johnson867bc392019-04-23 18:56:19 +00001071 if (Level == O0)
1072 return MPM;
1073
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001074 // Force any function attributes we want the rest of the pipeline to observe.
1075 MPM.addPass(ForceFunctionAttrsPass());
1076
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001077 // Add the core simplification pipeline.
Dehao Chen95f00302017-07-30 04:55:39 +00001078 MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink,
1079 DebugLogging));
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001080
1081 // Now add the optimization pipeline.
1082 MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
1083
1084 return MPM;
1085}
1086
1087ModulePassManager
Chandler Carruthe3f50642016-12-22 06:59:15 +00001088PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1089 bool DebugLogging) {
1090 assert(Level != O0 && "Must request optimizations for the default pipeline!");
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001091 // FIXME: We should use a customized pre-link pipeline!
Vedant Kumar47a0c9b2019-02-15 18:46:44 +00001092 return buildPerModuleDefaultPipeline(Level, DebugLogging,
Rong Xudb29a3a2019-03-04 20:21:27 +00001093 /* LTOPreLink */true);
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001094}
1095
Teresa Johnson28023db2018-07-19 14:51:32 +00001096ModulePassManager
1097PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
1098 ModuleSummaryIndex *ExportSummary) {
Chandler Carruthe3f50642016-12-22 06:59:15 +00001099 ModulePassManager MPM(DebugLogging);
1100
Teresa Johnson867bc392019-04-23 18:56:19 +00001101 if (Level == O0) {
1102 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1103 // metadata and intrinsics.
1104 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1105 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1106 return MPM;
1107 }
1108
Rong Xudb29a3a2019-03-04 20:21:27 +00001109 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
Xin Tong642c8d32018-11-15 18:06:42 +00001110 // Load sample profile before running the LTO optimization pipeline.
Rong Xudb29a3a2019-03-04 20:21:27 +00001111 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
Xin Tong642c8d32018-11-15 18:06:42 +00001112 PGOOpt->ProfileRemappingFile,
1113 false /* ThinLTOPhase::PreLink */));
Hiroshi Yamauchi09e539f2019-04-15 16:49:00 +00001114 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1115 // RequireAnalysisPass for PSI before subsequent non-module passes.
1116 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
Xin Tong642c8d32018-11-15 18:06:42 +00001117 }
1118
Davide Italiano089a9122017-01-24 00:57:39 +00001119 // Remove unused virtual tables to improve the quality of code generated by
1120 // whole-program devirtualization and bitset lowering.
1121 MPM.addPass(GlobalDCEPass());
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001122
Davide Italiano089a9122017-01-24 00:57:39 +00001123 // Force any function attributes we want the rest of the pipeline to observe.
1124 MPM.addPass(ForceFunctionAttrsPass());
Chandler Carruthe3f50642016-12-22 06:59:15 +00001125
Davide Italiano089a9122017-01-24 00:57:39 +00001126 // Do basic inference of function attributes from known properties of system
1127 // libraries and other oracles.
1128 MPM.addPass(InferFunctionAttrsPass());
1129
1130 if (Level > 1) {
Jun Bum Lim0c990072017-11-03 20:41:16 +00001131 FunctionPassManager EarlyFPM(DebugLogging);
1132 EarlyFPM.addPass(CallSiteSplittingPass());
1133 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1134
Davide Italiano089a9122017-01-24 00:57:39 +00001135 // Indirect call promotion. This should promote all the targets that are
1136 // left by the earlier promotion pass that promotes intra-module targets.
1137 // This two-step promotion is to save the compile time. For LTO, it should
1138 // produce the same result as if we only do promotion here.
Dehao Chen2f31d0d2017-06-29 23:33:05 +00001139 MPM.addPass(PGOIndirectCallPromotion(
Rong Xudb29a3a2019-03-04 20:21:27 +00001140 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
Davide Italiano089a9122017-01-24 00:57:39 +00001141 // Propagate constants at call sites into the functions they call. This
1142 // opens opportunities for globalopt (and inlining) by substituting function
1143 // pointers passed as arguments to direct uses of functions.
1144 MPM.addPass(IPSCCPPass());
Matthew Simpsoncb585582017-10-25 13:40:08 +00001145
1146 // Attach metadata to indirect call sites indicating the set of functions
1147 // they may target at run-time. This should follow IPSCCP.
1148 MPM.addPass(CalledValuePropagationPass());
Davide Italiano089a9122017-01-24 00:57:39 +00001149 }
1150
1151 // Now deduce any function attributes based in the current code.
1152 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1153 PostOrderFunctionAttrsPass()));
1154
1155 // Do RPO function attribute inference across the module to forward-propagate
1156 // attributes where applicable.
1157 // FIXME: Is this really an optimization rather than a canonicalization?
1158 MPM.addPass(ReversePostOrderFunctionAttrsPass());
1159
Eric Christopher721eaef2019-02-26 20:33:22 +00001160 // Use in-range annotations on GEP indices to split globals where beneficial.
Davide Italiano089a9122017-01-24 00:57:39 +00001161 MPM.addPass(GlobalSplitPass());
1162
1163 // Run whole program optimization of virtual call when the list of callees
1164 // is fixed.
Teresa Johnson28023db2018-07-19 14:51:32 +00001165 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
Davide Italiano089a9122017-01-24 00:57:39 +00001166
1167 // Stop here at -O1.
Teresa Johnson28023db2018-07-19 14:51:32 +00001168 if (Level == 1) {
1169 // The LowerTypeTestsPass needs to run to lower type metadata and the
1170 // type.test intrinsics. The pass does nothing if CFI is disabled.
1171 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
Davide Italiano089a9122017-01-24 00:57:39 +00001172 return MPM;
Teresa Johnson28023db2018-07-19 14:51:32 +00001173 }
Davide Italiano089a9122017-01-24 00:57:39 +00001174
1175 // Optimize globals to try and fold them into constants.
1176 MPM.addPass(GlobalOptPass());
1177
1178 // Promote any localized globals to SSA registers.
1179 MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1180
1181 // Linking modules together can lead to duplicate global constant, only
1182 // keep one copy of each constant.
1183 MPM.addPass(ConstantMergePass());
1184
1185 // Remove unused arguments from functions.
1186 MPM.addPass(DeadArgumentEliminationPass());
1187
1188 // Reduce the code after globalopt and ipsccp. Both can open up significant
1189 // simplification opportunities, and both can propagate functions through
1190 // function pointers. When this happens, we often have to resolve varargs
1191 // calls, etc, so let instcombine do this.
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001192 FunctionPassManager PeepholeFPM(DebugLogging);
Amjad Aboudf1f57a32018-01-25 12:06:32 +00001193 if (Level == O3)
1194 PeepholeFPM.addPass(AggressiveInstCombinePass());
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001195 PeepholeFPM.addPass(InstCombinePass());
1196 invokePeepholeEPCallbacks(PeepholeFPM, Level);
1197
1198 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
Davide Italiano089a9122017-01-24 00:57:39 +00001199
1200 // Note: historically, the PruneEH pass was run first to deduce nounwind and
1201 // generally clean up exception handling overhead. It isn't clear this is
1202 // valuable as the inliner doesn't currently care whether it is inlining an
1203 // invoke or a call.
1204 // Run the inliner now.
Easwaran Raman8249fac2017-06-28 13:33:49 +00001205 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1206 InlinerPass(getInlineParamsFromOptLevel(Level))));
Davide Italiano089a9122017-01-24 00:57:39 +00001207
1208 // Optimize globals again after we ran the inliner.
1209 MPM.addPass(GlobalOptPass());
1210
1211 // Garbage collect dead functions.
1212 // FIXME: Add ArgumentPromotion pass after once it's ported.
1213 MPM.addPass(GlobalDCEPass());
1214
1215 FunctionPassManager FPM(DebugLogging);
Davide Italiano089a9122017-01-24 00:57:39 +00001216 // The IPO Passes may leave cruft around. Clean up after them.
Davide Italiano089a9122017-01-24 00:57:39 +00001217 FPM.addPass(InstCombinePass());
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001218 invokePeepholeEPCallbacks(FPM, Level);
1219
Davide Italiano089a9122017-01-24 00:57:39 +00001220 FPM.addPass(JumpThreadingPass());
1221
Rong Xudb29a3a2019-03-04 20:21:27 +00001222 // Do a post inline PGO instrumentation and use pass. This is a context
1223 // sensitive PGO pass.
1224 if (PGOOpt) {
1225 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1226 addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
1227 /* IsCS */ true, PGOOpt->CSProfileGenFile,
1228 PGOOpt->ProfileRemappingFile);
1229 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1230 addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
1231 /* IsCS */ true, PGOOpt->ProfileFile,
1232 PGOOpt->ProfileRemappingFile);
1233 }
1234
Davide Italiano089a9122017-01-24 00:57:39 +00001235 // Break up allocas
1236 FPM.addPass(SROA());
1237
Robert Lougherf2158a82019-03-20 19:08:18 +00001238 // LTO provides additional opportunities for tailcall elimination due to
1239 // link-time inlining, and visibility of nocapture attribute.
1240 FPM.addPass(TailCallElimPass());
1241
Davide Italiano089a9122017-01-24 00:57:39 +00001242 // Run a few AA driver optimizations here and now to cleanup the code.
1243 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1244
1245 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1246 PostOrderFunctionAttrsPass()));
1247 // FIXME: here we run IP alias analysis in the legacy PM.
1248
1249 FunctionPassManager MainFPM;
1250
1251 // FIXME: once we fix LoopPass Manager, add LICM here.
1252 // FIXME: once we provide support for enabling MLSM, add it here.
Davide Italiano8a09b8e2017-05-22 23:41:40 +00001253 if (RunNewGVN)
1254 MainFPM.addPass(NewGVNPass());
1255 else
1256 MainFPM.addPass(GVN());
Davide Italiano089a9122017-01-24 00:57:39 +00001257
1258 // Remove dead memcpy()'s.
1259 MainFPM.addPass(MemCpyOptPass());
1260
1261 // Nuke dead stores.
1262 MainFPM.addPass(DSEPass());
1263
1264 // FIXME: at this point, we run a bunch of loop passes:
Eric Christopher86e2f162019-05-03 00:15:23 +00001265 // indVarSimplify, loopDeletion, loopInterchange, loopUnroll,
Davide Italiano089a9122017-01-24 00:57:39 +00001266 // loopVectorize. Enable them once the remaining issue with LPM
1267 // are sorted out.
1268
1269 MainFPM.addPass(InstCombinePass());
1270 MainFPM.addPass(SimplifyCFGPass());
1271 MainFPM.addPass(SCCPPass());
1272 MainFPM.addPass(InstCombinePass());
1273 MainFPM.addPass(BDCEPass());
1274
1275 // FIXME: We may want to run SLPVectorizer here.
1276 // After vectorization, assume intrinsics may tell us more
1277 // about pointer alignments.
1278#if 0
1279 MainFPM.add(AlignmentFromAssumptionsPass());
1280#endif
1281
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001282 // FIXME: Conditionally run LoadCombine here, after it's ported
1283 // (in case we still have this pass, given its questionable usefulness).
1284
Davide Italiano089a9122017-01-24 00:57:39 +00001285 MainFPM.addPass(InstCombinePass());
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001286 invokePeepholeEPCallbacks(MainFPM, Level);
Davide Italiano089a9122017-01-24 00:57:39 +00001287 MainFPM.addPass(JumpThreadingPass());
1288 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
1289
1290 // Create a function that performs CFI checks for cross-DSO calls with
1291 // targets in the current module.
1292 MPM.addPass(CrossDSOCFIPass());
1293
1294 // Lower type metadata and the type.test intrinsic. This pass supports
1295 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1296 // to be run at link time if CFI is enabled. This pass does nothing if
1297 // CFI is disabled.
Teresa Johnson28023db2018-07-19 14:51:32 +00001298 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
Davide Italiano089a9122017-01-24 00:57:39 +00001299
Vedant Kumar47a0c9b2019-02-15 18:46:44 +00001300 // Enable splitting late in the FullLTO post-link pipeline. This is done in
1301 // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1302 if (EnableHotColdSplit)
1303 MPM.addPass(HotColdSplittingPass());
1304
Davide Italiano089a9122017-01-24 00:57:39 +00001305 // Add late LTO optimization passes.
1306 // Delete basic blocks, which optimization passes may have killed.
1307 MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1308
1309 // Drop bodies of available eternally objects to improve GlobalDCE.
1310 MPM.addPass(EliminateAvailableExternallyPass());
1311
1312 // Now that we have optimized the program, discard unreachable functions.
1313 MPM.addPass(GlobalDCEPass());
1314
Eric Christopher86e2f162019-05-03 00:15:23 +00001315 // FIXME: Maybe enable MergeFuncs conditionally after it's ported.
Chandler Carruthe3f50642016-12-22 06:59:15 +00001316 return MPM;
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001317}
1318
Chandler Carruth060ad612016-12-23 20:38:19 +00001319AAManager PassBuilder::buildDefaultAAPipeline() {
1320 AAManager AA;
1321
1322 // The order in which these are registered determines their priority when
1323 // being queried.
1324
1325 // First we register the basic alias analysis that provides the majority of
1326 // per-function local AA logic. This is a stateless, on-demand local set of
1327 // AA techniques.
1328 AA.registerFunctionAnalysis<BasicAA>();
1329
1330 // Next we query fast, specialized alias analyses that wrap IR-embedded
1331 // information about aliasing.
1332 AA.registerFunctionAnalysis<ScopedNoAliasAA>();
1333 AA.registerFunctionAnalysis<TypeBasedAA>();
1334
1335 // Add support for querying global aliasing information when available.
Chandler Carruth534d6442016-12-24 05:11:17 +00001336 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1337 // analysis, all that the `AAManager` can do is query for any *cached*
Chandler Carruthe87fc8c2017-02-12 05:34:04 +00001338 // results from `GlobalsAA` through a readonly proxy.
Chandler Carruth060ad612016-12-23 20:38:19 +00001339 AA.registerModuleAnalysis<GlobalsAA>();
Chandler Carruth060ad612016-12-23 20:38:19 +00001340
1341 return AA;
1342}
1343
Chandler Carruth241bf242016-08-03 07:44:48 +00001344static Optional<int> parseRepeatPassName(StringRef Name) {
1345 if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
1346 return None;
1347 int Count;
1348 if (Name.getAsInteger(0, Count) || Count <= 0)
1349 return None;
1350 return Count;
1351}
1352
Chandler Carruth05ca5ac2016-12-28 11:07:33 +00001353static Optional<int> parseDevirtPassName(StringRef Name) {
1354 if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
1355 return None;
1356 int Count;
1357 if (Name.getAsInteger(0, Count) || Count <= 0)
1358 return None;
1359 return Count;
1360}
1361
Fedor Sergeevb7871402019-01-10 10:01:53 +00001362static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
1363 if (!Name.consume_front(PassName))
1364 return false;
1365 // normal pass name w/o parameters == default parameters
1366 if (Name.empty())
1367 return true;
1368 return Name.startswith("<") && Name.endswith(">");
1369}
1370
1371namespace {
1372
1373/// This performs customized parsing of pass name with parameters.
1374///
1375/// We do not need parametrization of passes in textual pipeline very often,
1376/// yet on a rare occasion ability to specify parameters right there can be
1377/// useful.
1378///
1379/// \p Name - parameterized specification of a pass from a textual pipeline
1380/// is a string in a form of :
1381/// PassName '<' parameter-list '>'
1382///
1383/// Parameter list is being parsed by the parser callable argument, \p Parser,
1384/// It takes a string-ref of parameters and returns either StringError or a
1385/// parameter list in a form of a custom parameters type, all wrapped into
1386/// Expected<> template class.
1387///
1388template <typename ParametersParseCallableT>
1389auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
1390 StringRef PassName) -> decltype(Parser(StringRef{})) {
1391 using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
1392
1393 StringRef Params = Name;
1394 if (!Params.consume_front(PassName)) {
1395 assert(false &&
1396 "unable to strip pass name from parametrized pass specification");
1397 }
1398 if (Params.empty())
1399 return ParametersT{};
1400 if (!Params.consume_front("<") || !Params.consume_back(">")) {
1401 assert(false && "invalid format for parametrized pass name");
1402 }
1403
1404 Expected<ParametersT> Result = Parser(Params);
1405 assert((Result || Result.template errorIsA<StringError>()) &&
1406 "Pass parameter parser can only return StringErrors.");
1407 return std::move(Result);
1408}
1409
1410/// Parser of parameters for LoopUnroll pass.
1411Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
1412 LoopUnrollOptions UnrollOpts;
1413 while (!Params.empty()) {
1414 StringRef ParamName;
1415 std::tie(ParamName, Params) = Params.split(';');
1416 int OptLevel = StringSwitch<int>(ParamName)
1417 .Case("O0", 0)
1418 .Case("O1", 1)
1419 .Case("O2", 2)
1420 .Case("O3", 3)
1421 .Default(-1);
1422 if (OptLevel >= 0) {
1423 UnrollOpts.setOptLevel(OptLevel);
1424 continue;
1425 }
1426
1427 bool Enable = !ParamName.consume_front("no-");
1428 if (ParamName == "partial") {
1429 UnrollOpts.setPartial(Enable);
1430 } else if (ParamName == "peeling") {
1431 UnrollOpts.setPeeling(Enable);
1432 } else if (ParamName == "runtime") {
1433 UnrollOpts.setRuntime(Enable);
1434 } else if (ParamName == "upperbound") {
1435 UnrollOpts.setUpperBound(Enable);
1436 } else {
1437 return make_error<StringError>(
1438 formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1439 inconvertibleErrorCode());
1440 }
1441 }
1442 return UnrollOpts;
1443}
1444
Philip Pfaffe0ee6a932019-02-04 21:02:49 +00001445Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
1446 MemorySanitizerOptions Result;
1447 while (!Params.empty()) {
1448 StringRef ParamName;
1449 std::tie(ParamName, Params) = Params.split(';');
1450
1451 if (ParamName == "recover") {
1452 Result.Recover = true;
1453 } else if (ParamName == "kernel") {
1454 Result.Kernel = true;
1455 } else if (ParamName.consume_front("track-origins=")) {
1456 if (ParamName.getAsInteger(0, Result.TrackOrigins))
1457 return make_error<StringError>(
1458 formatv("invalid argument to MemorySanitizer pass track-origins "
1459 "parameter: '{0}' ",
1460 ParamName)
1461 .str(),
1462 inconvertibleErrorCode());
1463 } else {
1464 return make_error<StringError>(
1465 formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
1466 .str(),
1467 inconvertibleErrorCode());
1468 }
1469 }
1470 return Result;
1471}
1472
Serguei Katkovf5432832019-04-15 08:57:53 +00001473/// Parser of parameters for SimplifyCFG pass.
1474Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
1475 SimplifyCFGOptions Result;
1476 while (!Params.empty()) {
1477 StringRef ParamName;
1478 std::tie(ParamName, Params) = Params.split(';');
1479
1480 bool Enable = !ParamName.consume_front("no-");
1481 if (ParamName == "forward-switch-cond") {
1482 Result.forwardSwitchCondToPhi(Enable);
1483 } else if (ParamName == "switch-to-lookup") {
1484 Result.convertSwitchToLookupTable(Enable);
1485 } else if (ParamName == "keep-loops") {
1486 Result.needCanonicalLoops(Enable);
1487 } else if (ParamName == "sink-common-insts") {
1488 Result.sinkCommonInsts(Enable);
1489 } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
1490 APInt BonusInstThreshold;
1491 if (ParamName.getAsInteger(0, BonusInstThreshold))
1492 return make_error<StringError>(
1493 formatv("invalid argument to SimplifyCFG pass bonus-threshold "
1494 "parameter: '{0}' ",
1495 ParamName).str(),
1496 inconvertibleErrorCode());
1497 Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
1498 } else {
1499 return make_error<StringError>(
1500 formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
1501 inconvertibleErrorCode());
1502 }
1503 }
1504 return Result;
1505}
1506
Serguei Katkovca6c03a2019-04-18 08:46:11 +00001507/// Parser of parameters for LoopVectorize pass.
1508Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
1509 LoopVectorizeOptions Opts;
1510 while (!Params.empty()) {
1511 StringRef ParamName;
1512 std::tie(ParamName, Params) = Params.split(';');
1513
1514 bool Enable = !ParamName.consume_front("no-");
1515 if (ParamName == "interleave-forced-only") {
1516 Opts.setInterleaveOnlyWhenForced(Enable);
1517 } else if (ParamName == "vectorize-forced-only") {
1518 Opts.setVectorizeOnlyWhenForced(Enable);
1519 } else {
1520 return make_error<StringError>(
1521 formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
1522 inconvertibleErrorCode());
1523 }
1524 }
1525 return Opts;
1526}
1527
Serguei Katkov40a3b962019-04-22 10:35:07 +00001528Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
1529 bool Result = false;
1530 while (!Params.empty()) {
1531 StringRef ParamName;
1532 std::tie(ParamName, Params) = Params.split(';');
1533
1534 bool Enable = !ParamName.consume_front("no-");
1535 if (ParamName == "nontrivial") {
1536 Result = Enable;
1537 } else {
1538 return make_error<StringError>(
1539 formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
1540 .str(),
1541 inconvertibleErrorCode());
1542 }
1543 }
1544 return Result;
1545}
Fedor Sergeevb7871402019-01-10 10:01:53 +00001546} // namespace
1547
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001548/// Tests whether a pass name starts with a valid prefix for a default pipeline
1549/// alias.
1550static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
1551 return Name.startswith("default") || Name.startswith("thinlto") ||
1552 Name.startswith("lto");
1553}
1554
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001555/// Tests whether registered callbacks will accept a given pass name.
1556///
1557/// When parsing a pipeline text, the type of the outermost pipeline may be
1558/// omitted, in which case the type is automatically determined from the first
1559/// pass name in the text. This may be a name that is handled through one of the
1560/// callbacks. We check this through the oridinary parsing callbacks by setting
1561/// up a dummy PassManager in order to not force the client to also handle this
1562/// type of query.
1563template <typename PassManagerT, typename CallbacksT>
1564static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
1565 if (!Callbacks.empty()) {
1566 PassManagerT DummyPM;
1567 for (auto &CB : Callbacks)
1568 if (CB(Name, DummyPM, {}))
1569 return true;
1570 }
1571 return false;
1572}
1573
1574template <typename CallbacksT>
1575static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001576 // Manually handle aliases for pre-configured pipeline fragments.
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001577 if (startsWithDefaultPipelineAliasPrefix(Name))
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001578 return DefaultAliasRegex.match(Name);
1579
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001580 // Explicitly handle pass manager names.
1581 if (Name == "module")
1582 return true;
1583 if (Name == "cgscc")
1584 return true;
1585 if (Name == "function")
1586 return true;
1587
Chandler Carruth241bf242016-08-03 07:44:48 +00001588 // Explicitly handle custom-parsed pass names.
1589 if (parseRepeatPassName(Name))
1590 return true;
1591
Chandler Carruth74a8a222016-06-17 07:15:29 +00001592#define MODULE_PASS(NAME, CREATE_PASS) \
1593 if (Name == NAME) \
1594 return true;
Chandler Carruth628503e2015-01-06 02:10:51 +00001595#define MODULE_ANALYSIS(NAME, CREATE_PASS) \
Chandler Carruth3472ffb2015-01-06 04:49:44 +00001596 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
Chandler Carruth628503e2015-01-06 02:10:51 +00001597 return true;
1598#include "PassRegistry.def"
1599
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001600 return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
Chandler Carruth66445382014-01-11 08:16:35 +00001601}
1602
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001603template <typename CallbacksT>
1604static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001605 // Explicitly handle pass manager names.
1606 if (Name == "cgscc")
1607 return true;
1608 if (Name == "function")
1609 return true;
1610
Chandler Carruth241bf242016-08-03 07:44:48 +00001611 // Explicitly handle custom-parsed pass names.
1612 if (parseRepeatPassName(Name))
1613 return true;
Chandler Carruth05ca5ac2016-12-28 11:07:33 +00001614 if (parseDevirtPassName(Name))
1615 return true;
Chandler Carruth241bf242016-08-03 07:44:48 +00001616
Chandler Carruth74a8a222016-06-17 07:15:29 +00001617#define CGSCC_PASS(NAME, CREATE_PASS) \
1618 if (Name == NAME) \
1619 return true;
Chandler Carruth628503e2015-01-06 02:10:51 +00001620#define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
Chandler Carruth3472ffb2015-01-06 04:49:44 +00001621 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
Chandler Carruth628503e2015-01-06 02:10:51 +00001622 return true;
1623#include "PassRegistry.def"
1624
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001625 return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
Chandler Carruth572e3402014-04-21 11:12:00 +00001626}
1627
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001628template <typename CallbacksT>
1629static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001630 // Explicitly handle pass manager names.
1631 if (Name == "function")
1632 return true;
1633 if (Name == "loop")
1634 return true;
1635
Chandler Carruth241bf242016-08-03 07:44:48 +00001636 // Explicitly handle custom-parsed pass names.
1637 if (parseRepeatPassName(Name))
1638 return true;
1639
Chandler Carruth74a8a222016-06-17 07:15:29 +00001640#define FUNCTION_PASS(NAME, CREATE_PASS) \
1641 if (Name == NAME) \
1642 return true;
Fedor Sergeevb7871402019-01-10 10:01:53 +00001643#define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
1644 if (checkParametrizedPassName(Name, NAME)) \
1645 return true;
Chandler Carruth628503e2015-01-06 02:10:51 +00001646#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
Chandler Carruth3472ffb2015-01-06 04:49:44 +00001647 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
Chandler Carruth628503e2015-01-06 02:10:51 +00001648 return true;
1649#include "PassRegistry.def"
1650
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001651 return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
Chandler Carruthd8330982014-01-12 09:34:22 +00001652}
1653
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001654template <typename CallbacksT>
1655static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001656 // Explicitly handle pass manager names.
1657 if (Name == "loop")
1658 return true;
1659
Chandler Carruth241bf242016-08-03 07:44:48 +00001660 // Explicitly handle custom-parsed pass names.
1661 if (parseRepeatPassName(Name))
1662 return true;
1663
Chandler Carruth74a8a222016-06-17 07:15:29 +00001664#define LOOP_PASS(NAME, CREATE_PASS) \
1665 if (Name == NAME) \
1666 return true;
Serguei Katkov40a3b962019-04-22 10:35:07 +00001667#define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
1668 if (checkParametrizedPassName(Name, NAME)) \
1669 return true;
Justin Bognereecc3c82016-02-25 07:23:08 +00001670#define LOOP_ANALYSIS(NAME, CREATE_PASS) \
1671 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1672 return true;
1673#include "PassRegistry.def"
1674
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001675 return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
Justin Bognereecc3c82016-02-25 07:23:08 +00001676}
1677
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001678Optional<std::vector<PassBuilder::PipelineElement>>
1679PassBuilder::parsePipelineText(StringRef Text) {
1680 std::vector<PipelineElement> ResultPipeline;
1681
1682 SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
1683 &ResultPipeline};
1684 for (;;) {
1685 std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
1686 size_t Pos = Text.find_first_of(",()");
1687 Pipeline.push_back({Text.substr(0, Pos), {}});
1688
1689 // If we have a single terminating name, we're done.
1690 if (Pos == Text.npos)
1691 break;
1692
1693 char Sep = Text[Pos];
1694 Text = Text.substr(Pos + 1);
1695 if (Sep == ',')
1696 // Just a name ending in a comma, continue.
1697 continue;
1698
1699 if (Sep == '(') {
1700 // Push the inner pipeline onto the stack to continue processing.
1701 PipelineStack.push_back(&Pipeline.back().InnerPipeline);
1702 continue;
1703 }
1704
1705 assert(Sep == ')' && "Bogus separator!");
1706 // When handling the close parenthesis, we greedily consume them to avoid
1707 // empty strings in the pipeline.
1708 do {
1709 // If we try to pop the outer pipeline we have unbalanced parentheses.
1710 if (PipelineStack.size() == 1)
1711 return None;
1712
1713 PipelineStack.pop_back();
1714 } while (Text.consume_front(")"));
1715
1716 // Check if we've finished parsing.
1717 if (Text.empty())
1718 break;
1719
1720 // Otherwise, the end of an inner pipeline always has to be followed by
1721 // a comma, and then we can continue.
1722 if (!Text.consume_front(","))
1723 return None;
1724 }
1725
1726 if (PipelineStack.size() > 1)
1727 // Unbalanced paretheses.
1728 return None;
1729
1730 assert(PipelineStack.back() == &ResultPipeline &&
1731 "Wrong pipeline at the bottom of the stack!");
1732 return {std::move(ResultPipeline)};
1733}
1734
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001735Error PassBuilder::parseModulePass(ModulePassManager &MPM,
1736 const PipelineElement &E,
1737 bool VerifyEachPass, bool DebugLogging) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001738 auto &Name = E.Name;
1739 auto &InnerPipeline = E.InnerPipeline;
1740
1741 // First handle complex passes like the pass managers which carry pipelines.
1742 if (!InnerPipeline.empty()) {
1743 if (Name == "module") {
1744 ModulePassManager NestedMPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001745 if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1746 VerifyEachPass, DebugLogging))
1747 return Err;
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001748 MPM.addPass(std::move(NestedMPM));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001749 return Error::success();
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001750 }
1751 if (Name == "cgscc") {
1752 CGSCCPassManager CGPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001753 if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass,
1754 DebugLogging))
1755 return Err;
Chandler Carruth19913b22017-08-11 05:47:13 +00001756 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001757 return Error::success();
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001758 }
1759 if (Name == "function") {
1760 FunctionPassManager FPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001761 if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1762 VerifyEachPass, DebugLogging))
1763 return Err;
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001764 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001765 return Error::success();
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001766 }
Chandler Carruth241bf242016-08-03 07:44:48 +00001767 if (auto Count = parseRepeatPassName(Name)) {
1768 ModulePassManager NestedMPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001769 if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1770 VerifyEachPass, DebugLogging))
1771 return Err;
Chandler Carrutha053a882016-08-04 03:52:53 +00001772 MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001773 return Error::success();
Chandler Carruth241bf242016-08-03 07:44:48 +00001774 }
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001775
1776 for (auto &C : ModulePipelineParsingCallbacks)
1777 if (C(Name, MPM, InnerPipeline))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001778 return Error::success();
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001779
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001780 // Normal passes can't have pipelines.
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001781 return make_error<StringError>(
1782 formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
1783 inconvertibleErrorCode());
1784 ;
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001785 }
1786
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001787 // Manually handle aliases for pre-configured pipeline fragments.
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001788 if (startsWithDefaultPipelineAliasPrefix(Name)) {
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001789 SmallVector<StringRef, 3> Matches;
1790 if (!DefaultAliasRegex.match(Name, &Matches))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001791 return make_error<StringError>(
1792 formatv("unknown default pipeline alias '{0}'", Name).str(),
1793 inconvertibleErrorCode());
1794
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001795 assert(Matches.size() == 3 && "Must capture two matched strings!");
1796
Jordan Rosef85a95f2016-07-25 18:34:51 +00001797 OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001798 .Case("O0", O0)
1799 .Case("O1", O1)
1800 .Case("O2", O2)
1801 .Case("O3", O3)
1802 .Case("Os", Os)
1803 .Case("Oz", Oz);
Chandler Carruthe3f50642016-12-22 06:59:15 +00001804 if (L == O0)
1805 // At O0 we do nothing at all!
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001806 return Error::success();
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001807
1808 if (Matches[1] == "default") {
Chandler Carruthe3f50642016-12-22 06:59:15 +00001809 MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging));
Chandler Carruth8b3be4e2017-06-01 11:39:39 +00001810 } else if (Matches[1] == "thinlto-pre-link") {
1811 MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging));
1812 } else if (Matches[1] == "thinlto") {
Teresa Johnson28023db2018-07-19 14:51:32 +00001813 MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr));
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001814 } else if (Matches[1] == "lto-pre-link") {
Chandler Carruthe3f50642016-12-22 06:59:15 +00001815 MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging));
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001816 } else {
1817 assert(Matches[1] == "lto" && "Not one of the matched options!");
Teresa Johnson28023db2018-07-19 14:51:32 +00001818 MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr));
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001819 }
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001820 return Error::success();
Chandler Carruth8b5a74192016-02-28 22:16:03 +00001821 }
1822
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001823 // Finally expand the basic registered passes from the .inc file.
Chandler Carruth58944182014-04-21 08:08:50 +00001824#define MODULE_PASS(NAME, CREATE_PASS) \
1825 if (Name == NAME) { \
1826 MPM.addPass(CREATE_PASS); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001827 return Error::success(); \
Chandler Carruth52eef882014-01-12 12:15:39 +00001828 }
Chandler Carruth628503e2015-01-06 02:10:51 +00001829#define MODULE_ANALYSIS(NAME, CREATE_PASS) \
1830 if (Name == "require<" NAME ">") { \
Chandler Carruth9b35e6d2016-08-19 18:36:06 +00001831 MPM.addPass( \
1832 RequireAnalysisPass< \
1833 std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001834 return Error::success(); \
Chandler Carruth3472ffb2015-01-06 04:49:44 +00001835 } \
1836 if (Name == "invalidate<" NAME ">") { \
Chandler Carruth470734b2016-02-26 12:30:18 +00001837 MPM.addPass(InvalidateAnalysisPass< \
1838 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001839 return Error::success(); \
Chandler Carruth628503e2015-01-06 02:10:51 +00001840 }
1841#include "PassRegistry.def"
1842
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001843 for (auto &C : ModulePipelineParsingCallbacks)
1844 if (C(Name, MPM, InnerPipeline))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001845 return Error::success();
1846 return make_error<StringError>(
1847 formatv("unknown module pass '{0}'", Name).str(),
1848 inconvertibleErrorCode());
Chandler Carruth66445382014-01-11 08:16:35 +00001849}
1850
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001851Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
1852 const PipelineElement &E, bool VerifyEachPass,
1853 bool DebugLogging) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001854 auto &Name = E.Name;
1855 auto &InnerPipeline = E.InnerPipeline;
1856
1857 // First handle complex passes like the pass managers which carry pipelines.
1858 if (!InnerPipeline.empty()) {
1859 if (Name == "cgscc") {
1860 CGSCCPassManager NestedCGPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001861 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1862 VerifyEachPass, DebugLogging))
1863 return Err;
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001864 // Add the nested pass manager with the appropriate adaptor.
1865 CGPM.addPass(std::move(NestedCGPM));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001866 return Error::success();
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001867 }
1868 if (Name == "function") {
1869 FunctionPassManager FPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001870 if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1871 VerifyEachPass, DebugLogging))
1872 return Err;
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001873 // Add the nested pass manager with the appropriate adaptor.
Chandler Carruth19913b22017-08-11 05:47:13 +00001874 CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001875 return Error::success();
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001876 }
Chandler Carruth241bf242016-08-03 07:44:48 +00001877 if (auto Count = parseRepeatPassName(Name)) {
1878 CGSCCPassManager NestedCGPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001879 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1880 VerifyEachPass, DebugLogging))
1881 return Err;
Chandler Carrutha053a882016-08-04 03:52:53 +00001882 CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001883 return Error::success();
Chandler Carruth241bf242016-08-03 07:44:48 +00001884 }
Chandler Carruth05ca5ac2016-12-28 11:07:33 +00001885 if (auto MaxRepetitions = parseDevirtPassName(Name)) {
1886 CGSCCPassManager NestedCGPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001887 if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1888 VerifyEachPass, DebugLogging))
1889 return Err;
Chandler Carruth19913b22017-08-11 05:47:13 +00001890 CGPM.addPass(
1891 createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001892 return Error::success();
Chandler Carruth05ca5ac2016-12-28 11:07:33 +00001893 }
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001894
1895 for (auto &C : CGSCCPipelineParsingCallbacks)
1896 if (C(Name, CGPM, InnerPipeline))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001897 return Error::success();
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001898
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001899 // Normal passes can't have pipelines.
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001900 return make_error<StringError>(
1901 formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
1902 inconvertibleErrorCode());
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001903 }
1904
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001905// Now expand the basic registered passes from the .inc file.
Chandler Carruth572e3402014-04-21 11:12:00 +00001906#define CGSCC_PASS(NAME, CREATE_PASS) \
1907 if (Name == NAME) { \
1908 CGPM.addPass(CREATE_PASS); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001909 return Error::success(); \
Chandler Carruth572e3402014-04-21 11:12:00 +00001910 }
Chandler Carruth628503e2015-01-06 02:10:51 +00001911#define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
1912 if (Name == "require<" NAME ">") { \
Chandler Carruth470734b2016-02-26 12:30:18 +00001913 CGPM.addPass(RequireAnalysisPass< \
Chandler Carruth9b35e6d2016-08-19 18:36:06 +00001914 std::remove_reference<decltype(CREATE_PASS)>::type, \
Chandler Carruth88823462016-08-24 09:37:14 +00001915 LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \
1916 CGSCCUpdateResult &>()); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001917 return Error::success(); \
Chandler Carruth3472ffb2015-01-06 04:49:44 +00001918 } \
1919 if (Name == "invalidate<" NAME ">") { \
Chandler Carruth470734b2016-02-26 12:30:18 +00001920 CGPM.addPass(InvalidateAnalysisPass< \
1921 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001922 return Error::success(); \
Chandler Carruth628503e2015-01-06 02:10:51 +00001923 }
1924#include "PassRegistry.def"
1925
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001926 for (auto &C : CGSCCPipelineParsingCallbacks)
1927 if (C(Name, CGPM, InnerPipeline))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001928 return Error::success();
1929 return make_error<StringError>(
1930 formatv("unknown cgscc pass '{0}'", Name).str(),
1931 inconvertibleErrorCode());
Chandler Carruth572e3402014-04-21 11:12:00 +00001932}
1933
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001934Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
1935 const PipelineElement &E,
1936 bool VerifyEachPass, bool DebugLogging) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001937 auto &Name = E.Name;
1938 auto &InnerPipeline = E.InnerPipeline;
1939
1940 // First handle complex passes like the pass managers which carry pipelines.
1941 if (!InnerPipeline.empty()) {
1942 if (Name == "function") {
1943 FunctionPassManager NestedFPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001944 if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
1945 VerifyEachPass, DebugLogging))
1946 return Err;
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001947 // Add the nested pass manager with the appropriate adaptor.
1948 FPM.addPass(std::move(NestedFPM));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001949 return Error::success();
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001950 }
1951 if (Name == "loop") {
1952 LoopPassManager LPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001953 if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass,
1954 DebugLogging))
1955 return Err;
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001956 // Add the nested pass manager with the appropriate adaptor.
Fedor Sergeev02e7f022017-12-29 08:16:06 +00001957 FPM.addPass(
1958 createFunctionToLoopPassAdaptor(std::move(LPM), DebugLogging));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001959 return Error::success();
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001960 }
Chandler Carruth241bf242016-08-03 07:44:48 +00001961 if (auto Count = parseRepeatPassName(Name)) {
1962 FunctionPassManager NestedFPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001963 if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
1964 VerifyEachPass, DebugLogging))
1965 return Err;
Chandler Carrutha053a882016-08-04 03:52:53 +00001966 FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001967 return Error::success();
Chandler Carruth241bf242016-08-03 07:44:48 +00001968 }
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001969
1970 for (auto &C : FunctionPipelineParsingCallbacks)
1971 if (C(Name, FPM, InnerPipeline))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001972 return Error::success();
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001973
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001974 // Normal passes can't have pipelines.
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001975 return make_error<StringError>(
1976 formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
1977 inconvertibleErrorCode());
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00001978 }
1979
Philip Pfaffe730f2f92017-07-10 10:57:55 +00001980// Now expand the basic registered passes from the .inc file.
Chandler Carruth58944182014-04-21 08:08:50 +00001981#define FUNCTION_PASS(NAME, CREATE_PASS) \
1982 if (Name == NAME) { \
1983 FPM.addPass(CREATE_PASS); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001984 return Error::success(); \
Chandler Carruth52eef882014-01-12 12:15:39 +00001985 }
Fedor Sergeevb7871402019-01-10 10:01:53 +00001986#define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
1987 if (checkParametrizedPassName(Name, NAME)) { \
1988 auto Params = parsePassParameters(PARSER, Name, NAME); \
1989 if (!Params) \
1990 return Params.takeError(); \
1991 FPM.addPass(CREATE_PASS(Params.get())); \
1992 return Error::success(); \
1993 }
Chandler Carruth628503e2015-01-06 02:10:51 +00001994#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
1995 if (Name == "require<" NAME ">") { \
Chandler Carruth9b35e6d2016-08-19 18:36:06 +00001996 FPM.addPass( \
1997 RequireAnalysisPass< \
1998 std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00001999 return Error::success(); \
Chandler Carruth3472ffb2015-01-06 04:49:44 +00002000 } \
2001 if (Name == "invalidate<" NAME ">") { \
Chandler Carruth470734b2016-02-26 12:30:18 +00002002 FPM.addPass(InvalidateAnalysisPass< \
2003 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002004 return Error::success(); \
Chandler Carruth628503e2015-01-06 02:10:51 +00002005 }
2006#include "PassRegistry.def"
2007
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002008 for (auto &C : FunctionPipelineParsingCallbacks)
2009 if (C(Name, FPM, InnerPipeline))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002010 return Error::success();
2011 return make_error<StringError>(
2012 formatv("unknown function pass '{0}'", Name).str(),
2013 inconvertibleErrorCode());
Chandler Carruthd8330982014-01-12 09:34:22 +00002014}
2015
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002016Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E,
2017 bool VerifyEachPass, bool DebugLogging) {
Chandler Carruth241bf242016-08-03 07:44:48 +00002018 StringRef Name = E.Name;
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002019 auto &InnerPipeline = E.InnerPipeline;
2020
2021 // First handle complex passes like the pass managers which carry pipelines.
2022 if (!InnerPipeline.empty()) {
2023 if (Name == "loop") {
2024 LoopPassManager NestedLPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002025 if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2026 VerifyEachPass, DebugLogging))
2027 return Err;
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002028 // Add the nested pass manager with the appropriate adaptor.
Chandler Carruthfdc6ba12016-08-03 09:14:03 +00002029 LPM.addPass(std::move(NestedLPM));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002030 return Error::success();
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002031 }
Chandler Carruth241bf242016-08-03 07:44:48 +00002032 if (auto Count = parseRepeatPassName(Name)) {
2033 LoopPassManager NestedLPM(DebugLogging);
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002034 if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2035 VerifyEachPass, DebugLogging))
2036 return Err;
Chandler Carrutha053a882016-08-04 03:52:53 +00002037 LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002038 return Error::success();
Chandler Carruth241bf242016-08-03 07:44:48 +00002039 }
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002040
2041 for (auto &C : LoopPipelineParsingCallbacks)
2042 if (C(Name, LPM, InnerPipeline))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002043 return Error::success();
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002044
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002045 // Normal passes can't have pipelines.
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002046 return make_error<StringError>(
2047 formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2048 inconvertibleErrorCode());
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002049 }
2050
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002051// Now expand the basic registered passes from the .inc file.
Justin Bognereecc3c82016-02-25 07:23:08 +00002052#define LOOP_PASS(NAME, CREATE_PASS) \
2053 if (Name == NAME) { \
Chandler Carruthfdc6ba12016-08-03 09:14:03 +00002054 LPM.addPass(CREATE_PASS); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002055 return Error::success(); \
Justin Bognereecc3c82016-02-25 07:23:08 +00002056 }
Serguei Katkov40a3b962019-04-22 10:35:07 +00002057#define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2058 if (checkParametrizedPassName(Name, NAME)) { \
2059 auto Params = parsePassParameters(PARSER, Name, NAME); \
2060 if (!Params) \
2061 return Params.takeError(); \
2062 LPM.addPass(CREATE_PASS(Params.get())); \
2063 return Error::success(); \
2064 }
Justin Bognereecc3c82016-02-25 07:23:08 +00002065#define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2066 if (Name == "require<" NAME ">") { \
Chandler Carruthfdc6ba12016-08-03 09:14:03 +00002067 LPM.addPass(RequireAnalysisPass< \
Chandler Carruth410eaeb2017-01-11 06:23:21 +00002068 std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \
2069 LoopAnalysisManager, LoopStandardAnalysisResults &, \
2070 LPMUpdater &>()); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002071 return Error::success(); \
Justin Bognereecc3c82016-02-25 07:23:08 +00002072 } \
2073 if (Name == "invalidate<" NAME ">") { \
Chandler Carruthfdc6ba12016-08-03 09:14:03 +00002074 LPM.addPass(InvalidateAnalysisPass< \
Chandler Carruth470734b2016-02-26 12:30:18 +00002075 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002076 return Error::success(); \
Justin Bognereecc3c82016-02-25 07:23:08 +00002077 }
2078#include "PassRegistry.def"
2079
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002080 for (auto &C : LoopPipelineParsingCallbacks)
2081 if (C(Name, LPM, InnerPipeline))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002082 return Error::success();
2083 return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2084 inconvertibleErrorCode());
Justin Bognereecc3c82016-02-25 07:23:08 +00002085}
2086
Chandler Carruthedf59962016-02-18 09:45:17 +00002087bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
Chandler Carruth45a9c202016-03-11 09:15:11 +00002088#define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2089 if (Name == NAME) { \
2090 AA.registerModuleAnalysis< \
2091 std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2092 return true; \
2093 }
Chandler Carruthedf59962016-02-18 09:45:17 +00002094#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2095 if (Name == NAME) { \
Chandler Carruth58dde8c2016-02-26 12:17:54 +00002096 AA.registerFunctionAnalysis< \
2097 std::remove_reference<decltype(CREATE_PASS)>::type>(); \
Chandler Carruthedf59962016-02-18 09:45:17 +00002098 return true; \
2099 }
2100#include "PassRegistry.def"
2101
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002102 for (auto &C : AAParsingCallbacks)
2103 if (C(Name, AA))
2104 return true;
Chandler Carruthedf59962016-02-18 09:45:17 +00002105 return false;
2106}
2107
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002108Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002109 ArrayRef<PipelineElement> Pipeline,
Chandler Carruth1ff77242015-03-07 09:02:36 +00002110 bool VerifyEachPass,
2111 bool DebugLogging) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002112 for (const auto &Element : Pipeline) {
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002113 if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging))
2114 return Err;
2115 // FIXME: No verifier support for Loop passes!
2116 }
2117 return Error::success();
2118}
2119
2120Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM,
2121 ArrayRef<PipelineElement> Pipeline,
2122 bool VerifyEachPass,
2123 bool DebugLogging) {
2124 for (const auto &Element : Pipeline) {
2125 if (auto Err =
2126 parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging))
2127 return Err;
2128 if (VerifyEachPass)
2129 FPM.addPass(VerifierPass());
2130 }
2131 return Error::success();
2132}
2133
2134Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2135 ArrayRef<PipelineElement> Pipeline,
2136 bool VerifyEachPass,
2137 bool DebugLogging) {
2138 for (const auto &Element : Pipeline) {
2139 if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging))
2140 return Err;
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002141 // FIXME: No verifier support for CGSCC passes!
Chandler Carruth572e3402014-04-21 11:12:00 +00002142 }
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002143 return Error::success();
Chandler Carruth572e3402014-04-21 11:12:00 +00002144}
2145
NAKAMURA Takumia42f5282016-05-16 10:13:37 +00002146void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
2147 FunctionAnalysisManager &FAM,
2148 CGSCCAnalysisManager &CGAM,
2149 ModuleAnalysisManager &MAM) {
2150 MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2151 MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
NAKAMURA Takumia42f5282016-05-16 10:13:37 +00002152 CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2153 FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2154 FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2155 FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2156 LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2157}
2158
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002159Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2160 ArrayRef<PipelineElement> Pipeline,
2161 bool VerifyEachPass,
2162 bool DebugLogging) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002163 for (const auto &Element : Pipeline) {
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002164 if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging))
2165 return Err;
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002166 if (VerifyEachPass)
2167 MPM.addPass(VerifierPass());
Chandler Carruth66445382014-01-11 08:16:35 +00002168 }
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002169 return Error::success();
Chandler Carruth66445382014-01-11 08:16:35 +00002170}
2171
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002172// Primary pass pipeline description parsing routine for a \c ModulePassManager
Chandler Carruth66445382014-01-11 08:16:35 +00002173// FIXME: Should this routine accept a TargetMachine or require the caller to
2174// pre-populate the analysis managers with target-specific stuff?
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002175Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
2176 StringRef PipelineText,
2177 bool VerifyEachPass, bool DebugLogging) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002178 auto Pipeline = parsePipelineText(PipelineText);
2179 if (!Pipeline || Pipeline->empty())
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002180 return make_error<StringError>(
2181 formatv("invalid pipeline '{0}'", PipelineText).str(),
2182 inconvertibleErrorCode());
Chandler Carruth66445382014-01-11 08:16:35 +00002183
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002184 // If the first name isn't at the module layer, wrap the pipeline up
2185 // automatically.
2186 StringRef FirstName = Pipeline->front().Name;
Chandler Carruth66445382014-01-11 08:16:35 +00002187
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002188 if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
2189 if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002190 Pipeline = {{"cgscc", std::move(*Pipeline)}};
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002191 } else if (isFunctionPassName(FirstName,
2192 FunctionPipelineParsingCallbacks)) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002193 Pipeline = {{"function", std::move(*Pipeline)}};
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002194 } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
Chandler Carruth6cb2ab22016-08-03 03:21:41 +00002195 Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002196 } else {
2197 for (auto &C : TopLevelPipelineParsingCallbacks)
2198 if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002199 return Error::success();
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002200
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002201 // Unknown pass or pipeline name!
2202 auto &InnerPipeline = Pipeline->front().InnerPipeline;
2203 return make_error<StringError>(
2204 formatv("unknown {0} name '{1}'",
2205 (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
2206 .str(),
2207 inconvertibleErrorCode());
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002208 }
Chandler Carruth572e3402014-04-21 11:12:00 +00002209 }
2210
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002211 if (auto Err =
2212 parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2213 return Err;
2214 return Error::success();
Chandler Carruth66445382014-01-11 08:16:35 +00002215}
Chandler Carruthedf59962016-02-18 09:45:17 +00002216
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002217// Primary pass pipeline description parsing routine for a \c CGSCCPassManager
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002218Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
2219 StringRef PipelineText,
2220 bool VerifyEachPass, bool DebugLogging) {
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002221 auto Pipeline = parsePipelineText(PipelineText);
2222 if (!Pipeline || Pipeline->empty())
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002223 return make_error<StringError>(
2224 formatv("invalid pipeline '{0}'", PipelineText).str(),
2225 inconvertibleErrorCode());
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002226
2227 StringRef FirstName = Pipeline->front().Name;
2228 if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002229 return make_error<StringError>(
2230 formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
2231 PipelineText)
2232 .str(),
2233 inconvertibleErrorCode());
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002234
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002235 if (auto Err =
2236 parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2237 return Err;
2238 return Error::success();
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002239}
2240
2241// Primary pass pipeline description parsing routine for a \c
2242// FunctionPassManager
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002243Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
2244 StringRef PipelineText,
2245 bool VerifyEachPass, bool DebugLogging) {
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002246 auto Pipeline = parsePipelineText(PipelineText);
2247 if (!Pipeline || Pipeline->empty())
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002248 return make_error<StringError>(
2249 formatv("invalid pipeline '{0}'", PipelineText).str(),
2250 inconvertibleErrorCode());
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002251
2252 StringRef FirstName = Pipeline->front().Name;
2253 if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002254 return make_error<StringError>(
2255 formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
2256 PipelineText)
2257 .str(),
2258 inconvertibleErrorCode());
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002259
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002260 if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass,
2261 DebugLogging))
2262 return Err;
2263 return Error::success();
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002264}
2265
2266// Primary pass pipeline description parsing routine for a \c LoopPassManager
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002267Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
2268 StringRef PipelineText,
2269 bool VerifyEachPass, bool DebugLogging) {
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002270 auto Pipeline = parsePipelineText(PipelineText);
2271 if (!Pipeline || Pipeline->empty())
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002272 return make_error<StringError>(
2273 formatv("invalid pipeline '{0}'", PipelineText).str(),
2274 inconvertibleErrorCode());
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002275
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002276 if (auto Err =
2277 parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2278 return Err;
2279
2280 return Error::success();
Philip Pfaffe730f2f92017-07-10 10:57:55 +00002281}
2282
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002283Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
Chandler Carruth060ad612016-12-23 20:38:19 +00002284 // If the pipeline just consists of the word 'default' just replace the AA
2285 // manager with our default one.
2286 if (PipelineText == "default") {
2287 AA = buildDefaultAAPipeline();
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002288 return Error::success();
Chandler Carruth060ad612016-12-23 20:38:19 +00002289 }
2290
Chandler Carruthedf59962016-02-18 09:45:17 +00002291 while (!PipelineText.empty()) {
2292 StringRef Name;
2293 std::tie(Name, PipelineText) = PipelineText.split(',');
2294 if (!parseAAPassName(AA, Name))
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002295 return make_error<StringError>(
2296 formatv("unknown alias analysis name '{0}'", Name).str(),
2297 inconvertibleErrorCode());
Chandler Carruthedf59962016-02-18 09:45:17 +00002298 }
2299
Fedor Sergeevbd6b2132018-10-17 10:36:23 +00002300 return Error::success();
Chandler Carruthedf59962016-02-18 09:45:17 +00002301}