blob: d54fb700200d4222fc59b404c3bc1b313fd018cf [file] [log] [blame]
Teresa Johnson2d5487c2016-04-11 13:58:45 +00001//===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass builds a ModuleSummaryIndex object for the module, to be written
11// to bitcode or LLVM assembly.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Analysis/ModuleSummaryAnalysis.h"
Eugene Zelenkobb1b2d02017-08-16 22:07:40 +000016#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/DenseSet.h"
Peter Collingbourne0c30f082016-12-20 21:12:28 +000018#include "llvm/ADT/MapVector.h"
Eugene Zelenkobb1b2d02017-08-16 22:07:40 +000019#include "llvm/ADT/STLExtras.h"
Peter Collingbourne0c30f082016-12-20 21:12:28 +000020#include "llvm/ADT/SetVector.h"
Eugene Zelenkobb1b2d02017-08-16 22:07:40 +000021#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/ADT/StringRef.h"
Teresa Johnson2d5487c2016-04-11 13:58:45 +000024#include "llvm/Analysis/BlockFrequencyInfo.h"
Teresa Johnson2d5487c2016-04-11 13:58:45 +000025#include "llvm/Analysis/BranchProbabilityInfo.h"
Teresa Johnsoncd21a642016-07-17 14:47:01 +000026#include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
Teresa Johnson2d5487c2016-04-11 13:58:45 +000027#include "llvm/Analysis/LoopInfo.h"
Piotr Padlewskid9830eb2016-09-26 20:37:32 +000028#include "llvm/Analysis/ProfileSummaryInfo.h"
Peter Collingbourne1b4137a72016-12-21 23:03:45 +000029#include "llvm/Analysis/TypeMetadataUtils.h"
Eugene Zelenkobb1b2d02017-08-16 22:07:40 +000030#include "llvm/IR/Attributes.h"
31#include "llvm/IR/BasicBlock.h"
Teresa Johnson2d5487c2016-04-11 13:58:45 +000032#include "llvm/IR/CallSite.h"
Eugene Zelenkobb1b2d02017-08-16 22:07:40 +000033#include "llvm/IR/Constant.h"
34#include "llvm/IR/Constants.h"
Teresa Johnson2d5487c2016-04-11 13:58:45 +000035#include "llvm/IR/Dominators.h"
Eugene Zelenkobb1b2d02017-08-16 22:07:40 +000036#include "llvm/IR/Function.h"
37#include "llvm/IR/GlobalAlias.h"
38#include "llvm/IR/GlobalValue.h"
39#include "llvm/IR/GlobalVariable.h"
40#include "llvm/IR/Instructions.h"
Teresa Johnson2d5487c2016-04-11 13:58:45 +000041#include "llvm/IR/IntrinsicInst.h"
Eugene Zelenkobb1b2d02017-08-16 22:07:40 +000042#include "llvm/IR/Intrinsics.h"
43#include "llvm/IR/Metadata.h"
44#include "llvm/IR/Module.h"
45#include "llvm/IR/ModuleSummaryIndex.h"
46#include "llvm/IR/Use.h"
47#include "llvm/IR/User.h"
Peter Collingbourne61781ac2017-03-31 00:08:24 +000048#include "llvm/Object/ModuleSymbolTable.h"
Eugene Zelenkobb1b2d02017-08-16 22:07:40 +000049#include "llvm/Object/SymbolicFile.h"
Teresa Johnson2d5487c2016-04-11 13:58:45 +000050#include "llvm/Pass.h"
Eugene Zelenkobb1b2d02017-08-16 22:07:40 +000051#include "llvm/Support/Casting.h"
52#include <algorithm>
53#include <cassert>
54#include <cstdint>
55#include <vector>
56
Teresa Johnson2d5487c2016-04-11 13:58:45 +000057using namespace llvm;
58
59#define DEBUG_TYPE "module-summary-analysis"
60
61// Walk through the operands of a given User via worklist iteration and populate
62// the set of GlobalValue references encountered. Invoked either on an
63// Instruction or a GlobalVariable (which walks its initializer).
Peter Collingbourne9667b912017-05-04 18:03:25 +000064static void findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
65 SetVector<ValueInfo> &RefEdges,
Teresa Johnson2d5487c2016-04-11 13:58:45 +000066 SmallPtrSet<const User *, 8> &Visited) {
67 SmallVector<const User *, 32> Worklist;
68 Worklist.push_back(CurUser);
69
70 while (!Worklist.empty()) {
71 const User *U = Worklist.pop_back_val();
72
73 if (!Visited.insert(U).second)
74 continue;
75
76 ImmutableCallSite CS(U);
77
78 for (const auto &OI : U->operands()) {
79 const User *Operand = dyn_cast<User>(OI);
80 if (!Operand)
81 continue;
82 if (isa<BlockAddress>(Operand))
83 continue;
Peter Collingbourne0c30f082016-12-20 21:12:28 +000084 if (auto *GV = dyn_cast<GlobalValue>(Operand)) {
Teresa Johnson2d5487c2016-04-11 13:58:45 +000085 // We have a reference to a global value. This should be added to
86 // the reference set unless it is a callee. Callees are handled
87 // specially by WriteFunction and are added to a separate list.
88 if (!(CS && CS.isCallee(&OI)))
Peter Collingbourne9667b912017-05-04 18:03:25 +000089 RefEdges.insert(Index.getOrInsertValueInfo(GV));
Teresa Johnson2d5487c2016-04-11 13:58:45 +000090 continue;
91 }
92 Worklist.push_back(Operand);
93 }
94 }
95}
96
Piotr Padlewskid9830eb2016-09-26 20:37:32 +000097static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount,
98 ProfileSummaryInfo *PSI) {
99 if (!PSI)
100 return CalleeInfo::HotnessType::Unknown;
101 if (PSI->isHotCount(ProfileCount))
102 return CalleeInfo::HotnessType::Hot;
103 if (PSI->isColdCount(ProfileCount))
104 return CalleeInfo::HotnessType::Cold;
105 return CalleeInfo::HotnessType::None;
106}
107
Teresa Johnson519465b2017-01-05 14:32:16 +0000108static bool isNonRenamableLocal(const GlobalValue &GV) {
109 return GV.hasSection() && GV.hasLocalLinkage();
110}
111
Peter Collingbournebe9ffaa2017-02-10 22:29:38 +0000112/// Determine whether this call has all constant integer arguments (excluding
113/// "this") and summarize it to VCalls or ConstVCalls as appropriate.
114static void addVCallToSet(DevirtCallSite Call, GlobalValue::GUID Guid,
115 SetVector<FunctionSummary::VFuncId> &VCalls,
116 SetVector<FunctionSummary::ConstVCall> &ConstVCalls) {
117 std::vector<uint64_t> Args;
118 // Start from the second argument to skip the "this" pointer.
119 for (auto &Arg : make_range(Call.CS.arg_begin() + 1, Call.CS.arg_end())) {
120 auto *CI = dyn_cast<ConstantInt>(Arg);
121 if (!CI || CI->getBitWidth() > 64) {
122 VCalls.insert({Guid, Call.Offset});
123 return;
124 }
125 Args.push_back(CI->getZExtValue());
126 }
127 ConstVCalls.insert({{Guid, Call.Offset}, std::move(Args)});
128}
129
130/// If this intrinsic call requires that we add information to the function
131/// summary, do so via the non-constant reference arguments.
132static void addIntrinsicToSummary(
133 const CallInst *CI, SetVector<GlobalValue::GUID> &TypeTests,
134 SetVector<FunctionSummary::VFuncId> &TypeTestAssumeVCalls,
135 SetVector<FunctionSummary::VFuncId> &TypeCheckedLoadVCalls,
136 SetVector<FunctionSummary::ConstVCall> &TypeTestAssumeConstVCalls,
137 SetVector<FunctionSummary::ConstVCall> &TypeCheckedLoadConstVCalls) {
138 switch (CI->getCalledFunction()->getIntrinsicID()) {
139 case Intrinsic::type_test: {
140 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
141 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
142 if (!TypeId)
143 break;
144 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
145
146 // Produce a summary from type.test intrinsics. We only summarize type.test
147 // intrinsics that are used other than by an llvm.assume intrinsic.
148 // Intrinsics that are assumed are relevant only to the devirtualization
149 // pass, not the type test lowering pass.
150 bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
151 auto *AssumeCI = dyn_cast<CallInst>(CIU.getUser());
152 if (!AssumeCI)
153 return true;
154 Function *F = AssumeCI->getCalledFunction();
155 return !F || F->getIntrinsicID() != Intrinsic::assume;
156 });
157 if (HasNonAssumeUses)
158 TypeTests.insert(Guid);
159
160 SmallVector<DevirtCallSite, 4> DevirtCalls;
161 SmallVector<CallInst *, 4> Assumes;
162 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI);
163 for (auto &Call : DevirtCalls)
164 addVCallToSet(Call, Guid, TypeTestAssumeVCalls,
165 TypeTestAssumeConstVCalls);
166
167 break;
168 }
169
170 case Intrinsic::type_checked_load: {
171 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
172 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
173 if (!TypeId)
174 break;
175 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
176
177 SmallVector<DevirtCallSite, 4> DevirtCalls;
178 SmallVector<Instruction *, 4> LoadedPtrs;
179 SmallVector<Instruction *, 4> Preds;
180 bool HasNonCallUses = false;
181 findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
182 HasNonCallUses, CI);
183 // Any non-call uses of the result of llvm.type.checked.load will
184 // prevent us from optimizing away the llvm.type.test.
185 if (HasNonCallUses)
186 TypeTests.insert(Guid);
187 for (auto &Call : DevirtCalls)
188 addVCallToSet(Call, Guid, TypeCheckedLoadVCalls,
189 TypeCheckedLoadConstVCalls);
190
191 break;
192 }
193 default:
194 break;
195 }
196}
197
Teresa Johnson519465b2017-01-05 14:32:16 +0000198static void
199computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
200 const Function &F, BlockFrequencyInfo *BFI,
Peter Collingbourne5e8b94c12017-09-01 16:24:02 +0000201 ProfileSummaryInfo *PSI, bool HasLocalsInUsedOrAsm,
Teresa Johnsone27b0582017-01-05 14:59:56 +0000202 DenseSet<GlobalValue::GUID> &CantBePromoted) {
Teresa Johnson02563cd2016-10-28 02:39:38 +0000203 // Summary not currently supported for anonymous functions, they should
204 // have been named.
205 assert(F.hasName());
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000206
207 unsigned NumInsts = 0;
208 // Map from callee ValueId to profile count. Used to accumulate profile
209 // counts for all static calls to a given callee.
Peter Collingbourne0c30f082016-12-20 21:12:28 +0000210 MapVector<ValueInfo, CalleeInfo> CallGraphEdges;
211 SetVector<ValueInfo> RefEdges;
Peter Collingbourne1b4137a72016-12-21 23:03:45 +0000212 SetVector<GlobalValue::GUID> TypeTests;
Peter Collingbournebe9ffaa2017-02-10 22:29:38 +0000213 SetVector<FunctionSummary::VFuncId> TypeTestAssumeVCalls,
214 TypeCheckedLoadVCalls;
215 SetVector<FunctionSummary::ConstVCall> TypeTestAssumeConstVCalls,
216 TypeCheckedLoadConstVCalls;
Teresa Johnsoncd21a642016-07-17 14:47:01 +0000217 ICallPromotionAnalysis ICallAnalysis;
Peter Collingbourne681fbb62017-09-07 05:35:35 +0000218 SmallPtrSet<const User *, 8> Visited;
219
220 // Add personality function, prefix data and prologue data to function's ref
221 // list.
222 findRefEdges(Index, &F, RefEdges, Visited);
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000223
Teresa Johnsond5033a42016-11-14 16:40:19 +0000224 bool HasInlineAsmMaybeReferencingInternal = false;
Benjamin Krameraa209152016-06-26 17:27:42 +0000225 for (const BasicBlock &BB : F)
226 for (const Instruction &I : BB) {
Piotr Padlewski442d38c2016-08-30 00:46:26 +0000227 if (isa<DbgInfoIntrinsic>(I))
228 continue;
229 ++NumInsts;
Peter Collingbourne9667b912017-05-04 18:03:25 +0000230 findRefEdges(Index, &I, RefEdges, Visited);
Piotr Padlewski442d38c2016-08-30 00:46:26 +0000231 auto CS = ImmutableCallSite(&I);
232 if (!CS)
233 continue;
Teresa Johnsonbf28c8f2016-10-30 05:40:44 +0000234
235 const auto *CI = dyn_cast<CallInst>(&I);
236 // Since we don't know exactly which local values are referenced in inline
Teresa Johnsond5033a42016-11-14 16:40:19 +0000237 // assembly, conservatively mark the function as possibly referencing
238 // a local value from inline assembly to ensure we don't export a
239 // reference (which would require renaming and promotion of the
240 // referenced value).
Peter Collingbourne5e8b94c12017-09-01 16:24:02 +0000241 if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
Teresa Johnsond5033a42016-11-14 16:40:19 +0000242 HasInlineAsmMaybeReferencingInternal = true;
Teresa Johnsonbf28c8f2016-10-30 05:40:44 +0000243
Teresa Johnson897bab92016-10-08 16:11:42 +0000244 auto *CalledValue = CS.getCalledValue();
Piotr Padlewski442d38c2016-08-30 00:46:26 +0000245 auto *CalledFunction = CS.getCalledFunction();
Volodymyr Sapsaia7396022017-11-10 00:47:47 +0000246 if (CalledValue && !CalledFunction) {
247 CalledValue = CalledValue->stripPointerCastsNoFollowAliases();
248 // Stripping pointer casts can reveal a called function.
249 CalledFunction = dyn_cast<Function>(CalledValue);
250 }
Teresa Johnson897bab92016-10-08 16:11:42 +0000251 // Check if this is an alias to a function. If so, get the
252 // called aliasee for the checks below.
253 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
254 assert(!CalledFunction && "Expected null called function in callsite for alias");
255 CalledFunction = dyn_cast<Function>(GA->getBaseObject());
256 }
Peter Collingbourne1b4137a72016-12-21 23:03:45 +0000257 // Check if this is a direct call to a known function or a known
258 // intrinsic, or an indirect call with profile data.
Piotr Padlewski442d38c2016-08-30 00:46:26 +0000259 if (CalledFunction) {
Peter Collingbournebe9ffaa2017-02-10 22:29:38 +0000260 if (CI && CalledFunction->isIntrinsic()) {
261 addIntrinsicToSummary(
262 CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
263 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls);
264 continue;
Peter Collingbourne1b4137a72016-12-21 23:03:45 +0000265 }
Teresa Johnson02563cd2016-10-28 02:39:38 +0000266 // We should have named any anonymous globals
267 assert(CalledFunction->hasName());
Easwaran Ramanf5f91602017-05-09 23:21:10 +0000268 auto ScaledCount = PSI->getProfileCount(&I, BFI);
Peter Collingbourne0c30f082016-12-20 21:12:28 +0000269 auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI)
270 : CalleeInfo::HotnessType::Unknown;
271
Teresa Johnson897bab92016-10-08 16:11:42 +0000272 // Use the original CalledValue, in case it was an alias. We want
273 // to record the call edge to the alias in that case. Eventually
274 // an alias summary will be created to associate the alias and
275 // aliasee.
Peter Collingbourne9667b912017-05-04 18:03:25 +0000276 CallGraphEdges[Index.getOrInsertValueInfo(
277 cast<GlobalValue>(CalledValue))]
278 .updateHotness(Hotness);
Piotr Padlewski442d38c2016-08-30 00:46:26 +0000279 } else {
Piotr Padlewski442d38c2016-08-30 00:46:26 +0000280 // Skip inline assembly calls.
281 if (CI && CI->isInlineAsm())
282 continue;
Volodymyr Sapsai8b46ff12017-11-17 18:28:05 +0000283 // Skip direct calls.
284 if (!CalledValue || isa<Constant>(CalledValue))
285 continue;
Piotr Padlewski442d38c2016-08-30 00:46:26 +0000286
287 uint32_t NumVals, NumCandidates;
288 uint64_t TotalCount;
289 auto CandidateProfileData =
290 ICallAnalysis.getPromotionCandidatesForInstruction(
291 &I, NumVals, TotalCount, NumCandidates);
292 for (auto &Candidate : CandidateProfileData)
Peter Collingbourne9667b912017-05-04 18:03:25 +0000293 CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
294 .updateHotness(getHotness(Candidate.Count, PSI));
Piotr Padlewski442d38c2016-08-30 00:46:26 +0000295 }
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000296 }
297
Dehao Chena60cdd32017-02-28 18:09:44 +0000298 // Explicit add hot edges to enforce importing for designated GUIDs for
299 // sample PGO, to enable the same inlines as the profiled optimized binary.
300 for (auto &I : F.getImportGUIDs())
Peter Collingbourne9667b912017-05-04 18:03:25 +0000301 CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
Dehao Chen64c46572017-07-07 21:01:00 +0000302 CalleeInfo::HotnessType::Critical);
Dehao Chena60cdd32017-02-28 18:09:44 +0000303
Teresa Johnson519465b2017-01-05 14:32:16 +0000304 bool NonRenamableLocal = isNonRenamableLocal(F);
305 bool NotEligibleForImport =
306 NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
307 // Inliner doesn't handle variadic functions.
308 // FIXME: refactor this to use the same code that inliner is using.
309 F.isVarArg();
Teresa Johnson6c475a72017-01-05 21:34:18 +0000310 GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport,
Sean Fertile4595a912017-11-04 17:04:39 +0000311 /* Live = */ false, F.isDSOLocal());
Charles Saternos75da10d2017-08-04 16:00:58 +0000312 FunctionSummary::FFlags FunFlags{
313 F.hasFnAttribute(Attribute::ReadNone),
314 F.hasFnAttribute(Attribute::ReadOnly),
315 F.hasFnAttribute(Attribute::NoRecurse),
316 F.returnDoesNotAlias(),
317 };
Peter Collingbourne0c30f082016-12-20 21:12:28 +0000318 auto FuncSummary = llvm::make_unique<FunctionSummary>(
Charles Saternos75da10d2017-08-04 16:00:58 +0000319 Flags, NumInsts, FunFlags, RefEdges.takeVector(),
320 CallGraphEdges.takeVector(), TypeTests.takeVector(),
321 TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(),
Peter Collingbournebe9ffaa2017-02-10 22:29:38 +0000322 TypeTestAssumeConstVCalls.takeVector(),
323 TypeCheckedLoadConstVCalls.takeVector());
Teresa Johnson519465b2017-01-05 14:32:16 +0000324 if (NonRenamableLocal)
325 CantBePromoted.insert(F.getGUID());
Chandler Carruthb7be5b62016-08-19 07:49:19 +0000326 Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary));
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000327}
328
Teresa Johnson519465b2017-01-05 14:32:16 +0000329static void
330computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V,
Teresa Johnsone27b0582017-01-05 14:59:56 +0000331 DenseSet<GlobalValue::GUID> &CantBePromoted) {
Peter Collingbourne0c30f082016-12-20 21:12:28 +0000332 SetVector<ValueInfo> RefEdges;
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000333 SmallPtrSet<const User *, 8> Visited;
Peter Collingbourne9667b912017-05-04 18:03:25 +0000334 findRefEdges(Index, &V, RefEdges, Visited);
Teresa Johnson519465b2017-01-05 14:32:16 +0000335 bool NonRenamableLocal = isNonRenamableLocal(V);
Teresa Johnson6c475a72017-01-05 21:34:18 +0000336 GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal,
Sean Fertile4595a912017-11-04 17:04:39 +0000337 /* Live = */ false, V.isDSOLocal());
Peter Collingbourne0c30f082016-12-20 21:12:28 +0000338 auto GVarSummary =
339 llvm::make_unique<GlobalVarSummary>(Flags, RefEdges.takeVector());
Teresa Johnson519465b2017-01-05 14:32:16 +0000340 if (NonRenamableLocal)
341 CantBePromoted.insert(V.getGUID());
Chandler Carruthb7be5b62016-08-19 07:49:19 +0000342 Index.addGlobalValueSummary(V.getName(), std::move(GVarSummary));
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000343}
344
Teresa Johnson519465b2017-01-05 14:32:16 +0000345static void
346computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A,
Teresa Johnsone27b0582017-01-05 14:59:56 +0000347 DenseSet<GlobalValue::GUID> &CantBePromoted) {
Teresa Johnson519465b2017-01-05 14:32:16 +0000348 bool NonRenamableLocal = isNonRenamableLocal(A);
Teresa Johnson6c475a72017-01-05 21:34:18 +0000349 GlobalValueSummary::GVFlags Flags(A.getLinkage(), NonRenamableLocal,
Sean Fertile4595a912017-11-04 17:04:39 +0000350 /* Live = */ false, A.isDSOLocal());
Teresa Johnsoncbdc5ff2017-09-13 17:10:24 +0000351 auto AS = llvm::make_unique<AliasSummary>(Flags);
Teresa Johnson02563cd2016-10-28 02:39:38 +0000352 auto *Aliasee = A.getBaseObject();
353 auto *AliaseeSummary = Index.getGlobalValueSummary(*Aliasee);
354 assert(AliaseeSummary && "Alias expects aliasee summary to be parsed");
355 AS->setAliasee(AliaseeSummary);
Teresa Johnson519465b2017-01-05 14:32:16 +0000356 if (NonRenamableLocal)
357 CantBePromoted.insert(A.getGUID());
Teresa Johnson02563cd2016-10-28 02:39:38 +0000358 Index.addGlobalValueSummary(A.getName(), std::move(AS));
359}
360
Teresa Johnson6c475a72017-01-05 21:34:18 +0000361// Set LiveRoot flag on entries matching the given value name.
362static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
Peter Collingbourne9667b912017-05-04 18:03:25 +0000363 if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name)))
364 for (auto &Summary : VI.getSummaryList())
Evgeniy Stepanov56584bb2017-06-01 20:30:06 +0000365 Summary->setLive(true);
Teresa Johnson6c475a72017-01-05 21:34:18 +0000366}
367
Chandler Carruthb7be5b62016-08-19 07:49:19 +0000368ModuleSummaryIndex llvm::buildModuleSummaryIndex(
369 const Module &M,
Piotr Padlewskid9830eb2016-09-26 20:37:32 +0000370 std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
371 ProfileSummaryInfo *PSI) {
Teresa Johnson94624ac2017-05-10 18:52:16 +0000372 assert(PSI);
Chandler Carruthb7be5b62016-08-19 07:49:19 +0000373 ModuleSummaryIndex Index;
Teresa Johnsonbf28c8f2016-10-30 05:40:44 +0000374
Teresa Johnsona0811452016-11-10 16:57:32 +0000375 // Identify the local values in the llvm.used and llvm.compiler.used sets,
376 // which should not be exported as they would then require renaming and
377 // promotion, but we may have opaque uses e.g. in inline asm. We collect them
378 // here because we use this information to mark functions containing inline
379 // assembly calls as not importable.
Mehdi Aminib6a11a72016-11-09 01:45:13 +0000380 SmallPtrSet<GlobalValue *, 8> LocalsUsed;
Teresa Johnsona0811452016-11-10 16:57:32 +0000381 SmallPtrSet<GlobalValue *, 8> Used;
382 // First collect those in the llvm.used set.
383 collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false);
Teresa Johnsona0811452016-11-10 16:57:32 +0000384 // Next collect those in the llvm.compiler.used set.
385 collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ true);
Teresa Johnsone27b0582017-01-05 14:59:56 +0000386 DenseSet<GlobalValue::GUID> CantBePromoted;
Teresa Johnsonbf28c8f2016-10-30 05:40:44 +0000387 for (auto *V : Used) {
Teresa Johnson519465b2017-01-05 14:32:16 +0000388 if (V->hasLocalLinkage()) {
Teresa Johnsonbf28c8f2016-10-30 05:40:44 +0000389 LocalsUsed.insert(V);
Teresa Johnson519465b2017-01-05 14:32:16 +0000390 CantBePromoted.insert(V->getGUID());
391 }
Teresa Johnsonbf28c8f2016-10-30 05:40:44 +0000392 }
Teresa Johnsonb35cc692016-04-20 14:39:45 +0000393
Peter Collingbourne5e8b94c12017-09-01 16:24:02 +0000394 bool HasLocalInlineAsmSymbol = false;
395 if (!M.getModuleInlineAsm().empty()) {
396 // Collect the local values defined by module level asm, and set up
397 // summaries for these symbols so that they can be marked as NoRename,
398 // to prevent export of any use of them in regular IR that would require
399 // renaming within the module level asm. Note we don't need to create a
400 // summary for weak or global defs, as they don't need to be flagged as
401 // NoRename, and defs in module level asm can't be imported anyway.
402 // Also, any values used but not defined within module level asm should
403 // be listed on the llvm.used or llvm.compiler.used global and marked as
404 // referenced from there.
405 ModuleSymbolTable::CollectAsmSymbols(
406 M, [&](StringRef Name, object::BasicSymbolRef::Flags Flags) {
407 // Symbols not marked as Weak or Global are local definitions.
408 if (Flags & (object::BasicSymbolRef::SF_Weak |
409 object::BasicSymbolRef::SF_Global))
410 return;
411 HasLocalInlineAsmSymbol = true;
412 GlobalValue *GV = M.getNamedValue(Name);
413 if (!GV)
414 return;
415 assert(GV->isDeclaration() && "Def in module asm already has definition");
416 GlobalValueSummary::GVFlags GVFlags(GlobalValue::InternalLinkage,
417 /* NotEligibleToImport = */ true,
Sean Fertile4595a912017-11-04 17:04:39 +0000418 /* Live = */ true,
419 /* Local */ GV->isDSOLocal());
Peter Collingbourne5e8b94c12017-09-01 16:24:02 +0000420 CantBePromoted.insert(GlobalValue::getGUID(Name));
421 // Create the appropriate summary type.
422 if (Function *F = dyn_cast<Function>(GV)) {
423 std::unique_ptr<FunctionSummary> Summary =
424 llvm::make_unique<FunctionSummary>(
425 GVFlags, 0,
426 FunctionSummary::FFlags{
427 F->hasFnAttribute(Attribute::ReadNone),
428 F->hasFnAttribute(Attribute::ReadOnly),
429 F->hasFnAttribute(Attribute::NoRecurse),
430 F->returnDoesNotAlias()},
431 ArrayRef<ValueInfo>{}, ArrayRef<FunctionSummary::EdgeTy>{},
432 ArrayRef<GlobalValue::GUID>{},
433 ArrayRef<FunctionSummary::VFuncId>{},
434 ArrayRef<FunctionSummary::VFuncId>{},
435 ArrayRef<FunctionSummary::ConstVCall>{},
436 ArrayRef<FunctionSummary::ConstVCall>{});
437 Index.addGlobalValueSummary(Name, std::move(Summary));
438 } else {
439 std::unique_ptr<GlobalVarSummary> Summary =
440 llvm::make_unique<GlobalVarSummary>(GVFlags,
441 ArrayRef<ValueInfo>{});
442 Index.addGlobalValueSummary(Name, std::move(Summary));
443 }
444 });
445 }
446
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000447 // Compute summaries for all functions defined in module, and save in the
448 // index.
Chandler Carruthb7be5b62016-08-19 07:49:19 +0000449 for (auto &F : M) {
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000450 if (F.isDeclaration())
451 continue;
452
453 BlockFrequencyInfo *BFI = nullptr;
454 std::unique_ptr<BlockFrequencyInfo> BFIPtr;
Chandler Carruthb7be5b62016-08-19 07:49:19 +0000455 if (GetBFICallback)
456 BFI = GetBFICallback(F);
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000457 else if (F.getEntryCount().hasValue()) {
458 LoopInfo LI{DominatorTree(const_cast<Function &>(F))};
459 BranchProbabilityInfo BPI{F, LI};
460 BFIPtr = llvm::make_unique<BlockFrequencyInfo>(F, BPI, LI);
461 BFI = BFIPtr.get();
462 }
463
Peter Collingbourne5e8b94c12017-09-01 16:24:02 +0000464 computeFunctionSummary(Index, M, F, BFI, PSI,
465 !LocalsUsed.empty() || HasLocalInlineAsmSymbol,
Teresa Johnson519465b2017-01-05 14:32:16 +0000466 CantBePromoted);
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000467 }
468
469 // Compute summaries for all variables defined in module, and save in the
470 // index.
Chandler Carruthb7be5b62016-08-19 07:49:19 +0000471 for (const GlobalVariable &G : M.globals()) {
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000472 if (G.isDeclaration())
473 continue;
Teresa Johnson519465b2017-01-05 14:32:16 +0000474 computeVariableSummary(Index, G, CantBePromoted);
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000475 }
Teresa Johnson02563cd2016-10-28 02:39:38 +0000476
477 // Compute summaries for all aliases defined in module, and save in the
478 // index.
479 for (const GlobalAlias &A : M.aliases())
Teresa Johnson519465b2017-01-05 14:32:16 +0000480 computeAliasSummary(Index, A, CantBePromoted);
Teresa Johnson02563cd2016-10-28 02:39:38 +0000481
Teresa Johnsonbf28c8f2016-10-30 05:40:44 +0000482 for (auto *V : LocalsUsed) {
483 auto *Summary = Index.getGlobalValueSummary(*V);
484 assert(Summary && "Missing summary for global value");
Teresa Johnson519465b2017-01-05 14:32:16 +0000485 Summary->setNotEligibleToImport();
Teresa Johnsonbf28c8f2016-10-30 05:40:44 +0000486 }
487
Teresa Johnson6c475a72017-01-05 21:34:18 +0000488 // The linker doesn't know about these LLVM produced values, so we need
489 // to flag them as live in the index to ensure index-based dead value
490 // analysis treats them as live roots of the analysis.
491 setLiveRoot(Index, "llvm.used");
492 setLiveRoot(Index, "llvm.compiler.used");
493 setLiveRoot(Index, "llvm.global_ctors");
494 setLiveRoot(Index, "llvm.global_dtors");
495 setLiveRoot(Index, "llvm.global.annotations");
496
Peter Collingbournee357fbd2017-06-08 23:01:49 +0000497 bool IsThinLTO = true;
498 if (auto *MD =
499 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
500 IsThinLTO = MD->getZExtValue();
501
Teresa Johnson519465b2017-01-05 14:32:16 +0000502 for (auto &GlobalList : Index) {
Peter Collingbourne9667b912017-05-04 18:03:25 +0000503 // Ignore entries for references that are undefined in the current module.
504 if (GlobalList.second.SummaryList.empty())
505 continue;
506
507 assert(GlobalList.second.SummaryList.size() == 1 &&
Teresa Johnson519465b2017-01-05 14:32:16 +0000508 "Expected module's index to have one summary per GUID");
Peter Collingbourne9667b912017-05-04 18:03:25 +0000509 auto &Summary = GlobalList.second.SummaryList[0];
Peter Collingbournee357fbd2017-06-08 23:01:49 +0000510 if (!IsThinLTO) {
511 Summary->setNotEligibleToImport();
512 continue;
513 }
514
Teresa Johnson519465b2017-01-05 14:32:16 +0000515 bool AllRefsCanBeExternallyReferenced =
516 llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
Peter Collingbourne9667b912017-05-04 18:03:25 +0000517 return !CantBePromoted.count(VI.getGUID());
Teresa Johnson519465b2017-01-05 14:32:16 +0000518 });
519 if (!AllRefsCanBeExternallyReferenced) {
520 Summary->setNotEligibleToImport();
521 continue;
522 }
523
524 if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
525 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
526 FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
Peter Collingbourne9667b912017-05-04 18:03:25 +0000527 return !CantBePromoted.count(Edge.first.getGUID());
Teresa Johnson519465b2017-01-05 14:32:16 +0000528 });
529 if (!AllCallsCanBeExternallyReferenced)
530 Summary->setNotEligibleToImport();
531 }
532 }
533
Chandler Carruthb7be5b62016-08-19 07:49:19 +0000534 return Index;
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000535}
536
Chandler Carruthdab4eae2016-11-23 17:53:26 +0000537AnalysisKey ModuleSummaryIndexAnalysis::Key;
Teresa Johnsonf93b2462016-08-12 13:53:02 +0000538
Chandler Carruthb7be5b62016-08-19 07:49:19 +0000539ModuleSummaryIndex
Teresa Johnsonf93b2462016-08-12 13:53:02 +0000540ModuleSummaryIndexAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
Piotr Padlewskid9830eb2016-09-26 20:37:32 +0000541 ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);
Teresa Johnsonf93b2462016-08-12 13:53:02 +0000542 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
Piotr Padlewskid9830eb2016-09-26 20:37:32 +0000543 return buildModuleSummaryIndex(
544 M,
545 [&FAM](const Function &F) {
546 return &FAM.getResult<BlockFrequencyAnalysis>(
547 *const_cast<Function *>(&F));
548 },
549 &PSI);
Teresa Johnsonf93b2462016-08-12 13:53:02 +0000550}
551
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000552char ModuleSummaryIndexWrapperPass::ID = 0;
Eugene Zelenkobb1b2d02017-08-16 22:07:40 +0000553
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000554INITIALIZE_PASS_BEGIN(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
555 "Module Summary Analysis", false, true)
556INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
Mehdi Amini89029482017-01-21 06:01:22 +0000557INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000558INITIALIZE_PASS_END(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
559 "Module Summary Analysis", false, true)
560
561ModulePass *llvm::createModuleSummaryIndexWrapperPass() {
562 return new ModuleSummaryIndexWrapperPass();
563}
564
565ModuleSummaryIndexWrapperPass::ModuleSummaryIndexWrapperPass()
566 : ModulePass(ID) {
567 initializeModuleSummaryIndexWrapperPassPass(*PassRegistry::getPassRegistry());
568}
569
570bool ModuleSummaryIndexWrapperPass::runOnModule(Module &M) {
Dehao Chen5461d8b2016-09-28 21:00:58 +0000571 auto &PSI = *getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
Piotr Padlewskid9830eb2016-09-26 20:37:32 +0000572 Index = buildModuleSummaryIndex(
573 M,
574 [this](const Function &F) {
575 return &(this->getAnalysis<BlockFrequencyInfoWrapperPass>(
576 *const_cast<Function *>(&F))
577 .getBFI());
578 },
579 &PSI);
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000580 return false;
581}
582
583bool ModuleSummaryIndexWrapperPass::doFinalization(Module &M) {
Chandler Carruthb7be5b62016-08-19 07:49:19 +0000584 Index.reset();
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000585 return false;
586}
587
588void ModuleSummaryIndexWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
589 AU.setPreservesAll();
590 AU.addRequired<BlockFrequencyInfoWrapperPass>();
Piotr Padlewskid9830eb2016-09-26 20:37:32 +0000591 AU.addRequired<ProfileSummaryInfoWrapperPass>();
Teresa Johnson2d5487c2016-04-11 13:58:45 +0000592}