blob: 8b3e4fc9fe1fa71f07e32bc020ffba9c79c0b620 [file] [log] [blame]
Justin Bogner61ba2e32014-12-08 18:02:35 +00001//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
Betul Buyukkurt6fac1742015-11-18 18:14:55 +000010// This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
11// It also builds the data structures and initialization code needed for
12// updating execution counts and emitting the profile at runtime.
Justin Bogner61ba2e32014-12-08 18:02:35 +000013//
14//===----------------------------------------------------------------------===//
15
Xinliang David Li69a00f02016-06-21 02:39:08 +000016#include "llvm/Transforms/InstrProfiling.h"
Justin Bogner61ba2e32014-12-08 18:02:35 +000017#include "llvm/ADT/Triple.h"
18#include "llvm/IR/IRBuilder.h"
19#include "llvm/IR/IntrinsicInst.h"
20#include "llvm/IR/Module.h"
Betul Buyukkurt6fac1742015-11-18 18:14:55 +000021#include "llvm/ProfileData/InstrProf.h"
Justin Bogner61ba2e32014-12-08 18:02:35 +000022#include "llvm/Transforms/Utils/ModuleUtils.h"
23
24using namespace llvm;
25
26#define DEBUG_TYPE "instrprof"
27
28namespace {
29
Xinliang David Lia82d6c02016-02-08 18:13:49 +000030cl::opt<bool> DoNameCompression("enable-name-compression",
31 cl::desc("Enable name string compression"),
32 cl::init(true));
33
Xinliang David Lib628dd32016-05-21 22:55:34 +000034cl::opt<bool> ValueProfileStaticAlloc(
35 "vp-static-alloc",
36 cl::desc("Do static counter allocation for value profiler"),
37 cl::init(true));
38cl::opt<double> NumCountersPerValueSite(
39 "vp-counters-per-site",
40 cl::desc("The average number of profile counters allocated "
41 "per value profiling site."),
42 // This is set to a very small value because in real programs, only
43 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
44 // For those sites with non-zero profile, the average number of targets
45 // is usually smaller than 2.
46 cl::init(1.0));
47
Xinliang David Lie6b89292016-04-18 17:47:38 +000048class InstrProfilingLegacyPass : public ModulePass {
49 InstrProfiling InstrProf;
50
Justin Bogner61ba2e32014-12-08 18:02:35 +000051public:
52 static char ID;
Xinliang David Lie6b89292016-04-18 17:47:38 +000053 InstrProfilingLegacyPass() : ModulePass(ID), InstrProf() {}
54 InstrProfilingLegacyPass(const InstrProfOptions &Options)
55 : ModulePass(ID), InstrProf(Options) {}
Mehdi Amini117296c2016-10-01 02:56:57 +000056 StringRef getPassName() const override {
Justin Bogner61ba2e32014-12-08 18:02:35 +000057 return "Frontend instrumentation-based coverage lowering";
58 }
59
Xinliang David Lie6b89292016-04-18 17:47:38 +000060 bool runOnModule(Module &M) override { return InstrProf.run(M); }
Justin Bogner61ba2e32014-12-08 18:02:35 +000061
62 void getAnalysisUsage(AnalysisUsage &AU) const override {
63 AU.setPreservesCFG();
64 }
Justin Bogner61ba2e32014-12-08 18:02:35 +000065};
66
67} // anonymous namespace
68
Sean Silvafd03ac62016-08-09 00:28:38 +000069PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
Xinliang David Lie6b89292016-04-18 17:47:38 +000070 if (!run(M))
71 return PreservedAnalyses::all();
72
73 return PreservedAnalyses::none();
74}
75
76char InstrProfilingLegacyPass::ID = 0;
77INITIALIZE_PASS(InstrProfilingLegacyPass, "instrprof",
Justin Bogner61ba2e32014-12-08 18:02:35 +000078 "Frontend instrumentation-based coverage lowering.", false,
79 false)
80
Xinliang David Li69a00f02016-06-21 02:39:08 +000081ModulePass *
82llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options) {
Xinliang David Lie6b89292016-04-18 17:47:38 +000083 return new InstrProfilingLegacyPass(Options);
Justin Bogner61ba2e32014-12-08 18:02:35 +000084}
85
Xinliang David Lie6b89292016-04-18 17:47:38 +000086bool InstrProfiling::isMachO() const {
87 return Triple(M->getTargetTriple()).isOSBinFormatMachO();
88}
89
90/// Get the section name for the counter variables.
91StringRef InstrProfiling::getCountersSection() const {
92 return getInstrProfCountersSectionName(isMachO());
93}
94
95/// Get the section name for the name variables.
96StringRef InstrProfiling::getNameSection() const {
97 return getInstrProfNameSectionName(isMachO());
98}
99
100/// Get the section name for the profile data variables.
101StringRef InstrProfiling::getDataSection() const {
102 return getInstrProfDataSectionName(isMachO());
103}
104
105/// Get the section name for the coverage mapping data.
106StringRef InstrProfiling::getCoverageSection() const {
107 return getInstrProfCoverageSectionName(isMachO());
108}
109
Xinliang David Li4ca17332016-09-18 18:34:07 +0000110static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
111 InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr);
112 if (Inc)
113 return Inc;
114 return dyn_cast<InstrProfIncrementInst>(Instr);
115}
116
Xinliang David Lie6b89292016-04-18 17:47:38 +0000117bool InstrProfiling::run(Module &M) {
Justin Bogner61ba2e32014-12-08 18:02:35 +0000118 bool MadeChange = false;
119
120 this->M = &M;
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000121 NamesVar = nullptr;
122 NamesSize = 0;
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000123 ProfileDataMap.clear();
Justin Bogner61ba2e32014-12-08 18:02:35 +0000124 UsedVars.clear();
125
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000126 // We did not know how many value sites there would be inside
127 // the instrumented function. This is counting the number of instrumented
128 // target value sites to enter it as field in the profile data variable.
Rong Xu294572f2016-01-19 18:29:54 +0000129 for (Function &F : M) {
130 InstrProfIncrementInst *FirstProfIncInst = nullptr;
Justin Bogner61ba2e32014-12-08 18:02:35 +0000131 for (BasicBlock &BB : F)
Rong Xu294572f2016-01-19 18:29:54 +0000132 for (auto I = BB.begin(), E = BB.end(); I != E; I++)
133 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000134 computeNumValueSiteCounts(Ind);
Rong Xu294572f2016-01-19 18:29:54 +0000135 else if (FirstProfIncInst == nullptr)
136 FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
137
138 // Value profiling intrinsic lowering requires per-function profile data
139 // variable to be created first.
140 if (FirstProfIncInst != nullptr)
141 static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
142 }
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000143
144 for (Function &F : M)
145 for (BasicBlock &BB : F)
146 for (auto I = BB.begin(), E = BB.end(); I != E;) {
147 auto Instr = I++;
Xinliang David Li4ca17332016-09-18 18:34:07 +0000148 InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
149 if (Inc) {
Justin Bogner61ba2e32014-12-08 18:02:35 +0000150 lowerIncrement(Inc);
151 MadeChange = true;
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000152 } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
153 lowerValueProfileInst(Ind);
154 MadeChange = true;
Justin Bogner61ba2e32014-12-08 18:02:35 +0000155 }
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000156 }
157
Xinliang David Li81056072016-01-07 20:05:49 +0000158 if (GlobalVariable *CoverageNamesVar =
Xinliang David Li440cd702016-01-20 00:24:36 +0000159 M.getNamedGlobal(getCoverageUnusedNamesVarName())) {
Xinliang David Li81056072016-01-07 20:05:49 +0000160 lowerCoverageData(CoverageNamesVar);
Justin Bognerd24e1852015-02-11 02:52:44 +0000161 MadeChange = true;
162 }
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000163
Justin Bogner61ba2e32014-12-08 18:02:35 +0000164 if (!MadeChange)
165 return false;
166
Xinliang David Lib628dd32016-05-21 22:55:34 +0000167 emitVNodes();
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000168 emitNameData();
Justin Bogner61ba2e32014-12-08 18:02:35 +0000169 emitRegistration();
170 emitRuntimeHook();
171 emitUses();
172 emitInitialization();
173 return true;
174}
175
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000176static Constant *getOrInsertValueProfilingCall(Module &M) {
Xinliang David Lic7673232015-11-22 00:22:07 +0000177 LLVMContext &Ctx = M.getContext();
178 auto *ReturnTy = Type::getVoidTy(M.getContext());
179 Type *ParamTypes[] = {
180#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
181#include "llvm/ProfileData/InstrProfData.inc"
182 };
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000183 auto *ValueProfilingCallTy =
Xinliang David Lic7673232015-11-22 00:22:07 +0000184 FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
Xinliang David Li924e0582015-11-22 05:42:31 +0000185 return M.getOrInsertFunction(getInstrProfValueProfFuncName(),
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000186 ValueProfilingCallTy);
187}
188
189void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
190
191 GlobalVariable *Name = Ind->getName();
192 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
193 uint64_t Index = Ind->getIndex()->getZExtValue();
194 auto It = ProfileDataMap.find(Name);
195 if (It == ProfileDataMap.end()) {
196 PerFunctionProfileData PD;
197 PD.NumValueSites[ValueKind] = Index + 1;
198 ProfileDataMap[Name] = PD;
199 } else if (It->second.NumValueSites[ValueKind] <= Index)
200 It->second.NumValueSites[ValueKind] = Index + 1;
201}
202
203void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
204
205 GlobalVariable *Name = Ind->getName();
206 auto It = ProfileDataMap.find(Name);
207 assert(It != ProfileDataMap.end() && It->second.DataVar &&
Xinliang David Li69a00f02016-06-21 02:39:08 +0000208 "value profiling detected in function with no counter incerement");
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000209
210 GlobalVariable *DataVar = It->second.DataVar;
211 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
212 uint64_t Index = Ind->getIndex()->getZExtValue();
213 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
214 Index += It->second.NumValueSites[Kind];
215
216 IRBuilder<> Builder(Ind);
Xinliang David Li69a00f02016-06-21 02:39:08 +0000217 Value *Args[3] = {Ind->getTargetValue(),
218 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
219 Builder.getInt32(Index)};
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000220 Ind->replaceAllUsesWith(
221 Builder.CreateCall(getOrInsertValueProfilingCall(*M), Args));
222 Ind->eraseFromParent();
223}
224
Justin Bogner61ba2e32014-12-08 18:02:35 +0000225void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
226 GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
227
Duncan P. N. Exon Smithe82c2862015-10-13 17:39:10 +0000228 IRBuilder<> Builder(Inc);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000229 uint64_t Index = Inc->getIndex()->getZExtValue();
Diego Novillob3029d22015-06-04 11:45:32 +0000230 Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
231 Value *Count = Builder.CreateLoad(Addr, "pgocount");
Xinliang David Lia754c472016-09-20 19:07:22 +0000232 Count = Builder.CreateAdd(Count, Inc->getStep());
Justin Bogner61ba2e32014-12-08 18:02:35 +0000233 Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
234 Inc->eraseFromParent();
235}
236
Xinliang David Li81056072016-01-07 20:05:49 +0000237void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
Justin Bognerd24e1852015-02-11 02:52:44 +0000238
Xinliang David Li81056072016-01-07 20:05:49 +0000239 ConstantArray *Names =
240 cast<ConstantArray>(CoverageNamesVar->getInitializer());
241 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
242 Constant *NC = Names->getOperand(I);
243 Value *V = NC->stripPointerCasts();
Justin Bognerd24e1852015-02-11 02:52:44 +0000244 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
245 GlobalVariable *Name = cast<GlobalVariable>(V);
246
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000247 Name->setLinkage(GlobalValue::PrivateLinkage);
248 ReferencedNames.push_back(Name);
Justin Bognerd24e1852015-02-11 02:52:44 +0000249 }
250}
251
Justin Bogner61ba2e32014-12-08 18:02:35 +0000252/// Get the name of a profiling variable for a particular function.
Xinliang David Li83bc4222015-10-22 20:32:12 +0000253static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
Xinliang David Lid1bab962015-12-12 17:28:03 +0000254 StringRef NamePrefix = getInstrProfNameVarPrefix();
255 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
Xinliang David Li83bc4222015-10-22 20:32:12 +0000256 return (Prefix + Name).str();
Justin Bogner61ba2e32014-12-08 18:02:35 +0000257}
258
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000259static inline bool shouldRecordFunctionAddr(Function *F) {
260 // Check the linkage
261 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
262 !F->hasAvailableExternallyLinkage())
263 return true;
Rong Xuaf5aeba2016-04-27 21:17:30 +0000264 // Prohibit function address recording if the function is both internal and
265 // COMDAT. This avoids the profile data variable referencing internal symbols
266 // in COMDAT.
267 if (F->hasLocalLinkage() && F->hasComdat())
268 return false;
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000269 // Check uses of this function for other than direct calls or invokes to it.
Xinliang David Li7008ce32016-06-02 16:33:41 +0000270 // Inline virtual functions have linkeOnceODR linkage. When a key method
271 // exists, the vtable will only be emitted in the TU where the key method
272 // is defined. In a TU where vtable is not available, the function won't
Xinliang David Li6c44e9e2016-06-03 23:02:28 +0000273 // be 'addresstaken'. If its address is not recorded here, the profile data
Xinliang David Li69a00f02016-06-21 02:39:08 +0000274 // with missing address may be picked by the linker leading to missing
Xinliang David Li6c44e9e2016-06-03 23:02:28 +0000275 // indirect call target info.
276 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000277}
278
Xinliang David Li985ff202016-02-27 23:11:30 +0000279static inline Comdat *getOrCreateProfileComdat(Module &M, Function &F,
Xinliang David Liab361ef2015-12-21 21:52:27 +0000280 InstrProfIncrementInst *Inc) {
Xinliang David Li985ff202016-02-27 23:11:30 +0000281 if (!needsComdatForCounter(F, M))
282 return nullptr;
283
Xinliang David Liab361ef2015-12-21 21:52:27 +0000284 // COFF format requires a COMDAT section to have a key symbol with the same
Vedant Kumar2d5b5d32016-02-03 23:22:43 +0000285 // name. The linker targeting COFF also requires that the COMDAT
Xinliang David Li5fe04552015-12-22 00:11:15 +0000286 // a section is associated to must precede the associating section. For this
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000287 // reason, we must choose the counter var's name as the name of the comdat.
Xinliang David Liab361ef2015-12-21 21:52:27 +0000288 StringRef ComdatPrefix = (Triple(M.getTargetTriple()).isOSBinFormatCOFF()
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000289 ? getInstrProfCountersVarPrefix()
Xinliang David Liab361ef2015-12-21 21:52:27 +0000290 : getInstrProfComdatPrefix());
291 return M.getOrInsertComdat(StringRef(getVarName(Inc, ComdatPrefix)));
292}
293
Xinliang David Lib628dd32016-05-21 22:55:34 +0000294static bool needsRuntimeRegistrationOfSectionRange(const Module &M) {
295 // Don't do this for Darwin. compiler-rt uses linker magic.
296 if (Triple(M.getTargetTriple()).isOSDarwin())
297 return false;
298
299 // Use linker script magic to get data/cnts/name start/end.
300 if (Triple(M.getTargetTriple()).isOSLinux() ||
301 Triple(M.getTargetTriple()).isOSFreeBSD() ||
302 Triple(M.getTargetTriple()).isPS4CPU())
303 return false;
304
305 return true;
306}
307
Justin Bogner61ba2e32014-12-08 18:02:35 +0000308GlobalVariable *
309InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
Xinliang David Li192c7482015-11-05 00:47:26 +0000310 GlobalVariable *NamePtr = Inc->getName();
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000311 auto It = ProfileDataMap.find(NamePtr);
312 PerFunctionProfileData PD;
313 if (It != ProfileDataMap.end()) {
314 if (It->second.RegionCounters)
315 return It->second.RegionCounters;
316 PD = It->second;
317 }
Justin Bogner61ba2e32014-12-08 18:02:35 +0000318
Wei Mi3cc92042015-09-23 22:40:45 +0000319 // Move the name variable to the right section. Place them in a COMDAT group
320 // if the associated function is a COMDAT. This will make sure that
321 // only one copy of counters of the COMDAT function will be emitted after
322 // linking.
Diego Novillodf4837b2015-05-27 19:34:01 +0000323 Function *Fn = Inc->getParent()->getParent();
Wei Mi3cc92042015-09-23 22:40:45 +0000324 Comdat *ProfileVarsComdat = nullptr;
Xinliang David Li985ff202016-02-27 23:11:30 +0000325 ProfileVarsComdat = getOrCreateProfileComdat(*M, *Fn, Inc);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000326
327 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
328 LLVMContext &Ctx = M->getContext();
329 ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
330
331 // Create the counters variable.
Xinliang David Li192c7482015-11-05 00:47:26 +0000332 auto *CounterPtr =
333 new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
Xinliang David Li83bc4222015-10-22 20:32:12 +0000334 Constant::getNullValue(CounterTy),
335 getVarName(Inc, getInstrProfCountersVarPrefix()));
Xinliang David Li192c7482015-11-05 00:47:26 +0000336 CounterPtr->setVisibility(NamePtr->getVisibility());
337 CounterPtr->setSection(getCountersSection());
338 CounterPtr->setAlignment(8);
339 CounterPtr->setComdat(ProfileVarsComdat);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000340
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000341 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
Xinliang David Lib628dd32016-05-21 22:55:34 +0000342 // Allocate statically the array of pointers to value profile nodes for
343 // the current function.
344 Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
345 if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(*M)) {
346
347 uint64_t NS = 0;
348 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
349 NS += PD.NumValueSites[Kind];
350 if (NS) {
351 ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
352
353 auto *ValuesVar =
354 new GlobalVariable(*M, ValuesTy, false, NamePtr->getLinkage(),
355 Constant::getNullValue(ValuesTy),
356 getVarName(Inc, getInstrProfValuesVarPrefix()));
357 ValuesVar->setVisibility(NamePtr->getVisibility());
358 ValuesVar->setSection(getInstrProfValuesSectionName(isMachO()));
359 ValuesVar->setAlignment(8);
360 ValuesVar->setComdat(ProfileVarsComdat);
361 ValuesPtrExpr =
362 ConstantExpr::getBitCast(ValuesVar, llvm::Type::getInt8PtrTy(Ctx));
363 }
364 }
365
366 // Create data variable.
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000367 auto *Int16Ty = Type::getInt16Ty(Ctx);
Xinliang David Lib628dd32016-05-21 22:55:34 +0000368 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
Xinliang David Li192c7482015-11-05 00:47:26 +0000369 Type *DataTypes[] = {
Xinliang David Li69a00f02016-06-21 02:39:08 +0000370#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
371#include "llvm/ProfileData/InstrProfData.inc"
Xinliang David Li192c7482015-11-05 00:47:26 +0000372 };
Justin Bogner61ba2e32014-12-08 18:02:35 +0000373 auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
Xinliang David Li192c7482015-11-05 00:47:26 +0000374
Xinliang David Li69a00f02016-06-21 02:39:08 +0000375 Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
376 ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
377 : ConstantPointerNull::get(Int8PtrTy);
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000378
Xinliang David Li69a00f02016-06-21 02:39:08 +0000379 Constant *Int16ArrayVals[IPVK_Last + 1];
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000380 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
381 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
382
Justin Bogner61ba2e32014-12-08 18:02:35 +0000383 Constant *DataVals[] = {
Xinliang David Li69a00f02016-06-21 02:39:08 +0000384#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
385#include "llvm/ProfileData/InstrProfData.inc"
Xinliang David Li192c7482015-11-05 00:47:26 +0000386 };
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000387 auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
Justin Bogner61ba2e32014-12-08 18:02:35 +0000388 ConstantStruct::get(DataTy, DataVals),
Xinliang David Li83bc4222015-10-22 20:32:12 +0000389 getVarName(Inc, getInstrProfDataVarPrefix()));
Xinliang David Li192c7482015-11-05 00:47:26 +0000390 Data->setVisibility(NamePtr->getVisibility());
Justin Bogner61ba2e32014-12-08 18:02:35 +0000391 Data->setSection(getDataSection());
Xinliang David Lic7c1f852015-11-23 18:02:59 +0000392 Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
Wei Mi3cc92042015-09-23 22:40:45 +0000393 Data->setComdat(ProfileVarsComdat);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000394
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000395 PD.RegionCounters = CounterPtr;
396 PD.DataVar = Data;
397 ProfileDataMap[NamePtr] = PD;
398
Justin Bogner61ba2e32014-12-08 18:02:35 +0000399 // Mark the data variable as used so that it isn't stripped out.
400 UsedVars.push_back(Data);
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000401 // Now that the linkage set by the FE has been passed to the data and counter
402 // variables, reset Name variable's linkage and visibility to private so that
403 // it can be removed later by the compiler.
404 NamePtr->setLinkage(GlobalValue::PrivateLinkage);
405 // Collect the referenced names to be used by emitNameData.
406 ReferencedNames.push_back(NamePtr);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000407
Xinliang David Li192c7482015-11-05 00:47:26 +0000408 return CounterPtr;
Justin Bogner61ba2e32014-12-08 18:02:35 +0000409}
410
Xinliang David Lib628dd32016-05-21 22:55:34 +0000411void InstrProfiling::emitVNodes() {
412 if (!ValueProfileStaticAlloc)
413 return;
Xinliang David Li8da773b2016-05-17 20:19:03 +0000414
Xinliang David Lib628dd32016-05-21 22:55:34 +0000415 // For now only support this on platforms that do
416 // not require runtime registration to discover
417 // named section start/end.
418 if (needsRuntimeRegistrationOfSectionRange(*M))
419 return;
Xinliang David Li8da773b2016-05-17 20:19:03 +0000420
Xinliang David Lib628dd32016-05-21 22:55:34 +0000421 size_t TotalNS = 0;
422 for (auto &PD : ProfileDataMap) {
423 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
424 TotalNS += PD.second.NumValueSites[Kind];
425 }
426
427 if (!TotalNS)
428 return;
429
430 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
Xinliang David Li69a00f02016-06-21 02:39:08 +0000431// Heuristic for small programs with very few total value sites.
432// The default value of vp-counters-per-site is chosen based on
433// the observation that large apps usually have a low percentage
434// of value sites that actually have any profile data, and thus
435// the average number of counters per site is low. For small
436// apps with very few sites, this may not be true. Bump up the
437// number of counters in this case.
Xinliang David Lie4520762016-05-23 19:29:26 +0000438#define INSTR_PROF_MIN_VAL_COUNTS 10
439 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
Xinliang David Li69a00f02016-06-21 02:39:08 +0000440 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
Xinliang David Lib628dd32016-05-21 22:55:34 +0000441
442 auto &Ctx = M->getContext();
443 Type *VNodeTypes[] = {
444#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
445#include "llvm/ProfileData/InstrProfData.inc"
446 };
447 auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
448
449 ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
450 auto *VNodesVar = new GlobalVariable(
451 *M, VNodesTy, false, llvm::GlobalValue::PrivateLinkage,
452 Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
453 VNodesVar->setSection(getInstrProfVNodesSectionName(isMachO()));
454 UsedVars.push_back(VNodesVar);
Xinliang David Li8da773b2016-05-17 20:19:03 +0000455}
456
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000457void InstrProfiling::emitNameData() {
458 std::string UncompressedData;
459
460 if (ReferencedNames.empty())
461 return;
462
463 std::string CompressedNameStr;
Vedant Kumar9152fd12016-05-19 03:54:45 +0000464 if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
Vedant Kumar43cba732016-05-03 16:53:17 +0000465 DoNameCompression)) {
Vedant Kumar9152fd12016-05-19 03:54:45 +0000466 llvm::report_fatal_error(toString(std::move(E)), false);
Vedant Kumar43cba732016-05-03 16:53:17 +0000467 }
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000468
469 auto &Ctx = M->getContext();
470 auto *NamesVal = llvm::ConstantDataArray::getString(
471 Ctx, StringRef(CompressedNameStr), false);
472 NamesVar = new llvm::GlobalVariable(*M, NamesVal->getType(), true,
473 llvm::GlobalValue::PrivateLinkage,
474 NamesVal, getInstrProfNamesVarName());
475 NamesSize = CompressedNameStr.size();
476 NamesVar->setSection(getNameSection());
477 UsedVars.push_back(NamesVar);
478}
479
Justin Bogner61ba2e32014-12-08 18:02:35 +0000480void InstrProfiling::emitRegistration() {
Xinliang David Li8da773b2016-05-17 20:19:03 +0000481 if (!needsRuntimeRegistrationOfSectionRange(*M))
Xinliang David Liaa0592c2015-10-19 04:17:10 +0000482 return;
Xinliang David Li3dd88172015-10-13 18:39:48 +0000483
Justin Bogner61ba2e32014-12-08 18:02:35 +0000484 // Construct the function.
485 auto *VoidTy = Type::getVoidTy(M->getContext());
486 auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000487 auto *Int64Ty = Type::getInt64Ty(M->getContext());
Justin Bogner61ba2e32014-12-08 18:02:35 +0000488 auto *RegisterFTy = FunctionType::get(VoidTy, false);
489 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000490 getInstrProfRegFuncsName(), M);
Peter Collingbourne96efdd62016-06-14 21:01:22 +0000491 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
Xinliang David Li69a00f02016-06-21 02:39:08 +0000492 if (Options.NoRedZone)
493 RegisterF->addFnAttr(Attribute::NoRedZone);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000494
Diego Novillob3029d22015-06-04 11:45:32 +0000495 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000496 auto *RuntimeRegisterF =
497 Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000498 getInstrProfRegFuncName(), M);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000499
500 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
501 for (Value *Data : UsedVars)
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000502 if (Data != NamesVar)
503 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
504
505 if (NamesVar) {
506 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
507 auto *NamesRegisterTy =
508 FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
509 auto *NamesRegisterF =
510 Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
511 getInstrProfNamesRegFuncName(), M);
512 IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
513 IRB.getInt64(NamesSize)});
514 }
515
Justin Bogner61ba2e32014-12-08 18:02:35 +0000516 IRB.CreateRetVoid();
517}
518
519void InstrProfiling::emitRuntimeHook() {
Justin Bogner61ba2e32014-12-08 18:02:35 +0000520
Xinliang David Li7a88ad62015-10-29 04:08:31 +0000521 // We expect the linker to be invoked with -u<hook_var> flag for linux,
522 // for which case there is no need to emit the user function.
523 if (Triple(M->getTargetTriple()).isOSLinux())
524 return;
525
Justin Bogner61ba2e32014-12-08 18:02:35 +0000526 // If the module's provided its own runtime, we don't need to do anything.
Xinliang David Li69a00f02016-06-21 02:39:08 +0000527 if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
528 return;
Justin Bogner61ba2e32014-12-08 18:02:35 +0000529
530 // Declare an external variable that will pull in the runtime initialization.
531 auto *Int32Ty = Type::getInt32Ty(M->getContext());
532 auto *Var =
533 new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000534 nullptr, getInstrProfRuntimeHookVarName());
Justin Bogner61ba2e32014-12-08 18:02:35 +0000535
536 // Make a function that uses it.
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000537 auto *User = Function::Create(FunctionType::get(Int32Ty, false),
538 GlobalValue::LinkOnceODRLinkage,
539 getInstrProfRuntimeHookVarUseFuncName(), M);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000540 User->addFnAttr(Attribute::NoInline);
Xinliang David Li69a00f02016-06-21 02:39:08 +0000541 if (Options.NoRedZone)
542 User->addFnAttr(Attribute::NoRedZone);
Justin Bogner2e427d42015-02-25 22:52:20 +0000543 User->setVisibility(GlobalValue::HiddenVisibility);
Xinliang David Lia2286082016-05-25 17:17:51 +0000544 if (Triple(M->getTargetTriple()).supportsCOMDAT())
Xinliang David Lif4edae62016-05-24 18:47:38 +0000545 User->setComdat(M->getOrInsertComdat(User->getName()));
Justin Bogner61ba2e32014-12-08 18:02:35 +0000546
547 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
548 auto *Load = IRB.CreateLoad(Var);
549 IRB.CreateRet(Load);
550
551 // Mark the user variable as used so that it isn't stripped out.
552 UsedVars.push_back(User);
553}
554
555void InstrProfiling::emitUses() {
Evgeniy Stepanovea6d49d2016-10-25 23:53:31 +0000556 if (!UsedVars.empty())
557 appendToUsed(*M, UsedVars);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000558}
559
560void InstrProfiling::emitInitialization() {
Vedant Kumarcd32eba2016-07-21 17:50:07 +0000561 StringRef InstrProfileOutput = Options.InstrProfileOutput;
Justin Bognerba1900c2015-04-30 23:49:23 +0000562
Xinliang David Li6f8c5042016-07-21 23:19:10 +0000563 if (!InstrProfileOutput.empty()) {
564 // Create variable for profile name.
565 Constant *ProfileNameConst =
566 ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
567 GlobalVariable *ProfileNameVar = new GlobalVariable(
568 *M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
569 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
570 Triple TT(M->getTargetTriple());
571 if (TT.supportsCOMDAT()) {
572 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
573 ProfileNameVar->setComdat(M->getOrInsertComdat(
574 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
575 }
576 }
577
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000578 Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName());
Xinliang David Li6f8c5042016-07-21 23:19:10 +0000579 if (!RegisterF)
Xinliang David Li69a00f02016-06-21 02:39:08 +0000580 return;
Justin Bogner61ba2e32014-12-08 18:02:35 +0000581
582 // Create the initialization function.
583 auto *VoidTy = Type::getVoidTy(M->getContext());
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000584 auto *F = Function::Create(FunctionType::get(VoidTy, false),
585 GlobalValue::InternalLinkage,
586 getInstrProfInitFuncName(), M);
Peter Collingbourne96efdd62016-06-14 21:01:22 +0000587 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000588 F->addFnAttr(Attribute::NoInline);
Xinliang David Li69a00f02016-06-21 02:39:08 +0000589 if (Options.NoRedZone)
590 F->addFnAttr(Attribute::NoRedZone);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000591
592 // Add the basic block and the necessary calls.
593 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
Justin Bognerba1900c2015-04-30 23:49:23 +0000594 if (RegisterF)
David Blaikieff6409d2015-05-18 22:13:54 +0000595 IRB.CreateCall(RegisterF, {});
Justin Bogner61ba2e32014-12-08 18:02:35 +0000596 IRB.CreateRetVoid();
597
598 appendToGlobalCtors(*M, F, 0);
599}