blob: 8b9abaddc84cd8630da36d8d3acae6825e308df0 [file] [log] [blame]
Nick Lewyckyd01d42e2008-11-02 05:52:50 +00001//===- MergeFunctions.cpp - Merge identical functions ---------------------===//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Nick Lewyckyd01d42e2008-11-02 05:52:50 +00006//
7//===----------------------------------------------------------------------===//
8//
9// This pass looks for equivalent functions that are mergable and folds them.
10//
Stepan Dyatkovskiy471eab32014-06-22 00:57:09 +000011// Order relation is defined on set of functions. It was made through
12// special function comparison procedure that returns
13// 0 when functions are equal,
14// -1 when Left function is less than right function, and
15// 1 for opposite case. We need total-ordering, so we need to maintain
16// four properties on the functions set:
17// a <= a (reflexivity)
18// if a <= b and b <= a then a = b (antisymmetry)
19// if a <= b and b <= c then a <= c (transitivity).
20// for all a and b: a <= b or b <= a (totality).
Nick Lewyckyd01d42e2008-11-02 05:52:50 +000021//
Stepan Dyatkovskiy471eab32014-06-22 00:57:09 +000022// Comparison iterates through each instruction in each basic block.
23// Functions are kept on binary tree. For each new function F we perform
24// lookup in binary tree.
25// In practice it works the following way:
26// -- We define Function* container class with custom "operator<" (FunctionPtr).
27// -- "FunctionPtr" instances are stored in std::set collection, so every
28// std::set::insert operation will give you result in log(N) time.
Fangrui Songf78650a2018-07-30 19:41:25 +000029//
JF Bastien5e4303d2015-08-15 01:18:18 +000030// As an optimization, a hash of the function structure is calculated first, and
31// two functions are only compared if they have the same hash. This hash is
32// cheap to compute, and has the property that if function F == G according to
33// the comparison function, then hash(F) == hash(G). This consistency property
34// is critical to ensuring all possible merging opportunities are exploited.
35// Collisions in the hash affect the speed of the pass but not the correctness
36// or determinism of the resulting transformation.
Nick Lewyckyd01d42e2008-11-02 05:52:50 +000037//
Nick Lewyckyd3c6dfe2010-05-13 05:48:45 +000038// When a match is found the functions are folded. If both functions are
39// overridable, we move the functionality into a new internal function and
40// leave two overridable thunks to it.
Nick Lewyckyd01d42e2008-11-02 05:52:50 +000041//
42//===----------------------------------------------------------------------===//
43//
44// Future work:
45//
Nick Lewyckyd01d42e2008-11-02 05:52:50 +000046// * virtual functions.
47//
48// Many functions have their address taken by the virtual function table for
49// the object they belong to. However, as long as it's only used for a lookup
Nick Lewyckyfbd27572010-08-08 05:04:23 +000050// and call, this is irrelevant, and we'd like to fold such functions.
Nick Lewyckyd01d42e2008-11-02 05:52:50 +000051//
Nick Lewyckyfbd27572010-08-08 05:04:23 +000052// * be smarter about bitcasts.
Nick Lewyckyd3c6dfe2010-05-13 05:48:45 +000053//
54// In order to fold functions, we will sometimes add either bitcast instructions
55// or bitcast constant expressions. Unfortunately, this can confound further
56// analysis since the two functions differ where one has a bitcast and the
Nick Lewyckyfbd27572010-08-08 05:04:23 +000057// other doesn't. We should learn to look through bitcasts.
Nick Lewyckyd3c6dfe2010-05-13 05:48:45 +000058//
Stepan Dyatkovskiy471eab32014-06-22 00:57:09 +000059// * Compare complex types with pointer types inside.
60// * Compare cross-reference cases.
61// * Compare complex expressions.
62//
63// All the three issues above could be described as ability to prove that
64// fA == fB == fC == fE == fF == fG in example below:
65//
66// void fA() {
67// fB();
68// }
69// void fB() {
70// fA();
71// }
72//
73// void fE() {
74// fF();
75// }
76// void fF() {
77// fG();
78// }
79// void fG() {
80// fE();
81// }
82//
83// Simplest cross-reference case (fA <--> fB) was implemented in previous
84// versions of MergeFunctions, though it presented only in two function pairs
85// in test-suite (that counts >50k functions)
86// Though possibility to detect complex cross-referencing (e.g.: A->B->C->D->A)
87// could cover much more cases.
88//
Nick Lewyckyd01d42e2008-11-02 05:52:50 +000089//===----------------------------------------------------------------------===//
90
Eugene Zelenkof27d1612017-10-19 21:21:30 +000091#include "llvm/ADT/ArrayRef.h"
Florian Hahna1cc8482018-06-12 11:16:56 +000092#include "llvm/ADT/SmallPtrSet.h"
Eugene Zelenkof27d1612017-10-19 21:21:30 +000093#include "llvm/ADT/SmallVector.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000094#include "llvm/ADT/Statistic.h"
Eugene Zelenkof27d1612017-10-19 21:21:30 +000095#include "llvm/IR/Argument.h"
96#include "llvm/IR/Attributes.h"
97#include "llvm/IR/BasicBlock.h"
Chandler Carruth219b89b2014-03-04 11:01:28 +000098#include "llvm/IR/CallSite.h"
Eugene Zelenkof27d1612017-10-19 21:21:30 +000099#include "llvm/IR/Constant.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000100#include "llvm/IR/Constants.h"
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000101#include "llvm/IR/DebugInfoMetadata.h"
102#include "llvm/IR/DebugLoc.h"
103#include "llvm/IR/DerivedTypes.h"
104#include "llvm/IR/Function.h"
105#include "llvm/IR/GlobalValue.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000106#include "llvm/IR/IRBuilder.h"
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000107#include "llvm/IR/InstrTypes.h"
108#include "llvm/IR/Instruction.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000109#include "llvm/IR/Instructions.h"
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000110#include "llvm/IR/IntrinsicInst.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000111#include "llvm/IR/Module.h"
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000112#include "llvm/IR/Type.h"
113#include "llvm/IR/Use.h"
114#include "llvm/IR/User.h"
115#include "llvm/IR/Value.h"
Chandler Carruth4220e9c2014-03-04 11:17:44 +0000116#include "llvm/IR/ValueHandle.h"
JF Bastien057292a2015-08-21 23:27:24 +0000117#include "llvm/IR/ValueMap.h"
Nick Lewyckyd01d42e2008-11-02 05:52:50 +0000118#include "llvm/Pass.h"
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000119#include "llvm/Support/Casting.h"
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000120#include "llvm/Support/CommandLine.h"
Nick Lewyckyd01d42e2008-11-02 05:52:50 +0000121#include "llvm/Support/Debug.h"
Daniel Dunbar0dd5e1e2009-07-25 00:23:56 +0000122#include "llvm/Support/raw_ostream.h"
Mehdi Aminib550cb12016-04-18 09:17:29 +0000123#include "llvm/Transforms/IPO.h"
Erik Eckstein4d6fb722016-11-11 21:15:13 +0000124#include "llvm/Transforms/Utils/FunctionComparator.h"
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000125#include <algorithm>
126#include <cassert>
127#include <iterator>
128#include <set>
129#include <utility>
Nick Lewycky68984ed2010-08-31 08:29:37 +0000130#include <vector>
Hans Wennborg083ca9b2015-10-06 23:24:35 +0000131
Nick Lewyckyd01d42e2008-11-02 05:52:50 +0000132using namespace llvm;
133
Chandler Carruth964daaa2014-04-22 02:55:47 +0000134#define DEBUG_TYPE "mergefunc"
135
Nick Lewyckyd01d42e2008-11-02 05:52:50 +0000136STATISTIC(NumFunctionsMerged, "Number of functions merged");
Nick Lewycky71972d42010-09-07 01:42:10 +0000137STATISTIC(NumThunksWritten, "Number of thunks generated");
Nikita Popov6f54fb02018-11-21 19:37:19 +0000138STATISTIC(NumAliasesWritten, "Number of aliases generated");
Nick Lewycky71972d42010-09-07 01:42:10 +0000139STATISTIC(NumDoubleWeak, "Number of new functions created");
Nick Lewyckyd01d42e2008-11-02 05:52:50 +0000140
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000141static cl::opt<unsigned> NumFunctionsForSanityCheck(
142 "mergefunc-sanity",
143 cl::desc("How many functions in module could be used for "
144 "MergeFunctions pass sanity check. "
145 "'0' disables this check. Works only with '-debug' key."),
146 cl::init(0), cl::Hidden);
147
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000148// Under option -mergefunc-preserve-debug-info we:
149// - Do not create a new function for a thunk.
150// - Retain the debug info for a thunk's parameters (and associated
151// instructions for the debug info) from the entry block.
152// Note: -debug will display the algorithm at work.
153// - Create debug-info for the call (to the shared implementation) made by
154// a thunk and its return value.
155// - Erase the rest of the function, retaining the (minimally sized) entry
156// block to create a thunk.
157// - Preserve a thunk's call site to point to the thunk even when both occur
158// within the same translation unit, to aid debugability. Note that this
159// behaviour differs from the underlying -mergefunc implementation which
160// modifies the thunk's call site to point to the shared implementation
161// when both occur within the same translation unit.
162static cl::opt<bool>
163 MergeFunctionsPDI("mergefunc-preserve-debug-info", cl::Hidden,
164 cl::init(false),
165 cl::desc("Preserve debug info in thunk when mergefunc "
166 "transformations are made."));
167
Nikita Popov6f54fb02018-11-21 19:37:19 +0000168static cl::opt<bool>
169 MergeFunctionsAliases("mergefunc-use-aliases", cl::Hidden,
170 cl::init(false),
171 cl::desc("Allow mergefunc to create aliases"));
172
Nick Lewyckyf3a07ec2010-09-05 09:00:32 +0000173namespace {
Nick Lewycky00959372010-09-05 08:22:49 +0000174
Stepan Dyatkovskiyfe134cd2014-09-10 10:08:25 +0000175class FunctionNode {
Arnold Schwaighofer0302da62015-06-09 00:03:29 +0000176 mutable AssertingVH<Function> F;
JF Bastien5e4303d2015-08-15 01:18:18 +0000177 FunctionComparator::FunctionHash Hash;
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000178
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000179public:
JF Bastien5e4303d2015-08-15 01:18:18 +0000180 // Note the hash is recalculated potentially multiple times, but it is cheap.
JF Bastien057292a2015-08-21 23:27:24 +0000181 FunctionNode(Function *F)
182 : F(F), Hash(FunctionComparator::functionHash(*F)) {}
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000183
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000184 Function *getFunc() const { return F; }
JF Bastien057292a2015-08-21 23:27:24 +0000185 FunctionComparator::FunctionHash getHash() const { return Hash; }
Arnold Schwaighofer0302da62015-06-09 00:03:29 +0000186
187 /// Replace the reference to the function F by the function G, assuming their
188 /// implementations are equal.
189 void replaceBy(Function *G) const {
Arnold Schwaighofer0302da62015-06-09 00:03:29 +0000190 F = G;
191 }
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000192};
Nick Lewycky564fcca2011-01-28 07:36:21 +0000193
194/// MergeFunctions finds functions which will generate identical machine code,
195/// by considering all pointer types to be equivalent. Once identified,
196/// MergeFunctions will fold them by replacing a call to one to a call to a
197/// bitcast of the other.
Nick Lewycky564fcca2011-01-28 07:36:21 +0000198class MergeFunctions : public ModulePass {
199public:
200 static char ID;
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000201
Nick Lewycky564fcca2011-01-28 07:36:21 +0000202 MergeFunctions()
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000203 : ModulePass(ID), FnTree(FunctionNodeCmp(&GlobalNumbers)) {
Nick Lewycky564fcca2011-01-28 07:36:21 +0000204 initializeMergeFunctionsPass(*PassRegistry::getPassRegistry());
205 }
206
Craig Topper3e4c6972014-03-05 09:10:37 +0000207 bool runOnModule(Module &M) override;
Nick Lewycky564fcca2011-01-28 07:36:21 +0000208
209private:
JF Bastien057292a2015-08-21 23:27:24 +0000210 // The function comparison operator is provided here so that FunctionNodes do
211 // not need to become larger with another pointer.
212 class FunctionNodeCmp {
213 GlobalNumberState* GlobalNumbers;
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000214
JF Bastien057292a2015-08-21 23:27:24 +0000215 public:
216 FunctionNodeCmp(GlobalNumberState* GN) : GlobalNumbers(GN) {}
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000217
JF Bastien057292a2015-08-21 23:27:24 +0000218 bool operator()(const FunctionNode &LHS, const FunctionNode &RHS) const {
219 // Order first by hashes, then full function comparison.
220 if (LHS.getHash() != RHS.getHash())
221 return LHS.getHash() < RHS.getHash();
222 FunctionComparator FCmp(LHS.getFunc(), RHS.getFunc(), GlobalNumbers);
223 return FCmp.compare() == -1;
224 }
225 };
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000226 using FnTreeType = std::set<FunctionNode, FunctionNodeCmp>;
JF Bastien057292a2015-08-21 23:27:24 +0000227
228 GlobalNumberState GlobalNumbers;
Nick Lewycky564fcca2011-01-28 07:36:21 +0000229
230 /// A work queue of functions that may have been modified and should be
231 /// analyzed again.
Sanjoy Dase6bca0e2017-05-01 17:07:49 +0000232 std::vector<WeakTrackingVH> Deferred;
Nick Lewycky564fcca2011-01-28 07:36:21 +0000233
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000234#ifndef NDEBUG
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000235 /// Checks the rules of order relation introduced among functions set.
236 /// Returns true, if sanity check has been passed, and false if failed.
Sanjoy Dase6bca0e2017-05-01 17:07:49 +0000237 bool doSanityCheck(std::vector<WeakTrackingVH> &Worklist);
Davide Italianob6681e22017-04-28 19:39:45 +0000238#endif
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000239
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000240 /// Insert a ComparableFunction into the FnTree, or merge it away if it's
Nick Lewycky564fcca2011-01-28 07:36:21 +0000241 /// equal to one that's already present.
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000242 bool insert(Function *NewFunction);
Nick Lewycky564fcca2011-01-28 07:36:21 +0000243
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000244 /// Remove a Function from the FnTree and queue it up for a second sweep of
Nick Lewycky564fcca2011-01-28 07:36:21 +0000245 /// analysis.
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000246 void remove(Function *F);
Nick Lewycky564fcca2011-01-28 07:36:21 +0000247
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000248 /// Find the functions that use this Value and remove them from FnTree and
Nick Lewycky564fcca2011-01-28 07:36:21 +0000249 /// queue the functions.
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000250 void removeUsers(Value *V);
Nick Lewycky564fcca2011-01-28 07:36:21 +0000251
252 /// Replace all direct calls of Old with calls of New. Will bitcast New if
253 /// necessary to make types match.
254 void replaceDirectCallers(Function *Old, Function *New);
255
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000256 /// Merge two equivalent functions. Upon completion, G may be deleted, or may
257 /// be converted into a thunk. In either case, it should never be visited
258 /// again.
259 void mergeTwoFunctions(Function *F, Function *G);
Nick Lewycky564fcca2011-01-28 07:36:21 +0000260
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000261 /// Fill PDIUnrelatedWL with instructions from the entry block that are
262 /// unrelated to parameter related debug info.
263 void filterInstsUnrelatedToPDI(BasicBlock *GEntryBlock,
264 std::vector<Instruction *> &PDIUnrelatedWL);
265
266 /// Erase the rest of the CFG (i.e. barring the entry block).
267 void eraseTail(Function *G);
268
269 /// Erase the instructions in PDIUnrelatedWL as they are unrelated to the
270 /// parameter debug info, from the entry block.
271 void eraseInstsUnrelatedToPDI(std::vector<Instruction *> &PDIUnrelatedWL);
272
273 /// Replace G with a simple tail call to bitcast(F). Also (unless
274 /// MergeFunctionsPDI holds) replace direct uses of G with bitcast(F),
275 /// delete G.
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000276 void writeThunk(Function *F, Function *G);
Nick Lewycky564fcca2011-01-28 07:36:21 +0000277
Nikita Popov6f54fb02018-11-21 19:37:19 +0000278 // Replace G with an alias to F (deleting function G)
279 void writeAlias(Function *F, Function *G);
280
Vedant Kumarb537b942019-01-19 02:46:22 +0000281 // Replace G with an alias to F if possible, or a thunk to F if possible.
282 // Returns false if neither is the case.
Nikita Popov6f54fb02018-11-21 19:37:19 +0000283 bool writeThunkOrAlias(Function *F, Function *G);
284
Arnold Schwaighofer0302da62015-06-09 00:03:29 +0000285 /// Replace function F with function G in the function tree.
JF Bastien3a4ad612015-09-02 23:55:23 +0000286 void replaceFunctionInTree(const FunctionNode &FN, Function *G);
Arnold Schwaighofer0302da62015-06-09 00:03:29 +0000287
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000288 /// The set of all distinct functions. Use the insert() and remove() methods
JF Bastien3a4ad612015-09-02 23:55:23 +0000289 /// to modify it. The map allows efficient lookup and deferring of Functions.
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000290 FnTreeType FnTree;
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000291
JF Bastien3a4ad612015-09-02 23:55:23 +0000292 // Map functions to the iterators of the FunctionNode which contains them
293 // in the FnTree. This must be updated carefully whenever the FnTree is
294 // modified, i.e. in insert(), remove(), and replaceFunctionInTree(), to avoid
295 // dangling iterators into FnTree. The invariant that preserves this is that
296 // there is exactly one mapping F -> FN for each FunctionNode FN in FnTree.
whitequark73cb9782018-11-08 03:58:01 +0000297 DenseMap<AssertingVH<Function>, FnTreeType::iterator> FNodesInTree;
Nick Lewycky564fcca2011-01-28 07:36:21 +0000298};
299
Hans Wennborg083ca9b2015-10-06 23:24:35 +0000300} // end anonymous namespace
Nick Lewycky564fcca2011-01-28 07:36:21 +0000301
302char MergeFunctions::ID = 0;
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000303
Nick Lewycky564fcca2011-01-28 07:36:21 +0000304INITIALIZE_PASS(MergeFunctions, "mergefunc", "Merge Functions", false, false)
305
306ModulePass *llvm::createMergeFunctionsPass() {
307 return new MergeFunctions();
308}
309
Davide Italianob6681e22017-04-28 19:39:45 +0000310#ifndef NDEBUG
Sanjoy Dase6bca0e2017-05-01 17:07:49 +0000311bool MergeFunctions::doSanityCheck(std::vector<WeakTrackingVH> &Worklist) {
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000312 if (const unsigned Max = NumFunctionsForSanityCheck) {
313 unsigned TripleNumber = 0;
314 bool Valid = true;
315
316 dbgs() << "MERGEFUNC-SANITY: Started for first " << Max << " functions.\n";
317
318 unsigned i = 0;
Sanjoy Dase6bca0e2017-05-01 17:07:49 +0000319 for (std::vector<WeakTrackingVH>::iterator I = Worklist.begin(),
320 E = Worklist.end();
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000321 I != E && i < Max; ++I, ++i) {
322 unsigned j = i;
Sanjoy Dase6bca0e2017-05-01 17:07:49 +0000323 for (std::vector<WeakTrackingVH>::iterator J = I; J != E && j < Max;
324 ++J, ++j) {
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000325 Function *F1 = cast<Function>(*I);
326 Function *F2 = cast<Function>(*J);
JF Bastien057292a2015-08-21 23:27:24 +0000327 int Res1 = FunctionComparator(F1, F2, &GlobalNumbers).compare();
328 int Res2 = FunctionComparator(F2, F1, &GlobalNumbers).compare();
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000329
330 // If F1 <= F2, then F2 >= F1, otherwise report failure.
331 if (Res1 != -Res2) {
332 dbgs() << "MERGEFUNC-SANITY: Non-symmetric; triple: " << TripleNumber
333 << "\n";
Matthias Braun194ded52017-01-28 06:53:55 +0000334 dbgs() << *F1 << '\n' << *F2 << '\n';
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000335 Valid = false;
336 }
337
338 if (Res1 == 0)
339 continue;
340
341 unsigned k = j;
Sanjoy Dase6bca0e2017-05-01 17:07:49 +0000342 for (std::vector<WeakTrackingVH>::iterator K = J; K != E && k < Max;
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000343 ++k, ++K, ++TripleNumber) {
344 if (K == J)
345 continue;
346
347 Function *F3 = cast<Function>(*K);
JF Bastien057292a2015-08-21 23:27:24 +0000348 int Res3 = FunctionComparator(F1, F3, &GlobalNumbers).compare();
349 int Res4 = FunctionComparator(F2, F3, &GlobalNumbers).compare();
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000350
351 bool Transitive = true;
352
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000353 if (Res1 != 0 && Res1 == Res4) {
Stepan Dyatkovskiy0b588012014-06-21 19:07:51 +0000354 // F1 > F2, F2 > F3 => F1 > F3
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000355 Transitive = Res3 == Res1;
Stepan Dyatkovskiy0b588012014-06-21 19:07:51 +0000356 } else if (Res3 != 0 && Res3 == -Res4) {
357 // F1 > F3, F3 > F2 => F1 > F2
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000358 Transitive = Res3 == Res1;
Stepan Dyatkovskiy0b588012014-06-21 19:07:51 +0000359 } else if (Res4 != 0 && -Res3 == Res4) {
360 // F2 > F3, F3 > F1 => F2 > F1
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000361 Transitive = Res4 == -Res1;
362 }
363
364 if (!Transitive) {
365 dbgs() << "MERGEFUNC-SANITY: Non-transitive; triple: "
366 << TripleNumber << "\n";
367 dbgs() << "Res1, Res3, Res4: " << Res1 << ", " << Res3 << ", "
368 << Res4 << "\n";
Matthias Braun194ded52017-01-28 06:53:55 +0000369 dbgs() << *F1 << '\n' << *F2 << '\n' << *F3 << '\n';
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000370 Valid = false;
371 }
372 }
373 }
374 }
375
376 dbgs() << "MERGEFUNC-SANITY: " << (Valid ? "Passed." : "Failed.") << "\n";
377 return Valid;
378 }
379 return true;
380}
Davide Italianob6681e22017-04-28 19:39:45 +0000381#endif
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000382
Vedant Kumara9906c12019-01-17 02:15:05 +0000383/// Check whether \p F is eligible for function merging.
384static bool isEligibleForMerging(Function &F) {
Vedant Kumarb537b942019-01-19 02:46:22 +0000385 return !F.isDeclaration() && !F.hasAvailableExternallyLinkage();
Vedant Kumara9906c12019-01-17 02:15:05 +0000386}
387
Nick Lewycky564fcca2011-01-28 07:36:21 +0000388bool MergeFunctions::runOnModule(Module &M) {
Andrew Kayloraa641a52016-04-22 22:06:11 +0000389 if (skipModule(M))
390 return false;
391
Nick Lewycky564fcca2011-01-28 07:36:21 +0000392 bool Changed = false;
Nick Lewycky564fcca2011-01-28 07:36:21 +0000393
JF Bastien5e4303d2015-08-15 01:18:18 +0000394 // All functions in the module, ordered by hash. Functions with a unique
395 // hash value are easily eliminated.
396 std::vector<std::pair<FunctionComparator::FunctionHash, Function *>>
397 HashedFuncs;
398 for (Function &Func : M) {
Vedant Kumara9906c12019-01-17 02:15:05 +0000399 if (isEligibleForMerging(Func)) {
JF Bastien5e4303d2015-08-15 01:18:18 +0000400 HashedFuncs.push_back({FunctionComparator::functionHash(Func), &Func});
Fangrui Songf78650a2018-07-30 19:41:25 +0000401 }
Nick Lewycky564fcca2011-01-28 07:36:21 +0000402 }
Nick Lewycky564fcca2011-01-28 07:36:21 +0000403
Fangrui Songefd94c52019-04-23 14:51:27 +0000404 llvm::stable_sort(HashedFuncs, less_first());
JF Bastien5e4303d2015-08-15 01:18:18 +0000405
406 auto S = HashedFuncs.begin();
407 for (auto I = HashedFuncs.begin(), IE = HashedFuncs.end(); I != IE; ++I) {
408 // If the hash value matches the previous value or the next one, we must
409 // consider merging it. Otherwise it is dropped and never considered again.
410 if ((I != S && std::prev(I)->first == I->first) ||
411 (std::next(I) != IE && std::next(I)->first == I->first) ) {
Sanjoy Dase6bca0e2017-05-01 17:07:49 +0000412 Deferred.push_back(WeakTrackingVH(I->second));
JF Bastien5e4303d2015-08-15 01:18:18 +0000413 }
414 }
Fangrui Songf78650a2018-07-30 19:41:25 +0000415
Nick Lewycky564fcca2011-01-28 07:36:21 +0000416 do {
Sanjoy Dase6bca0e2017-05-01 17:07:49 +0000417 std::vector<WeakTrackingVH> Worklist;
Nick Lewycky564fcca2011-01-28 07:36:21 +0000418 Deferred.swap(Worklist);
419
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000420 LLVM_DEBUG(doSanityCheck(Worklist));
Stepan Dyatkovskiya77f3d82014-06-21 18:58:11 +0000421
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000422 LLVM_DEBUG(dbgs() << "size of module: " << M.size() << '\n');
423 LLVM_DEBUG(dbgs() << "size of worklist: " << Worklist.size() << '\n');
Nick Lewycky564fcca2011-01-28 07:36:21 +0000424
Erik Eckstein0c48dd82016-05-31 17:20:23 +0000425 // Insert functions and merge them.
Sanjoy Dase6bca0e2017-05-01 17:07:49 +0000426 for (WeakTrackingVH &I : Worklist) {
Benjamin Kramer135f7352016-06-26 12:28:59 +0000427 if (!I)
428 continue;
429 Function *F = cast<Function>(I);
Erik Eckstein0c48dd82016-05-31 17:20:23 +0000430 if (!F->isDeclaration() && !F->hasAvailableExternallyLinkage()) {
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000431 Changed |= insert(F);
Nick Lewycky564fcca2011-01-28 07:36:21 +0000432 }
433 }
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000434 LLVM_DEBUG(dbgs() << "size of FnTree: " << FnTree.size() << '\n');
Nick Lewycky564fcca2011-01-28 07:36:21 +0000435 } while (!Deferred.empty());
436
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000437 FnTree.clear();
whitequark73cb9782018-11-08 03:58:01 +0000438 FNodesInTree.clear();
Arnold Schwaighofer0591c5d2015-10-05 17:26:36 +0000439 GlobalNumbers.clear();
Nick Lewycky564fcca2011-01-28 07:36:21 +0000440
441 return Changed;
442}
443
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000444// Replace direct callers of Old with New.
Nick Lewyckyf1cec162011-01-25 08:56:50 +0000445void MergeFunctions::replaceDirectCallers(Function *Old, Function *New) {
446 Constant *BitcastNew = ConstantExpr::getBitCast(New, Old->getType());
Chandler Carruthcdf47882014-03-09 03:16:01 +0000447 for (auto UI = Old->use_begin(), UE = Old->use_end(); UI != UE;) {
448 Use *U = &*UI;
Nick Lewyckyf1cec162011-01-25 08:56:50 +0000449 ++UI;
Chandler Carruthcdf47882014-03-09 03:16:01 +0000450 CallSite CS(U->getUser());
451 if (CS && CS.isCallee(U)) {
Arnold Schwaighofer36512332015-07-21 17:07:07 +0000452 // Transfer the called function's attributes to the call site. Due to the
JF Bastienfa946232015-09-10 18:08:35 +0000453 // bitcast we will 'lose' ABI changing attributes because the 'called
Arnold Schwaighofer36512332015-07-21 17:07:07 +0000454 // function' is no longer a Function* but the bitcast. Code that looks up
455 // the attributes from the called function will fail.
JF Bastienfa946232015-09-10 18:08:35 +0000456
457 // FIXME: This is not actually true, at least not anymore. The callsite
458 // will always have the same ABI affecting attributes as the callee,
459 // because otherwise the original input has UB. Note that Old and New
460 // always have matching ABI, so no attributes need to be changed.
461 // Transferring other attributes may help other optimizations, but that
462 // should be done uniformly and not in this ad-hoc way.
Arnold Schwaighofer36512332015-07-21 17:07:07 +0000463 auto &Context = New->getContext();
Reid Klecknerf021fab2017-04-13 23:12:13 +0000464 auto NewPAL = New->getAttributes();
465 SmallVector<AttributeSet, 4> NewArgAttrs;
466 for (unsigned argIdx = 0; argIdx < CS.arg_size(); argIdx++)
467 NewArgAttrs.push_back(NewPAL.getParamAttributes(argIdx));
468 // Don't transfer attributes from the function to the callee. Function
469 // attributes typically aren't relevant to the calling convention or ABI.
470 CS.setAttributes(AttributeList::get(Context, /*FnAttrs=*/AttributeSet(),
471 NewPAL.getRetAttributes(),
472 NewArgAttrs));
Arnold Schwaighofer36512332015-07-21 17:07:07 +0000473
Vedant Kumar08fe7e02019-01-11 17:56:21 +0000474 remove(CS.getInstruction()->getFunction());
Chandler Carruthcdf47882014-03-09 03:16:01 +0000475 U->set(BitcastNew);
Nick Lewyckyf1cec162011-01-25 08:56:50 +0000476 }
477 }
478}
479
Stepan Dyatkovskiydc2c4b42013-09-17 09:36:11 +0000480// Helper for writeThunk,
481// Selects proper bitcast operation,
Alp Tokercb402912014-01-24 17:20:08 +0000482// but a bit simpler then CastInst::getCastOpcode.
Mehdi Aminiba9fba82016-03-13 21:05:13 +0000483static Value *createCast(IRBuilder<> &Builder, Value *V, Type *DestTy) {
Stepan Dyatkovskiydc2c4b42013-09-17 09:36:11 +0000484 Type *SrcTy = V->getType();
Carlo Kok307625c2014-04-30 17:53:04 +0000485 if (SrcTy->isStructTy()) {
486 assert(DestTy->isStructTy());
487 assert(SrcTy->getStructNumElements() == DestTy->getStructNumElements());
488 Value *Result = UndefValue::get(DestTy);
489 for (unsigned int I = 0, E = SrcTy->getStructNumElements(); I < E; ++I) {
490 Value *Element = createCast(
Craig Toppere1d12942014-08-27 05:25:25 +0000491 Builder, Builder.CreateExtractValue(V, makeArrayRef(I)),
Carlo Kok307625c2014-04-30 17:53:04 +0000492 DestTy->getStructElementType(I));
493
494 Result =
Craig Toppere1d12942014-08-27 05:25:25 +0000495 Builder.CreateInsertValue(Result, Element, makeArrayRef(I));
Carlo Kok307625c2014-04-30 17:53:04 +0000496 }
497 return Result;
498 }
499 assert(!DestTy->isStructTy());
Stepan Dyatkovskiydc2c4b42013-09-17 09:36:11 +0000500 if (SrcTy->isIntegerTy() && DestTy->isPointerTy())
501 return Builder.CreateIntToPtr(V, DestTy);
502 else if (SrcTy->isPointerTy() && DestTy->isIntegerTy())
503 return Builder.CreatePtrToInt(V, DestTy);
504 else
505 return Builder.CreateBitCast(V, DestTy);
506}
507
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000508// Erase the instructions in PDIUnrelatedWL as they are unrelated to the
509// parameter debug info, from the entry block.
510void MergeFunctions::eraseInstsUnrelatedToPDI(
511 std::vector<Instruction *> &PDIUnrelatedWL) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000512 LLVM_DEBUG(
513 dbgs() << " Erasing instructions (in reverse order of appearance in "
514 "entry block) unrelated to parameter debug info from entry "
515 "block: {\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000516 while (!PDIUnrelatedWL.empty()) {
517 Instruction *I = PDIUnrelatedWL.back();
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000518 LLVM_DEBUG(dbgs() << " Deleting Instruction: ");
519 LLVM_DEBUG(I->print(dbgs()));
520 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000521 I->eraseFromParent();
522 PDIUnrelatedWL.pop_back();
523 }
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000524 LLVM_DEBUG(dbgs() << " } // Done erasing instructions unrelated to parameter "
525 "debug info from entry block. \n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000526}
527
528// Reduce G to its entry block.
529void MergeFunctions::eraseTail(Function *G) {
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000530 std::vector<BasicBlock *> WorklistBB;
531 for (Function::iterator BBI = std::next(G->begin()), BBE = G->end();
532 BBI != BBE; ++BBI) {
533 BBI->dropAllReferences();
534 WorklistBB.push_back(&*BBI);
535 }
536 while (!WorklistBB.empty()) {
537 BasicBlock *BB = WorklistBB.back();
538 BB->eraseFromParent();
539 WorklistBB.pop_back();
540 }
541}
542
543// We are interested in the following instructions from the entry block as being
544// related to parameter debug info:
545// - @llvm.dbg.declare
546// - stores from the incoming parameters to locations on the stack-frame
547// - allocas that create these locations on the stack-frame
548// - @llvm.dbg.value
549// - the entry block's terminator
550// The rest are unrelated to debug info for the parameters; fill up
551// PDIUnrelatedWL with such instructions.
552void MergeFunctions::filterInstsUnrelatedToPDI(
553 BasicBlock *GEntryBlock, std::vector<Instruction *> &PDIUnrelatedWL) {
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000554 std::set<Instruction *> PDIRelated;
555 for (BasicBlock::iterator BI = GEntryBlock->begin(), BIE = GEntryBlock->end();
556 BI != BIE; ++BI) {
557 if (auto *DVI = dyn_cast<DbgValueInst>(&*BI)) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000558 LLVM_DEBUG(dbgs() << " Deciding: ");
559 LLVM_DEBUG(BI->print(dbgs()));
560 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000561 DILocalVariable *DILocVar = DVI->getVariable();
562 if (DILocVar->isParameter()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000563 LLVM_DEBUG(dbgs() << " Include (parameter): ");
564 LLVM_DEBUG(BI->print(dbgs()));
565 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000566 PDIRelated.insert(&*BI);
567 } else {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000568 LLVM_DEBUG(dbgs() << " Delete (!parameter): ");
569 LLVM_DEBUG(BI->print(dbgs()));
570 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000571 }
572 } else if (auto *DDI = dyn_cast<DbgDeclareInst>(&*BI)) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000573 LLVM_DEBUG(dbgs() << " Deciding: ");
574 LLVM_DEBUG(BI->print(dbgs()));
575 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000576 DILocalVariable *DILocVar = DDI->getVariable();
577 if (DILocVar->isParameter()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000578 LLVM_DEBUG(dbgs() << " Parameter: ");
579 LLVM_DEBUG(DILocVar->print(dbgs()));
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000580 AllocaInst *AI = dyn_cast_or_null<AllocaInst>(DDI->getAddress());
581 if (AI) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000582 LLVM_DEBUG(dbgs() << " Processing alloca users: ");
583 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000584 for (User *U : AI->users()) {
585 if (StoreInst *SI = dyn_cast<StoreInst>(U)) {
586 if (Value *Arg = SI->getValueOperand()) {
587 if (dyn_cast<Argument>(Arg)) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000588 LLVM_DEBUG(dbgs() << " Include: ");
589 LLVM_DEBUG(AI->print(dbgs()));
590 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000591 PDIRelated.insert(AI);
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000592 LLVM_DEBUG(dbgs() << " Include (parameter): ");
593 LLVM_DEBUG(SI->print(dbgs()));
594 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000595 PDIRelated.insert(SI);
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000596 LLVM_DEBUG(dbgs() << " Include: ");
597 LLVM_DEBUG(BI->print(dbgs()));
598 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000599 PDIRelated.insert(&*BI);
600 } else {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000601 LLVM_DEBUG(dbgs() << " Delete (!parameter): ");
602 LLVM_DEBUG(SI->print(dbgs()));
603 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000604 }
605 }
606 } else {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000607 LLVM_DEBUG(dbgs() << " Defer: ");
608 LLVM_DEBUG(U->print(dbgs()));
609 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000610 }
611 }
612 } else {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000613 LLVM_DEBUG(dbgs() << " Delete (alloca NULL): ");
614 LLVM_DEBUG(BI->print(dbgs()));
615 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000616 }
617 } else {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000618 LLVM_DEBUG(dbgs() << " Delete (!parameter): ");
619 LLVM_DEBUG(BI->print(dbgs()));
620 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000621 }
Chandler Carruth93cf2ea2018-10-18 00:37:37 +0000622 } else if (BI->isTerminator() && &*BI == GEntryBlock->getTerminator()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000623 LLVM_DEBUG(dbgs() << " Will Include Terminator: ");
624 LLVM_DEBUG(BI->print(dbgs()));
625 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000626 PDIRelated.insert(&*BI);
627 } else {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000628 LLVM_DEBUG(dbgs() << " Defer: ");
629 LLVM_DEBUG(BI->print(dbgs()));
630 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000631 }
632 }
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000633 LLVM_DEBUG(
634 dbgs()
635 << " Report parameter debug info related/related instructions: {\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000636 for (BasicBlock::iterator BI = GEntryBlock->begin(), BE = GEntryBlock->end();
637 BI != BE; ++BI) {
638
639 Instruction *I = &*BI;
640 if (PDIRelated.find(I) == PDIRelated.end()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000641 LLVM_DEBUG(dbgs() << " !PDIRelated: ");
642 LLVM_DEBUG(I->print(dbgs()));
643 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000644 PDIUnrelatedWL.push_back(I);
645 } else {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000646 LLVM_DEBUG(dbgs() << " PDIRelated: ");
647 LLVM_DEBUG(I->print(dbgs()));
648 LLVM_DEBUG(dbgs() << "\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000649 }
650 }
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000651 LLVM_DEBUG(dbgs() << " }\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000652}
653
Vedant Kumarb537b942019-01-19 02:46:22 +0000654/// Whether this function may be replaced by a forwarding thunk.
655static bool canCreateThunkFor(Function *F) {
656 if (F->isVarArg())
657 return false;
658
659 // Don't merge tiny functions using a thunk, since it can just end up
660 // making the function larger.
whitequark8f0ab252018-05-15 11:31:07 +0000661 if (F->size() == 1) {
662 if (F->front().size() <= 2) {
Vedant Kumarb537b942019-01-19 02:46:22 +0000663 LLVM_DEBUG(dbgs() << "canCreateThunkFor: " << F->getName()
Nicola Zaghen03d0b912018-05-23 15:09:29 +0000664 << " is too small to bother creating a thunk for\n");
whitequark8f0ab252018-05-15 11:31:07 +0000665 return false;
666 }
667 }
668 return true;
669}
670
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000671// Replace G with a simple tail call to bitcast(F). Also (unless
672// MergeFunctionsPDI holds) replace direct uses of G with bitcast(F),
673// delete G. Under MergeFunctionsPDI, we use G itself for creating
674// the thunk as we preserve the debug info (and associated instructions)
675// from G's entry block pertaining to G's incoming arguments which are
676// passed on as corresponding arguments in the call that G makes to F.
677// For better debugability, under MergeFunctionsPDI, we do not modify G's
678// call sites to point to F even when within the same translation unit.
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000679void MergeFunctions::writeThunk(Function *F, Function *G) {
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000680 BasicBlock *GEntryBlock = nullptr;
681 std::vector<Instruction *> PDIUnrelatedWL;
682 BasicBlock *BB = nullptr;
683 Function *NewG = nullptr;
684 if (MergeFunctionsPDI) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000685 LLVM_DEBUG(dbgs() << "writeThunk: (MergeFunctionsPDI) Do not create a new "
686 "function as thunk; retain original: "
687 << G->getName() << "()\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000688 GEntryBlock = &G->getEntryBlock();
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000689 LLVM_DEBUG(
690 dbgs() << "writeThunk: (MergeFunctionsPDI) filter parameter related "
691 "debug info for "
692 << G->getName() << "() {\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000693 filterInstsUnrelatedToPDI(GEntryBlock, PDIUnrelatedWL);
694 GEntryBlock->getTerminator()->eraseFromParent();
695 BB = GEntryBlock;
696 } else {
Dylan McKayf920da02018-12-18 09:52:52 +0000697 NewG = Function::Create(G->getFunctionType(), G->getLinkage(),
698 G->getAddressSpace(), "", G->getParent());
Saleem Abdulrasoolb96d9b32019-04-19 01:48:36 +0000699 NewG->setComdat(G->getComdat());
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000700 BB = BasicBlock::Create(F->getContext(), "", NewG);
701 }
Nick Lewyckye04dc222009-06-12 08:04:51 +0000702
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000703 IRBuilder<> Builder(BB);
704 Function *H = MergeFunctionsPDI ? G : NewG;
Nick Lewyckyd3c6dfe2010-05-13 05:48:45 +0000705 SmallVector<Value *, 16> Args;
Nick Lewyckye04dc222009-06-12 08:04:51 +0000706 unsigned i = 0;
Chris Lattner229907c2011-07-18 04:54:35 +0000707 FunctionType *FFTy = F->getFunctionType();
Eugene Zelenkof27d1612017-10-19 21:21:30 +0000708 for (Argument &AI : H->args()) {
Duncan P. N. Exon Smith17323402015-10-13 17:51:03 +0000709 Args.push_back(createCast(Builder, &AI, FFTy->getParamType(i)));
Nick Lewyckye04dc222009-06-12 08:04:51 +0000710 ++i;
711 }
712
Jay Foad5bd375a2011-07-15 08:37:34 +0000713 CallInst *CI = Builder.CreateCall(F, Args);
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000714 ReturnInst *RI = nullptr;
Nick Lewyckye04dc222009-06-12 08:04:51 +0000715 CI->setTailCall();
Nick Lewyckyd5bf51f2009-06-12 16:04:00 +0000716 CI->setCallingConv(F->getCallingConv());
JF Bastienfa946232015-09-10 18:08:35 +0000717 CI->setAttributes(F->getAttributes());
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000718 if (H->getReturnType()->isVoidTy()) {
719 RI = Builder.CreateRetVoid();
Nick Lewyckye04dc222009-06-12 08:04:51 +0000720 } else {
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000721 RI = Builder.CreateRet(createCast(Builder, CI, H->getReturnType()));
Nick Lewyckye04dc222009-06-12 08:04:51 +0000722 }
723
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000724 if (MergeFunctionsPDI) {
725 DISubprogram *DIS = G->getSubprogram();
726 if (DIS) {
727 DebugLoc CIDbgLoc = DebugLoc::get(DIS->getScopeLine(), 0, DIS);
728 DebugLoc RIDbgLoc = DebugLoc::get(DIS->getScopeLine(), 0, DIS);
729 CI->setDebugLoc(CIDbgLoc);
730 RI->setDebugLoc(RIDbgLoc);
731 } else {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000732 LLVM_DEBUG(
733 dbgs() << "writeThunk: (MergeFunctionsPDI) No DISubprogram for "
734 << G->getName() << "()\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000735 }
736 eraseTail(G);
737 eraseInstsUnrelatedToPDI(PDIUnrelatedWL);
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000738 LLVM_DEBUG(
739 dbgs() << "} // End of parameter related debug info filtering for: "
740 << G->getName() << "()\n");
Anmol P. Paralkar910dc8d2017-01-21 02:02:56 +0000741 } else {
742 NewG->copyAttributesFrom(G);
743 NewG->takeName(G);
744 removeUsers(G);
745 G->replaceAllUsesWith(NewG);
746 G->eraseFromParent();
747 }
Nick Lewycky71972d42010-09-07 01:42:10 +0000748
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000749 LLVM_DEBUG(dbgs() << "writeThunk: " << H->getName() << '\n');
Nick Lewycky71972d42010-09-07 01:42:10 +0000750 ++NumThunksWritten;
Nick Lewyckye04dc222009-06-12 08:04:51 +0000751}
752
Nikita Popov6f54fb02018-11-21 19:37:19 +0000753// Whether this function may be replaced by an alias
754static bool canCreateAliasFor(Function *F) {
755 if (!MergeFunctionsAliases || !F->hasGlobalUnnamedAddr())
756 return false;
757
758 // We should only see linkages supported by aliases here
759 assert(F->hasLocalLinkage() || F->hasExternalLinkage()
760 || F->hasWeakLinkage() || F->hasLinkOnceLinkage());
761 return true;
762}
763
764// Replace G with an alias to F (deleting function G)
765void MergeFunctions::writeAlias(Function *F, Function *G) {
766 Constant *BitcastF = ConstantExpr::getBitCast(F, G->getType());
767 PointerType *PtrType = G->getType();
768 auto *GA = GlobalAlias::create(
769 PtrType->getElementType(), PtrType->getAddressSpace(),
770 G->getLinkage(), "", BitcastF, G->getParent());
771
Guillaume Chatelet0e620112019-10-15 11:24:36 +0000772 F->setAlignment(MaybeAlign(std::max(F->getAlignment(), G->getAlignment())));
Nikita Popov6f54fb02018-11-21 19:37:19 +0000773 GA->takeName(G);
774 GA->setVisibility(G->getVisibility());
775 GA->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
776
777 removeUsers(G);
778 G->replaceAllUsesWith(GA);
779 G->eraseFromParent();
780
781 LLVM_DEBUG(dbgs() << "writeAlias: " << GA->getName() << '\n');
782 ++NumAliasesWritten;
783}
784
785// Replace G with an alias to F if possible, or a thunk to F if
786// profitable. Returns false if neither is the case.
787bool MergeFunctions::writeThunkOrAlias(Function *F, Function *G) {
788 if (canCreateAliasFor(G)) {
789 writeAlias(F, G);
790 return true;
791 }
Vedant Kumarb537b942019-01-19 02:46:22 +0000792 if (canCreateThunkFor(F)) {
Nikita Popov6f54fb02018-11-21 19:37:19 +0000793 writeThunk(F, G);
794 return true;
795 }
796 return false;
797}
798
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000799// Merge two equivalent functions. Upon completion, Function G is deleted.
800void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {
Sanjoy Das5ce32722016-04-08 00:48:30 +0000801 if (F->isInterposable()) {
802 assert(G->isInterposable());
Nick Lewyckyd3c6dfe2010-05-13 05:48:45 +0000803
Nikita Popov6f54fb02018-11-21 19:37:19 +0000804 // Both writeThunkOrAlias() calls below must succeed, either because we can
805 // create aliases for G and NewF, or because a thunk for F is profitable.
806 // F here has the same signature as NewF below, so that's what we check.
Vedant Kumarb537b942019-01-19 02:46:22 +0000807 if (!canCreateThunkFor(F) &&
808 (!canCreateAliasFor(F) || !canCreateAliasFor(G)))
whitequark8f0ab252018-05-15 11:31:07 +0000809 return;
whitequark8f0ab252018-05-15 11:31:07 +0000810
Arnold Schwaighofer7e226272015-06-09 18:19:17 +0000811 // Make them both thunks to the same internal function.
Dylan McKayf920da02018-12-18 09:52:52 +0000812 Function *NewF = Function::Create(F->getFunctionType(), F->getLinkage(),
813 F->getAddressSpace(), "", F->getParent());
Nikita Popov6f54fb02018-11-21 19:37:19 +0000814 NewF->copyAttributesFrom(F);
815 NewF->takeName(F);
Arnold Schwaighofer7e226272015-06-09 18:19:17 +0000816 removeUsers(F);
Nikita Popov6f54fb02018-11-21 19:37:19 +0000817 F->replaceAllUsesWith(NewF);
Arnold Schwaighofer7e226272015-06-09 18:19:17 +0000818
Guillaume Chatelet0e620112019-10-15 11:24:36 +0000819 MaybeAlign MaxAlignment(std::max(G->getAlignment(), NewF->getAlignment()));
Arnold Schwaighofer7e226272015-06-09 18:19:17 +0000820
Nikita Popov6f54fb02018-11-21 19:37:19 +0000821 writeThunkOrAlias(F, G);
822 writeThunkOrAlias(F, NewF);
Nick Lewycky71972d42010-09-07 01:42:10 +0000823
Arnold Schwaighofer7e226272015-06-09 18:19:17 +0000824 F->setAlignment(MaxAlignment);
825 F->setLinkage(GlobalValue::PrivateLinkage);
Nick Lewycky71972d42010-09-07 01:42:10 +0000826 ++NumDoubleWeak;
whitequark8f0ab252018-05-15 11:31:07 +0000827 ++NumFunctionsMerged;
Nick Lewyckyf216f69a2010-08-06 07:21:30 +0000828 } else {
whitequark8f0ab252018-05-15 11:31:07 +0000829 // For better debugability, under MergeFunctionsPDI, we do not modify G's
830 // call sites to point to F even when within the same translation unit.
831 if (!G->isInterposable() && !MergeFunctionsPDI) {
832 if (G->hasGlobalUnnamedAddr()) {
833 // G might have been a key in our GlobalNumberState, and it's illegal
834 // to replace a key in ValueMap<GlobalValue *> with a non-global.
835 GlobalNumbers.erase(G);
836 // If G's address is not significant, replace it entirely.
837 Constant *BitcastF = ConstantExpr::getBitCast(F, G->getType());
whitequark3580ac62018-11-08 03:57:55 +0000838 removeUsers(G);
whitequark8f0ab252018-05-15 11:31:07 +0000839 G->replaceAllUsesWith(BitcastF);
840 } else {
841 // Redirect direct callers of G to F. (See note on MergeFunctionsPDI
842 // above).
843 replaceDirectCallers(G, F);
844 }
845 }
Nick Lewycky3c6d34a2008-11-02 16:46:26 +0000846
whitequark8f0ab252018-05-15 11:31:07 +0000847 // If G was internal then we may have replaced all uses of G with F. If so,
848 // stop here and delete G. There's no need for a thunk. (See note on
849 // MergeFunctionsPDI above).
Vedant Kumaree10ef72019-01-11 17:56:35 +0000850 if (G->isDiscardableIfUnused() && G->use_empty() && !MergeFunctionsPDI) {
whitequark8f0ab252018-05-15 11:31:07 +0000851 G->eraseFromParent();
852 ++NumFunctionsMerged;
853 return;
854 }
855
Nikita Popov6f54fb02018-11-21 19:37:19 +0000856 if (writeThunkOrAlias(F, G)) {
857 ++NumFunctionsMerged;
whitequark8f0ab252018-05-15 11:31:07 +0000858 }
whitequark8f0ab252018-05-15 11:31:07 +0000859 }
Nick Lewyckyd01d42e2008-11-02 05:52:50 +0000860}
861
JF Bastien3a4ad612015-09-02 23:55:23 +0000862/// Replace function F by function G.
863void MergeFunctions::replaceFunctionInTree(const FunctionNode &FN,
Arnold Schwaighofer0302da62015-06-09 00:03:29 +0000864 Function *G) {
JF Bastien3a4ad612015-09-02 23:55:23 +0000865 Function *F = FN.getFunc();
JF Bastien057292a2015-08-21 23:27:24 +0000866 assert(FunctionComparator(F, G, &GlobalNumbers).compare() == 0 &&
867 "The two functions must be equal");
Fangrui Songf78650a2018-07-30 19:41:25 +0000868
JF Bastien3a4ad612015-09-02 23:55:23 +0000869 auto I = FNodesInTree.find(F);
870 assert(I != FNodesInTree.end() && "F should be in FNodesInTree");
871 assert(FNodesInTree.count(G) == 0 && "FNodesInTree should not contain G");
Fangrui Songf78650a2018-07-30 19:41:25 +0000872
JF Bastien3a4ad612015-09-02 23:55:23 +0000873 FnTreeType::iterator IterToFNInFnTree = I->second;
874 assert(&(*IterToFNInFnTree) == &FN && "F should map to FN in FNodesInTree.");
875 // Remove F -> FN and insert G -> FN
876 FNodesInTree.erase(I);
877 FNodesInTree.insert({G, IterToFNInFnTree});
878 // Replace F with G in FN, which is stored inside the FnTree.
879 FN.replaceBy(G);
Arnold Schwaighofer0302da62015-06-09 00:03:29 +0000880}
881
whitequark73cb9782018-11-08 03:58:01 +0000882// Ordering for functions that are equal under FunctionComparator
883static bool isFuncOrderCorrect(const Function *F, const Function *G) {
884 if (F->isInterposable() != G->isInterposable()) {
885 // Strong before weak, because the weak function may call the strong
886 // one, but not the other way around.
887 return !F->isInterposable();
888 }
889 if (F->hasLocalLinkage() != G->hasLocalLinkage()) {
890 // External before local, because we definitely have to keep the external
891 // function, but may be able to drop the local one.
892 return !F->hasLocalLinkage();
893 }
894 // Impose a total order (by name) on the replacement of functions. This is
895 // important when operating on more than one module independently to prevent
896 // cycles of thunks calling each other when the modules are linked together.
897 return F->getName() <= G->getName();
898}
899
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000900// Insert a ComparableFunction into the FnTree, or merge it away if equal to one
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000901// that was already inserted.
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000902bool MergeFunctions::insert(Function *NewFunction) {
903 std::pair<FnTreeType::iterator, bool> Result =
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000904 FnTree.insert(FunctionNode(NewFunction));
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000905
Nick Lewycky292e78c2011-02-09 06:32:02 +0000906 if (Result.second) {
JF Bastien3a4ad612015-09-02 23:55:23 +0000907 assert(FNodesInTree.count(NewFunction) == 0);
908 FNodesInTree.insert({NewFunction, Result.first});
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000909 LLVM_DEBUG(dbgs() << "Inserting as unique: " << NewFunction->getName()
910 << '\n');
Nick Lewycky00959372010-09-05 08:22:49 +0000911 return false;
Nick Lewycky292e78c2011-02-09 06:32:02 +0000912 }
Nick Lewyckyfbd27572010-08-08 05:04:23 +0000913
Stepan Dyatkovskiyfe134cd2014-09-10 10:08:25 +0000914 const FunctionNode &OldF = *Result.first;
Nick Lewycky00959372010-09-05 08:22:49 +0000915
whitequark73cb9782018-11-08 03:58:01 +0000916 if (!isFuncOrderCorrect(OldF.getFunc(), NewFunction)) {
Erik Eckstein0c48dd82016-05-31 17:20:23 +0000917 // Swap the two functions.
918 Function *F = OldF.getFunc();
919 replaceFunctionInTree(*Result.first, NewFunction);
920 NewFunction = F;
921 assert(OldF.getFunc() != F && "Must have swapped the functions.");
922 }
Nick Lewycky00959372010-09-05 08:22:49 +0000923
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000924 LLVM_DEBUG(dbgs() << " " << OldF.getFunc()->getName()
925 << " == " << NewFunction->getName() << '\n');
Nick Lewycky00959372010-09-05 08:22:49 +0000926
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000927 Function *DeleteF = NewFunction;
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000928 mergeTwoFunctions(OldF.getFunc(), DeleteF);
Nick Lewycky00959372010-09-05 08:22:49 +0000929 return true;
Nick Lewyckyfbd27572010-08-08 05:04:23 +0000930}
Nick Lewyckyd01d42e2008-11-02 05:52:50 +0000931
Stepan Dyatkovskiyf4af8552014-06-21 20:54:36 +0000932// Remove a function from FnTree. If it was already in FnTree, add
933// it to Deferred so that we'll look at it in the next round.
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000934void MergeFunctions::remove(Function *F) {
JF Bastien3a4ad612015-09-02 23:55:23 +0000935 auto I = FNodesInTree.find(F);
936 if (I != FNodesInTree.end()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000937 LLVM_DEBUG(dbgs() << "Deferred " << F->getName() << ".\n");
JF Bastien3a4ad612015-09-02 23:55:23 +0000938 FnTree.erase(I->second);
939 // I->second has been invalidated, remove it from the FNodesInTree map to
940 // preserve the invariant.
941 FNodesInTree.erase(I);
Benjamin Kramerf5e2fc42015-05-29 19:43:39 +0000942 Deferred.emplace_back(F);
Nick Lewycky0464d1d2010-08-31 05:53:05 +0000943 }
Nick Lewycky4e250c82011-01-02 02:46:33 +0000944}
Nick Lewycky00959372010-09-05 08:22:49 +0000945
Nick Lewyckycfb284c2011-01-28 08:43:14 +0000946// For each instruction used by the value, remove() the function that contains
947// the instruction. This should happen right before a call to RAUW.
948void MergeFunctions::removeUsers(Value *V) {
Fangrui Song884f5572019-04-19 07:57:51 +0000949 for (User *U : V->users())
950 if (auto *I = dyn_cast<Instruction>(U))
951 remove(I->getFunction());
Nick Lewycky00959372010-09-05 08:22:49 +0000952}