blob: d82e54509f4564868e1495a2c29b0fd58fa12921 [file] [log] [blame]
Chris Lattner166f2262004-11-22 22:00:25 +00001//===-- JITEmitter.cpp - Write machine code to executable memory ----------===//
Misha Brukmanf976c852005-04-21 22:55:34 +00002//
John Criswellb576c942003-10-20 19:43:21 +00003// The LLVM Compiler Infrastructure
4//
5// This file was developed by the LLVM research group and is distributed under
6// the University of Illinois Open Source License. See LICENSE.TXT for details.
Misha Brukmanf976c852005-04-21 22:55:34 +00007//
John Criswellb576c942003-10-20 19:43:21 +00008//===----------------------------------------------------------------------===//
Chris Lattnerbd199fb2002-12-24 00:01:05 +00009//
Chris Lattner5be478f2004-11-20 03:46:14 +000010// This file defines a MachineCodeEmitter object that is used by the JIT to
11// write machine code to memory and remember where relocatable values are.
Chris Lattnerbd199fb2002-12-24 00:01:05 +000012//
13//===----------------------------------------------------------------------===//
14
Chris Lattner3785fad2003-08-05 17:00:32 +000015#define DEBUG_TYPE "jit"
Chris Lattner4d326fa2003-12-20 01:46:27 +000016#include "JIT.h"
Chris Lattner2c0a6a12003-11-30 04:23:21 +000017#include "llvm/Constant.h"
18#include "llvm/Module.h"
Chris Lattner5b3a4552005-03-17 15:38:16 +000019#include "llvm/Type.h"
Chris Lattnerbd199fb2002-12-24 00:01:05 +000020#include "llvm/CodeGen/MachineCodeEmitter.h"
21#include "llvm/CodeGen/MachineFunction.h"
Chris Lattner1cc08382003-01-13 01:00:12 +000022#include "llvm/CodeGen/MachineConstantPool.h"
Chris Lattner5be478f2004-11-20 03:46:14 +000023#include "llvm/CodeGen/MachineRelocation.h"
Chris Lattner1cc08382003-01-13 01:00:12 +000024#include "llvm/Target/TargetData.h"
Chris Lattner5be478f2004-11-20 03:46:14 +000025#include "llvm/Target/TargetJITInfo.h"
Reid Spencer551ccae2004-09-01 22:55:40 +000026#include "llvm/Support/Debug.h"
27#include "llvm/ADT/Statistic.h"
Reid Spencer52b0ba62004-09-11 04:31:03 +000028#include "llvm/System/Memory.h"
Andrew Lenhartha00269b2005-07-29 23:40:16 +000029#include <list>
30#include <algorithm>
Chris Lattnerc19aade2003-12-08 08:06:28 +000031using namespace llvm;
Brian Gaeked0fde302003-11-11 22:41:34 +000032
Chris Lattnerbd199fb2002-12-24 00:01:05 +000033namespace {
Chris Lattnere7386562003-10-20 05:45:49 +000034 Statistic<> NumBytes("jit", "Number of bytes of machine code compiled");
Chris Lattnere884dc22005-07-20 16:29:20 +000035 Statistic<> NumRelos("jit", "Number of relocations applied");
Chris Lattner4d326fa2003-12-20 01:46:27 +000036 JIT *TheJIT = 0;
Chris Lattner54266522004-11-20 23:57:07 +000037}
Chris Lattnerbd199fb2002-12-24 00:01:05 +000038
Chris Lattner54266522004-11-20 23:57:07 +000039
40//===----------------------------------------------------------------------===//
41// JITMemoryManager code.
42//
43namespace {
Chris Lattner688506d2003-08-14 18:35:27 +000044 /// JITMemoryManager - Manage memory for the JIT code generation in a logical,
45 /// sane way. This splits a large block of MAP_NORESERVE'd memory into two
46 /// sections, one for function stubs, one for the functions themselves. We
47 /// have to do this because we may need to emit a function stub while in the
48 /// middle of emitting a function, and we don't know how large the function we
49 /// are emitting is. This never bothers to release the memory, because when
50 /// we are ready to destroy the JIT, the program exits.
51 class JITMemoryManager {
Andrew Lenhartha00269b2005-07-29 23:40:16 +000052 std::list<sys::MemoryBlock> Blocks; // List of blocks allocated by the JIT
Chris Lattner688506d2003-08-14 18:35:27 +000053 unsigned char *FunctionBase; // Start of the function body area
Andrew Lenhartha00269b2005-07-29 23:40:16 +000054 unsigned char *GlobalBase; // Start of the Global area
55 unsigned char *ConstantBase; // Memory allocated for constant pools
56 unsigned char *CurStubPtr, *CurFunctionPtr, *CurConstantPtr, *CurGlobalPtr;
Andrew Lenharth16ec33c2005-07-22 20:48:12 +000057 unsigned char *GOTBase; //Target Specific reserved memory
Andrew Lenhartha00269b2005-07-29 23:40:16 +000058
59 // centralize memory block allocation
60 sys::MemoryBlock getNewMemoryBlock(unsigned size);
Chris Lattner688506d2003-08-14 18:35:27 +000061 public:
Andrew Lenharth16ec33c2005-07-22 20:48:12 +000062 JITMemoryManager(bool useGOT);
Reid Spencer4af3da62004-12-13 16:04:04 +000063 ~JITMemoryManager();
Misha Brukmanf976c852005-04-21 22:55:34 +000064
Chris Lattner688506d2003-08-14 18:35:27 +000065 inline unsigned char *allocateStub(unsigned StubSize);
Chris Lattner281a6012005-01-10 18:23:22 +000066 inline unsigned char *allocateConstant(unsigned ConstantSize,
67 unsigned Alignment);
Andrew Lenharth6a974612005-07-28 12:44:13 +000068 inline unsigned char* allocateGlobal(unsigned Size,
69 unsigned Alignment);
Chris Lattner688506d2003-08-14 18:35:27 +000070 inline unsigned char *startFunctionBody();
Chris Lattner281a6012005-01-10 18:23:22 +000071 inline void endFunctionBody(unsigned char *FunctionEnd);
Andrew Lenharth16ec33c2005-07-22 20:48:12 +000072 inline unsigned char* getGOTBase() const;
73
74 inline bool isManagingGOT() const;
Chris Lattner688506d2003-08-14 18:35:27 +000075 };
76}
77
Andrew Lenharth16ec33c2005-07-22 20:48:12 +000078JITMemoryManager::JITMemoryManager(bool useGOT) {
Andrew Lenhartha00269b2005-07-29 23:40:16 +000079 // Allocate a 16M block of memory for functions
80 sys::MemoryBlock FunBlock = getNewMemoryBlock(16 << 20);
81 // Allocate a 1M block of memory for Constants
82 sys::MemoryBlock ConstBlock = getNewMemoryBlock(1 << 20);
83 // Allocate a 1M Block of memory for Globals
84 sys::MemoryBlock GVBlock = getNewMemoryBlock(1 << 20);
Andrew Lenharth16ec33c2005-07-22 20:48:12 +000085
Andrew Lenhartha00269b2005-07-29 23:40:16 +000086 Blocks.push_front(FunBlock);
87 Blocks.push_front(ConstBlock);
88 Blocks.push_front(GVBlock);
Chris Lattner688506d2003-08-14 18:35:27 +000089
Andrew Lenhartha00269b2005-07-29 23:40:16 +000090 FunctionBase = reinterpret_cast<unsigned char*>(FunBlock.base());
91 ConstantBase = reinterpret_cast<unsigned char*>(ConstBlock.base());
92 GlobalBase = reinterpret_cast<unsigned char*>(GVBlock.base());
Chris Lattner281a6012005-01-10 18:23:22 +000093
Andrew Lenhartha00269b2005-07-29 23:40:16 +000094 //Allocate the GOT just like a global array
95 GOTBase = NULL;
96 if (useGOT)
97 GOTBase = allocateGlobal(sizeof(void*) * 8192, 8);
98
99 // Allocate stubs backwards from the base, allocate functions forward
100 // from the base.
101 CurStubPtr = CurFunctionPtr = FunctionBase + 512*1024;// Use 512k for stubs
102
103 CurConstantPtr = ConstantBase + ConstBlock.size();
104 CurGlobalPtr = GlobalBase + GVBlock.size();
Chris Lattner688506d2003-08-14 18:35:27 +0000105}
106
Reid Spencer4af3da62004-12-13 16:04:04 +0000107JITMemoryManager::~JITMemoryManager() {
Andrew Lenhartha00269b2005-07-29 23:40:16 +0000108 for (std::list<sys::MemoryBlock>::iterator ib = Blocks.begin(), ie = Blocks.end();
109 ib != ie; ++ib)
110 sys::Memory::ReleaseRWX(*ib);
111 Blocks.clear();
Reid Spencer4af3da62004-12-13 16:04:04 +0000112}
113
Chris Lattner688506d2003-08-14 18:35:27 +0000114unsigned char *JITMemoryManager::allocateStub(unsigned StubSize) {
115 CurStubPtr -= StubSize;
Andrew Lenhartha00269b2005-07-29 23:40:16 +0000116 if (CurStubPtr < FunctionBase) {
117 //FIXME: allocate a new block
Chris Lattner688506d2003-08-14 18:35:27 +0000118 std::cerr << "JIT ran out of memory for function stubs!\n";
119 abort();
120 }
121 return CurStubPtr;
122}
123
Chris Lattner281a6012005-01-10 18:23:22 +0000124unsigned char *JITMemoryManager::allocateConstant(unsigned ConstantSize,
125 unsigned Alignment) {
126 // Reserve space and align pointer.
127 CurConstantPtr -= ConstantSize;
128 CurConstantPtr =
129 (unsigned char *)((intptr_t)CurConstantPtr & ~((intptr_t)Alignment - 1));
130
Andrew Lenhartha00269b2005-07-29 23:40:16 +0000131 if (CurConstantPtr < ConstantBase) {
132 //Either allocate another MB or 2xConstantSize
133 sys::MemoryBlock ConstBlock = getNewMemoryBlock(2 * ConstantSize);
134 ConstantBase = reinterpret_cast<unsigned char*>(ConstBlock.base());
135 CurConstantPtr = ConstantBase + ConstBlock.size();
136 return allocateConstant(ConstantSize, Alignment);
Chris Lattner281a6012005-01-10 18:23:22 +0000137 }
138 return CurConstantPtr;
139}
140
Andrew Lenharth6a974612005-07-28 12:44:13 +0000141unsigned char *JITMemoryManager::allocateGlobal(unsigned Size,
142 unsigned Alignment) {
Andrew Lenhartha00269b2005-07-29 23:40:16 +0000143 // Reserve space and align pointer.
144 CurGlobalPtr -= Size;
145 CurGlobalPtr =
146 (unsigned char *)((intptr_t)CurGlobalPtr & ~((intptr_t)Alignment - 1));
Andrew Lenharth6a974612005-07-28 12:44:13 +0000147
Andrew Lenhartha00269b2005-07-29 23:40:16 +0000148 if (CurGlobalPtr < GlobalBase) {
149 //Either allocate another MB or 2xSize
150 sys::MemoryBlock GVBlock = getNewMemoryBlock(2 * Size);
151 GlobalBase = reinterpret_cast<unsigned char*>(GVBlock.base());
152 CurGlobalPtr = GlobalBase + GVBlock.size();
153 return allocateGlobal(Size, Alignment);
Andrew Lenharth6a974612005-07-28 12:44:13 +0000154 }
Andrew Lenhartha00269b2005-07-29 23:40:16 +0000155 return CurGlobalPtr;
Andrew Lenharth6a974612005-07-28 12:44:13 +0000156}
157
Chris Lattner688506d2003-08-14 18:35:27 +0000158unsigned char *JITMemoryManager::startFunctionBody() {
Chris Lattner5be478f2004-11-20 03:46:14 +0000159 // Round up to an even multiple of 8 bytes, this should eventually be target
Chris Lattner688506d2003-08-14 18:35:27 +0000160 // specific.
Chris Lattner5be478f2004-11-20 03:46:14 +0000161 return (unsigned char*)(((intptr_t)CurFunctionPtr + 7) & ~7);
Chris Lattner688506d2003-08-14 18:35:27 +0000162}
163
164void JITMemoryManager::endFunctionBody(unsigned char *FunctionEnd) {
165 assert(FunctionEnd > CurFunctionPtr);
166 CurFunctionPtr = FunctionEnd;
167}
168
Andrew Lenharth16ec33c2005-07-22 20:48:12 +0000169unsigned char* JITMemoryManager::getGOTBase() const {
170 return GOTBase;
171}
172
173bool JITMemoryManager::isManagingGOT() const {
174 return GOTBase != NULL;
175}
176
Andrew Lenhartha00269b2005-07-29 23:40:16 +0000177sys::MemoryBlock JITMemoryManager::getNewMemoryBlock(unsigned size) {
178 const sys::MemoryBlock* BOld = 0;
179 if (Blocks.size())
180 BOld = &Blocks.front();
181 //never allocate less than 1 MB
182 sys::MemoryBlock B;
183 try {
184 B = sys::Memory::AllocateRWX(std::max(((unsigned)1 << 20), size), BOld);
185 } catch (std::string& err) {
186 std::cerr << "Allocation failed when allocating new memory in the JIT\n";
187 std::cerr << err << "\n";
188 abort();
189 }
190 Blocks.push_front(B);
191 return B;
192}
193
Chris Lattner54266522004-11-20 23:57:07 +0000194//===----------------------------------------------------------------------===//
195// JIT lazy compilation code.
196//
197namespace {
Reid Spenceree448632005-07-12 15:51:55 +0000198 class JITResolverState {
199 private:
200 /// FunctionToStubMap - Keep track of the stub created for a particular
201 /// function so that we can reuse them if necessary.
202 std::map<Function*, void*> FunctionToStubMap;
203
204 /// StubToFunctionMap - Keep track of the function that each stub
205 /// corresponds to.
206 std::map<void*, Function*> StubToFunctionMap;
Jeff Cohen00b168892005-07-27 06:12:32 +0000207
Reid Spenceree448632005-07-12 15:51:55 +0000208 public:
209 std::map<Function*, void*>& getFunctionToStubMap(const MutexGuard& locked) {
210 assert(locked.holds(TheJIT->lock));
211 return FunctionToStubMap;
212 }
Jeff Cohen00b168892005-07-27 06:12:32 +0000213
Reid Spenceree448632005-07-12 15:51:55 +0000214 std::map<void*, Function*>& getStubToFunctionMap(const MutexGuard& locked) {
215 assert(locked.holds(TheJIT->lock));
216 return StubToFunctionMap;
217 }
218 };
Jeff Cohen00b168892005-07-27 06:12:32 +0000219
Chris Lattner54266522004-11-20 23:57:07 +0000220 /// JITResolver - Keep track of, and resolve, call sites for functions that
221 /// have not yet been compiled.
222 class JITResolver {
Chris Lattner5e225582004-11-21 03:37:42 +0000223 /// MCE - The MachineCodeEmitter to use to emit stubs with.
Chris Lattner54266522004-11-20 23:57:07 +0000224 MachineCodeEmitter &MCE;
225
Chris Lattner5e225582004-11-21 03:37:42 +0000226 /// LazyResolverFn - The target lazy resolver function that we actually
227 /// rewrite instructions to use.
228 TargetJITInfo::LazyResolverFn LazyResolverFn;
229
Reid Spenceree448632005-07-12 15:51:55 +0000230 JITResolverState state;
Chris Lattner54266522004-11-20 23:57:07 +0000231
Chris Lattnerd91ff7c2005-04-18 01:44:27 +0000232 /// ExternalFnToStubMap - This is the equivalent of FunctionToStubMap for
233 /// external functions.
234 std::map<void*, void*> ExternalFnToStubMap;
Andrew Lenharth6a974612005-07-28 12:44:13 +0000235
236 //map addresses to indexes in the GOT
237 std::map<void*, unsigned> revGOTMap;
238 unsigned nextGOTIndex;
239
Chris Lattner54266522004-11-20 23:57:07 +0000240 public:
Andrew Lenharth6a974612005-07-28 12:44:13 +0000241 JITResolver(MachineCodeEmitter &mce) : MCE(mce), nextGOTIndex(0) {
Chris Lattner5e225582004-11-21 03:37:42 +0000242 LazyResolverFn =
243 TheJIT->getJITInfo().getLazyResolverFunction(JITCompilerFn);
244 }
Chris Lattner54266522004-11-20 23:57:07 +0000245
246 /// getFunctionStub - This returns a pointer to a function stub, creating
247 /// one on demand as needed.
248 void *getFunctionStub(Function *F);
249
Chris Lattnerd91ff7c2005-04-18 01:44:27 +0000250 /// getExternalFunctionStub - Return a stub for the function at the
251 /// specified address, created lazily on demand.
252 void *getExternalFunctionStub(void *FnAddr);
253
Chris Lattner5e225582004-11-21 03:37:42 +0000254 /// AddCallbackAtLocation - If the target is capable of rewriting an
255 /// instruction without the use of a stub, record the location of the use so
256 /// we know which function is being used at the location.
257 void *AddCallbackAtLocation(Function *F, void *Location) {
Reid Spenceree448632005-07-12 15:51:55 +0000258 MutexGuard locked(TheJIT->lock);
Chris Lattner5e225582004-11-21 03:37:42 +0000259 /// Get the target-specific JIT resolver function.
Reid Spenceree448632005-07-12 15:51:55 +0000260 state.getStubToFunctionMap(locked)[Location] = F;
Chris Lattner5e225582004-11-21 03:37:42 +0000261 return (void*)LazyResolverFn;
262 }
263
Andrew Lenharth6a974612005-07-28 12:44:13 +0000264 /// getGOTIndexForAddress - Return a new or existing index in the GOT for
265 /// and address. This function only manages slots, it does not manage the
266 /// contents of the slots or the memory associated with the GOT.
267 unsigned getGOTIndexForAddr(void* addr);
268
Chris Lattner54266522004-11-20 23:57:07 +0000269 /// JITCompilerFn - This function is called to resolve a stub to a compiled
270 /// address. If the LLVM Function corresponding to the stub has not yet
271 /// been compiled, this function compiles it first.
272 static void *JITCompilerFn(void *Stub);
273 };
274}
275
276/// getJITResolver - This function returns the one instance of the JIT resolver.
277///
278static JITResolver &getJITResolver(MachineCodeEmitter *MCE = 0) {
279 static JITResolver TheJITResolver(*MCE);
280 return TheJITResolver;
281}
282
283/// getFunctionStub - This returns a pointer to a function stub, creating
284/// one on demand as needed.
285void *JITResolver::getFunctionStub(Function *F) {
Reid Spenceree448632005-07-12 15:51:55 +0000286 MutexGuard locked(TheJIT->lock);
287
Chris Lattner54266522004-11-20 23:57:07 +0000288 // If we already have a stub for this function, recycle it.
Reid Spenceree448632005-07-12 15:51:55 +0000289 void *&Stub = state.getFunctionToStubMap(locked)[F];
Chris Lattner54266522004-11-20 23:57:07 +0000290 if (Stub) return Stub;
291
Chris Lattnerb43dbdc2004-11-22 07:24:43 +0000292 // Call the lazy resolver function unless we already KNOW it is an external
293 // function, in which case we just skip the lazy resolution step.
294 void *Actual = (void*)LazyResolverFn;
Chris Lattner69435702005-02-20 18:43:35 +0000295 if (F->isExternal() && F->hasExternalLinkage())
Chris Lattnerb43dbdc2004-11-22 07:24:43 +0000296 Actual = TheJIT->getPointerToFunction(F);
Misha Brukmanf976c852005-04-21 22:55:34 +0000297
Chris Lattner54266522004-11-20 23:57:07 +0000298 // Otherwise, codegen a new stub. For now, the stub will call the lazy
299 // resolver function.
Chris Lattnerb43dbdc2004-11-22 07:24:43 +0000300 Stub = TheJIT->getJITInfo().emitFunctionStub(Actual, MCE);
301
Chris Lattner69435702005-02-20 18:43:35 +0000302 if (Actual != (void*)LazyResolverFn) {
Chris Lattnerb43dbdc2004-11-22 07:24:43 +0000303 // If we are getting the stub for an external function, we really want the
304 // address of the stub in the GlobalAddressMap for the JIT, not the address
305 // of the external function.
306 TheJIT->updateGlobalMapping(F, Stub);
307 }
Chris Lattner54266522004-11-20 23:57:07 +0000308
Chris Lattnercb479412004-11-21 03:44:32 +0000309 DEBUG(std::cerr << "JIT: Stub emitted at [" << Stub << "] for function '"
Chris Lattner6f717202004-11-22 21:48:33 +0000310 << F->getName() << "'\n");
Chris Lattnercb479412004-11-21 03:44:32 +0000311
Chris Lattner54266522004-11-20 23:57:07 +0000312 // Finally, keep track of the stub-to-Function mapping so that the
313 // JITCompilerFn knows which function to compile!
Reid Spenceree448632005-07-12 15:51:55 +0000314 state.getStubToFunctionMap(locked)[Stub] = F;
Chris Lattner54266522004-11-20 23:57:07 +0000315 return Stub;
316}
317
Chris Lattnerd91ff7c2005-04-18 01:44:27 +0000318/// getExternalFunctionStub - Return a stub for the function at the
319/// specified address, created lazily on demand.
320void *JITResolver::getExternalFunctionStub(void *FnAddr) {
321 // If we already have a stub for this function, recycle it.
322 void *&Stub = ExternalFnToStubMap[FnAddr];
323 if (Stub) return Stub;
324
325 Stub = TheJIT->getJITInfo().emitFunctionStub(FnAddr, MCE);
326 DEBUG(std::cerr << "JIT: Stub emitted at [" << Stub
327 << "] for external function at '" << FnAddr << "'\n");
328 return Stub;
329}
330
Andrew Lenharth6a974612005-07-28 12:44:13 +0000331unsigned JITResolver::getGOTIndexForAddr(void* addr) {
332 unsigned idx = revGOTMap[addr];
333 if (!idx) {
334 idx = ++nextGOTIndex;
335 revGOTMap[addr] = idx;
336 DEBUG(std::cerr << "Adding GOT entry " << idx
337 << " for addr " << addr << "\n");
338 // ((void**)MemMgr.getGOTBase())[idx] = addr;
339 }
340 return idx;
341}
Chris Lattnerd91ff7c2005-04-18 01:44:27 +0000342
Chris Lattner54266522004-11-20 23:57:07 +0000343/// JITCompilerFn - This function is called when a lazy compilation stub has
344/// been entered. It looks up which function this stub corresponds to, compiles
345/// it if necessary, then returns the resultant function pointer.
346void *JITResolver::JITCompilerFn(void *Stub) {
347 JITResolver &JR = getJITResolver();
Misha Brukmanf976c852005-04-21 22:55:34 +0000348
Reid Spenceree448632005-07-12 15:51:55 +0000349 MutexGuard locked(TheJIT->lock);
350
Chris Lattner54266522004-11-20 23:57:07 +0000351 // The address given to us for the stub may not be exactly right, it might be
352 // a little bit after the stub. As such, use upper_bound to find it.
353 std::map<void*, Function*>::iterator I =
Reid Spenceree448632005-07-12 15:51:55 +0000354 JR.state.getStubToFunctionMap(locked).upper_bound(Stub);
355 assert(I != JR.state.getStubToFunctionMap(locked).begin() && "This is not a known stub!");
Chris Lattner54266522004-11-20 23:57:07 +0000356 Function *F = (--I)->second;
357
Reid Spenceree448632005-07-12 15:51:55 +0000358 // We might like to remove the stub from the StubToFunction map.
359 // We can't do that! Multiple threads could be stuck, waiting to acquire the
360 // lock above. As soon as the 1st function finishes compiling the function,
361 // the next one will be released, and needs to be able to find the function it needs
362 // to call.
363 //JR.state.getStubToFunctionMap(locked).erase(I);
Chris Lattner54266522004-11-20 23:57:07 +0000364
Chris Lattnercb479412004-11-21 03:44:32 +0000365 DEBUG(std::cerr << "JIT: Lazily resolving function '" << F->getName()
Chris Lattner54266522004-11-20 23:57:07 +0000366 << "' In stub ptr = " << Stub << " actual ptr = "
367 << I->first << "\n");
368
369 void *Result = TheJIT->getPointerToFunction(F);
370
371 // We don't need to reuse this stub in the future, as F is now compiled.
Reid Spenceree448632005-07-12 15:51:55 +0000372 JR.state.getFunctionToStubMap(locked).erase(F);
Chris Lattner54266522004-11-20 23:57:07 +0000373
374 // FIXME: We could rewrite all references to this stub if we knew them.
Andrew Lenharth6a974612005-07-28 12:44:13 +0000375
376 // What we will do is set the compiled function address to map to the
377 // same GOT entry as the stub so that later clients may update the GOT
378 // if they see it still using the stub address.
379 // Note: this is done so the Resolver doesn't have to manage GOT memory
380 // Do this without allocating map space if the target isn't using a GOT
381 if(JR.revGOTMap.find(Stub) != JR.revGOTMap.end())
382 JR.revGOTMap[Result] = JR.revGOTMap[Stub];
383
Chris Lattner54266522004-11-20 23:57:07 +0000384 return Result;
385}
Chris Lattner688506d2003-08-14 18:35:27 +0000386
387
Chris Lattnere518b712004-12-05 07:19:16 +0000388// getPointerToFunctionOrStub - If the specified function has been
389// code-gen'd, return a pointer to the function. If not, compile it, or use
390// a stub to implement lazy compilation if available.
391//
392void *JIT::getPointerToFunctionOrStub(Function *F) {
393 // If we have already code generated the function, just return the address.
394 if (void *Addr = getPointerToGlobalIfAvailable(F))
395 return Addr;
396
397 // Get a stub if the target supports it
398 return getJITResolver(MCE).getFunctionStub(F);
399}
400
401
402
Chris Lattner54266522004-11-20 23:57:07 +0000403//===----------------------------------------------------------------------===//
Chris Lattner166f2262004-11-22 22:00:25 +0000404// JITEmitter code.
Chris Lattner54266522004-11-20 23:57:07 +0000405//
Chris Lattner688506d2003-08-14 18:35:27 +0000406namespace {
Chris Lattner166f2262004-11-22 22:00:25 +0000407 /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is
408 /// used to output functions to memory for execution.
409 class JITEmitter : public MachineCodeEmitter {
Chris Lattner688506d2003-08-14 18:35:27 +0000410 JITMemoryManager MemMgr;
411
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000412 // CurBlock - The start of the current block of memory. CurByte - The
413 // current byte being emitted to.
Chris Lattner6125fdd2003-05-09 03:30:07 +0000414 unsigned char *CurBlock, *CurByte;
415
416 // When outputting a function stub in the context of some other function, we
417 // save CurBlock and CurByte here.
418 unsigned char *SavedCurBlock, *SavedCurByte;
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000419
420 // ConstantPoolAddresses - Contains the location for each entry in the
421 // constant pool.
Chris Lattner1cc08382003-01-13 01:00:12 +0000422 std::vector<void*> ConstantPoolAddresses;
Chris Lattner5be478f2004-11-20 03:46:14 +0000423
424 /// Relocations - These are the relocations that the function needs, as
425 /// emitted.
426 std::vector<MachineRelocation> Relocations;
Andrew Lenharth16ec33c2005-07-22 20:48:12 +0000427
Chris Lattnerbd199fb2002-12-24 00:01:05 +0000428 public:
Andrew Lenharth16ec33c2005-07-22 20:48:12 +0000429 JITEmitter(JIT &jit)
Andrew Lenharth6a974612005-07-28 12:44:13 +0000430 :MemMgr(jit.getJITInfo().needsGOT())
Andrew Lenharth16ec33c2005-07-22 20:48:12 +0000431 {
Jeff Cohen00b168892005-07-27 06:12:32 +0000432 TheJIT = &jit;
433 DEBUG(std::cerr <<
434 (MemMgr.isManagingGOT() ? "JIT is managing GOT\n"
Andrew Lenharth16ec33c2005-07-22 20:48:12 +0000435 : "JIT is not managing GOT\n"));
436 }
Chris Lattnerbd199fb2002-12-24 00:01:05 +0000437
438 virtual void startFunction(MachineFunction &F);
439 virtual void finishFunction(MachineFunction &F);
Chris Lattner1cc08382003-01-13 01:00:12 +0000440 virtual void emitConstantPool(MachineConstantPool *MCP);
Chris Lattner54266522004-11-20 23:57:07 +0000441 virtual void startFunctionStub(unsigned StubSize);
442 virtual void* finishFunctionStub(const Function *F);
Chris Lattnerbd199fb2002-12-24 00:01:05 +0000443 virtual void emitByte(unsigned char B);
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000444 virtual void emitWord(unsigned W);
Brian Gaekeaea1b582004-04-23 17:11:14 +0000445 virtual void emitWordAt(unsigned W, unsigned *Ptr);
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000446
Chris Lattner5be478f2004-11-20 03:46:14 +0000447 virtual void addRelocation(const MachineRelocation &MR) {
448 Relocations.push_back(MR);
449 }
450
451 virtual uint64_t getCurrentPCValue();
452 virtual uint64_t getCurrentPCOffset();
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000453 virtual uint64_t getConstantPoolEntryAddress(unsigned Entry);
Andrew Lenharth6a974612005-07-28 12:44:13 +0000454 virtual unsigned char* allocateGlobal(unsigned size, unsigned alignment);
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000455
Chris Lattner54266522004-11-20 23:57:07 +0000456 private:
Chris Lattner5e225582004-11-21 03:37:42 +0000457 void *getPointerToGlobal(GlobalValue *GV, void *Reference, bool NoNeedStub);
Chris Lattnerbd199fb2002-12-24 00:01:05 +0000458 };
459}
460
Chris Lattner4d326fa2003-12-20 01:46:27 +0000461MachineCodeEmitter *JIT::createEmitter(JIT &jit) {
Chris Lattner166f2262004-11-22 22:00:25 +0000462 return new JITEmitter(jit);
Chris Lattnerbd199fb2002-12-24 00:01:05 +0000463}
464
Chris Lattner166f2262004-11-22 22:00:25 +0000465void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference,
466 bool DoesntNeedStub) {
Chris Lattner54266522004-11-20 23:57:07 +0000467 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) {
468 /// FIXME: If we straightened things out, this could actually emit the
469 /// global immediately instead of queuing it for codegen later!
Chris Lattner54266522004-11-20 23:57:07 +0000470 return TheJIT->getOrEmitGlobalVariable(GV);
471 }
472
473 // If we have already compiled the function, return a pointer to its body.
474 Function *F = cast<Function>(V);
475 void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F);
476 if (ResultPtr) return ResultPtr;
477
Chris Lattner532343b2004-11-30 17:41:49 +0000478 if (F->hasExternalLinkage() && F->isExternal()) {
Chris Lattner54266522004-11-20 23:57:07 +0000479 // If this is an external function pointer, we can force the JIT to
480 // 'compile' it, which really just adds it to the map.
Chris Lattnerb43dbdc2004-11-22 07:24:43 +0000481 if (DoesntNeedStub)
482 return TheJIT->getPointerToFunction(F);
483
484 return getJITResolver(this).getFunctionStub(F);
Chris Lattner54266522004-11-20 23:57:07 +0000485 }
486
Chris Lattner5e225582004-11-21 03:37:42 +0000487 // Okay, the function has not been compiled yet, if the target callback
488 // mechanism is capable of rewriting the instruction directly, prefer to do
489 // that instead of emitting a stub.
490 if (DoesntNeedStub)
491 return getJITResolver(this).AddCallbackAtLocation(F, Reference);
492
Chris Lattner54266522004-11-20 23:57:07 +0000493 // Otherwise, we have to emit a lazy resolving stub.
494 return getJITResolver(this).getFunctionStub(F);
495}
496
Chris Lattner166f2262004-11-22 22:00:25 +0000497void JITEmitter::startFunction(MachineFunction &F) {
Chris Lattner688506d2003-08-14 18:35:27 +0000498 CurByte = CurBlock = MemMgr.startFunctionBody();
Chris Lattner4d326fa2003-12-20 01:46:27 +0000499 TheJIT->addGlobalMapping(F.getFunction(), CurBlock);
Chris Lattnerbd199fb2002-12-24 00:01:05 +0000500}
501
Chris Lattner166f2262004-11-22 22:00:25 +0000502void JITEmitter::finishFunction(MachineFunction &F) {
Chris Lattner688506d2003-08-14 18:35:27 +0000503 MemMgr.endFunctionBody(CurByte);
Chris Lattnerbd199fb2002-12-24 00:01:05 +0000504 NumBytes += CurByte-CurBlock;
505
Chris Lattner5be478f2004-11-20 03:46:14 +0000506 if (!Relocations.empty()) {
Chris Lattnere884dc22005-07-20 16:29:20 +0000507 NumRelos += Relocations.size();
508
Chris Lattner5be478f2004-11-20 03:46:14 +0000509 // Resolve the relocations to concrete pointers.
510 for (unsigned i = 0, e = Relocations.size(); i != e; ++i) {
511 MachineRelocation &MR = Relocations[i];
512 void *ResultPtr;
Chris Lattnerd91ff7c2005-04-18 01:44:27 +0000513 if (MR.isString()) {
Chris Lattner5be478f2004-11-20 03:46:14 +0000514 ResultPtr = TheJIT->getPointerToNamedFunction(MR.getString());
Misha Brukmanf976c852005-04-21 22:55:34 +0000515
Chris Lattnerd91ff7c2005-04-18 01:44:27 +0000516 // If the target REALLY wants a stub for this function, emit it now.
517 if (!MR.doesntNeedFunctionStub())
518 ResultPtr = getJITResolver(this).getExternalFunctionStub(ResultPtr);
Jeff Cohen00b168892005-07-27 06:12:32 +0000519 } else if (MR.isGlobalValue())
Chris Lattner5e225582004-11-21 03:37:42 +0000520 ResultPtr = getPointerToGlobal(MR.getGlobalValue(),
521 CurBlock+MR.getMachineCodeOffset(),
522 MR.doesntNeedFunctionStub());
Andrew Lenharth16ec33c2005-07-22 20:48:12 +0000523 else //ConstantPoolIndex
Jeff Cohen00b168892005-07-27 06:12:32 +0000524 ResultPtr =
Chris Lattnerd6bbac52005-07-25 23:42:58 +0000525 (void*)(intptr_t)getConstantPoolEntryAddress(MR.getConstantPoolIndex());
Jeff Cohen00b168892005-07-27 06:12:32 +0000526
Chris Lattner5be478f2004-11-20 03:46:14 +0000527 MR.setResultPointer(ResultPtr);
Andrew Lenharth16ec33c2005-07-22 20:48:12 +0000528
Andrew Lenharth6a974612005-07-28 12:44:13 +0000529 // if we are managing the GOT and the relocation wants an index,
530 // give it one
531 if (MemMgr.isManagingGOT() && !MR.isConstantPoolIndex() &&
532 MR.isGOTRelative()) {
533 unsigned idx = getJITResolver(this).getGOTIndexForAddr(ResultPtr);
534 MR.setGOTIndex(idx);
535 if (((void**)MemMgr.getGOTBase())[idx] != ResultPtr) {
536 DEBUG(std::cerr << "GOT was out of date for " << ResultPtr
537 << " pointing at " << ((void**)MemMgr.getGOTBase())[idx] << "\n");
538 ((void**)MemMgr.getGOTBase())[idx] = ResultPtr;
539 }
Andrew Lenharth16ec33c2005-07-22 20:48:12 +0000540 }
Chris Lattner5be478f2004-11-20 03:46:14 +0000541 }
542
543 TheJIT->getJITInfo().relocate(CurBlock, &Relocations[0],
Andrew Lenharth16ec33c2005-07-22 20:48:12 +0000544 Relocations.size(), MemMgr.getGOTBase());
Chris Lattner5be478f2004-11-20 03:46:14 +0000545 }
546
Andrew Lenharth6a974612005-07-28 12:44:13 +0000547 //Update the GOT entry for F to point to the new code.
548 if(MemMgr.isManagingGOT()) {
549 unsigned idx = getJITResolver(this).getGOTIndexForAddr((void*)CurBlock);
550 if (((void**)MemMgr.getGOTBase())[idx] != (void*)CurBlock) {
551 DEBUG(std::cerr << "GOT was out of date for " << (void*)CurBlock
552 << " pointing at " << ((void**)MemMgr.getGOTBase())[idx] << "\n");
553 ((void**)MemMgr.getGOTBase())[idx] = (void*)CurBlock;
554 }
555 }
556
Chris Lattnercb479412004-11-21 03:44:32 +0000557 DEBUG(std::cerr << "JIT: Finished CodeGen of [" << (void*)CurBlock
Misha Brukman1d440852003-06-06 06:52:35 +0000558 << "] Function: " << F.getFunction()->getName()
Chris Lattner5be478f2004-11-20 03:46:14 +0000559 << ": " << CurByte-CurBlock << " bytes of text, "
560 << Relocations.size() << " relocations\n");
561 Relocations.clear();
Andrew Lenharth16ec33c2005-07-22 20:48:12 +0000562 ConstantPoolAddresses.clear();
Chris Lattnerbd199fb2002-12-24 00:01:05 +0000563}
564
Chris Lattner166f2262004-11-22 22:00:25 +0000565void JITEmitter::emitConstantPool(MachineConstantPool *MCP) {
Chris Lattner1cc08382003-01-13 01:00:12 +0000566 const std::vector<Constant*> &Constants = MCP->getConstants();
Chris Lattner2c0a6a12003-11-30 04:23:21 +0000567 if (Constants.empty()) return;
568
Chris Lattner1cc08382003-01-13 01:00:12 +0000569 for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
Chris Lattner2c0a6a12003-11-30 04:23:21 +0000570 const Type *Ty = Constants[i]->getType();
Chris Lattnera8101c12005-01-08 20:07:03 +0000571 unsigned Size = (unsigned)TheJIT->getTargetData().getTypeSize(Ty);
Chris Lattner4d326fa2003-12-20 01:46:27 +0000572 unsigned Alignment = TheJIT->getTargetData().getTypeAlignment(Ty);
Chris Lattner2c0a6a12003-11-30 04:23:21 +0000573
Chris Lattner281a6012005-01-10 18:23:22 +0000574 void *Addr = MemMgr.allocateConstant(Size, Alignment);
Chris Lattner4d326fa2003-12-20 01:46:27 +0000575 TheJIT->InitializeMemory(Constants[i], Addr);
Misha Brukman91de3522003-11-30 00:50:53 +0000576 ConstantPoolAddresses.push_back(Addr);
Chris Lattner1cc08382003-01-13 01:00:12 +0000577 }
578}
579
Chris Lattner166f2262004-11-22 22:00:25 +0000580void JITEmitter::startFunctionStub(unsigned StubSize) {
Chris Lattner6125fdd2003-05-09 03:30:07 +0000581 SavedCurBlock = CurBlock; SavedCurByte = CurByte;
Chris Lattner688506d2003-08-14 18:35:27 +0000582 CurByte = CurBlock = MemMgr.allocateStub(StubSize);
Chris Lattner6125fdd2003-05-09 03:30:07 +0000583}
584
Chris Lattner166f2262004-11-22 22:00:25 +0000585void *JITEmitter::finishFunctionStub(const Function *F) {
Chris Lattner6125fdd2003-05-09 03:30:07 +0000586 NumBytes += CurByte-CurBlock;
Chris Lattner6125fdd2003-05-09 03:30:07 +0000587 std::swap(CurBlock, SavedCurBlock);
588 CurByte = SavedCurByte;
589 return SavedCurBlock;
590}
591
Chris Lattner166f2262004-11-22 22:00:25 +0000592void JITEmitter::emitByte(unsigned char B) {
Chris Lattnerbd199fb2002-12-24 00:01:05 +0000593 *CurByte++ = B; // Write the byte to memory
594}
595
Chris Lattner166f2262004-11-22 22:00:25 +0000596void JITEmitter::emitWord(unsigned W) {
Chris Lattner688506d2003-08-14 18:35:27 +0000597 // This won't work if the endianness of the host and target don't agree! (For
598 // a JIT this can't happen though. :)
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000599 *(unsigned*)CurByte = W;
600 CurByte += sizeof(unsigned);
601}
602
Chris Lattner166f2262004-11-22 22:00:25 +0000603void JITEmitter::emitWordAt(unsigned W, unsigned *Ptr) {
Brian Gaekeaea1b582004-04-23 17:11:14 +0000604 *Ptr = W;
605}
606
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000607// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry
608// in the constant pool that was last emitted with the 'emitConstantPool'
609// method.
610//
Chris Lattner166f2262004-11-22 22:00:25 +0000611uint64_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) {
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000612 assert(ConstantNum < ConstantPoolAddresses.size() &&
Misha Brukman3c944972005-04-22 04:08:30 +0000613 "Invalid ConstantPoolIndex!");
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000614 return (intptr_t)ConstantPoolAddresses[ConstantNum];
615}
616
Andrew Lenharth6a974612005-07-28 12:44:13 +0000617unsigned char* JITEmitter::allocateGlobal(unsigned size, unsigned alignment)
618{
619 return MemMgr.allocateGlobal(size, alignment);
620}
621
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000622// getCurrentPCValue - This returns the address that the next emitted byte
623// will be output to.
624//
Chris Lattner166f2262004-11-22 22:00:25 +0000625uint64_t JITEmitter::getCurrentPCValue() {
Chris Lattnerbba1b6d2003-06-01 23:24:36 +0000626 return (intptr_t)CurByte;
627}
628
Chris Lattner166f2262004-11-22 22:00:25 +0000629uint64_t JITEmitter::getCurrentPCOffset() {
Chris Lattner5be478f2004-11-20 03:46:14 +0000630 return (intptr_t)CurByte-(intptr_t)CurBlock;
631}
632
Misha Brukmand69c1e62003-07-28 19:09:06 +0000633// getPointerToNamedFunction - This function is used as a global wrapper to
Chris Lattner4d326fa2003-12-20 01:46:27 +0000634// JIT::getPointerToNamedFunction for the purpose of resolving symbols when
Misha Brukmand69c1e62003-07-28 19:09:06 +0000635// bugpoint is debugging the JIT. In that scenario, we are loading an .so and
636// need to resolve function(s) that are being mis-codegenerated, so we need to
637// resolve their addresses at runtime, and this is the way to do it.
638extern "C" {
639 void *getPointerToNamedFunction(const char *Name) {
Chris Lattner4d326fa2003-12-20 01:46:27 +0000640 Module &M = TheJIT->getModule();
Misha Brukmand69c1e62003-07-28 19:09:06 +0000641 if (Function *F = M.getNamedFunction(Name))
Chris Lattner4d326fa2003-12-20 01:46:27 +0000642 return TheJIT->getPointerToFunction(F);
643 return TheJIT->getPointerToNamedFunction(Name);
Misha Brukmand69c1e62003-07-28 19:09:06 +0000644 }
645}