Add an option to allocate JITed global data separately from code. By
default, this option is not enabled to support clients who rely on
this behavior.
Fixes http://llvm.org/PR4483
A patch to allocate additional memory for globals after we run out is
forthcoming.
Patch by Reid Kleckner!
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@75059 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/ExecutionEngine/JIT/JIT.cpp b/lib/ExecutionEngine/JIT/JIT.cpp
index 785b5b9..55ce16c 100644
--- a/lib/ExecutionEngine/JIT/JIT.cpp
+++ b/lib/ExecutionEngine/JIT/JIT.cpp
@@ -197,8 +197,10 @@
ExecutionEngine *ExecutionEngine::createJIT(ModuleProvider *MP,
std::string *ErrorStr,
JITMemoryManager *JMM,
- CodeGenOpt::Level OptLevel) {
- ExecutionEngine *EE = JIT::createJIT(MP, ErrorStr, JMM, OptLevel);
+ CodeGenOpt::Level OptLevel,
+ bool GVsWithCode) {
+ ExecutionEngine *EE = JIT::createJIT(MP, ErrorStr, JMM, OptLevel,
+ GVsWithCode);
if (!EE) return 0;
// Make sure we can resolve symbols in the program as well. The zero arg
@@ -208,8 +210,8 @@
}
JIT::JIT(ModuleProvider *MP, TargetMachine &tm, TargetJITInfo &tji,
- JITMemoryManager *JMM, CodeGenOpt::Level OptLevel)
- : ExecutionEngine(MP), TM(tm), TJI(tji) {
+ JITMemoryManager *JMM, CodeGenOpt::Level OptLevel, bool GVsWithCode)
+ : ExecutionEngine(MP), TM(tm), TJI(tji), AllocateGVsWithCode(GVsWithCode) {
setTargetData(TM.getTargetData());
jitstate = new JITState(MP);
@@ -677,37 +679,11 @@
}
addGlobalMapping(GV, Ptr);
} else {
- // GlobalVariable's which are not "constant" will cause trouble in a server
- // situation. It's returned in the same block of memory as code which may
- // not be writable.
- if (isGVCompilationDisabled() && !GV->isConstant()) {
- cerr << "Compilation of non-internal GlobalValue is disabled!\n";
- abort();
- }
// If the global hasn't been emitted to memory yet, allocate space and
- // emit it into memory. It goes in the same array as the generated
- // code, jump tables, etc.
- const Type *GlobalType = GV->getType()->getElementType();
- size_t S = getTargetData()->getTypeAllocSize(GlobalType);
- size_t A = getTargetData()->getPreferredAlignment(GV);
- if (GV->isThreadLocal()) {
- MutexGuard locked(lock);
- Ptr = TJI.allocateThreadLocalMemory(S);
- } else if (TJI.allocateSeparateGVMemory()) {
- if (A <= 8) {
- Ptr = malloc(S);
- } else {
- // Allocate S+A bytes of memory, then use an aligned pointer within that
- // space.
- Ptr = malloc(S+A);
- unsigned MisAligned = ((intptr_t)Ptr & (A-1));
- Ptr = (char*)Ptr + (MisAligned ? (A-MisAligned) : 0);
- }
- } else {
- Ptr = JCE->allocateSpace(S, A);
- }
+ // emit it into memory.
+ Ptr = getMemoryForGV(GV);
addGlobalMapping(GV, Ptr);
- EmitGlobalVariable(GV);
+ EmitGlobalVariable(GV); // Initialize the variable.
}
return Ptr;
}
@@ -742,14 +718,42 @@
/// on the target.
///
char* JIT::getMemoryForGV(const GlobalVariable* GV) {
- const Type *ElTy = GV->getType()->getElementType();
- size_t GVSize = (size_t)getTargetData()->getTypeAllocSize(ElTy);
+ char *Ptr;
+
+ // GlobalVariable's which are not "constant" will cause trouble in a server
+ // situation. It's returned in the same block of memory as code which may
+ // not be writable.
+ if (isGVCompilationDisabled() && !GV->isConstant()) {
+ cerr << "Compilation of non-internal GlobalValue is disabled!\n";
+ abort();
+ }
+
+ // Some applications require globals and code to live together, so they may
+ // be allocated into the same buffer, but in general globals are allocated
+ // through the memory manager which puts them near the code but not in the
+ // same buffer.
+ const Type *GlobalType = GV->getType()->getElementType();
+ size_t S = getTargetData()->getTypeAllocSize(GlobalType);
+ size_t A = getTargetData()->getPreferredAlignment(GV);
if (GV->isThreadLocal()) {
MutexGuard locked(lock);
- return TJI.allocateThreadLocalMemory(GVSize);
+ Ptr = TJI.allocateThreadLocalMemory(S);
+ } else if (TJI.allocateSeparateGVMemory()) {
+ if (A <= 8) {
+ Ptr = (char*)malloc(S);
+ } else {
+ // Allocate S+A bytes of memory, then use an aligned pointer within that
+ // space.
+ Ptr = (char*)malloc(S+A);
+ unsigned MisAligned = ((intptr_t)Ptr & (A-1));
+ Ptr = Ptr + (MisAligned ? (A-MisAligned) : 0);
+ }
+ } else if (AllocateGVsWithCode) {
+ Ptr = (char*)JCE->allocateSpace(S, A);
} else {
- return new char[GVSize];
+ Ptr = (char*)JCE->allocateGlobal(S, A);
}
+ return Ptr;
}
void JIT::addPendingFunction(Function *F) {
diff --git a/lib/ExecutionEngine/JIT/JIT.h b/lib/ExecutionEngine/JIT/JIT.h
index 66417a7..c992509 100644
--- a/lib/ExecutionEngine/JIT/JIT.h
+++ b/lib/ExecutionEngine/JIT/JIT.h
@@ -55,10 +55,16 @@
JITCodeEmitter *JCE; // JCE object
std::vector<JITEventListener*> EventListeners;
+ /// AllocateGVsWithCode - Some applications require that global variables and
+ /// code be allocated into the same region of memory, in which case this flag
+ /// should be set to true. Doing so breaks freeMachineCodeForFunction.
+ bool AllocateGVsWithCode;
+
JITState *jitstate;
- JIT(ModuleProvider *MP, TargetMachine &tm, TargetJITInfo &tji,
- JITMemoryManager *JMM, CodeGenOpt::Level OptLevel);
+ JIT(ModuleProvider *MP, TargetMachine &tm, TargetJITInfo &tji,
+ JITMemoryManager *JMM, CodeGenOpt::Level OptLevel,
+ bool AllocateGVsWithCode);
public:
~JIT();
@@ -75,8 +81,9 @@
///
static ExecutionEngine *create(ModuleProvider *MP, std::string *Err,
CodeGenOpt::Level OptLevel =
- CodeGenOpt::Default) {
- return createJIT(MP, Err, 0, OptLevel);
+ CodeGenOpt::Default,
+ bool AllocateGVsWithCode = true) {
+ return createJIT(MP, Err, 0, OptLevel, AllocateGVsWithCode);
}
virtual void addModuleProvider(ModuleProvider *MP);
@@ -151,9 +158,11 @@
/// getCodeEmitter - Return the code emitter this JIT is emitting into.
JITCodeEmitter *getCodeEmitter() const { return JCE; }
- static ExecutionEngine *createJIT(ModuleProvider *MP, std::string *Err,
+ static ExecutionEngine *createJIT(ModuleProvider *MP,
+ std::string *Err,
JITMemoryManager *JMM,
- CodeGenOpt::Level OptLevel);
+ CodeGenOpt::Level OptLevel,
+ bool AllocateGVsWithCode);
// Run the JIT on F and return information about the generated code
diff --git a/lib/ExecutionEngine/JIT/JITEmitter.cpp b/lib/ExecutionEngine/JIT/JITEmitter.cpp
index 8fe7ab8..cfc3680 100644
--- a/lib/ExecutionEngine/JIT/JITEmitter.cpp
+++ b/lib/ExecutionEngine/JIT/JITEmitter.cpp
@@ -527,6 +527,11 @@
/// allocate a new one of the given size.
virtual void *allocateSpace(uintptr_t Size, unsigned Alignment);
+ /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace,
+ /// this method does not allocate memory in the current output buffer,
+ /// because a global may live longer than the current function.
+ virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment);
+
virtual void addRelocation(const MachineRelocation &MR) {
Relocations.push_back(MR);
}
@@ -1161,6 +1166,11 @@
return CurBufferPtr;
}
+void* JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) {
+ // Delegate this call through the memory manager.
+ return MemMgr->allocateGlobal(Size, Alignment);
+}
+
void JITEmitter::emitConstantPool(MachineConstantPool *MCP) {
if (TheJIT->getJITInfo().hasCustomConstantPool())
return;
diff --git a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp b/lib/ExecutionEngine/JIT/JITMemoryManager.cpp
index 70ccdcc..9820493 100644
--- a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp
+++ b/lib/ExecutionEngine/JIT/JITMemoryManager.cpp
@@ -251,6 +251,8 @@
/// middle of emitting a function, and we don't know how large the function we
/// are emitting is.
class VISIBILITY_HIDDEN DefaultJITMemoryManager : public JITMemoryManager {
+ bool PoisonMemory; // Whether to poison freed memory.
+
std::vector<sys::MemoryBlock> Blocks; // Memory blocks allocated by the JIT
FreeRangeHeader *FreeMemoryList; // Circular list of free blocks.
@@ -258,6 +260,7 @@
MemoryRangeHeader *CurBlock;
uint8_t *CurStubPtr, *StubBase;
+ uint8_t *CurGlobalPtr, *GlobalEnd;
uint8_t *GOTBase; // Target Specific reserved memory
void *DlsymTable; // Stub external symbol information
@@ -324,7 +327,7 @@
CurBlock = FreeMemoryList;
FreeMemoryList = FreeMemoryList->AllocateBlock();
- uint8_t *result = (uint8_t *)CurBlock+1;
+ uint8_t *result = (uint8_t *)(CurBlock + 1);
if (Alignment == 0) Alignment = 1;
result = (uint8_t*)(((intptr_t)result+Alignment-1) &
@@ -336,6 +339,30 @@
return result;
}
+ /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace,
+ /// this method does not touch the current block and can be called at any
+ /// time.
+ uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) {
+ uint8_t *Result = CurGlobalPtr;
+
+ // Align the pointer.
+ if (Alignment == 0) Alignment = 1;
+ Result = (uint8_t*)(((uintptr_t)Result + Alignment-1) &
+ ~(uintptr_t)(Alignment-1));
+
+ // Move the current global pointer forward.
+ CurGlobalPtr += Result - CurGlobalPtr + Size;
+
+ // Check for overflow.
+ if (CurGlobalPtr > GlobalEnd) {
+ // FIXME: Allocate more memory.
+ fprintf(stderr, "JIT ran out of memory for globals!\n");
+ abort();
+ }
+
+ return Result;
+ }
+
/// startExceptionTable - Use startFunctionBody to allocate memory for the
/// function's exception table.
uint8_t* startExceptionTable(const Function* F, uintptr_t &ActualSize) {
@@ -375,12 +402,12 @@
// Find the block that is allocated for this function.
MemoryRangeHeader *MemRange = I->second;
assert(MemRange->ThisAllocated && "Block isn't allocated!");
-
+
// Fill the buffer with garbage!
-#ifndef NDEBUG
- memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
-#endif
-
+ if (PoisonMemory) {
+ memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
+ }
+
// Free the memory.
FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
@@ -393,12 +420,12 @@
// Find the block that is allocated for this function.
MemRange = I->second;
assert(MemRange->ThisAllocated && "Block isn't allocated!");
-
+
// Fill the buffer with garbage!
-#ifndef NDEBUG
- memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
-#endif
-
+ if (PoisonMemory) {
+ memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
+ }
+
// Free the memory.
FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
@@ -420,10 +447,22 @@
for (unsigned i = 0, e = Blocks.size(); i != e; ++i)
sys::Memory::setExecutable(Blocks[i]);
}
+
+ /// setPoisonMemory - Controls whether we write garbage over freed memory.
+ ///
+ void setPoisonMemory(bool poison) {
+ PoisonMemory = poison;
+ }
};
}
DefaultJITMemoryManager::DefaultJITMemoryManager() {
+#ifdef NDEBUG
+ PoisonMemory = true;
+#else
+ PoisonMemory = false;
+#endif
+
// Allocate a 16M block of memory for functions.
#if defined(__APPLE__) && defined(__arm__)
sys::MemoryBlock MemBlock = getNewMemoryBlock(4 << 20);
@@ -433,11 +472,13 @@
uint8_t *MemBase = static_cast<uint8_t*>(MemBlock.base());
- // Allocate stubs backwards from the base, allocate functions forward
- // from the base.
+ // Allocate stubs backwards to the base, globals forward from the stubs, and
+ // functions forward after globals.
StubBase = MemBase;
CurStubPtr = MemBase + 512*1024; // Use 512k for stubs, working backwards.
-
+ CurGlobalPtr = CurStubPtr; // Use 2M for globals, working forwards.
+ GlobalEnd = CurGlobalPtr + 2*1024*1024;
+
// We set up the memory chunk with 4 mem regions, like this:
// [ START
// [ Free #0 ] -> Large space to allocate functions from.
@@ -474,7 +515,7 @@
// Add a FreeRangeHeader to the start of the function body region, indicating
// that the space is free. Mark the previous block allocated so we never look
// at it.
- FreeRangeHeader *Mem0 = (FreeRangeHeader*)CurStubPtr;
+ FreeRangeHeader *Mem0 = (FreeRangeHeader*)GlobalEnd;
Mem0->ThisAllocated = 0;
Mem0->PrevAllocated = 1;
Mem0->BlockSize = (char*)Mem1-(char*)Mem0;
@@ -522,7 +563,7 @@
sys::MemoryBlock DefaultJITMemoryManager::getNewMemoryBlock(unsigned size) {
// Allocate a new block close to the last one.
- const sys::MemoryBlock *BOld = Blocks.empty() ? 0 : &Blocks.front();
+ const sys::MemoryBlock *BOld = Blocks.empty() ? 0 : &Blocks.back();
std::string ErrMsg;
sys::MemoryBlock B = sys::Memory::AllocateRWX(size, BOld, &ErrMsg);
if (B.base() == 0) {
diff --git a/lib/ExecutionEngine/JIT/TargetSelect.cpp b/lib/ExecutionEngine/JIT/TargetSelect.cpp
index 0f20819..24dd013 100644
--- a/lib/ExecutionEngine/JIT/TargetSelect.cpp
+++ b/lib/ExecutionEngine/JIT/TargetSelect.cpp
@@ -43,7 +43,8 @@
///
ExecutionEngine *JIT::createJIT(ModuleProvider *MP, std::string *ErrorStr,
JITMemoryManager *JMM,
- CodeGenOpt::Level OptLevel) {
+ CodeGenOpt::Level OptLevel,
+ bool AllocateGVsWithCode) {
const TargetMachineRegistry::entry *TheArch = MArch;
if (TheArch == 0) {
std::string Error;
@@ -75,7 +76,7 @@
// If the target supports JIT code generation, return a new JIT now.
if (TargetJITInfo *TJ = Target->getJITInfo())
- return new JIT(MP, *Target, *TJ, JMM, OptLevel);
+ return new JIT(MP, *Target, *TJ, JMM, OptLevel, AllocateGVsWithCode);
if (ErrorStr)
*ErrorStr = "target does not support JIT code generation";