Lang Hames | 4821c24 | 2016-05-07 03:36:38 +0000 | [diff] [blame] | 1 | //===------------- OrcABISupport.cpp - ABI specific support code ----------===// |
Lang Hames | 4df7ba7 | 2015-10-26 06:40:28 +0000 | [diff] [blame] | 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | |
Lang Hames | 4821c24 | 2016-05-07 03:36:38 +0000 | [diff] [blame] | 10 | #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" |
Lang Hames | 98c2ac1 | 2015-10-19 17:43:51 +0000 | [diff] [blame] | 11 | #include "llvm/Support/Process.h" |
Lang Hames | 93de2a1 | 2015-01-23 21:25:00 +0000 | [diff] [blame] | 12 | |
Lang Hames | 93de2a1 | 2015-01-23 21:25:00 +0000 | [diff] [blame] | 13 | namespace llvm { |
Lang Hames | e738061 | 2015-02-21 20:44:36 +0000 | [diff] [blame] | 14 | namespace orc { |
Lang Hames | 93de2a1 | 2015-01-23 21:25:00 +0000 | [diff] [blame] | 15 | |
Lang Hames | f74e93b | 2016-04-29 21:32:00 +0000 | [diff] [blame] | 16 | void OrcAArch64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, |
| 17 | void *CallbackMgr) { |
| 18 | |
| 19 | const uint32_t ResolverCode[] = { |
| 20 | // resolver_entry: |
Lang Hames | 2307f40 | 2016-05-01 00:14:45 +0000 | [diff] [blame] | 21 | 0xa9bf47fd, // 0x000: stp x29, x17, [sp, #-16]! |
| 22 | 0x910003fd, // 0x004: mov x29, sp |
| 23 | 0xa9bf73fb, // 0x008: stp x27, x28, [sp, #-16]! |
| 24 | 0xa9bf6bf9, // 0x00c: stp x25, x26, [sp, #-16]! |
| 25 | 0xa9bf63f7, // 0x010: stp x23, x24, [sp, #-16]! |
| 26 | 0xa9bf5bf5, // 0x014: stp x21, x22, [sp, #-16]! |
| 27 | 0xa9bf53f3, // 0x018: stp x19, x20, [sp, #-16]! |
| 28 | 0xa9bf3fee, // 0x01c: stp x14, x15, [sp, #-16]! |
| 29 | 0xa9bf37ec, // 0x020: stp x12, x13, [sp, #-16]! |
| 30 | 0xa9bf2fea, // 0x024: stp x10, x11, [sp, #-16]! |
| 31 | 0xa9bf27e8, // 0x028: stp x8, x9, [sp, #-16]! |
| 32 | 0xa9bf1fe6, // 0x02c: stp x6, x7, [sp, #-16]! |
| 33 | 0xa9bf17e4, // 0x030: stp x4, x5, [sp, #-16]! |
| 34 | 0xa9bf0fe2, // 0x034: stp x2, x3, [sp, #-16]! |
| 35 | 0xa9bf07e0, // 0x038: stp x0, x1, [sp, #-16]! |
| 36 | 0xadbf7ffe, // 0x03c: stp q30, q31, [sp, #-32]! |
| 37 | 0xadbf77fc, // 0x040: stp q28, q29, [sp, #-32]! |
| 38 | 0xadbf6ffa, // 0x044: stp q26, q27, [sp, #-32]! |
| 39 | 0xadbf67f8, // 0x048: stp q24, q25, [sp, #-32]! |
| 40 | 0xadbf5ff6, // 0x04c: stp q22, q23, [sp, #-32]! |
| 41 | 0xadbf57f4, // 0x050: stp q20, q21, [sp, #-32]! |
| 42 | 0xadbf4ff2, // 0x054: stp q18, q19, [sp, #-32]! |
| 43 | 0xadbf47f0, // 0x058: stp q16, q17, [sp, #-32]! |
| 44 | 0xadbf3fee, // 0x05c: stp q14, q15, [sp, #-32]! |
| 45 | 0xadbf37ec, // 0x060: stp q12, q13, [sp, #-32]! |
| 46 | 0xadbf2fea, // 0x064: stp q10, q11, [sp, #-32]! |
| 47 | 0xadbf27e8, // 0x068: stp q8, q9, [sp, #-32]! |
| 48 | 0xadbf1fe6, // 0x06c: stp q6, q7, [sp, #-32]! |
| 49 | 0xadbf17e4, // 0x070: stp q4, q5, [sp, #-32]! |
| 50 | 0xadbf0fe2, // 0x074: stp q2, q3, [sp, #-32]! |
| 51 | 0xadbf07e0, // 0x078: stp q0, q1, [sp, #-32]! |
| 52 | 0x580004e0, // 0x07c: ldr x0, Lcallbackmgr |
| 53 | 0xaa1e03e1, // 0x080: mov x1, x30 |
| 54 | 0xd1003021, // 0x084: sub x1, x1, #12 |
| 55 | 0x58000442, // 0x088: ldr x2, Lreentry_fn_ptr |
| 56 | 0xd63f0040, // 0x08c: blr x2 |
| 57 | 0xaa0003f1, // 0x090: mov x17, x0 |
| 58 | 0xacc107e0, // 0x094: ldp q0, q1, [sp], #32 |
| 59 | 0xacc10fe2, // 0x098: ldp q2, q3, [sp], #32 |
| 60 | 0xacc117e4, // 0x09c: ldp q4, q5, [sp], #32 |
| 61 | 0xacc11fe6, // 0x0a0: ldp q6, q7, [sp], #32 |
| 62 | 0xacc127e8, // 0x0a4: ldp q8, q9, [sp], #32 |
| 63 | 0xacc12fea, // 0x0a8: ldp q10, q11, [sp], #32 |
| 64 | 0xacc137ec, // 0x0ac: ldp q12, q13, [sp], #32 |
| 65 | 0xacc13fee, // 0x0b0: ldp q14, q15, [sp], #32 |
| 66 | 0xacc147f0, // 0x0b4: ldp q16, q17, [sp], #32 |
| 67 | 0xacc14ff2, // 0x0b8: ldp q18, q19, [sp], #32 |
| 68 | 0xacc157f4, // 0x0bc: ldp q20, q21, [sp], #32 |
| 69 | 0xacc15ff6, // 0x0c0: ldp q22, q23, [sp], #32 |
| 70 | 0xacc167f8, // 0x0c4: ldp q24, q25, [sp], #32 |
| 71 | 0xacc16ffa, // 0x0c8: ldp q26, q27, [sp], #32 |
| 72 | 0xacc177fc, // 0x0cc: ldp q28, q29, [sp], #32 |
| 73 | 0xacc17ffe, // 0x0d0: ldp q30, q31, [sp], #32 |
| 74 | 0xa8c107e0, // 0x0d4: ldp x0, x1, [sp], #16 |
| 75 | 0xa8c10fe2, // 0x0d8: ldp x2, x3, [sp], #16 |
| 76 | 0xa8c117e4, // 0x0dc: ldp x4, x5, [sp], #16 |
| 77 | 0xa8c11fe6, // 0x0e0: ldp x6, x7, [sp], #16 |
| 78 | 0xa8c127e8, // 0x0e4: ldp x8, x9, [sp], #16 |
| 79 | 0xa8c12fea, // 0x0e8: ldp x10, x11, [sp], #16 |
| 80 | 0xa8c137ec, // 0x0ec: ldp x12, x13, [sp], #16 |
| 81 | 0xa8c13fee, // 0x0f0: ldp x14, x15, [sp], #16 |
| 82 | 0xa8c153f3, // 0x0f4: ldp x19, x20, [sp], #16 |
| 83 | 0xa8c15bf5, // 0x0f8: ldp x21, x22, [sp], #16 |
| 84 | 0xa8c163f7, // 0x0fc: ldp x23, x24, [sp], #16 |
| 85 | 0xa8c16bf9, // 0x100: ldp x25, x26, [sp], #16 |
| 86 | 0xa8c173fb, // 0x104: ldp x27, x28, [sp], #16 |
| 87 | 0xa8c17bfd, // 0x108: ldp x29, x30, [sp], #16 |
| 88 | 0xd65f0220, // 0x10c: ret x17 |
| 89 | 0x01234567, // 0x110: Lreentry_fn_ptr: |
| 90 | 0xdeadbeef, // 0x114: .quad 0 |
| 91 | 0x98765432, // 0x118: Lcallbackmgr: |
| 92 | 0xcafef00d // 0x11c: .quad 0 |
Lang Hames | f74e93b | 2016-04-29 21:32:00 +0000 | [diff] [blame] | 93 | }; |
| 94 | |
Lang Hames | 2307f40 | 2016-05-01 00:14:45 +0000 | [diff] [blame] | 95 | const unsigned ReentryFnAddrOffset = 0x110; |
| 96 | const unsigned CallbackMgrAddrOffset = 0x118; |
Lang Hames | f74e93b | 2016-04-29 21:32:00 +0000 | [diff] [blame] | 97 | |
| 98 | memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); |
| 99 | memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); |
| 100 | memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, |
| 101 | sizeof(CallbackMgr)); |
| 102 | } |
| 103 | |
| 104 | void OrcAArch64::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, |
| 105 | unsigned NumTrampolines) { |
| 106 | |
| 107 | unsigned OffsetToPtr = alignTo(NumTrampolines * TrampolineSize, 8); |
| 108 | |
| 109 | memcpy(TrampolineMem + OffsetToPtr, &ResolverAddr, sizeof(void *)); |
| 110 | |
| 111 | // OffsetToPtr is actually the offset from the PC for the 2nd instruction, so |
| 112 | // subtract 32-bits. |
| 113 | OffsetToPtr -= 4; |
| 114 | |
| 115 | uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem); |
| 116 | |
| 117 | for (unsigned I = 0; I < NumTrampolines; ++I, OffsetToPtr -= TrampolineSize) { |
| 118 | Trampolines[3 * I + 0] = 0xaa1e03f1; // mov x17, x30 |
Lang Hames | d4d6a1a | 2017-09-15 06:50:19 +0000 | [diff] [blame] | 119 | Trampolines[3 * I + 1] = 0x58000010 | (OffsetToPtr << 3); // adr x16, Lptr |
Lang Hames | f74e93b | 2016-04-29 21:32:00 +0000 | [diff] [blame] | 120 | Trampolines[3 * I + 2] = 0xd63f0200; // blr x16 |
| 121 | } |
| 122 | |
| 123 | } |
| 124 | |
| 125 | Error OrcAArch64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, |
| 126 | unsigned MinStubs, |
| 127 | void *InitialPtrVal) { |
| 128 | // Stub format is: |
| 129 | // |
| 130 | // .section __orc_stubs |
| 131 | // stub1: |
| 132 | // ldr x0, ptr1 ; PC-rel load of ptr1 |
| 133 | // br x0 ; Jump to resolver |
| 134 | // stub2: |
| 135 | // ldr x0, ptr2 ; PC-rel load of ptr2 |
| 136 | // br x0 ; Jump to resolver |
| 137 | // |
| 138 | // ... |
| 139 | // |
| 140 | // .section __orc_ptrs |
| 141 | // ptr1: |
| 142 | // .quad 0x0 |
| 143 | // ptr2: |
| 144 | // .quad 0x0 |
| 145 | // |
| 146 | // ... |
| 147 | |
| 148 | const unsigned StubSize = IndirectStubsInfo::StubSize; |
| 149 | |
| 150 | // Emit at least MinStubs, rounded up to fill the pages allocated. |
| 151 | unsigned PageSize = sys::Process::getPageSize(); |
| 152 | unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; |
| 153 | unsigned NumStubs = (NumPages * PageSize) / StubSize; |
| 154 | |
| 155 | // Allocate memory for stubs and pointers in one call. |
| 156 | std::error_code EC; |
| 157 | auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( |
| 158 | 2 * NumPages * PageSize, nullptr, |
| 159 | sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); |
| 160 | |
| 161 | if (EC) |
| 162 | return errorCodeToError(EC); |
| 163 | |
| 164 | // Create separate MemoryBlocks representing the stubs and pointers. |
| 165 | sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); |
| 166 | sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + |
| 167 | NumPages * PageSize, |
| 168 | NumPages * PageSize); |
| 169 | |
| 170 | // Populate the stubs page stubs and mark it executable. |
| 171 | uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlock.base()); |
| 172 | uint64_t PtrOffsetField = static_cast<uint64_t>(NumPages * PageSize) |
| 173 | << 3; |
| 174 | |
| 175 | for (unsigned I = 0; I < NumStubs; ++I) |
| 176 | Stub[I] = 0xd61f020058000010 | PtrOffsetField; |
| 177 | |
| 178 | if (auto EC = sys::Memory::protectMappedMemory( |
| 179 | StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) |
| 180 | return errorCodeToError(EC); |
| 181 | |
| 182 | // Initialize all pointers to point at FailureAddress. |
| 183 | void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); |
| 184 | for (unsigned I = 0; I < NumStubs; ++I) |
| 185 | Ptr[I] = InitialPtrVal; |
| 186 | |
| 187 | StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); |
| 188 | |
| 189 | return Error::success(); |
| 190 | } |
| 191 | |
Lang Hames | 4821c24 | 2016-05-07 03:36:38 +0000 | [diff] [blame] | 192 | void OrcX86_64_Base::writeTrampolines(uint8_t *TrampolineMem, |
| 193 | void *ResolverAddr, |
| 194 | unsigned NumTrampolines) { |
Rafael Espindola | 2f34463 | 2015-11-03 16:25:20 +0000 | [diff] [blame] | 195 | |
Rafael Espindola | e63e018 | 2015-11-03 16:40:37 +0000 | [diff] [blame] | 196 | unsigned OffsetToPtr = NumTrampolines * TrampolineSize; |
| 197 | |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 198 | memcpy(TrampolineMem + OffsetToPtr, &ResolverAddr, sizeof(void *)); |
Rafael Espindola | e63e018 | 2015-11-03 16:40:37 +0000 | [diff] [blame] | 199 | |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 200 | uint64_t *Trampolines = reinterpret_cast<uint64_t *>(TrampolineMem); |
Rafael Espindola | e63e018 | 2015-11-03 16:40:37 +0000 | [diff] [blame] | 201 | uint64_t CallIndirPCRel = 0xf1c40000000015ff; |
| 202 | |
| 203 | for (unsigned I = 0; I < NumTrampolines; ++I, OffsetToPtr -= TrampolineSize) |
| 204 | Trampolines[I] = CallIndirPCRel | ((OffsetToPtr - 6) << 16); |
Lang Hames | 2754714 | 2015-02-17 01:18:38 +0000 | [diff] [blame] | 205 | } |
| 206 | |
Lang Hames | 4821c24 | 2016-05-07 03:36:38 +0000 | [diff] [blame] | 207 | Error OrcX86_64_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, |
| 208 | unsigned MinStubs, |
| 209 | void *InitialPtrVal) { |
Lang Hames | 98c2ac1 | 2015-10-19 17:43:51 +0000 | [diff] [blame] | 210 | // Stub format is: |
| 211 | // |
| 212 | // .section __orc_stubs |
| 213 | // stub1: |
| 214 | // jmpq *ptr1(%rip) |
| 215 | // .byte 0xC4 ; <- Invalid opcode padding. |
| 216 | // .byte 0xF1 |
| 217 | // stub2: |
| 218 | // jmpq *ptr2(%rip) |
| 219 | // |
| 220 | // ... |
| 221 | // |
| 222 | // .section __orc_ptrs |
| 223 | // ptr1: |
| 224 | // .quad 0x0 |
| 225 | // ptr2: |
| 226 | // .quad 0x0 |
| 227 | // |
| 228 | // ... |
| 229 | |
| 230 | const unsigned StubSize = IndirectStubsInfo::StubSize; |
| 231 | |
| 232 | // Emit at least MinStubs, rounded up to fill the pages allocated. |
| 233 | unsigned PageSize = sys::Process::getPageSize(); |
| 234 | unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; |
| 235 | unsigned NumStubs = (NumPages * PageSize) / StubSize; |
| 236 | |
| 237 | // Allocate memory for stubs and pointers in one call. |
| 238 | std::error_code EC; |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 239 | auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( |
| 240 | 2 * NumPages * PageSize, nullptr, |
| 241 | sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); |
Lang Hames | 98c2ac1 | 2015-10-19 17:43:51 +0000 | [diff] [blame] | 242 | |
| 243 | if (EC) |
Lang Hames | ef5a0ee | 2016-04-25 19:56:45 +0000 | [diff] [blame] | 244 | return errorCodeToError(EC); |
Lang Hames | 98c2ac1 | 2015-10-19 17:43:51 +0000 | [diff] [blame] | 245 | |
| 246 | // Create separate MemoryBlocks representing the stubs and pointers. |
Lang Hames | 5796eb2 | 2015-10-31 00:55:32 +0000 | [diff] [blame] | 247 | sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 248 | sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + |
| 249 | NumPages * PageSize, |
Lang Hames | 98c2ac1 | 2015-10-19 17:43:51 +0000 | [diff] [blame] | 250 | NumPages * PageSize); |
| 251 | |
| 252 | // Populate the stubs page stubs and mark it executable. |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 253 | uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlock.base()); |
| 254 | uint64_t PtrOffsetField = static_cast<uint64_t>(NumPages * PageSize - 6) |
| 255 | << 16; |
Lang Hames | 98c2ac1 | 2015-10-19 17:43:51 +0000 | [diff] [blame] | 256 | for (unsigned I = 0; I < NumStubs; ++I) |
| 257 | Stub[I] = 0xF1C40000000025ff | PtrOffsetField; |
| 258 | |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 259 | if (auto EC = sys::Memory::protectMappedMemory( |
| 260 | StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) |
Lang Hames | ef5a0ee | 2016-04-25 19:56:45 +0000 | [diff] [blame] | 261 | return errorCodeToError(EC); |
Lang Hames | 98c2ac1 | 2015-10-19 17:43:51 +0000 | [diff] [blame] | 262 | |
| 263 | // Initialize all pointers to point at FailureAddress. |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 264 | void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); |
Lang Hames | 98c2ac1 | 2015-10-19 17:43:51 +0000 | [diff] [blame] | 265 | for (unsigned I = 0; I < NumStubs; ++I) |
| 266 | Ptr[I] = InitialPtrVal; |
| 267 | |
Lang Hames | e28b118 | 2016-02-02 19:31:15 +0000 | [diff] [blame] | 268 | StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); |
Lang Hames | 98c2ac1 | 2015-10-19 17:43:51 +0000 | [diff] [blame] | 269 | |
Lang Hames | ef5a0ee | 2016-04-25 19:56:45 +0000 | [diff] [blame] | 270 | return Error::success(); |
Lang Hames | 98c2ac1 | 2015-10-19 17:43:51 +0000 | [diff] [blame] | 271 | } |
| 272 | |
Lang Hames | 4821c24 | 2016-05-07 03:36:38 +0000 | [diff] [blame] | 273 | void OrcX86_64_SysV::writeResolverCode(uint8_t *ResolverMem, |
| 274 | JITReentryFn ReentryFn, |
| 275 | void *CallbackMgr) { |
| 276 | |
| 277 | const uint8_t ResolverCode[] = { |
| 278 | // resolver_entry: |
| 279 | 0x55, // 0x00: pushq %rbp |
| 280 | 0x48, 0x89, 0xe5, // 0x01: movq %rsp, %rbp |
| 281 | 0x50, // 0x04: pushq %rax |
| 282 | 0x53, // 0x05: pushq %rbx |
| 283 | 0x51, // 0x06: pushq %rcx |
| 284 | 0x52, // 0x07: pushq %rdx |
| 285 | 0x56, // 0x08: pushq %rsi |
| 286 | 0x57, // 0x09: pushq %rdi |
| 287 | 0x41, 0x50, // 0x0a: pushq %r8 |
| 288 | 0x41, 0x51, // 0x0c: pushq %r9 |
| 289 | 0x41, 0x52, // 0x0e: pushq %r10 |
| 290 | 0x41, 0x53, // 0x10: pushq %r11 |
| 291 | 0x41, 0x54, // 0x12: pushq %r12 |
| 292 | 0x41, 0x55, // 0x14: pushq %r13 |
| 293 | 0x41, 0x56, // 0x16: pushq %r14 |
| 294 | 0x41, 0x57, // 0x18: pushq %r15 |
| 295 | 0x48, 0x81, 0xec, 0x08, 0x02, 0x00, 0x00, // 0x1a: subq 0x208, %rsp |
| 296 | 0x48, 0x0f, 0xae, 0x04, 0x24, // 0x21: fxsave64 (%rsp) |
| 297 | 0x48, 0xbf, // 0x26: movabsq <CBMgr>, %rdi |
| 298 | |
| 299 | // 0x28: Callback manager addr. |
| 300 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, |
| 301 | |
| 302 | 0x48, 0x8b, 0x75, 0x08, // 0x30: movq 8(%rbp), %rsi |
| 303 | 0x48, 0x83, 0xee, 0x06, // 0x34: subq $6, %rsi |
| 304 | 0x48, 0xb8, // 0x38: movabsq <REntry>, %rax |
| 305 | |
| 306 | // 0x3a: JIT re-entry fn addr: |
| 307 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, |
| 308 | |
| 309 | 0xff, 0xd0, // 0x42: callq *%rax |
| 310 | 0x48, 0x89, 0x45, 0x08, // 0x44: movq %rax, 8(%rbp) |
| 311 | 0x48, 0x0f, 0xae, 0x0c, 0x24, // 0x48: fxrstor64 (%rsp) |
| 312 | 0x48, 0x81, 0xc4, 0x08, 0x02, 0x00, 0x00, // 0x4d: addq 0x208, %rsp |
| 313 | 0x41, 0x5f, // 0x54: popq %r15 |
| 314 | 0x41, 0x5e, // 0x56: popq %r14 |
| 315 | 0x41, 0x5d, // 0x58: popq %r13 |
| 316 | 0x41, 0x5c, // 0x5a: popq %r12 |
| 317 | 0x41, 0x5b, // 0x5c: popq %r11 |
| 318 | 0x41, 0x5a, // 0x5e: popq %r10 |
| 319 | 0x41, 0x59, // 0x60: popq %r9 |
| 320 | 0x41, 0x58, // 0x62: popq %r8 |
| 321 | 0x5f, // 0x64: popq %rdi |
| 322 | 0x5e, // 0x65: popq %rsi |
| 323 | 0x5a, // 0x66: popq %rdx |
| 324 | 0x59, // 0x67: popq %rcx |
| 325 | 0x5b, // 0x68: popq %rbx |
| 326 | 0x58, // 0x69: popq %rax |
| 327 | 0x5d, // 0x6a: popq %rbp |
| 328 | 0xc3, // 0x6b: retq |
| 329 | }; |
| 330 | |
| 331 | const unsigned ReentryFnAddrOffset = 0x3a; |
| 332 | const unsigned CallbackMgrAddrOffset = 0x28; |
| 333 | |
| 334 | memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); |
| 335 | memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); |
| 336 | memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, |
| 337 | sizeof(CallbackMgr)); |
| 338 | } |
| 339 | |
| 340 | void OrcX86_64_Win32::writeResolverCode(uint8_t *ResolverMem, |
| 341 | JITReentryFn ReentryFn, |
| 342 | void *CallbackMgr) { |
| 343 | |
| 344 | // resolverCode is similar to OrcX86_64 with differences specific to windows x64 calling convention: |
| 345 | // arguments go into rcx, rdx and come in reverse order, shadow space allocation on stack |
| 346 | const uint8_t ResolverCode[] = { |
| 347 | // resolver_entry: |
| 348 | 0x55, // 0x00: pushq %rbp |
| 349 | 0x48, 0x89, 0xe5, // 0x01: movq %rsp, %rbp |
| 350 | 0x50, // 0x04: pushq %rax |
| 351 | 0x53, // 0x05: pushq %rbx |
| 352 | 0x51, // 0x06: pushq %rcx |
| 353 | 0x52, // 0x07: pushq %rdx |
| 354 | 0x56, // 0x08: pushq %rsi |
| 355 | 0x57, // 0x09: pushq %rdi |
| 356 | 0x41, 0x50, // 0x0a: pushq %r8 |
| 357 | 0x41, 0x51, // 0x0c: pushq %r9 |
| 358 | 0x41, 0x52, // 0x0e: pushq %r10 |
| 359 | 0x41, 0x53, // 0x10: pushq %r11 |
| 360 | 0x41, 0x54, // 0x12: pushq %r12 |
| 361 | 0x41, 0x55, // 0x14: pushq %r13 |
| 362 | 0x41, 0x56, // 0x16: pushq %r14 |
| 363 | 0x41, 0x57, // 0x18: pushq %r15 |
| 364 | 0x48, 0x81, 0xec, 0x08, 0x02, 0x00, 0x00, // 0x1a: subq 0x208, %rsp |
| 365 | 0x48, 0x0f, 0xae, 0x04, 0x24, // 0x21: fxsave64 (%rsp) |
| 366 | |
| 367 | 0x48, 0xb9, // 0x26: movabsq <CBMgr>, %rcx |
| 368 | // 0x28: Callback manager addr. |
| 369 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, |
| 370 | |
| 371 | 0x48, 0x8B, 0x55, 0x08, // 0x30: mov rdx, [rbp+0x8] |
| 372 | 0x48, 0x83, 0xea, 0x06, // 0x34: sub rdx, 0x6 |
| 373 | |
| 374 | 0x48, 0xb8, // 0x38: movabsq <REntry>, %rax |
| 375 | // 0x3a: JIT re-entry fn addr: |
| 376 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, |
| 377 | |
| 378 | // 0x42: sub rsp, 0x20 (Allocate shadow space) |
| 379 | 0x48, 0x83, 0xEC, 0x20, |
| 380 | 0xff, 0xd0, // 0x46: callq *%rax |
| 381 | |
| 382 | // 0x48: add rsp, 0x20 (Free shadow space) |
| 383 | 0x48, 0x83, 0xC4, 0x20, |
| 384 | |
| 385 | 0x48, 0x89, 0x45, 0x08, // 0x4C: movq %rax, 8(%rbp) |
| 386 | 0x48, 0x0f, 0xae, 0x0c, 0x24, // 0x50: fxrstor64 (%rsp) |
| 387 | 0x48, 0x81, 0xc4, 0x08, 0x02, 0x00, 0x00, // 0x55: addq 0x208, %rsp |
| 388 | 0x41, 0x5f, // 0x5C: popq %r15 |
| 389 | 0x41, 0x5e, // 0x5E: popq %r14 |
| 390 | 0x41, 0x5d, // 0x60: popq %r13 |
| 391 | 0x41, 0x5c, // 0x62: popq %r12 |
| 392 | 0x41, 0x5b, // 0x64: popq %r11 |
| 393 | 0x41, 0x5a, // 0x66: popq %r10 |
| 394 | 0x41, 0x59, // 0x68: popq %r9 |
| 395 | 0x41, 0x58, // 0x6a: popq %r8 |
| 396 | 0x5f, // 0x6c: popq %rdi |
| 397 | 0x5e, // 0x6d: popq %rsi |
| 398 | 0x5a, // 0x6e: popq %rdx |
| 399 | 0x59, // 0x6f: popq %rcx |
| 400 | 0x5b, // 0x70: popq %rbx |
| 401 | 0x58, // 0x71: popq %rax |
| 402 | 0x5d, // 0x72: popq %rbp |
| 403 | 0xc3, // 0x73: retq |
| 404 | }; |
| 405 | |
| 406 | |
| 407 | const unsigned ReentryFnAddrOffset = 0x3a; |
| 408 | const unsigned CallbackMgrAddrOffset = 0x28; |
| 409 | |
| 410 | memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); |
| 411 | memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); |
| 412 | memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, |
| 413 | sizeof(CallbackMgr)); |
| 414 | } |
| 415 | |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 416 | void OrcI386::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, |
| 417 | void *CallbackMgr) { |
| 418 | |
| 419 | const uint8_t ResolverCode[] = { |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 420 | // resolver_entry: |
| 421 | 0x55, // 0x00: pushl %ebp |
| 422 | 0x89, 0xe5, // 0x01: movl %esp, %ebp |
| 423 | 0x54, // 0x03: pushl %esp |
| 424 | 0x83, 0xe4, 0xf0, // 0x04: andl $-0x10, %esp |
| 425 | 0x50, // 0x07: pushl %eax |
| 426 | 0x53, // 0x08: pushl %ebx |
| 427 | 0x51, // 0x09: pushl %ecx |
| 428 | 0x52, // 0x0a: pushl %edx |
| 429 | 0x56, // 0x0b: pushl %esi |
| 430 | 0x57, // 0x0c: pushl %edi |
| 431 | 0x81, 0xec, 0x18, 0x02, 0x00, 0x00, // 0x0d: subl $0x218, %esp |
| 432 | 0x0f, 0xae, 0x44, 0x24, 0x10, // 0x13: fxsave 0x10(%esp) |
| 433 | 0x8b, 0x75, 0x04, // 0x18: movl 0x4(%ebp), %esi |
| 434 | 0x83, 0xee, 0x05, // 0x1b: subl $0x5, %esi |
| 435 | 0x89, 0x74, 0x24, 0x04, // 0x1e: movl %esi, 0x4(%esp) |
| 436 | 0xc7, 0x04, 0x24, 0x00, 0x00, 0x00, |
| 437 | 0x00, // 0x22: movl <cbmgr>, (%esp) |
| 438 | 0xb8, 0x00, 0x00, 0x00, 0x00, // 0x29: movl <reentry>, %eax |
| 439 | 0xff, 0xd0, // 0x2e: calll *%eax |
| 440 | 0x89, 0x45, 0x04, // 0x30: movl %eax, 0x4(%ebp) |
| 441 | 0x0f, 0xae, 0x4c, 0x24, 0x10, // 0x33: fxrstor 0x10(%esp) |
| 442 | 0x81, 0xc4, 0x18, 0x02, 0x00, 0x00, // 0x38: addl $0x218, %esp |
| 443 | 0x5f, // 0x3e: popl %edi |
| 444 | 0x5e, // 0x3f: popl %esi |
| 445 | 0x5a, // 0x40: popl %edx |
| 446 | 0x59, // 0x41: popl %ecx |
| 447 | 0x5b, // 0x42: popl %ebx |
| 448 | 0x58, // 0x43: popl %eax |
| 449 | 0x8b, 0x65, 0xfc, // 0x44: movl -0x4(%ebp), %esp |
| 450 | 0x5d, // 0x48: popl %ebp |
| 451 | 0xc3 // 0x49: retl |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 452 | }; |
| 453 | |
Lang Hames | e1fd99c | 2016-02-21 22:50:26 +0000 | [diff] [blame] | 454 | const unsigned ReentryFnAddrOffset = 0x2a; |
| 455 | const unsigned CallbackMgrAddrOffset = 0x25; |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 456 | |
| 457 | memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); |
| 458 | memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); |
| 459 | memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, |
| 460 | sizeof(CallbackMgr)); |
| 461 | } |
| 462 | |
| 463 | void OrcI386::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, |
| 464 | unsigned NumTrampolines) { |
| 465 | |
| 466 | uint64_t CallRelImm = 0xF1C4C400000000e8; |
| 467 | uint64_t Resolver = reinterpret_cast<uint64_t>(ResolverAddr); |
| 468 | uint64_t ResolverRel = |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 469 | Resolver - reinterpret_cast<uint64_t>(TrampolineMem) - 5; |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 470 | |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 471 | uint64_t *Trampolines = reinterpret_cast<uint64_t *>(TrampolineMem); |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 472 | for (unsigned I = 0; I < NumTrampolines; ++I, ResolverRel -= TrampolineSize) |
| 473 | Trampolines[I] = CallRelImm | (ResolverRel << 8); |
| 474 | } |
| 475 | |
Lang Hames | ef5a0ee | 2016-04-25 19:56:45 +0000 | [diff] [blame] | 476 | Error OrcI386::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 477 | unsigned MinStubs, void *InitialPtrVal) { |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 478 | // Stub format is: |
| 479 | // |
| 480 | // .section __orc_stubs |
| 481 | // stub1: |
| 482 | // jmpq *ptr1 |
| 483 | // .byte 0xC4 ; <- Invalid opcode padding. |
| 484 | // .byte 0xF1 |
| 485 | // stub2: |
| 486 | // jmpq *ptr2 |
| 487 | // |
| 488 | // ... |
| 489 | // |
| 490 | // .section __orc_ptrs |
| 491 | // ptr1: |
| 492 | // .quad 0x0 |
| 493 | // ptr2: |
| 494 | // .quad 0x0 |
| 495 | // |
| 496 | // ... |
| 497 | |
| 498 | const unsigned StubSize = IndirectStubsInfo::StubSize; |
| 499 | |
| 500 | // Emit at least MinStubs, rounded up to fill the pages allocated. |
| 501 | unsigned PageSize = sys::Process::getPageSize(); |
| 502 | unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; |
| 503 | unsigned NumStubs = (NumPages * PageSize) / StubSize; |
| 504 | |
| 505 | // Allocate memory for stubs and pointers in one call. |
| 506 | std::error_code EC; |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 507 | auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( |
| 508 | 2 * NumPages * PageSize, nullptr, |
| 509 | sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 510 | |
| 511 | if (EC) |
Lang Hames | ef5a0ee | 2016-04-25 19:56:45 +0000 | [diff] [blame] | 512 | return errorCodeToError(EC); |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 513 | |
| 514 | // Create separate MemoryBlocks representing the stubs and pointers. |
| 515 | sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 516 | sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + |
| 517 | NumPages * PageSize, |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 518 | NumPages * PageSize); |
| 519 | |
| 520 | // Populate the stubs page stubs and mark it executable. |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 521 | uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlock.base()); |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 522 | uint64_t PtrAddr = reinterpret_cast<uint64_t>(PtrsBlock.base()); |
| 523 | for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 4) |
| 524 | Stub[I] = 0xF1C40000000025ff | (PtrAddr << 16); |
| 525 | |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 526 | if (auto EC = sys::Memory::protectMappedMemory( |
| 527 | StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) |
Lang Hames | ef5a0ee | 2016-04-25 19:56:45 +0000 | [diff] [blame] | 528 | return errorCodeToError(EC); |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 529 | |
| 530 | // Initialize all pointers to point at FailureAddress. |
Lang Hames | 1fa0e0e | 2016-04-25 21:21:20 +0000 | [diff] [blame] | 531 | void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 532 | for (unsigned I = 0; I < NumStubs; ++I) |
| 533 | Ptr[I] = InitialPtrVal; |
| 534 | |
| 535 | StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); |
| 536 | |
Lang Hames | ef5a0ee | 2016-04-25 19:56:45 +0000 | [diff] [blame] | 537 | return Error::success(); |
Lang Hames | 4f8194e | 2016-02-10 01:02:33 +0000 | [diff] [blame] | 538 | } |
| 539 | |
Petar Jovanovic | 5abf4bb | 2018-09-11 13:10:04 +0000 | [diff] [blame] | 540 | void OrcMips32_Base::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, |
| 541 | void *CallbackMgr, bool isBigEndian) { |
| 542 | |
| 543 | const uint32_t ResolverCode[] = { |
| 544 | // resolver_entry: |
| 545 | 0x27bdff98, // 0x00: addiu $sp,$sp,-104 |
| 546 | 0xafa20000, // 0x04: sw $v0,0($sp) |
| 547 | 0xafa30004, // 0x08: sw $v1,4($sp) |
| 548 | 0xafa40008, // 0x0c: sw $a0,8($sp) |
| 549 | 0xafa5000c, // 0x10: sw $a1,12($sp) |
| 550 | 0xafa60010, // 0x14: sw $a2,16($sp) |
| 551 | 0xafa70014, // 0x18: sw $a3,20($sp) |
| 552 | 0xafb00018, // 0x1c: sw $s0,24($sp) |
| 553 | 0xafb1001c, // 0x20: sw $s1,28($sp) |
| 554 | 0xafb20020, // 0x24: sw $s2,32($sp) |
| 555 | 0xafb30024, // 0x28: sw $s3,36($sp) |
| 556 | 0xafb40028, // 0x2c: sw $s4,40($sp) |
| 557 | 0xafb5002c, // 0x30: sw $s5,44($sp) |
| 558 | 0xafb60030, // 0x34: sw $s6,48($sp) |
| 559 | 0xafb70034, // 0x38: sw $s7,52($sp) |
| 560 | 0xafa80038, // 0x3c: sw $t0,56($sp) |
| 561 | 0xafa9003c, // 0x40: sw $t1,60($sp) |
| 562 | 0xafaa0040, // 0x44: sw $t2,64($sp) |
| 563 | 0xafab0044, // 0x48: sw $t3,68($sp) |
| 564 | 0xafac0048, // 0x4c: sw $t4,72($sp) |
| 565 | 0xafad004c, // 0x50: sw $t5,76($sp) |
| 566 | 0xafae0050, // 0x54: sw $t6,80($sp) |
| 567 | 0xafaf0054, // 0x58: sw $t7,84($sp) |
| 568 | 0xafb80058, // 0x5c: sw $t8,88($sp) |
| 569 | 0xafb9005c, // 0x60: sw $t9,92($sp) |
| 570 | 0xafbe0060, // 0x64: sw $fp,96($sp) |
| 571 | 0xafbf0064, // 0x68: sw $ra,100($sp) |
| 572 | |
| 573 | // Callback manager addr. |
| 574 | 0x00000000, // 0x6c: lui $a0,callbackmgr |
| 575 | 0x00000000, // 0x70: addiu $a0,$a0,callbackmgr |
| 576 | |
| 577 | 0x03e02825, // 0x74: move $a1, $ra |
| 578 | 0x24a5ffec, // 0x78: addiu $a1,$a1,-20 |
| 579 | |
| 580 | // JIT re-entry fn addr: |
| 581 | 0x00000000, // 0x7c: lui $t9,reentry |
| 582 | 0x00000000, // 0x80: addiu $t9,$t9,reentry |
| 583 | |
| 584 | 0x0320f809, // 0x84: jalr $t9 |
| 585 | 0x00000000, // 0x88: nop |
| 586 | 0x8fbf0064, // 0x8c: lw $ra,100($sp) |
| 587 | 0x8fbe0060, // 0x90: lw $fp,96($sp) |
| 588 | 0x8fb9005c, // 0x94: lw $t9,92($sp) |
| 589 | 0x8fb80058, // 0x98: lw $t8,88($sp) |
| 590 | 0x8faf0054, // 0x9c: lw $t7,84($sp) |
| 591 | 0x8fae0050, // 0xa0: lw $t6,80($sp) |
| 592 | 0x8fad004c, // 0xa4: lw $t5,76($sp) |
| 593 | 0x8fac0048, // 0xa8: lw $t4,72($sp) |
| 594 | 0x8fab0044, // 0xac: lw $t3,68($sp) |
| 595 | 0x8faa0040, // 0xb0: lw $t2,64($sp) |
| 596 | 0x8fa9003c, // 0xb4: lw $t1,60($sp) |
| 597 | 0x8fa80038, // 0xb8: lw $t0,56($sp) |
| 598 | 0x8fb70034, // 0xbc: lw $s7,52($sp) |
| 599 | 0x8fb60030, // 0xc0: lw $s6,48($sp) |
| 600 | 0x8fb5002c, // 0xc4: lw $s5,44($sp) |
| 601 | 0x8fb40028, // 0xc8: lw $s4,40($sp) |
| 602 | 0x8fb30024, // 0xcc: lw $s3,36($sp) |
| 603 | 0x8fb20020, // 0xd0: lw $s2,32($sp) |
| 604 | 0x8fb1001c, // 0xd4: lw $s1,28($sp) |
| 605 | 0x8fb00018, // 0xd8: lw $s0,24($sp) |
| 606 | 0x8fa70014, // 0xdc: lw $a3,20($sp) |
| 607 | 0x8fa70014, // 0xe0: lw $a3,20($sp) |
| 608 | 0x8fa60010, // 0xe4: lw $a2,16($sp) |
| 609 | 0x8fa5000c, // 0xe8: lw $a1,12($sp) |
| 610 | 0x8fa40008, // 0xec: lw $a0,8($sp) |
| 611 | 0x27bd0068, // 0xf4: addiu $sp,$sp,104 |
| 612 | 0x0300f825, // 0xf8: move $ra, $t8 |
| 613 | 0x00000000 // 0xfc: jr $v0/v1 |
| 614 | }; |
| 615 | |
| 616 | |
| 617 | const unsigned ReentryFnAddrOffset = 0x7c; // JIT re-entry fn addr lui |
| 618 | const unsigned CallbackMgrAddrOffset = 0x6c; // Callback manager addr lui |
| 619 | const unsigned offsett = 0xfc; |
| 620 | |
| 621 | memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); |
| 622 | |
| 623 | //Depending on endian return value will be in v0 or v1. |
| 624 | uint32_t JumpV0 = 0x00400008; |
| 625 | uint32_t JumpV1 = 0x00600008; |
| 626 | |
| 627 | if(isBigEndian == true) |
| 628 | memcpy(ResolverMem + offsett, &JumpV1, |
| 629 | sizeof(JumpV1)); |
| 630 | else |
| 631 | memcpy(ResolverMem + offsett, &JumpV0, |
| 632 | sizeof(JumpV0)); |
| 633 | |
| 634 | uint64_t CallMgrAddr = reinterpret_cast<uint64_t>(CallbackMgr); |
| 635 | uint32_t CallMgrLUi = 0x3c040000 | (((CallMgrAddr + 0x8000) >> 16) & 0xFFFF); |
| 636 | uint32_t CallMgrADDiu = 0x24840000 | ((CallMgrAddr) & 0xFFFF); |
| 637 | memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi, |
| 638 | sizeof(CallMgrLUi)); |
| 639 | memcpy(ResolverMem + (CallbackMgrAddrOffset + 4), &CallMgrADDiu, |
| 640 | sizeof(CallMgrADDiu)); |
| 641 | |
| 642 | uint64_t ReentryAddr = reinterpret_cast<uint64_t>(ReentryFn); |
| 643 | uint32_t ReentryLUi = 0x3c190000 | (((ReentryAddr + 0x8000) >> 16) & 0xFFFF); |
| 644 | uint32_t ReentryADDiu = 0x27390000 | ((ReentryAddr) & 0xFFFF); |
| 645 | memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi, |
| 646 | sizeof(ReentryLUi)); |
| 647 | memcpy(ResolverMem + (ReentryFnAddrOffset + 4), &ReentryADDiu, |
| 648 | sizeof(ReentryADDiu)); |
| 649 | } |
| 650 | |
| 651 | void OrcMips32_Base::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, |
| 652 | unsigned NumTrampolines) { |
| 653 | |
| 654 | uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem); |
| 655 | uint64_t ResolveAddr = reinterpret_cast<uint64_t>(ResolverAddr); |
| 656 | uint32_t RHiAddr = ((ResolveAddr + 0x8000) >> 16); |
| 657 | |
| 658 | for (unsigned I = 0; I < NumTrampolines; ++I) { |
| 659 | Trampolines[5 * I + 0] = 0x03e0c025; // move $t8,$ra |
| 660 | Trampolines[5 * I + 1] = 0x3c190000 | (RHiAddr & 0xFFFF); // lui $t9,resolveAddr |
| 661 | Trampolines[5 * I + 2] = 0x27390000 | (ResolveAddr & 0xFFFF); // addiu $t9,$t9,resolveAddr |
| 662 | Trampolines[5 * I + 3] = 0x0320f809; // jalr $t9 |
| 663 | Trampolines[5 * I + 4] = 0x00000000; // nop |
| 664 | } |
| 665 | |
| 666 | } |
| 667 | |
| 668 | Error OrcMips32_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, |
| 669 | unsigned MinStubs, |
| 670 | void *InitialPtrVal) { |
| 671 | // Stub format is: |
| 672 | // |
| 673 | // .section __orc_stubs |
| 674 | // stub1: |
| 675 | // lui $t9, ptr1 |
| 676 | // lw $t9, %lo(ptr1)($t9) |
| 677 | // jr $t9 |
| 678 | // stub2: |
| 679 | // lui $t9, ptr2 |
| 680 | // lw $t9,%lo(ptr1)($t9) |
| 681 | // jr $t9 |
| 682 | // |
| 683 | // ... |
| 684 | // |
| 685 | // .section __orc_ptrs |
| 686 | // ptr1: |
| 687 | // .word 0x0 |
| 688 | // ptr2: |
| 689 | // .word 0x0 |
| 690 | // |
| 691 | // ... |
| 692 | |
| 693 | const unsigned StubSize = IndirectStubsInfo::StubSize; |
| 694 | |
| 695 | // Emit at least MinStubs, rounded up to fill the pages allocated. |
| 696 | unsigned PageSize = sys::Process::getPageSize(); |
| 697 | unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; |
| 698 | unsigned NumStubs = (NumPages * PageSize) / StubSize; |
| 699 | |
| 700 | // Allocate memory for stubs and pointers in one call. |
| 701 | std::error_code EC; |
| 702 | auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( |
| 703 | 2 * NumPages * PageSize, nullptr, |
| 704 | sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); |
| 705 | |
| 706 | if (EC) |
| 707 | return errorCodeToError(EC); |
| 708 | |
| 709 | // Create separate MemoryBlocks representing the stubs and pointers. |
| 710 | sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); |
| 711 | sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + |
| 712 | NumPages * PageSize, |
| 713 | NumPages * PageSize); |
| 714 | |
| 715 | // Populate the stubs page stubs and mark it executable. |
| 716 | uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlock.base()); |
| 717 | uint64_t PtrAddr = reinterpret_cast<uint64_t>(Stub) + NumPages * PageSize; |
| 718 | |
| 719 | for (unsigned I = 0; I < NumStubs; ++I) { |
| 720 | uint32_t HiAddr = ((PtrAddr + 0x8000) >> 16); |
| 721 | Stub[4 * I + 0] = 0x3c190000 | (HiAddr & 0xFFFF); // lui $t9,ptr1 |
| 722 | Stub[4 * I + 1] = 0x8f390000 | (PtrAddr & 0xFFFF); // lw $t9,%lo(ptr1)($t9) |
| 723 | Stub[4 * I + 2] = 0x03200008; // jr $t9 |
| 724 | Stub[4 * I + 3] = 0x00000000; // nop |
| 725 | PtrAddr += 4; |
| 726 | } |
| 727 | |
| 728 | if (auto EC = sys::Memory::protectMappedMemory( |
| 729 | StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) |
| 730 | return errorCodeToError(EC); |
| 731 | |
| 732 | // Initialize all pointers to point at FailureAddress. |
| 733 | void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); |
| 734 | for (unsigned I = 0; I < NumStubs; ++I) |
| 735 | Ptr[I] = InitialPtrVal; |
| 736 | |
| 737 | StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); |
| 738 | |
| 739 | return Error::success(); |
| 740 | } |
| 741 | |
| 742 | void OrcMips64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, |
| 743 | void *CallbackMgr) { |
| 744 | |
| 745 | const uint32_t ResolverCode[] = { |
| 746 | //resolver_entry: |
| 747 | 0x67bdff30, // 0x00: daddiu $sp,$sp,-208 |
| 748 | 0xffa20000, // 0x04: sd v0,0(sp) |
| 749 | 0xffa30008, // 0x08: sd v1,8(sp) |
| 750 | 0xffa40010, // 0x0c: sd a0,16(sp) |
| 751 | 0xffa50018, // 0x10: sd a1,24(sp) |
| 752 | 0xffa60020, // 0x14: sd a2,32(sp) |
| 753 | 0xffa70028, // 0x18: sd a3,40(sp) |
| 754 | 0xffa80030, // 0x1c: sd a4,48(sp) |
| 755 | 0xffa90038, // 0x20: sd a5,56(sp) |
| 756 | 0xffaa0040, // 0x24: sd a6,64(sp) |
| 757 | 0xffab0048, // 0x28: sd a7,72(sp) |
| 758 | 0xffac0050, // 0x2c: sd t0,80(sp) |
| 759 | 0xffad0058, // 0x30: sd t1,88(sp) |
| 760 | 0xffae0060, // 0x34: sd t2,96(sp) |
| 761 | 0xffaf0068, // 0x38: sd t3,104(sp) |
| 762 | 0xffb00070, // 0x3c: sd s0,112(sp) |
| 763 | 0xffb10078, // 0x40: sd s1,120(sp) |
| 764 | 0xffb20080, // 0x44: sd s2,128(sp) |
| 765 | 0xffb30088, // 0x48: sd s3,136(sp) |
| 766 | 0xffb40090, // 0x4c: sd s4,144(sp) |
| 767 | 0xffb50098, // 0x50: sd s5,152(sp) |
| 768 | 0xffb600a0, // 0x54: sd s6,160(sp) |
| 769 | 0xffb700a8, // 0x58: sd s7,168(sp) |
| 770 | 0xffb800b0, // 0x5c: sd t8,176(sp) |
| 771 | 0xffb900b8, // 0x60: sd t9,184(sp) |
| 772 | 0xffbe00c0, // 0x64: sd s8,192(sp) |
| 773 | 0xffbf00c8, // 0x68: sd ra,200(sp) |
| 774 | |
| 775 | // Callback manager addr. |
| 776 | 0x00000000, // 0x6c: lui $a0,heighest(callbackmgr) |
| 777 | 0x00000000, // 0x70: daddiu $a0,$a0,heigher(callbackmgr) |
| 778 | 0x00000000, // 0x74: dsll $a0,$a0,16 |
| 779 | 0x00000000, // 0x78: daddiu $a0,$a0,hi(callbackmgr) |
| 780 | 0x00000000, // 0x7c: dsll $a0,$a0,16 |
| 781 | 0x00000000, // 0x80: daddiu $a0,$a0,lo(callbackmgr) |
| 782 | |
| 783 | 0x03e02825, // 0x84: move $a1, $ra |
| 784 | 0x64a5ffdc, // 0x88: daddiu $a1,$a1,-36 |
| 785 | |
| 786 | // JIT re-entry fn addr: |
| 787 | 0x00000000, // 0x8c: lui $t9,reentry |
| 788 | 0x00000000, // 0x90: daddiu $t9,$t9,reentry |
| 789 | 0x00000000, // 0x94: dsll $t9,$t9, |
| 790 | 0x00000000, // 0x98: daddiu $t9,$t9, |
| 791 | 0x00000000, // 0x9c: dsll $t9,$t9, |
| 792 | 0x00000000, // 0xa0: daddiu $t9,$t9, |
| 793 | 0x0320f809, // 0xa4: jalr $t9 |
| 794 | 0x00000000, // 0xa8: nop |
| 795 | 0xdfbf00c8, // 0xac: ld ra, 200(sp) |
| 796 | 0xdfbe00c0, // 0xb0: ld s8, 192(sp) |
| 797 | 0xdfb900b8, // 0xb4: ld t9, 184(sp) |
| 798 | 0xdfb800b0, // 0xb8: ld t8, 176(sp) |
| 799 | 0xdfb700a8, // 0xbc: ld s7, 168(sp) |
| 800 | 0xdfb600a0, // 0xc0: ld s6, 160(sp) |
| 801 | 0xdfb50098, // 0xc4: ld s5, 152(sp) |
| 802 | 0xdfb40090, // 0xc8: ld s4, 144(sp) |
| 803 | 0xdfb30088, // 0xcc: ld s3, 136(sp) |
| 804 | 0xdfb20080, // 0xd0: ld s2, 128(sp) |
| 805 | 0xdfb10078, // 0xd4: ld s1, 120(sp) |
| 806 | 0xdfb00070, // 0xd8: ld s0, 112(sp) |
| 807 | 0xdfaf0068, // 0xdc: ld t3, 104(sp) |
| 808 | 0xdfae0060, // 0xe0: ld t2, 96(sp) |
| 809 | 0xdfad0058, // 0xe4: ld t1, 88(sp) |
| 810 | 0xdfac0050, // 0xe8: ld t0, 80(sp) |
| 811 | 0xdfab0048, // 0xec: ld a7, 72(sp) |
| 812 | 0xdfaa0040, // 0xf0: ld a6, 64(sp) |
| 813 | 0xdfa90038, // 0xf4: ld a5, 56(sp) |
| 814 | 0xdfa80030, // 0xf8: ld a4, 48(sp) |
| 815 | 0xdfa70028, // 0xfc: ld a3, 40(sp) |
| 816 | 0xdfa60020, // 0x100: ld a2, 32(sp) |
| 817 | 0xdfa50018, // 0x104: ld a1, 24(sp) |
| 818 | 0xdfa40010, // 0x108: ld a0, 16(sp) |
| 819 | 0xdfa30008, // 0x10c: ld v1, 8(sp) |
| 820 | 0x67bd00d0, // 0x110: daddiu $sp,$sp,208 |
| 821 | 0x0300f825, // 0x114: move $ra, $t8 |
| 822 | 0x00400008 // 0x118: jr $v0 |
| 823 | }; |
| 824 | |
| 825 | const unsigned ReentryFnAddrOffset = 0x8c; // JIT re-entry fn addr lui |
| 826 | const unsigned CallbackMgrAddrOffset = 0x6c; // Callback manager addr lui |
| 827 | |
| 828 | memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); |
| 829 | |
| 830 | |
| 831 | uint64_t CallMgrAddr = reinterpret_cast<uint64_t>(CallbackMgr); |
| 832 | |
| 833 | uint32_t CallMgrLUi = |
| 834 | 0x3c040000 | (((CallMgrAddr + 0x800080008000) >> 48) & 0xFFFF); |
| 835 | uint32_t CallMgrDADDiu = |
| 836 | 0x64840000 | (((CallMgrAddr + 0x80008000) >> 32) & 0xFFFF); |
| 837 | uint32_t CallMgrDSLL = 0x00042438; |
| 838 | uint32_t CallMgrDADDiu2 = |
| 839 | 0x64840000 | ((((CallMgrAddr + 0x8000) >> 16) & 0xFFFF)); |
| 840 | uint32_t CallMgrDSLL2 = 0x00042438; |
| 841 | uint32_t CallMgrDADDiu3 = 0x64840000 | ((CallMgrAddr) & 0xFFFF); |
| 842 | |
| 843 | memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi, |
| 844 | sizeof(CallMgrLUi)); |
| 845 | memcpy(ResolverMem + (CallbackMgrAddrOffset + 4), &CallMgrDADDiu, |
| 846 | sizeof(CallMgrDADDiu)); |
| 847 | memcpy(ResolverMem + (CallbackMgrAddrOffset + 8), &CallMgrDSLL, |
| 848 | sizeof(CallMgrDSLL)); |
| 849 | memcpy(ResolverMem + (CallbackMgrAddrOffset + 12), &CallMgrDADDiu2, |
| 850 | sizeof(CallMgrDADDiu2)); |
| 851 | memcpy(ResolverMem + (CallbackMgrAddrOffset + 16), &CallMgrDSLL2, |
| 852 | sizeof(CallMgrDSLL2)); |
| 853 | memcpy(ResolverMem + (CallbackMgrAddrOffset + 20), &CallMgrDADDiu3, |
| 854 | sizeof(CallMgrDADDiu3)); |
| 855 | |
| 856 | uint64_t ReentryAddr = reinterpret_cast<uint64_t>(ReentryFn); |
| 857 | |
| 858 | uint32_t ReentryLUi = |
| 859 | 0x3c190000 | (((ReentryAddr + 0x800080008000) >> 48) & 0xFFFF); |
| 860 | |
| 861 | uint32_t ReentryDADDiu = |
| 862 | 0x67390000 | (((ReentryAddr + 0x80008000) >> 32) & 0xFFFF); |
| 863 | |
| 864 | uint32_t ReentryDSLL = 0x0019cc38; |
| 865 | |
| 866 | uint32_t ReentryDADDiu2 = |
| 867 | 0x67390000 | (((ReentryAddr + 0x8000) >> 16) & 0xFFFF); |
| 868 | |
| 869 | uint32_t ReentryDSLL2 = 0x0019cc38; |
| 870 | |
| 871 | uint32_t ReentryDADDiu3 = 0x67390000 | ((ReentryAddr) & 0xFFFF); |
| 872 | |
| 873 | memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi, |
| 874 | sizeof(ReentryLUi)); |
| 875 | memcpy(ResolverMem + (ReentryFnAddrOffset + 4), &ReentryDADDiu, |
| 876 | sizeof(ReentryDADDiu)); |
| 877 | memcpy(ResolverMem + (ReentryFnAddrOffset + 8), &ReentryDSLL, |
| 878 | sizeof(ReentryDSLL)); |
| 879 | memcpy(ResolverMem + (ReentryFnAddrOffset + 12), &ReentryDADDiu2, |
| 880 | sizeof(ReentryDADDiu2)); |
| 881 | memcpy(ResolverMem + (ReentryFnAddrOffset + 16), &ReentryDSLL2, |
| 882 | sizeof(ReentryDSLL2)); |
| 883 | memcpy(ResolverMem + (ReentryFnAddrOffset + 20), &ReentryDADDiu3, |
| 884 | sizeof(ReentryDADDiu3)); |
| 885 | } |
| 886 | |
| 887 | void OrcMips64::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, |
| 888 | unsigned NumTrampolines) { |
| 889 | |
| 890 | uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem); |
| 891 | uint64_t ResolveAddr = reinterpret_cast<uint64_t>(ResolverAddr); |
| 892 | |
| 893 | uint64_t HeighestAddr = ((ResolveAddr + 0x800080008000) >> 48); |
| 894 | uint64_t HeigherAddr = ((ResolveAddr + 0x80008000) >> 32); |
| 895 | uint64_t HiAddr = ((ResolveAddr + 0x8000) >> 16); |
| 896 | |
| 897 | for (unsigned I = 0; I < NumTrampolines; ++I) { |
| 898 | Trampolines[10 * I + 0] = 0x03e0c025; // move $t8,$ra |
| 899 | Trampolines[10 * I + 1] = 0x3c190000 | (HeighestAddr & 0xFFFF); // lui $t9,resolveAddr |
| 900 | Trampolines[10 * I + 2] = 0x67390000 | (HeigherAddr & 0xFFFF); // daddiu $t9,$t9,%higher(resolveAddr) |
| 901 | Trampolines[10 * I + 3] = 0x0019cc38; // dsll $t9,$t9,16 |
| 902 | Trampolines[10 * I + 4] = 0x67390000 | (HiAddr & 0xFFFF); // daddiu $t9,$t9,%hi(ptr) |
| 903 | Trampolines[10 * I + 5] = 0x0019cc38; // dsll $t9,$t9,16 |
| 904 | Trampolines[10 * I + 6] = 0x67390000 | (ResolveAddr & 0xFFFF); // daddiu $t9,$t9,%lo(ptr) |
| 905 | Trampolines[10 * I + 7] = 0x0320f809; // jalr $t9 |
| 906 | Trampolines[10 * I + 8] = 0x00000000; // nop |
| 907 | Trampolines[10 * I + 9] = 0x00000000; // nop |
| 908 | } |
| 909 | |
| 910 | } |
| 911 | |
| 912 | Error OrcMips64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, |
| 913 | unsigned MinStubs, |
| 914 | void *InitialPtrVal) { |
| 915 | // Stub format is: |
| 916 | // |
| 917 | // .section __orc_stubs |
| 918 | // stub1: |
| 919 | // lui $t9,ptr1 |
| 920 | // dsll $t9,$t9,16 |
| 921 | // daddiu $t9,$t9,%hi(ptr) |
| 922 | // dsll $t9,$t9,16 |
| 923 | // ld $t9,%lo(ptr) |
| 924 | // jr $t9 |
| 925 | // stub2: |
| 926 | // lui $t9,ptr1 |
| 927 | // dsll $t9,$t9,16 |
| 928 | // daddiu $t9,$t9,%hi(ptr) |
| 929 | // dsll $t9,$t9,16 |
| 930 | // ld $t9,%lo(ptr) |
| 931 | // jr $t9 |
| 932 | // |
| 933 | // ... |
| 934 | // |
| 935 | // .section __orc_ptrs |
| 936 | // ptr1: |
| 937 | // .dword 0x0 |
| 938 | // ptr2: |
| 939 | // .dword 0x0 |
| 940 | // |
| 941 | // ... |
| 942 | const unsigned StubSize = IndirectStubsInfo::StubSize; |
| 943 | |
| 944 | // Emit at least MinStubs, rounded up to fill the pages allocated. |
| 945 | unsigned PageSize = sys::Process::getPageSize(); |
| 946 | unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; |
| 947 | unsigned NumStubs = (NumPages * PageSize) / StubSize; |
| 948 | |
| 949 | // Allocate memory for stubs and pointers in one call. |
| 950 | std::error_code EC; |
| 951 | auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( |
| 952 | 2 * NumPages * PageSize, nullptr, |
| 953 | sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); |
| 954 | |
| 955 | if (EC) |
| 956 | return errorCodeToError(EC); |
| 957 | |
| 958 | // Create separate MemoryBlocks representing the stubs and pointers. |
| 959 | sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); |
| 960 | sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + |
| 961 | NumPages * PageSize, |
| 962 | NumPages * PageSize); |
| 963 | |
| 964 | // Populate the stubs page stubs and mark it executable. |
| 965 | uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlock.base()); |
| 966 | uint64_t PtrAddr = reinterpret_cast<uint64_t>(PtrsBlock.base()); |
| 967 | |
| 968 | for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 8) { |
| 969 | uint64_t HeighestAddr = ((PtrAddr + 0x800080008000) >> 48); |
| 970 | uint64_t HeigherAddr = ((PtrAddr + 0x80008000) >> 32); |
| 971 | uint64_t HiAddr = ((PtrAddr + 0x8000) >> 16); |
| 972 | Stub[8 * I + 0] = 0x3c190000 | (HeighestAddr & 0xFFFF); // lui $t9,ptr1 |
| 973 | Stub[8 * I + 1] = 0x67390000 | (HeigherAddr & 0xFFFF); // daddiu $t9,$t9,%higher(ptr) |
| 974 | Stub[8 * I + 2] = 0x0019cc38; // dsll $t9,$t9,16 |
| 975 | Stub[8 * I + 3] = 0x67390000 | (HiAddr & 0xFFFF); // daddiu $t9,$t9,%hi(ptr) |
| 976 | Stub[8 * I + 4] = 0x0019cc38; // dsll $t9,$t9,16 |
| 977 | Stub[8 * I + 5] = 0xdf390000 | (PtrAddr & 0xFFFF); // ld $t9,%lo(ptr) |
| 978 | Stub[8 * I + 6] = 0x03200008; // jr $t9 |
| 979 | Stub[8 * I + 7] = 0x00000000; // nop |
| 980 | } |
| 981 | |
| 982 | if (auto EC = sys::Memory::protectMappedMemory( |
| 983 | StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) |
| 984 | return errorCodeToError(EC); |
| 985 | |
| 986 | // Initialize all pointers to point at FailureAddress. |
| 987 | void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); |
| 988 | for (unsigned I = 0; I < NumStubs; ++I) |
| 989 | Ptr[I] = InitialPtrVal; |
| 990 | |
| 991 | StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); |
| 992 | |
| 993 | return Error::success(); |
| 994 | } |
Lang Hames | e738061 | 2015-02-21 20:44:36 +0000 | [diff] [blame] | 995 | } // End namespace orc. |
| 996 | } // End namespace llvm. |