Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2015 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #include "linker/arm64/relative_patcher_arm64.h" |
| 18 | |
| 19 | #include "arch/arm64/instruction_set_features_arm64.h" |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 20 | #include "art_method.h" |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 21 | #include "compiled_method.h" |
| 22 | #include "driver/compiler_driver.h" |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 23 | #include "utils/arm64/assembler_arm64.h" |
| 24 | #include "oat.h" |
| 25 | #include "output_stream.h" |
| 26 | |
| 27 | namespace art { |
| 28 | namespace linker { |
| 29 | |
| 30 | Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider, |
| 31 | const Arm64InstructionSetFeatures* features) |
| 32 | : ArmBaseRelativePatcher(provider, kArm64, CompileThunkCode(), |
| 33 | kMaxPositiveDisplacement, kMaxNegativeDisplacement), |
| 34 | fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()), |
| 35 | reserved_adrp_thunks_(0u), |
| 36 | processed_adrp_thunks_(0u) { |
| 37 | if (fix_cortex_a53_843419_) { |
| 38 | adrp_thunk_locations_.reserve(16u); |
| 39 | current_method_thunks_.reserve(16u * kAdrpThunkSize); |
| 40 | } |
| 41 | } |
| 42 | |
| 43 | uint32_t Arm64RelativePatcher::ReserveSpace(uint32_t offset, |
Vladimir Marko | 4d23c9d | 2015-04-01 23:03:09 +0100 | [diff] [blame] | 44 | const CompiledMethod* compiled_method, |
| 45 | MethodReference method_ref) { |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 46 | if (!fix_cortex_a53_843419_) { |
| 47 | DCHECK(adrp_thunk_locations_.empty()); |
Vladimir Marko | 4d23c9d | 2015-04-01 23:03:09 +0100 | [diff] [blame] | 48 | return ReserveSpaceInternal(offset, compiled_method, method_ref, 0u); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 49 | } |
| 50 | |
| 51 | // Add thunks for previous method if any. |
| 52 | if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { |
| 53 | size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; |
| 54 | offset = CompiledMethod::AlignCode(offset, kArm64) + kAdrpThunkSize * num_adrp_thunks; |
| 55 | reserved_adrp_thunks_ = adrp_thunk_locations_.size(); |
| 56 | } |
| 57 | |
| 58 | // Count the number of ADRP insns as the upper bound on the number of thunks needed |
| 59 | // and use it to reserve space for other linker patches. |
| 60 | size_t num_adrp = 0u; |
Vladimir Marko | 71b0ddf | 2015-04-02 19:45:06 +0100 | [diff] [blame] | 61 | DCHECK(compiled_method != nullptr); |
| 62 | for (const LinkerPatch& patch : compiled_method->GetPatches()) { |
| 63 | if (patch.Type() == kLinkerPatchDexCacheArray && |
| 64 | patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch |
| 65 | ++num_adrp; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 66 | } |
| 67 | } |
Vladimir Marko | 4d23c9d | 2015-04-01 23:03:09 +0100 | [diff] [blame] | 68 | offset = ReserveSpaceInternal(offset, compiled_method, method_ref, kAdrpThunkSize * num_adrp); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 69 | if (num_adrp == 0u) { |
| 70 | return offset; |
| 71 | } |
| 72 | |
| 73 | // Now that we have the actual offset where the code will be placed, locate the ADRP insns |
| 74 | // that actually require the thunk. |
| 75 | uint32_t quick_code_offset = compiled_method->AlignCode(offset) + sizeof(OatQuickMethodHeader); |
| 76 | ArrayRef<const uint8_t> code(*compiled_method->GetQuickCode()); |
| 77 | uint32_t thunk_offset = compiled_method->AlignCode(quick_code_offset + code.size()); |
| 78 | DCHECK(compiled_method != nullptr); |
| 79 | for (const LinkerPatch& patch : compiled_method->GetPatches()) { |
| 80 | if (patch.Type() == kLinkerPatchDexCacheArray && |
| 81 | patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch |
| 82 | uint32_t patch_offset = quick_code_offset + patch.LiteralOffset(); |
| 83 | if (NeedsErratum843419Thunk(code, patch.LiteralOffset(), patch_offset)) { |
| 84 | adrp_thunk_locations_.emplace_back(patch_offset, thunk_offset); |
| 85 | thunk_offset += kAdrpThunkSize; |
| 86 | } |
| 87 | } |
| 88 | } |
| 89 | return offset; |
| 90 | } |
| 91 | |
Vladimir Marko | 71b0ddf | 2015-04-02 19:45:06 +0100 | [diff] [blame] | 92 | uint32_t Arm64RelativePatcher::ReserveSpaceEnd(uint32_t offset) { |
| 93 | if (!fix_cortex_a53_843419_) { |
| 94 | DCHECK(adrp_thunk_locations_.empty()); |
| 95 | } else { |
| 96 | // Add thunks for the last method if any. |
| 97 | if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { |
| 98 | size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; |
| 99 | offset = CompiledMethod::AlignCode(offset, kArm64) + kAdrpThunkSize * num_adrp_thunks; |
| 100 | reserved_adrp_thunks_ = adrp_thunk_locations_.size(); |
| 101 | } |
| 102 | } |
| 103 | return ArmBaseRelativePatcher::ReserveSpaceEnd(offset); |
| 104 | } |
| 105 | |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 106 | uint32_t Arm64RelativePatcher::WriteThunks(OutputStream* out, uint32_t offset) { |
| 107 | if (fix_cortex_a53_843419_) { |
| 108 | if (!current_method_thunks_.empty()) { |
| 109 | uint32_t aligned_offset = CompiledMethod::AlignCode(offset, kArm64); |
| 110 | if (kIsDebugBuild) { |
| 111 | CHECK(IsAligned<kAdrpThunkSize>(current_method_thunks_.size())); |
| 112 | size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; |
| 113 | CHECK_LE(num_thunks, processed_adrp_thunks_); |
| 114 | for (size_t i = 0u; i != num_thunks; ++i) { |
| 115 | const auto& entry = adrp_thunk_locations_[processed_adrp_thunks_ - num_thunks + i]; |
| 116 | CHECK_EQ(entry.second, aligned_offset + i * kAdrpThunkSize); |
| 117 | } |
| 118 | } |
| 119 | uint32_t aligned_code_delta = aligned_offset - offset; |
| 120 | if (aligned_code_delta != 0u && !WriteCodeAlignment(out, aligned_code_delta)) { |
| 121 | return 0u; |
| 122 | } |
| 123 | if (!WriteMiscThunk(out, ArrayRef<const uint8_t>(current_method_thunks_))) { |
| 124 | return 0u; |
| 125 | } |
| 126 | offset = aligned_offset + current_method_thunks_.size(); |
| 127 | current_method_thunks_.clear(); |
| 128 | } |
| 129 | } |
| 130 | return ArmBaseRelativePatcher::WriteThunks(out, offset); |
| 131 | } |
| 132 | |
| 133 | void Arm64RelativePatcher::PatchCall(std::vector<uint8_t>* code, uint32_t literal_offset, |
| 134 | uint32_t patch_offset, uint32_t target_offset) { |
| 135 | DCHECK_LE(literal_offset + 4u, code->size()); |
| 136 | DCHECK_EQ(literal_offset & 3u, 0u); |
| 137 | DCHECK_EQ(patch_offset & 3u, 0u); |
| 138 | DCHECK_EQ(target_offset & 3u, 0u); |
| 139 | uint32_t displacement = CalculateDisplacement(patch_offset, target_offset & ~1u); |
| 140 | DCHECK_EQ(displacement & 3u, 0u); |
| 141 | DCHECK((displacement >> 27) == 0u || (displacement >> 27) == 31u); // 28-bit signed. |
| 142 | uint32_t insn = (displacement & 0x0fffffffu) >> 2; |
| 143 | insn |= 0x94000000; // BL |
| 144 | |
| 145 | // Check that we're just overwriting an existing BL. |
| 146 | DCHECK_EQ(GetInsn(code, literal_offset) & 0xfc000000u, 0x94000000u); |
| 147 | // Write the new BL. |
| 148 | SetInsn(code, literal_offset, insn); |
| 149 | } |
| 150 | |
| 151 | void Arm64RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code, |
| 152 | const LinkerPatch& patch, |
| 153 | uint32_t patch_offset, |
| 154 | uint32_t target_offset) { |
| 155 | DCHECK_EQ(patch_offset & 3u, 0u); |
| 156 | DCHECK_EQ(target_offset & 3u, 0u); |
| 157 | uint32_t literal_offset = patch.LiteralOffset(); |
| 158 | uint32_t insn = GetInsn(code, literal_offset); |
| 159 | uint32_t pc_insn_offset = patch.PcInsnOffset(); |
| 160 | uint32_t disp = target_offset - ((patch_offset - literal_offset + pc_insn_offset) & ~0xfffu); |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 161 | bool wide = (insn & 0x40000000) != 0; |
| 162 | uint32_t shift = wide ? 3u : 2u; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 163 | if (literal_offset == pc_insn_offset) { |
| 164 | // Check it's an ADRP with imm == 0 (unset). |
| 165 | DCHECK_EQ((insn & 0xffffffe0u), 0x90000000u) |
| 166 | << literal_offset << ", " << pc_insn_offset << ", 0x" << std::hex << insn; |
| 167 | if (fix_cortex_a53_843419_ && processed_adrp_thunks_ != adrp_thunk_locations_.size() && |
| 168 | adrp_thunk_locations_[processed_adrp_thunks_].first == patch_offset) { |
| 169 | DCHECK(NeedsErratum843419Thunk(ArrayRef<const uint8_t>(*code), |
| 170 | literal_offset, patch_offset)); |
| 171 | uint32_t thunk_offset = adrp_thunk_locations_[processed_adrp_thunks_].second; |
| 172 | uint32_t adrp_disp = target_offset - (thunk_offset & ~0xfffu); |
| 173 | uint32_t adrp = PatchAdrp(insn, adrp_disp); |
| 174 | |
| 175 | uint32_t out_disp = thunk_offset - patch_offset; |
| 176 | DCHECK_EQ(out_disp & 3u, 0u); |
| 177 | DCHECK((out_disp >> 27) == 0u || (out_disp >> 27) == 31u); // 28-bit signed. |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 178 | insn = (out_disp & 0x0fffffffu) >> shift; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 179 | insn |= 0x14000000; // B <thunk> |
| 180 | |
| 181 | uint32_t back_disp = -out_disp; |
| 182 | DCHECK_EQ(back_disp & 3u, 0u); |
| 183 | DCHECK((back_disp >> 27) == 0u || (back_disp >> 27) == 31u); // 28-bit signed. |
| 184 | uint32_t b_back = (back_disp & 0x0fffffffu) >> 2; |
| 185 | b_back |= 0x14000000; // B <back> |
| 186 | size_t thunks_code_offset = current_method_thunks_.size(); |
| 187 | current_method_thunks_.resize(thunks_code_offset + kAdrpThunkSize); |
| 188 | SetInsn(¤t_method_thunks_, thunks_code_offset, adrp); |
| 189 | SetInsn(¤t_method_thunks_, thunks_code_offset + 4u, b_back); |
| 190 | static_assert(kAdrpThunkSize == 2 * 4u, "thunk has 2 instructions"); |
| 191 | |
| 192 | processed_adrp_thunks_ += 1u; |
| 193 | } else { |
| 194 | insn = PatchAdrp(insn, disp); |
| 195 | } |
| 196 | // Write the new ADRP (or B to the erratum 843419 thunk). |
| 197 | SetInsn(code, literal_offset, insn); |
| 198 | } else { |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 199 | // LDR 32-bit or 64-bit with imm12 == 0 (unset). |
| 200 | DCHECK_EQ(insn & 0xbffffc00, 0xb9400000) << insn; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 201 | if (kIsDebugBuild) { |
| 202 | uint32_t adrp = GetInsn(code, pc_insn_offset); |
| 203 | if ((adrp & 0x9f000000u) != 0x90000000u) { |
| 204 | CHECK(fix_cortex_a53_843419_); |
| 205 | CHECK_EQ(adrp & 0xfc000000u, 0x14000000u); // B <thunk> |
| 206 | CHECK(IsAligned<kAdrpThunkSize>(current_method_thunks_.size())); |
| 207 | size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; |
| 208 | CHECK_LE(num_thunks, processed_adrp_thunks_); |
| 209 | uint32_t b_offset = patch_offset - literal_offset + pc_insn_offset; |
| 210 | for (size_t i = processed_adrp_thunks_ - num_thunks; ; ++i) { |
| 211 | CHECK_NE(i, processed_adrp_thunks_); |
| 212 | if (adrp_thunk_locations_[i].first == b_offset) { |
| 213 | size_t idx = num_thunks - (processed_adrp_thunks_ - i); |
| 214 | adrp = GetInsn(¤t_method_thunks_, idx * kAdrpThunkSize); |
| 215 | break; |
| 216 | } |
| 217 | } |
| 218 | } |
| 219 | CHECK_EQ(adrp & 0x9f00001fu, // Check that pc_insn_offset points |
| 220 | 0x90000000 | ((insn >> 5) & 0x1fu)); // to ADRP with matching register. |
| 221 | } |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 222 | uint32_t imm12 = (disp & 0xfffu) >> shift; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 223 | insn = (insn & ~(0xfffu << 10)) | (imm12 << 10); |
| 224 | SetInsn(code, literal_offset, insn); |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | std::vector<uint8_t> Arm64RelativePatcher::CompileThunkCode() { |
| 229 | // The thunk just uses the entry point in the ArtMethod. This works even for calls |
| 230 | // to the generic JNI and interpreter trampolines. |
| 231 | arm64::Arm64Assembler assembler; |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 232 | Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset( |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 233 | kArm64PointerSize).Int32Value()); |
| 234 | assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0)); |
| 235 | // Ensure we emit the literal pool. |
Vladimir Marko | fbeb4ae | 2015-06-16 11:32:01 +0000 | [diff] [blame^] | 236 | assembler.EmitSlowPaths(); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 237 | std::vector<uint8_t> thunk_code(assembler.CodeSize()); |
| 238 | MemoryRegion code(thunk_code.data(), thunk_code.size()); |
| 239 | assembler.FinalizeInstructions(code); |
| 240 | return thunk_code; |
| 241 | } |
| 242 | |
| 243 | uint32_t Arm64RelativePatcher::PatchAdrp(uint32_t adrp, uint32_t disp) { |
| 244 | return (adrp & 0x9f00001fu) | // Clear offset bits, keep ADRP with destination reg. |
| 245 | // Bottom 12 bits are ignored, the next 2 lowest bits are encoded in bits 29-30. |
| 246 | ((disp & 0x00003000u) << (29 - 12)) | |
| 247 | // The next 16 bits are encoded in bits 5-22. |
| 248 | ((disp & 0xffffc000u) >> (12 + 2 - 5)) | |
| 249 | // Since the target_offset is based on the beginning of the oat file and the |
| 250 | // image space precedes the oat file, the target_offset into image space will |
| 251 | // be negative yet passed as uint32_t. Therefore we limit the displacement |
| 252 | // to +-2GiB (rather than the maximim +-4GiB) and determine the sign bit from |
| 253 | // the highest bit of the displacement. This is encoded in bit 23. |
| 254 | ((disp & 0x80000000u) >> (31 - 23)); |
| 255 | } |
| 256 | |
| 257 | bool Arm64RelativePatcher::NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, |
| 258 | uint32_t literal_offset, |
| 259 | uint32_t patch_offset) { |
| 260 | DCHECK_EQ(patch_offset & 0x3u, 0u); |
| 261 | if ((patch_offset & 0xff8) == 0xff8) { // ...ff8 or ...ffc |
| 262 | uint32_t adrp = GetInsn(code, literal_offset); |
| 263 | DCHECK_EQ(adrp & 0xff000000, 0x90000000); |
Matteo Franchin | 97e2f26 | 2015-04-02 15:49:06 +0100 | [diff] [blame] | 264 | uint32_t next_offset = patch_offset + 4u; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 265 | uint32_t next_insn = GetInsn(code, literal_offset + 4u); |
Matteo Franchin | 97e2f26 | 2015-04-02 15:49:06 +0100 | [diff] [blame] | 266 | |
| 267 | // Below we avoid patching sequences where the adrp is followed by a load which can easily |
| 268 | // be proved to be aligned. |
| 269 | |
| 270 | // First check if the next insn is the LDR using the result of the ADRP. |
| 271 | // LDR <Wt>, [<Xn>, #pimm], where <Xn> == ADRP destination reg. |
| 272 | if ((next_insn & 0xffc00000) == 0xb9400000 && |
| 273 | (((next_insn >> 5) ^ adrp) & 0x1f) == 0) { |
| 274 | return false; |
| 275 | } |
| 276 | |
| 277 | // LDR <Wt>, <label> is always aligned and thus it doesn't cause boundary crossing. |
| 278 | if ((next_insn & 0xff000000) == 0x18000000) { |
| 279 | return false; |
| 280 | } |
| 281 | |
| 282 | // LDR <Xt>, <label> is aligned iff the pc + displacement is a multiple of 8. |
| 283 | if ((next_insn & 0xff000000) == 0x58000000) { |
| 284 | bool is_aligned_load = (((next_offset >> 2) ^ (next_insn >> 5)) & 1) == 0; |
| 285 | return !is_aligned_load; |
| 286 | } |
| 287 | |
| 288 | // LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned loads, as SP is |
| 289 | // guaranteed to be 128-bits aligned and <pimm> is multiple of the load size. |
| 290 | if ((next_insn & 0xbfc003e0) == 0xb94003e0) { |
| 291 | return false; |
| 292 | } |
| 293 | return true; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 294 | } |
| 295 | return false; |
| 296 | } |
| 297 | |
| 298 | void Arm64RelativePatcher::SetInsn(std::vector<uint8_t>* code, uint32_t offset, uint32_t value) { |
| 299 | DCHECK_LE(offset + 4u, code->size()); |
| 300 | DCHECK_EQ(offset & 3u, 0u); |
| 301 | uint8_t* addr = &(*code)[offset]; |
| 302 | addr[0] = (value >> 0) & 0xff; |
| 303 | addr[1] = (value >> 8) & 0xff; |
| 304 | addr[2] = (value >> 16) & 0xff; |
| 305 | addr[3] = (value >> 24) & 0xff; |
| 306 | } |
| 307 | |
| 308 | uint32_t Arm64RelativePatcher::GetInsn(ArrayRef<const uint8_t> code, uint32_t offset) { |
| 309 | DCHECK_LE(offset + 4u, code.size()); |
| 310 | DCHECK_EQ(offset & 3u, 0u); |
| 311 | const uint8_t* addr = &code[offset]; |
| 312 | return |
| 313 | (static_cast<uint32_t>(addr[0]) << 0) + |
| 314 | (static_cast<uint32_t>(addr[1]) << 8) + |
| 315 | (static_cast<uint32_t>(addr[2]) << 16)+ |
| 316 | (static_cast<uint32_t>(addr[3]) << 24); |
| 317 | } |
| 318 | |
| 319 | template <typename Alloc> |
| 320 | uint32_t Arm64RelativePatcher::GetInsn(std::vector<uint8_t, Alloc>* code, uint32_t offset) { |
| 321 | return GetInsn(ArrayRef<const uint8_t>(*code), offset); |
| 322 | } |
| 323 | |
| 324 | } // namespace linker |
| 325 | } // namespace art |