Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2015 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #include "linker/arm64/relative_patcher_arm64.h" |
| 18 | |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 19 | #include "arch/arm64/asm_support_arm64.h" |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 20 | #include "arch/arm64/instruction_set_features_arm64.h" |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 21 | #include "art_method.h" |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 22 | #include "base/bit_utils.h" |
David Sehr | 3215fff | 2018-04-03 17:10:12 -0700 | [diff] [blame] | 23 | #include "base/malloc_arena_pool.h" |
Vladimir Marko | d8dbc8d | 2017-09-20 13:37:47 +0100 | [diff] [blame] | 24 | #include "compiled_method-inl.h" |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 25 | #include "driver/compiler_driver.h" |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 26 | #include "entrypoints/quick/quick_entrypoints_enum.h" |
Andreas Gampe | 09659c2 | 2017-09-18 18:23:32 -0700 | [diff] [blame] | 27 | #include "heap_poisoning.h" |
Vladimir Marko | d8dbc8d | 2017-09-20 13:37:47 +0100 | [diff] [blame] | 28 | #include "linker/linker_patch.h" |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 29 | #include "lock_word.h" |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 30 | #include "mirror/array-inl.h" |
Andreas Gampe | 8cf9cb3 | 2017-07-19 09:28:38 -0700 | [diff] [blame] | 31 | #include "mirror/object.h" |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 32 | #include "oat.h" |
Nicolas Geoffray | 524e7ea | 2015-10-16 17:13:34 +0100 | [diff] [blame] | 33 | #include "oat_quick_method_header.h" |
Vladimir Marko | 66d691d | 2017-04-07 17:53:39 +0100 | [diff] [blame] | 34 | #include "read_barrier.h" |
David Srbecky | 2faab00 | 2019-02-12 16:35:48 +0000 | [diff] [blame] | 35 | #include "stream/output_stream.h" |
Vladimir Marko | 131980f | 2015-12-03 18:29:23 +0000 | [diff] [blame] | 36 | #include "utils/arm64/assembler_arm64.h" |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 37 | |
| 38 | namespace art { |
| 39 | namespace linker { |
| 40 | |
Vladimir Marko | cac5a7e | 2016-02-22 10:39:50 +0000 | [diff] [blame] | 41 | namespace { |
| 42 | |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 43 | // Maximum positive and negative displacement for method call measured from the patch location. |
| 44 | // (Signed 28 bit displacement with the last two bits 0 has range [-2^27, 2^27-4] measured from |
| 45 | // the ARM64 PC pointing to the BL.) |
| 46 | constexpr uint32_t kMaxMethodCallPositiveDisplacement = (1u << 27) - 4u; |
| 47 | constexpr uint32_t kMaxMethodCallNegativeDisplacement = (1u << 27); |
| 48 | |
| 49 | // Maximum positive and negative displacement for a conditional branch measured from the patch |
| 50 | // location. (Signed 21 bit displacement with the last two bits 0 has range [-2^20, 2^20-4] |
| 51 | // measured from the ARM64 PC pointing to the B.cond.) |
| 52 | constexpr uint32_t kMaxBcondPositiveDisplacement = (1u << 20) - 4u; |
| 53 | constexpr uint32_t kMaxBcondNegativeDisplacement = (1u << 20); |
| 54 | |
| 55 | // The ADRP thunk for erratum 843419 is 2 instructions, i.e. 8 bytes. |
| 56 | constexpr uint32_t kAdrpThunkSize = 8u; |
| 57 | |
Vladimir Marko | cac5a7e | 2016-02-22 10:39:50 +0000 | [diff] [blame] | 58 | inline bool IsAdrpPatch(const LinkerPatch& patch) { |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 59 | switch (patch.GetType()) { |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 60 | case LinkerPatch::Type::kCallRelative: |
Vladimir Marko | f667508 | 2019-05-17 12:05:28 +0100 | [diff] [blame] | 61 | case LinkerPatch::Type::kCallEntrypoint: |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 62 | case LinkerPatch::Type::kBakerReadBarrierBranch: |
| 63 | return false; |
Vladimir Marko | 6fd1606 | 2018-06-26 11:02:04 +0100 | [diff] [blame] | 64 | case LinkerPatch::Type::kIntrinsicReference: |
Vladimir Marko | b066d43 | 2018-01-03 13:14:37 +0000 | [diff] [blame] | 65 | case LinkerPatch::Type::kDataBimgRelRo: |
Vladimir Marko | 19d7d50 | 2017-05-24 13:04:14 +0100 | [diff] [blame] | 66 | case LinkerPatch::Type::kMethodRelative: |
Vladimir Marko | 0eb882b | 2017-05-15 13:39:18 +0100 | [diff] [blame] | 67 | case LinkerPatch::Type::kMethodBssEntry: |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 68 | case LinkerPatch::Type::kTypeRelative: |
| 69 | case LinkerPatch::Type::kTypeBssEntry: |
| 70 | case LinkerPatch::Type::kStringRelative: |
| 71 | case LinkerPatch::Type::kStringBssEntry: |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 72 | return patch.LiteralOffset() == patch.PcInsnOffset(); |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | inline uint32_t MaxExtraSpace(size_t num_adrp, size_t code_size) { |
| 77 | if (num_adrp == 0u) { |
| 78 | return 0u; |
| 79 | } |
Vladimir Marko | 33bff25 | 2017-11-01 14:35:42 +0000 | [diff] [blame] | 80 | uint32_t alignment_bytes = |
| 81 | CompiledMethod::AlignCode(code_size, InstructionSet::kArm64) - code_size; |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 82 | return kAdrpThunkSize * num_adrp + alignment_bytes; |
Vladimir Marko | cac5a7e | 2016-02-22 10:39:50 +0000 | [diff] [blame] | 83 | } |
| 84 | |
| 85 | } // anonymous namespace |
| 86 | |
Vladimir Marko | ca1e038 | 2018-04-11 09:58:41 +0000 | [diff] [blame] | 87 | Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider, |
| 88 | RelativePatcherTargetProvider* target_provider, |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 89 | const Arm64InstructionSetFeatures* features) |
Vladimir Marko | ca1e038 | 2018-04-11 09:58:41 +0000 | [diff] [blame] | 90 | : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kArm64), |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 91 | fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()), |
| 92 | reserved_adrp_thunks_(0u), |
| 93 | processed_adrp_thunks_(0u) { |
| 94 | if (fix_cortex_a53_843419_) { |
| 95 | adrp_thunk_locations_.reserve(16u); |
| 96 | current_method_thunks_.reserve(16u * kAdrpThunkSize); |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | uint32_t Arm64RelativePatcher::ReserveSpace(uint32_t offset, |
Vladimir Marko | 4d23c9d | 2015-04-01 23:03:09 +0100 | [diff] [blame] | 101 | const CompiledMethod* compiled_method, |
| 102 | MethodReference method_ref) { |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 103 | if (!fix_cortex_a53_843419_) { |
| 104 | DCHECK(adrp_thunk_locations_.empty()); |
Vladimir Marko | 4d23c9d | 2015-04-01 23:03:09 +0100 | [diff] [blame] | 105 | return ReserveSpaceInternal(offset, compiled_method, method_ref, 0u); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 106 | } |
| 107 | |
| 108 | // Add thunks for previous method if any. |
| 109 | if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { |
| 110 | size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; |
Vladimir Marko | 33bff25 | 2017-11-01 14:35:42 +0000 | [diff] [blame] | 111 | offset = CompiledMethod::AlignCode(offset, InstructionSet::kArm64) + |
| 112 | kAdrpThunkSize * num_adrp_thunks; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 113 | reserved_adrp_thunks_ = adrp_thunk_locations_.size(); |
| 114 | } |
| 115 | |
| 116 | // Count the number of ADRP insns as the upper bound on the number of thunks needed |
| 117 | // and use it to reserve space for other linker patches. |
| 118 | size_t num_adrp = 0u; |
Vladimir Marko | 71b0ddf | 2015-04-02 19:45:06 +0100 | [diff] [blame] | 119 | DCHECK(compiled_method != nullptr); |
| 120 | for (const LinkerPatch& patch : compiled_method->GetPatches()) { |
Vladimir Marko | cac5a7e | 2016-02-22 10:39:50 +0000 | [diff] [blame] | 121 | if (IsAdrpPatch(patch)) { |
Vladimir Marko | 71b0ddf | 2015-04-02 19:45:06 +0100 | [diff] [blame] | 122 | ++num_adrp; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 123 | } |
| 124 | } |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 125 | ArrayRef<const uint8_t> code = compiled_method->GetQuickCode(); |
| 126 | uint32_t max_extra_space = MaxExtraSpace(num_adrp, code.size()); |
| 127 | offset = ReserveSpaceInternal(offset, compiled_method, method_ref, max_extra_space); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 128 | if (num_adrp == 0u) { |
| 129 | return offset; |
| 130 | } |
| 131 | |
| 132 | // Now that we have the actual offset where the code will be placed, locate the ADRP insns |
| 133 | // that actually require the thunk. |
Vladimir Marko | 0c737df | 2016-08-01 16:33:16 +0100 | [diff] [blame] | 134 | uint32_t quick_code_offset = compiled_method->AlignCode(offset + sizeof(OatQuickMethodHeader)); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 135 | uint32_t thunk_offset = compiled_method->AlignCode(quick_code_offset + code.size()); |
| 136 | DCHECK(compiled_method != nullptr); |
| 137 | for (const LinkerPatch& patch : compiled_method->GetPatches()) { |
Vladimir Marko | cac5a7e | 2016-02-22 10:39:50 +0000 | [diff] [blame] | 138 | if (IsAdrpPatch(patch)) { |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 139 | uint32_t patch_offset = quick_code_offset + patch.LiteralOffset(); |
| 140 | if (NeedsErratum843419Thunk(code, patch.LiteralOffset(), patch_offset)) { |
| 141 | adrp_thunk_locations_.emplace_back(patch_offset, thunk_offset); |
| 142 | thunk_offset += kAdrpThunkSize; |
| 143 | } |
| 144 | } |
| 145 | } |
| 146 | return offset; |
| 147 | } |
| 148 | |
Vladimir Marko | 71b0ddf | 2015-04-02 19:45:06 +0100 | [diff] [blame] | 149 | uint32_t Arm64RelativePatcher::ReserveSpaceEnd(uint32_t offset) { |
| 150 | if (!fix_cortex_a53_843419_) { |
| 151 | DCHECK(adrp_thunk_locations_.empty()); |
| 152 | } else { |
| 153 | // Add thunks for the last method if any. |
| 154 | if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { |
| 155 | size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; |
Vladimir Marko | 33bff25 | 2017-11-01 14:35:42 +0000 | [diff] [blame] | 156 | offset = CompiledMethod::AlignCode(offset, InstructionSet::kArm64) + |
| 157 | kAdrpThunkSize * num_adrp_thunks; |
Vladimir Marko | 71b0ddf | 2015-04-02 19:45:06 +0100 | [diff] [blame] | 158 | reserved_adrp_thunks_ = adrp_thunk_locations_.size(); |
| 159 | } |
| 160 | } |
| 161 | return ArmBaseRelativePatcher::ReserveSpaceEnd(offset); |
| 162 | } |
| 163 | |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 164 | uint32_t Arm64RelativePatcher::WriteThunks(OutputStream* out, uint32_t offset) { |
| 165 | if (fix_cortex_a53_843419_) { |
| 166 | if (!current_method_thunks_.empty()) { |
Vladimir Marko | 33bff25 | 2017-11-01 14:35:42 +0000 | [diff] [blame] | 167 | uint32_t aligned_offset = CompiledMethod::AlignCode(offset, InstructionSet::kArm64); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 168 | if (kIsDebugBuild) { |
Roland Levillain | 14d9057 | 2015-07-16 10:52:26 +0100 | [diff] [blame] | 169 | CHECK_ALIGNED(current_method_thunks_.size(), kAdrpThunkSize); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 170 | size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; |
| 171 | CHECK_LE(num_thunks, processed_adrp_thunks_); |
| 172 | for (size_t i = 0u; i != num_thunks; ++i) { |
| 173 | const auto& entry = adrp_thunk_locations_[processed_adrp_thunks_ - num_thunks + i]; |
| 174 | CHECK_EQ(entry.second, aligned_offset + i * kAdrpThunkSize); |
| 175 | } |
| 176 | } |
| 177 | uint32_t aligned_code_delta = aligned_offset - offset; |
| 178 | if (aligned_code_delta != 0u && !WriteCodeAlignment(out, aligned_code_delta)) { |
| 179 | return 0u; |
| 180 | } |
| 181 | if (!WriteMiscThunk(out, ArrayRef<const uint8_t>(current_method_thunks_))) { |
| 182 | return 0u; |
| 183 | } |
| 184 | offset = aligned_offset + current_method_thunks_.size(); |
| 185 | current_method_thunks_.clear(); |
| 186 | } |
| 187 | } |
| 188 | return ArmBaseRelativePatcher::WriteThunks(out, offset); |
| 189 | } |
| 190 | |
Vladimir Marko | 944da60 | 2016-02-19 12:27:55 +0000 | [diff] [blame] | 191 | void Arm64RelativePatcher::PatchCall(std::vector<uint8_t>* code, |
| 192 | uint32_t literal_offset, |
Vladimir Marko | f667508 | 2019-05-17 12:05:28 +0100 | [diff] [blame] | 193 | uint32_t patch_offset, |
| 194 | uint32_t target_offset) { |
| 195 | DCHECK_ALIGNED(literal_offset, 4u); |
| 196 | DCHECK_ALIGNED(patch_offset, 4u); |
| 197 | DCHECK_ALIGNED(target_offset, 4u); |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 198 | uint32_t displacement = CalculateMethodCallDisplacement(patch_offset, target_offset & ~1u); |
Vladimir Marko | f667508 | 2019-05-17 12:05:28 +0100 | [diff] [blame] | 199 | PatchBl(code, literal_offset, displacement); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 200 | } |
| 201 | |
Vladimir Marko | cac5a7e | 2016-02-22 10:39:50 +0000 | [diff] [blame] | 202 | void Arm64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, |
| 203 | const LinkerPatch& patch, |
| 204 | uint32_t patch_offset, |
| 205 | uint32_t target_offset) { |
Vladimir Marko | f667508 | 2019-05-17 12:05:28 +0100 | [diff] [blame] | 206 | DCHECK_ALIGNED(patch_offset, 4u); |
| 207 | DCHECK_ALIGNED(target_offset, 4u); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 208 | uint32_t literal_offset = patch.LiteralOffset(); |
| 209 | uint32_t insn = GetInsn(code, literal_offset); |
| 210 | uint32_t pc_insn_offset = patch.PcInsnOffset(); |
| 211 | uint32_t disp = target_offset - ((patch_offset - literal_offset + pc_insn_offset) & ~0xfffu); |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 212 | bool wide = (insn & 0x40000000) != 0; |
| 213 | uint32_t shift = wide ? 3u : 2u; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 214 | if (literal_offset == pc_insn_offset) { |
| 215 | // Check it's an ADRP with imm == 0 (unset). |
| 216 | DCHECK_EQ((insn & 0xffffffe0u), 0x90000000u) |
| 217 | << literal_offset << ", " << pc_insn_offset << ", 0x" << std::hex << insn; |
| 218 | if (fix_cortex_a53_843419_ && processed_adrp_thunks_ != adrp_thunk_locations_.size() && |
| 219 | adrp_thunk_locations_[processed_adrp_thunks_].first == patch_offset) { |
| 220 | DCHECK(NeedsErratum843419Thunk(ArrayRef<const uint8_t>(*code), |
| 221 | literal_offset, patch_offset)); |
| 222 | uint32_t thunk_offset = adrp_thunk_locations_[processed_adrp_thunks_].second; |
| 223 | uint32_t adrp_disp = target_offset - (thunk_offset & ~0xfffu); |
| 224 | uint32_t adrp = PatchAdrp(insn, adrp_disp); |
| 225 | |
| 226 | uint32_t out_disp = thunk_offset - patch_offset; |
| 227 | DCHECK_EQ(out_disp & 3u, 0u); |
| 228 | DCHECK((out_disp >> 27) == 0u || (out_disp >> 27) == 31u); // 28-bit signed. |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 229 | insn = (out_disp & 0x0fffffffu) >> shift; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 230 | insn |= 0x14000000; // B <thunk> |
| 231 | |
| 232 | uint32_t back_disp = -out_disp; |
| 233 | DCHECK_EQ(back_disp & 3u, 0u); |
| 234 | DCHECK((back_disp >> 27) == 0u || (back_disp >> 27) == 31u); // 28-bit signed. |
| 235 | uint32_t b_back = (back_disp & 0x0fffffffu) >> 2; |
| 236 | b_back |= 0x14000000; // B <back> |
| 237 | size_t thunks_code_offset = current_method_thunks_.size(); |
| 238 | current_method_thunks_.resize(thunks_code_offset + kAdrpThunkSize); |
| 239 | SetInsn(¤t_method_thunks_, thunks_code_offset, adrp); |
| 240 | SetInsn(¤t_method_thunks_, thunks_code_offset + 4u, b_back); |
| 241 | static_assert(kAdrpThunkSize == 2 * 4u, "thunk has 2 instructions"); |
| 242 | |
| 243 | processed_adrp_thunks_ += 1u; |
| 244 | } else { |
| 245 | insn = PatchAdrp(insn, disp); |
| 246 | } |
| 247 | // Write the new ADRP (or B to the erratum 843419 thunk). |
| 248 | SetInsn(code, literal_offset, insn); |
| 249 | } else { |
Vladimir Marko | cac5a7e | 2016-02-22 10:39:50 +0000 | [diff] [blame] | 250 | if ((insn & 0xfffffc00) == 0x91000000) { |
| 251 | // ADD immediate, 64-bit with imm12 == 0 (unset). |
Hiroshi Yamauchi | a5b7557 | 2016-04-18 16:05:21 -0700 | [diff] [blame] | 252 | if (!kEmitCompilerReadBarrier) { |
Vladimir Marko | 6fd1606 | 2018-06-26 11:02:04 +0100 | [diff] [blame] | 253 | DCHECK(patch.GetType() == LinkerPatch::Type::kIntrinsicReference || |
| 254 | patch.GetType() == LinkerPatch::Type::kMethodRelative || |
Vladimir Marko | 0eb882b | 2017-05-15 13:39:18 +0100 | [diff] [blame] | 255 | patch.GetType() == LinkerPatch::Type::kTypeRelative || |
| 256 | patch.GetType() == LinkerPatch::Type::kStringRelative) << patch.GetType(); |
Hiroshi Yamauchi | a5b7557 | 2016-04-18 16:05:21 -0700 | [diff] [blame] | 257 | } else { |
Vladimir Marko | ea4c126 | 2017-02-06 19:59:33 +0000 | [diff] [blame] | 258 | // With the read barrier (non-Baker) enabled, it could be kStringBssEntry or kTypeBssEntry. |
Vladimir Marko | 6fd1606 | 2018-06-26 11:02:04 +0100 | [diff] [blame] | 259 | DCHECK(patch.GetType() == LinkerPatch::Type::kIntrinsicReference || |
| 260 | patch.GetType() == LinkerPatch::Type::kMethodRelative || |
Vladimir Marko | dbb7f5b | 2016-03-30 13:23:58 +0100 | [diff] [blame] | 261 | patch.GetType() == LinkerPatch::Type::kTypeRelative || |
Vladimir Marko | 0eb882b | 2017-05-15 13:39:18 +0100 | [diff] [blame] | 262 | patch.GetType() == LinkerPatch::Type::kStringRelative || |
| 263 | patch.GetType() == LinkerPatch::Type::kTypeBssEntry || |
| 264 | patch.GetType() == LinkerPatch::Type::kStringBssEntry) << patch.GetType(); |
Hiroshi Yamauchi | a5b7557 | 2016-04-18 16:05:21 -0700 | [diff] [blame] | 265 | } |
Vladimir Marko | cac5a7e | 2016-02-22 10:39:50 +0000 | [diff] [blame] | 266 | shift = 0u; // No shift for ADD. |
| 267 | } else { |
Vladimir Marko | aad75c6 | 2016-10-03 08:46:48 +0000 | [diff] [blame] | 268 | // LDR/STR 32-bit or 64-bit with imm12 == 0 (unset). |
Vladimir Marko | b066d43 | 2018-01-03 13:14:37 +0000 | [diff] [blame] | 269 | DCHECK(patch.GetType() == LinkerPatch::Type::kDataBimgRelRo || |
| 270 | patch.GetType() == LinkerPatch::Type::kMethodBssEntry || |
Vladimir Marko | 6bec91c | 2017-01-09 15:03:12 +0000 | [diff] [blame] | 271 | patch.GetType() == LinkerPatch::Type::kTypeBssEntry || |
Vladimir Marko | aad75c6 | 2016-10-03 08:46:48 +0000 | [diff] [blame] | 272 | patch.GetType() == LinkerPatch::Type::kStringBssEntry) << patch.GetType(); |
| 273 | DCHECK_EQ(insn & 0xbfbffc00, 0xb9000000) << std::hex << insn; |
Vladimir Marko | cac5a7e | 2016-02-22 10:39:50 +0000 | [diff] [blame] | 274 | } |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 275 | if (kIsDebugBuild) { |
| 276 | uint32_t adrp = GetInsn(code, pc_insn_offset); |
| 277 | if ((adrp & 0x9f000000u) != 0x90000000u) { |
| 278 | CHECK(fix_cortex_a53_843419_); |
| 279 | CHECK_EQ(adrp & 0xfc000000u, 0x14000000u); // B <thunk> |
Roland Levillain | 14d9057 | 2015-07-16 10:52:26 +0100 | [diff] [blame] | 280 | CHECK_ALIGNED(current_method_thunks_.size(), kAdrpThunkSize); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 281 | size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; |
| 282 | CHECK_LE(num_thunks, processed_adrp_thunks_); |
| 283 | uint32_t b_offset = patch_offset - literal_offset + pc_insn_offset; |
| 284 | for (size_t i = processed_adrp_thunks_ - num_thunks; ; ++i) { |
| 285 | CHECK_NE(i, processed_adrp_thunks_); |
| 286 | if (adrp_thunk_locations_[i].first == b_offset) { |
| 287 | size_t idx = num_thunks - (processed_adrp_thunks_ - i); |
| 288 | adrp = GetInsn(¤t_method_thunks_, idx * kAdrpThunkSize); |
| 289 | break; |
| 290 | } |
| 291 | } |
| 292 | } |
| 293 | CHECK_EQ(adrp & 0x9f00001fu, // Check that pc_insn_offset points |
| 294 | 0x90000000 | ((insn >> 5) & 0x1fu)); // to ADRP with matching register. |
| 295 | } |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 296 | uint32_t imm12 = (disp & 0xfffu) >> shift; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 297 | insn = (insn & ~(0xfffu << 10)) | (imm12 << 10); |
| 298 | SetInsn(code, literal_offset, insn); |
| 299 | } |
| 300 | } |
| 301 | |
Vladimir Marko | f667508 | 2019-05-17 12:05:28 +0100 | [diff] [blame] | 302 | void Arm64RelativePatcher::PatchEntrypointCall(std::vector<uint8_t>* code, |
| 303 | const LinkerPatch& patch, |
| 304 | uint32_t patch_offset) { |
| 305 | DCHECK_ALIGNED(patch_offset, 4u); |
| 306 | ThunkKey key = GetEntrypointCallKey(patch); |
| 307 | uint32_t target_offset = GetThunkTargetOffset(key, patch_offset); |
| 308 | uint32_t displacement = target_offset - patch_offset; |
| 309 | PatchBl(code, patch.LiteralOffset(), displacement); |
| 310 | } |
| 311 | |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 312 | void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* code, |
| 313 | const LinkerPatch& patch, |
| 314 | uint32_t patch_offset) { |
| 315 | DCHECK_ALIGNED(patch_offset, 4u); |
| 316 | uint32_t literal_offset = patch.LiteralOffset(); |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 317 | uint32_t insn = GetInsn(code, literal_offset); |
| 318 | DCHECK_EQ(insn & 0xffffffe0u, 0xb5000000); // CBNZ Xt, +0 (unpatched) |
Vladimir Marko | 0a51fc3 | 2017-05-02 13:12:02 +0100 | [diff] [blame] | 319 | ThunkKey key = GetBakerThunkKey(patch); |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 320 | uint32_t target_offset = GetThunkTargetOffset(key, patch_offset); |
| 321 | DCHECK_ALIGNED(target_offset, 4u); |
| 322 | uint32_t disp = target_offset - patch_offset; |
| 323 | DCHECK((disp >> 20) == 0u || (disp >> 20) == 4095u); // 21-bit signed. |
| 324 | insn |= (disp << (5 - 2)) & 0x00ffffe0u; // Shift bits 2-20 to 5-23. |
| 325 | SetInsn(code, literal_offset, insn); |
| 326 | } |
| 327 | |
Vladimir Marko | 0a51fc3 | 2017-05-02 13:12:02 +0100 | [diff] [blame] | 328 | uint32_t Arm64RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) { |
| 329 | switch (key.GetType()) { |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 330 | case ThunkType::kMethodCall: |
Vladimir Marko | f667508 | 2019-05-17 12:05:28 +0100 | [diff] [blame] | 331 | case ThunkType::kEntrypointCall: |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 332 | return kMaxMethodCallPositiveDisplacement; |
Vladimir Marko | 0a51fc3 | 2017-05-02 13:12:02 +0100 | [diff] [blame] | 333 | case ThunkType::kBakerReadBarrier: |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 334 | return kMaxBcondPositiveDisplacement; |
| 335 | } |
| 336 | } |
| 337 | |
Vladimir Marko | 0a51fc3 | 2017-05-02 13:12:02 +0100 | [diff] [blame] | 338 | uint32_t Arm64RelativePatcher::MaxNegativeDisplacement(const ThunkKey& key) { |
| 339 | switch (key.GetType()) { |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 340 | case ThunkType::kMethodCall: |
Vladimir Marko | f667508 | 2019-05-17 12:05:28 +0100 | [diff] [blame] | 341 | case ThunkType::kEntrypointCall: |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 342 | return kMaxMethodCallNegativeDisplacement; |
Vladimir Marko | 0a51fc3 | 2017-05-02 13:12:02 +0100 | [diff] [blame] | 343 | case ThunkType::kBakerReadBarrier: |
Vladimir Marko | f4f2daa | 2017-03-20 18:26:59 +0000 | [diff] [blame] | 344 | return kMaxBcondNegativeDisplacement; |
| 345 | } |
| 346 | } |
| 347 | |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 348 | uint32_t Arm64RelativePatcher::PatchAdrp(uint32_t adrp, uint32_t disp) { |
| 349 | return (adrp & 0x9f00001fu) | // Clear offset bits, keep ADRP with destination reg. |
| 350 | // Bottom 12 bits are ignored, the next 2 lowest bits are encoded in bits 29-30. |
| 351 | ((disp & 0x00003000u) << (29 - 12)) | |
| 352 | // The next 16 bits are encoded in bits 5-22. |
| 353 | ((disp & 0xffffc000u) >> (12 + 2 - 5)) | |
| 354 | // Since the target_offset is based on the beginning of the oat file and the |
| 355 | // image space precedes the oat file, the target_offset into image space will |
| 356 | // be negative yet passed as uint32_t. Therefore we limit the displacement |
| 357 | // to +-2GiB (rather than the maximim +-4GiB) and determine the sign bit from |
| 358 | // the highest bit of the displacement. This is encoded in bit 23. |
| 359 | ((disp & 0x80000000u) >> (31 - 23)); |
| 360 | } |
| 361 | |
Vladimir Marko | f667508 | 2019-05-17 12:05:28 +0100 | [diff] [blame] | 362 | void Arm64RelativePatcher::PatchBl(std::vector<uint8_t>* code, |
| 363 | uint32_t literal_offset, |
| 364 | uint32_t displacement) { |
| 365 | DCHECK_ALIGNED(displacement, 4u); |
| 366 | DCHECK((displacement >> 27) == 0u || (displacement >> 27) == 31u); // 28-bit signed. |
| 367 | uint32_t insn = (displacement & 0x0fffffffu) >> 2; |
| 368 | insn |= 0x94000000; // BL |
| 369 | |
| 370 | // Check that we're just overwriting an existing BL. |
| 371 | DCHECK_EQ(GetInsn(code, literal_offset) & 0xfc000000u, 0x94000000u); |
| 372 | // Write the new BL. |
| 373 | SetInsn(code, literal_offset, insn); |
| 374 | } |
| 375 | |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 376 | bool Arm64RelativePatcher::NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, |
| 377 | uint32_t literal_offset, |
| 378 | uint32_t patch_offset) { |
| 379 | DCHECK_EQ(patch_offset & 0x3u, 0u); |
| 380 | if ((patch_offset & 0xff8) == 0xff8) { // ...ff8 or ...ffc |
| 381 | uint32_t adrp = GetInsn(code, literal_offset); |
Vladimir Marko | cac5a7e | 2016-02-22 10:39:50 +0000 | [diff] [blame] | 382 | DCHECK_EQ(adrp & 0x9f000000, 0x90000000); |
Matteo Franchin | 97e2f26 | 2015-04-02 15:49:06 +0100 | [diff] [blame] | 383 | uint32_t next_offset = patch_offset + 4u; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 384 | uint32_t next_insn = GetInsn(code, literal_offset + 4u); |
Matteo Franchin | 97e2f26 | 2015-04-02 15:49:06 +0100 | [diff] [blame] | 385 | |
| 386 | // Below we avoid patching sequences where the adrp is followed by a load which can easily |
| 387 | // be proved to be aligned. |
| 388 | |
| 389 | // First check if the next insn is the LDR using the result of the ADRP. |
| 390 | // LDR <Wt>, [<Xn>, #pimm], where <Xn> == ADRP destination reg. |
| 391 | if ((next_insn & 0xffc00000) == 0xb9400000 && |
| 392 | (((next_insn >> 5) ^ adrp) & 0x1f) == 0) { |
| 393 | return false; |
| 394 | } |
| 395 | |
Vladimir Marko | 19d7d50 | 2017-05-24 13:04:14 +0100 | [diff] [blame] | 396 | // And since LinkerPatch::Type::k{Method,Type,String}Relative is using the result |
| 397 | // of the ADRP for an ADD immediate, check for that as well. We generalize a bit |
| 398 | // to include ADD/ADDS/SUB/SUBS immediate that either uses the ADRP destination |
| 399 | // or stores the result to a different register. |
Vladimir Marko | cac5a7e | 2016-02-22 10:39:50 +0000 | [diff] [blame] | 400 | if ((next_insn & 0x1f000000) == 0x11000000 && |
| 401 | ((((next_insn >> 5) ^ adrp) & 0x1f) == 0 || ((next_insn ^ adrp) & 0x1f) != 0)) { |
| 402 | return false; |
| 403 | } |
| 404 | |
Matteo Franchin | 97e2f26 | 2015-04-02 15:49:06 +0100 | [diff] [blame] | 405 | // LDR <Wt>, <label> is always aligned and thus it doesn't cause boundary crossing. |
| 406 | if ((next_insn & 0xff000000) == 0x18000000) { |
| 407 | return false; |
| 408 | } |
| 409 | |
| 410 | // LDR <Xt>, <label> is aligned iff the pc + displacement is a multiple of 8. |
| 411 | if ((next_insn & 0xff000000) == 0x58000000) { |
| 412 | bool is_aligned_load = (((next_offset >> 2) ^ (next_insn >> 5)) & 1) == 0; |
| 413 | return !is_aligned_load; |
| 414 | } |
| 415 | |
| 416 | // LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned loads, as SP is |
| 417 | // guaranteed to be 128-bits aligned and <pimm> is multiple of the load size. |
| 418 | if ((next_insn & 0xbfc003e0) == 0xb94003e0) { |
| 419 | return false; |
| 420 | } |
| 421 | return true; |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 422 | } |
| 423 | return false; |
| 424 | } |
| 425 | |
| 426 | void Arm64RelativePatcher::SetInsn(std::vector<uint8_t>* code, uint32_t offset, uint32_t value) { |
| 427 | DCHECK_LE(offset + 4u, code->size()); |
Vladimir Marko | f667508 | 2019-05-17 12:05:28 +0100 | [diff] [blame] | 428 | DCHECK_ALIGNED(offset, 4u); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 429 | uint8_t* addr = &(*code)[offset]; |
| 430 | addr[0] = (value >> 0) & 0xff; |
| 431 | addr[1] = (value >> 8) & 0xff; |
| 432 | addr[2] = (value >> 16) & 0xff; |
| 433 | addr[3] = (value >> 24) & 0xff; |
| 434 | } |
| 435 | |
| 436 | uint32_t Arm64RelativePatcher::GetInsn(ArrayRef<const uint8_t> code, uint32_t offset) { |
| 437 | DCHECK_LE(offset + 4u, code.size()); |
Vladimir Marko | f667508 | 2019-05-17 12:05:28 +0100 | [diff] [blame] | 438 | DCHECK_ALIGNED(offset, 4u); |
Vladimir Marko | b163bb7 | 2015-03-31 21:49:49 +0100 | [diff] [blame] | 439 | const uint8_t* addr = &code[offset]; |
| 440 | return |
| 441 | (static_cast<uint32_t>(addr[0]) << 0) + |
| 442 | (static_cast<uint32_t>(addr[1]) << 8) + |
| 443 | (static_cast<uint32_t>(addr[2]) << 16)+ |
| 444 | (static_cast<uint32_t>(addr[3]) << 24); |
| 445 | } |
| 446 | |
| 447 | template <typename Alloc> |
| 448 | uint32_t Arm64RelativePatcher::GetInsn(std::vector<uint8_t, Alloc>* code, uint32_t offset) { |
| 449 | return GetInsn(ArrayRef<const uint8_t>(*code), offset); |
| 450 | } |
| 451 | |
| 452 | } // namespace linker |
| 453 | } // namespace art |