| //===-- memtag.h ------------------------------------------------*- C++ -*-===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #ifndef SCUDO_MEMTAG_H_ |
| #define SCUDO_MEMTAG_H_ |
| |
| #include "internal_defs.h" |
| |
| #if SCUDO_LINUX |
| #include <sys/auxv.h> |
| #include <sys/prctl.h> |
| #endif |
| |
| namespace scudo { |
| |
| void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask, uptr *TaggedBegin, |
| uptr *TaggedEnd); |
| |
| #if defined(__aarch64__) || defined(SCUDO_FUZZ) |
| |
| // We assume that Top-Byte Ignore is enabled if the architecture supports memory |
| // tagging. Not all operating systems enable TBI, so we only claim architectural |
| // support for memory tagging if the operating system enables TBI. |
| #if SCUDO_LINUX |
| inline constexpr bool archSupportsMemoryTagging() { return true; } |
| #else |
| inline constexpr bool archSupportsMemoryTagging() { return false; } |
| #endif |
| |
| inline constexpr uptr archMemoryTagGranuleSize() { return 16; } |
| |
| inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); } |
| |
| inline uint8_t extractTag(uptr Ptr) { return (Ptr >> 56) & 0xf; } |
| |
| #else |
| |
| inline constexpr bool archSupportsMemoryTagging() { return false; } |
| |
| inline uptr archMemoryTagGranuleSize() { |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| inline uptr untagPointer(uptr Ptr) { |
| (void)Ptr; |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| inline uint8_t extractTag(uptr Ptr) { |
| (void)Ptr; |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| #endif |
| |
| #if defined(__aarch64__) |
| |
| #if SCUDO_LINUX |
| |
| inline bool systemSupportsMemoryTagging() { |
| #ifndef HWCAP2_MTE |
| #define HWCAP2_MTE (1 << 18) |
| #endif |
| return getauxval(AT_HWCAP2) & HWCAP2_MTE; |
| } |
| |
| inline bool systemDetectsMemoryTagFaultsTestOnly() { |
| #ifndef PR_GET_TAGGED_ADDR_CTRL |
| #define PR_GET_TAGGED_ADDR_CTRL 56 |
| #endif |
| #ifndef PR_MTE_TCF_SHIFT |
| #define PR_MTE_TCF_SHIFT 1 |
| #endif |
| #ifndef PR_MTE_TCF_NONE |
| #define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT) |
| #endif |
| #ifndef PR_MTE_TCF_MASK |
| #define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT) |
| #endif |
| return (static_cast<unsigned long>( |
| prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) & |
| PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE; |
| } |
| |
| #else // !SCUDO_LINUX |
| |
| inline bool systemSupportsMemoryTagging() { return false; } |
| |
| inline bool systemDetectsMemoryTagFaultsTestOnly() { return false; } |
| |
| #endif // SCUDO_LINUX |
| |
| inline void disableMemoryTagChecksTestOnly() { |
| __asm__ __volatile__(".arch_extension memtag; msr tco, #1"); |
| } |
| |
| inline void enableMemoryTagChecksTestOnly() { |
| __asm__ __volatile__(".arch_extension memtag; msr tco, #0"); |
| } |
| |
| class ScopedDisableMemoryTagChecks { |
| size_t PrevTCO; |
| |
| public: |
| ScopedDisableMemoryTagChecks() { |
| __asm__ __volatile__(".arch_extension memtag; mrs %0, tco; msr tco, #1" |
| : "=r"(PrevTCO)); |
| } |
| |
| ~ScopedDisableMemoryTagChecks() { |
| __asm__ __volatile__(".arch_extension memtag; msr tco, %0" |
| : |
| : "r"(PrevTCO)); |
| } |
| }; |
| |
| inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) { |
| uptr TaggedPtr; |
| __asm__ __volatile__( |
| ".arch_extension memtag; irg %[TaggedPtr], %[Ptr], %[ExcludeMask]" |
| : [TaggedPtr] "=r"(TaggedPtr) |
| : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask)); |
| return TaggedPtr; |
| } |
| |
| inline uptr addFixedTag(uptr Ptr, uptr Tag) { return Ptr | (Tag << 56); } |
| |
| inline uptr storeTags(uptr Begin, uptr End) { |
| DCHECK(Begin % 16 == 0); |
| if (Begin != End) { |
| __asm__ __volatile__( |
| R"( |
| .arch_extension memtag |
| |
| 1: |
| stzg %[Cur], [%[Cur]], #16 |
| cmp %[Cur], %[End] |
| b.lt 1b |
| )" |
| : [Cur] "+&r"(Begin) |
| : [End] "r"(End) |
| : "memory"); |
| } |
| return Begin; |
| } |
| |
| inline void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask, |
| uptr BlockEnd) { |
| // Prepare the granule before the chunk to store the chunk header by setting |
| // its tag to 0. Normally its tag will already be 0, but in the case where a |
| // chunk holding a low alignment allocation is reused for a higher alignment |
| // allocation, the chunk may already have a non-zero tag from the previous |
| // allocation. |
| __asm__ __volatile__(".arch_extension memtag; stg %0, [%0, #-16]" |
| : |
| : "r"(Ptr) |
| : "memory"); |
| |
| uptr TaggedBegin, TaggedEnd; |
| setRandomTag(Ptr, Size, ExcludeMask, &TaggedBegin, &TaggedEnd); |
| |
| // Finally, set the tag of the granule past the end of the allocation to 0, |
| // to catch linear overflows even if a previous larger allocation used the |
| // same block and tag. Only do this if the granule past the end is in our |
| // block, because this would otherwise lead to a SEGV if the allocation |
| // covers the entire block and our block is at the end of a mapping. The tag |
| // of the next block's header granule will be set to 0, so it will serve the |
| // purpose of catching linear overflows in this case. |
| uptr UntaggedEnd = untagPointer(TaggedEnd); |
| if (UntaggedEnd != BlockEnd) |
| __asm__ __volatile__(".arch_extension memtag; stg %0, [%0]" |
| : |
| : "r"(UntaggedEnd) |
| : "memory"); |
| return reinterpret_cast<void *>(TaggedBegin); |
| } |
| |
| inline void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr BlockEnd) { |
| uptr RoundOldPtr = roundUpTo(OldPtr, 16); |
| if (RoundOldPtr >= NewPtr) { |
| // If the allocation is shrinking we just need to set the tag past the end |
| // of the allocation to 0. See explanation in prepareTaggedChunk above. |
| uptr RoundNewPtr = untagPointer(roundUpTo(NewPtr, 16)); |
| if (RoundNewPtr != BlockEnd) |
| __asm__ __volatile__(".arch_extension memtag; stg %0, [%0]" |
| : |
| : "r"(RoundNewPtr) |
| : "memory"); |
| return; |
| } |
| |
| __asm__ __volatile__(R"( |
| .arch_extension memtag |
| |
| // Set the memory tag of the region |
| // [roundUpTo(OldPtr, 16), roundUpTo(NewPtr, 16)) |
| // to the pointer tag stored in OldPtr. |
| 1: |
| stzg %[Cur], [%[Cur]], #16 |
| cmp %[Cur], %[End] |
| b.lt 1b |
| |
| // Finally, set the tag of the granule past the end of the allocation to 0. |
| and %[Cur], %[Cur], #(1 << 56) - 1 |
| cmp %[Cur], %[BlockEnd] |
| b.eq 2f |
| stg %[Cur], [%[Cur]] |
| |
| 2: |
| )" |
| : [Cur] "+&r"(RoundOldPtr), [End] "+&r"(NewPtr) |
| : [BlockEnd] "r"(BlockEnd) |
| : "memory"); |
| } |
| |
| inline uptr loadTag(uptr Ptr) { |
| uptr TaggedPtr = Ptr; |
| __asm__ __volatile__(".arch_extension memtag; ldg %0, [%0]" |
| : "+r"(TaggedPtr) |
| : |
| : "memory"); |
| return TaggedPtr; |
| } |
| |
| #else |
| |
| inline bool systemSupportsMemoryTagging() { |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| inline bool systemDetectsMemoryTagFaultsTestOnly() { |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| inline void disableMemoryTagChecksTestOnly() { |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| inline void enableMemoryTagChecksTestOnly() { |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| struct ScopedDisableMemoryTagChecks { |
| ScopedDisableMemoryTagChecks() {} |
| }; |
| |
| inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) { |
| (void)Ptr; |
| (void)ExcludeMask; |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| inline uptr addFixedTag(uptr Ptr, uptr Tag) { |
| (void)Ptr; |
| (void)Tag; |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| inline uptr storeTags(uptr Begin, uptr End) { |
| (void)Begin; |
| (void)End; |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| inline void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask, |
| uptr BlockEnd) { |
| (void)Ptr; |
| (void)Size; |
| (void)ExcludeMask; |
| (void)BlockEnd; |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| inline void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr BlockEnd) { |
| (void)OldPtr; |
| (void)NewPtr; |
| (void)BlockEnd; |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| inline uptr loadTag(uptr Ptr) { |
| (void)Ptr; |
| UNREACHABLE("memory tagging not supported"); |
| } |
| |
| #endif |
| |
| inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask, |
| uptr *TaggedBegin, uptr *TaggedEnd) { |
| *TaggedBegin = selectRandomTag(reinterpret_cast<uptr>(Ptr), ExcludeMask); |
| *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size); |
| } |
| |
| inline void *untagPointer(void *Ptr) { |
| return reinterpret_cast<void *>(untagPointer(reinterpret_cast<uptr>(Ptr))); |
| } |
| |
| template <typename Config> |
| inline constexpr bool allocatorSupportsMemoryTagging() { |
| return archSupportsMemoryTagging() && Config::MaySupportMemoryTagging; |
| } |
| |
| } // namespace scudo |
| |
| #endif |