| /* |
| * Copyright (C) 2019 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #include "src/profiling/memory/shared_ring_buffer.h" |
| |
| #include <atomic> |
| #include <type_traits> |
| |
| #include <inttypes.h> |
| #include <sys/mman.h> |
| #include <sys/stat.h> |
| #include <unistd.h> |
| |
| #include "perfetto/base/build_config.h" |
| #include "perfetto/base/scoped_file.h" |
| #include "perfetto/base/temp_file.h" |
| |
| #if PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID) |
| #include <linux/memfd.h> |
| #include <sys/syscall.h> |
| #endif |
| |
| namespace perfetto { |
| namespace profiling { |
| |
| namespace { |
| |
| constexpr auto kMetaPageSize = base::kPageSize; |
| constexpr auto kAlignment = 8; // 64 bits to use aligned memcpy(). |
| constexpr auto kHeaderSize = kAlignment; |
| constexpr auto kGuardSize = base::kPageSize * 1024 * 16; // 64 MB. |
| |
| } // namespace |
| |
| void ScopedSpinlock::LockSlow(Mode mode) { |
| // Slowpath. |
| for (size_t attempt = 0; mode == Mode::Blocking || attempt < 1024 * 10; |
| attempt++) { |
| if (!lock_->load(std::memory_order_relaxed) && |
| PERFETTO_LIKELY(!lock_->exchange(true, std::memory_order_acquire))) { |
| locked_ = true; |
| return; |
| } |
| if (attempt && attempt % 1024 == 0) |
| usleep(1000); |
| } |
| } |
| |
| ScopedSpinlock::ScopedSpinlock(ScopedSpinlock&& other) noexcept |
| : lock_(other.lock_), locked_(other.locked_) { |
| other.locked_ = false; |
| } |
| |
| ScopedSpinlock& ScopedSpinlock::operator=(ScopedSpinlock&& other) { |
| if (this != &other) { |
| this->~ScopedSpinlock(); |
| new (this) ScopedSpinlock(std::move(other)); |
| } |
| return *this; |
| } |
| |
| ScopedSpinlock::~ScopedSpinlock() { |
| Unlock(); |
| } |
| |
| SharedRingBuffer::SharedRingBuffer(CreateFlag, size_t size) { |
| size_t size_with_meta = size + kMetaPageSize; |
| // TODO(primiano): this is copy/pasted from posix_shared_memory.cc . Refactor. |
| base::ScopedFile fd; |
| #if PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID) |
| bool is_memfd = false; |
| fd.reset(static_cast<int>(syscall(__NR_memfd_create, "heaprofd_ringbuf", |
| MFD_CLOEXEC | MFD_ALLOW_SEALING))); |
| is_memfd = !!fd; |
| |
| if (!fd) { |
| // TODO: if this fails on Android we should fall back on ashmem. |
| PERFETTO_DPLOG("memfd_create() failed"); |
| } |
| #endif |
| |
| if (!fd) |
| fd = base::TempFile::CreateUnlinked().ReleaseFD(); |
| |
| PERFETTO_CHECK(fd); |
| int res = ftruncate(fd.get(), static_cast<off_t>(size_with_meta)); |
| PERFETTO_CHECK(res == 0); |
| #if PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID) |
| if (is_memfd) { |
| res = fcntl(*fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_SEAL); |
| PERFETTO_DCHECK(res == 0); |
| } |
| #endif |
| Initialize(std::move(fd)); |
| } |
| |
| SharedRingBuffer::~SharedRingBuffer() { |
| static_assert(std::is_trivially_constructible<MetadataPage>::value, |
| "MetadataPage must be trivially constructible"); |
| static_assert(std::is_trivially_destructible<MetadataPage>::value, |
| "MetadataPage must be trivially destructible"); |
| |
| if (is_valid()) { |
| size_t outer_size = kMetaPageSize + size_ * 2 + kGuardSize; |
| munmap(meta_, outer_size); |
| } |
| } |
| |
| void SharedRingBuffer::Initialize(base::ScopedFile mem_fd) { |
| struct stat stat_buf = {}; |
| int res = fstat(*mem_fd, &stat_buf); |
| PERFETTO_CHECK(res == 0 && stat_buf.st_size > 0); |
| auto size_with_meta = static_cast<size_t>(stat_buf.st_size); |
| auto size = size_with_meta - kMetaPageSize; |
| |
| // |size_with_meta| must be a power of two number of pages + 1 page (for |
| // metadata). |
| if (size_with_meta < 2 * base::kPageSize || size % base::kPageSize || |
| (size & (size - 1))) { |
| PERFETTO_ELOG("SharedRingBuffer size is invalid (%zu)", size_with_meta); |
| return; |
| } |
| |
| // First of all reserve the whole virtual region to fit the buffer twice |
| // + metadata page + red zone at the end. |
| size_t outer_size = kMetaPageSize + size * 2 + kGuardSize; |
| uint8_t* region = reinterpret_cast<uint8_t*>( |
| mmap(nullptr, outer_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); |
| if (region == MAP_FAILED) { |
| PERFETTO_PLOG("mmap(PROT_NONE) failed"); |
| return; |
| } |
| |
| // Map first the whole buffer (including the initial metadata page) @ off=0. |
| void* reg1 = mmap(region, size_with_meta, PROT_READ | PROT_WRITE, |
| MAP_SHARED | MAP_FIXED, *mem_fd, 0); |
| |
| // Then map again the buffer, skipping the metadata page. The final result is: |
| // [ METADATA ] [ RING BUFFER SHMEM ] [ RING BUFFER SHMEM ] |
| void* reg2 = mmap(region + size_with_meta, size, PROT_READ | PROT_WRITE, |
| MAP_SHARED | MAP_FIXED, *mem_fd, |
| /*offset=*/kMetaPageSize); |
| |
| if (reg1 != region || reg2 != region + size_with_meta) { |
| PERFETTO_PLOG("mmap(MAP_SHARED) failed"); |
| munmap(region, outer_size); |
| return; |
| } |
| size_ = size; |
| meta_ = new (region) MetadataPage(); |
| mem_ = region + kMetaPageSize; |
| mem_fd_ = std::move(mem_fd); |
| } |
| |
| SharedRingBuffer::Buffer SharedRingBuffer::BeginWrite( |
| const ScopedSpinlock& spinlock, |
| size_t size) { |
| PERFETTO_DCHECK(spinlock.locked()); |
| Buffer result; |
| |
| if (IsCorrupt()) |
| return result; |
| |
| const uint64_t size_with_header = |
| base::AlignUp<kAlignment>(size + kHeaderSize); |
| if (size_with_header > write_avail(spinlock)) { |
| meta_->num_writes_failed++; |
| return result; |
| } |
| |
| uint8_t* wr_ptr = at(meta_->write_pos); |
| |
| result.size = size; |
| result.data = wr_ptr + kHeaderSize; |
| meta_->write_pos += size_with_header; |
| meta_->bytes_written += size; |
| meta_->num_writes_succeeded++; |
| // By making this a release store, we can save grabbing the spinlock in |
| // EndWrite. |
| reinterpret_cast<std::atomic<uint32_t>*>(wr_ptr)->store( |
| 0, std::memory_order_release); |
| return result; |
| } |
| |
| void SharedRingBuffer::EndWrite(Buffer buf) { |
| uint8_t* wr_ptr = buf.data - kHeaderSize; |
| PERFETTO_DCHECK(reinterpret_cast<uintptr_t>(wr_ptr) % kAlignment == 0); |
| reinterpret_cast<std::atomic<uint32_t>*>(wr_ptr)->store( |
| static_cast<uint32_t>(buf.size), std::memory_order_release); |
| } |
| |
| SharedRingBuffer::Buffer SharedRingBuffer::BeginRead() { |
| ScopedSpinlock spinlock(&meta_->spinlock, ScopedSpinlock::Mode::Blocking); |
| |
| if (IsCorrupt()) { |
| meta_->num_reads_failed++; |
| return Buffer(); |
| } |
| |
| uint64_t avail_read = read_avail(spinlock); |
| |
| if (avail_read < kHeaderSize) |
| return Buffer(); // No data |
| |
| uint8_t* rd_ptr = at(meta_->read_pos); |
| PERFETTO_DCHECK(reinterpret_cast<uintptr_t>(rd_ptr) % kAlignment == 0); |
| const size_t size = reinterpret_cast<std::atomic<uint32_t>*>(rd_ptr)->load( |
| std::memory_order_acquire); |
| if (size == 0) |
| return Buffer(); |
| const size_t size_with_header = base::AlignUp<kAlignment>(size + kHeaderSize); |
| |
| if (size_with_header > avail_read) { |
| PERFETTO_ELOG( |
| "Corrupted header detected, size=%zu" |
| ", read_avail=%zu, rd=%" PRIu64 ", wr=%" PRIu64, |
| size, read_avail(spinlock), meta_->read_pos, meta_->write_pos); |
| meta_->num_reads_failed++; |
| return Buffer(); |
| } |
| |
| rd_ptr += kHeaderSize; |
| PERFETTO_DCHECK(reinterpret_cast<uintptr_t>(rd_ptr) % kAlignment == 0); |
| return Buffer(rd_ptr, size); |
| } |
| |
| void SharedRingBuffer::EndRead(Buffer buf) { |
| if (!buf) |
| return; |
| ScopedSpinlock spinlock(&meta_->spinlock, ScopedSpinlock::Mode::Blocking); |
| size_t size_with_header = base::AlignUp<kAlignment>(buf.size + kHeaderSize); |
| meta_->read_pos += size_with_header; |
| } |
| |
| bool SharedRingBuffer::IsCorrupt() { |
| if (meta_->write_pos < meta_->read_pos || |
| meta_->write_pos - meta_->read_pos > size_ || |
| meta_->write_pos % kAlignment || meta_->read_pos % kAlignment) { |
| PERFETTO_ELOG("Ring buffer corrupted, rd=%" PRIu64 ", wr=%" PRIu64 |
| ", size=%zu", |
| meta_->read_pos, meta_->write_pos, size_); |
| return true; |
| } |
| return false; |
| } |
| |
| SharedRingBuffer::SharedRingBuffer(SharedRingBuffer&& other) noexcept { |
| *this = std::move(other); |
| } |
| |
| SharedRingBuffer& SharedRingBuffer::operator=(SharedRingBuffer&& other) { |
| mem_fd_ = std::move(other.mem_fd_); |
| std::tie(meta_, mem_, size_) = std::tie(other.meta_, other.mem_, other.size_); |
| std::tie(other.meta_, other.mem_, other.size_) = |
| std::make_tuple(nullptr, nullptr, 0); |
| return *this; |
| } |
| |
| // static |
| base::Optional<SharedRingBuffer> SharedRingBuffer::Create(size_t size) { |
| auto buf = SharedRingBuffer(CreateFlag(), size); |
| if (!buf.is_valid()) |
| return base::nullopt; |
| return base::make_optional(std::move(buf)); |
| } |
| |
| // static |
| base::Optional<SharedRingBuffer> SharedRingBuffer::Attach( |
| base::ScopedFile mem_fd) { |
| auto buf = SharedRingBuffer(AttachFlag(), std::move(mem_fd)); |
| if (!buf.is_valid()) |
| return base::nullopt; |
| return base::make_optional(std::move(buf)); |
| } |
| |
| } // namespace profiling |
| } // namespace perfetto |