ART: Fix valgrind
Allow ValgrindMallocSpace wrapper for RosAlloc.Requires refactoring,
as ValgrindMallocSpace was bound to the signature of DlMallocSpace.
Also turn of native stack dumping when running under Valgrind to
work around b/18119146.
Ritzperf before and after
Mean 3190.725 3082.475
Standard Error 11.68407 10.37911
Mode 3069 2980
Median 3182.5 3051.5
Variance 16382.117 12927.125
Standard Deviation 127.99264 113.69751
Kurtosis 1.1065632 0.3657799
Skewness 0.9013805 0.9117792
Range 644 528
Minimum 2991 2928
Maximum 3635 3456
Count 120 120
Bug: 18119146
Change-Id: I25558ea7cb578406011dede9d3d0bdbfee4ff4d5
diff --git a/runtime/gc/allocator/rosalloc-inl.h b/runtime/gc/allocator/rosalloc-inl.h
index dd419a4..f6c9d3c 100644
--- a/runtime/gc/allocator/rosalloc-inl.h
+++ b/runtime/gc/allocator/rosalloc-inl.h
@@ -23,6 +23,10 @@
namespace gc {
namespace allocator {
+inline ALWAYS_INLINE bool RosAlloc::ShouldCheckZeroMemory() {
+ return kCheckZeroMemory && !running_on_valgrind_;
+}
+
template<bool kThreadSafe>
inline ALWAYS_INLINE void* RosAlloc::Alloc(Thread* self, size_t size, size_t* bytes_allocated) {
if (UNLIKELY(size > kLargeSizeThreshold)) {
@@ -35,7 +39,7 @@
m = AllocFromRunThreadUnsafe(self, size, bytes_allocated);
}
// Check if the returned memory is really all zero.
- if (kCheckZeroMemory && m != nullptr) {
+ if (ShouldCheckZeroMemory() && m != nullptr) {
uint8_t* bytes = reinterpret_cast<uint8_t*>(m);
for (size_t i = 0; i < size; ++i) {
DCHECK_EQ(bytes[i], 0);
diff --git a/runtime/gc/allocator/rosalloc.cc b/runtime/gc/allocator/rosalloc.cc
index f9d6a51..991b956 100644
--- a/runtime/gc/allocator/rosalloc.cc
+++ b/runtime/gc/allocator/rosalloc.cc
@@ -14,13 +14,15 @@
* limitations under the License.
*/
+#include "rosalloc.h"
+
#include "base/mutex-inl.h"
+#include "gc/space/valgrind_settings.h"
#include "mirror/class-inl.h"
#include "mirror/object.h"
#include "mirror/object-inl.h"
#include "thread-inl.h"
#include "thread_list.h"
-#include "rosalloc.h"
#include <map>
#include <list>
@@ -47,13 +49,15 @@
reinterpret_cast<RosAlloc::Run*>(dedicated_full_run_storage_);
RosAlloc::RosAlloc(void* base, size_t capacity, size_t max_capacity,
- PageReleaseMode page_release_mode, size_t page_release_size_threshold)
+ PageReleaseMode page_release_mode, bool running_on_valgrind,
+ size_t page_release_size_threshold)
: base_(reinterpret_cast<uint8_t*>(base)), footprint_(capacity),
capacity_(capacity), max_capacity_(max_capacity),
lock_("rosalloc global lock", kRosAllocGlobalLock),
bulk_free_lock_("rosalloc bulk free lock", kRosAllocBulkFreeLock),
page_release_mode_(page_release_mode),
- page_release_size_threshold_(page_release_size_threshold) {
+ page_release_size_threshold_(page_release_size_threshold),
+ running_on_valgrind_(running_on_valgrind) {
DCHECK_EQ(RoundUp(capacity, kPageSize), capacity);
DCHECK_EQ(RoundUp(max_capacity, kPageSize), max_capacity);
CHECK_LE(capacity, max_capacity);
@@ -317,7 +321,7 @@
}
const size_t byte_size = num_pages * kPageSize;
if (already_zero) {
- if (kCheckZeroMemory) {
+ if (ShouldCheckZeroMemory()) {
const uintptr_t* word_ptr = reinterpret_cast<uintptr_t*>(ptr);
for (size_t i = 0; i < byte_size / sizeof(uintptr_t); ++i) {
CHECK_EQ(word_ptr[i], 0U) << "words don't match at index " << i;
@@ -471,7 +475,7 @@
<< "(" << std::dec << (num_pages * kPageSize) << ")";
}
// Check if the returned memory is really all zero.
- if (kCheckZeroMemory) {
+ if (ShouldCheckZeroMemory()) {
CHECK_EQ(total_bytes % sizeof(uintptr_t), 0U);
const uintptr_t* words = reinterpret_cast<uintptr_t*>(r);
for (size_t i = 0; i < total_bytes / sizeof(uintptr_t); ++i) {
@@ -1433,7 +1437,7 @@
return stream.str();
}
-size_t RosAlloc::UsableSize(void* ptr) {
+size_t RosAlloc::UsableSize(const void* ptr) {
DCHECK_LE(base_, ptr);
DCHECK_LT(ptr, base_ + footprint_);
size_t pm_idx = RoundDownToPageMapIndex(ptr);
@@ -1470,7 +1474,7 @@
Run* run = reinterpret_cast<Run*>(base_ + pm_idx * kPageSize);
DCHECK_EQ(run->magic_num_, kMagicNum);
size_t idx = run->size_bracket_idx_;
- size_t offset_from_slot_base = reinterpret_cast<uint8_t*>(ptr)
+ size_t offset_from_slot_base = reinterpret_cast<const uint8_t*>(ptr)
- (reinterpret_cast<uint8_t*>(run) + headerSizes[idx]);
DCHECK_EQ(offset_from_slot_base % bracketSizes[idx], static_cast<size_t>(0));
return IndexToBracketSize(idx);
@@ -1915,10 +1919,15 @@
num_pages++;
idx++;
}
- void* start = base_ + i * kPageSize;
+ uint8_t* start = base_ + i * kPageSize;
+ if (running_on_valgrind_) {
+ start += ::art::gc::space::kDefaultValgrindRedZoneBytes;
+ }
mirror::Object* obj = reinterpret_cast<mirror::Object*>(start);
size_t obj_size = obj->SizeOf();
- CHECK_GT(obj_size, kLargeSizeThreshold)
+ CHECK_GT(obj_size +
+ (running_on_valgrind_ ? 2 * ::art::gc::space::kDefaultValgrindRedZoneBytes : 0),
+ kLargeSizeThreshold)
<< "A rosalloc large object size must be > " << kLargeSizeThreshold;
CHECK_EQ(num_pages, RoundUp(obj_size, kPageSize) / kPageSize)
<< "A rosalloc large object size " << obj_size
@@ -1986,11 +1995,11 @@
}
// Call Verify() here for the lock order.
for (auto& run : runs) {
- run->Verify(self, this);
+ run->Verify(self, this, running_on_valgrind_);
}
}
-void RosAlloc::Run::Verify(Thread* self, RosAlloc* rosalloc) {
+void RosAlloc::Run::Verify(Thread* self, RosAlloc* rosalloc, bool running_on_valgrind) {
DCHECK_EQ(magic_num_, kMagicNum) << "Bad magic number : " << Dump();
const size_t idx = size_bracket_idx_;
CHECK_LT(idx, kNumOfSizeBrackets) << "Out of range size bracket index : " << Dump();
@@ -2073,6 +2082,9 @@
}
// Check each slot.
size_t slots = 0;
+ size_t valgrind_modifier = running_on_valgrind ?
+ 2 * ::art::gc::space::kDefaultValgrindRedZoneBytes :
+ 0U;
for (size_t v = 0; v < num_vec; v++, slots += 32) {
DCHECK_GE(num_slots, slots) << "Out of bounds";
uint32_t vec = alloc_bit_map_[v];
@@ -2085,14 +2097,17 @@
bool is_thread_local_freed = IsThreadLocal() && ((thread_local_free_vec >> i) & 0x1) != 0;
if (is_allocated && !is_thread_local_freed) {
uint8_t* slot_addr = slot_base + (slots + i) * bracket_size;
+ if (running_on_valgrind) {
+ slot_addr += ::art::gc::space::kDefaultValgrindRedZoneBytes;
+ }
mirror::Object* obj = reinterpret_cast<mirror::Object*>(slot_addr);
size_t obj_size = obj->SizeOf();
- CHECK_LE(obj_size, kLargeSizeThreshold)
+ CHECK_LE(obj_size + valgrind_modifier, kLargeSizeThreshold)
<< "A run slot contains a large object " << Dump();
- CHECK_EQ(SizeToIndex(obj_size), idx)
+ CHECK_EQ(SizeToIndex(obj_size + valgrind_modifier), idx)
<< PrettyTypeOf(obj) << " "
- << "obj_size=" << obj_size << ", idx=" << idx << " "
- << "A run slot contains an object with wrong size " << Dump();
+ << "obj_size=" << obj_size << "(" << obj_size + valgrind_modifier << "), idx=" << idx
+ << " A run slot contains an object with wrong size " << Dump();
}
}
}
@@ -2162,6 +2177,11 @@
// In the debug build, the first page of a free page run
// contains a magic number for debugging. Exclude it.
start += kPageSize;
+
+ // Single pages won't be released.
+ if (start == end) {
+ return 0;
+ }
}
if (!kMadviseZeroes) {
// TODO: Do this when we resurrect the page instead.
diff --git a/runtime/gc/allocator/rosalloc.h b/runtime/gc/allocator/rosalloc.h
index 2a0bf10..431686a 100644
--- a/runtime/gc/allocator/rosalloc.h
+++ b/runtime/gc/allocator/rosalloc.h
@@ -249,7 +249,7 @@
// Dump the run metadata for debugging.
std::string Dump();
// Verify for debugging.
- void Verify(Thread* self, RosAlloc* rosalloc)
+ void Verify(Thread* self, RosAlloc* rosalloc, bool running_on_valgrind)
EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_)
EXCLUSIVE_LOCKS_REQUIRED(Locks::thread_list_lock_);
@@ -360,13 +360,14 @@
// Returns the page map index from an address. Requires that the
// address is page size aligned.
size_t ToPageMapIndex(const void* addr) const {
- DCHECK(base_ <= addr && addr < base_ + capacity_);
+ DCHECK_LE(base_, addr);
+ DCHECK_LT(addr, base_ + capacity_);
size_t byte_offset = reinterpret_cast<const uint8_t*>(addr) - base_;
DCHECK_EQ(byte_offset % static_cast<size_t>(kPageSize), static_cast<size_t>(0));
return byte_offset / kPageSize;
}
// Returns the page map index from an address with rounding.
- size_t RoundDownToPageMapIndex(void* addr) const {
+ size_t RoundDownToPageMapIndex(const void* addr) const {
DCHECK(base_ <= addr && addr < reinterpret_cast<uint8_t*>(base_) + capacity_);
return (reinterpret_cast<uintptr_t>(addr) - reinterpret_cast<uintptr_t>(base_)) / kPageSize;
}
@@ -377,6 +378,10 @@
// If true, check that the returned memory is actually zero.
static constexpr bool kCheckZeroMemory = kIsDebugBuild;
+ // Valgrind protects memory, so do not check memory when running under valgrind. In a normal
+ // build with kCheckZeroMemory the whole test should be optimized away.
+ // TODO: Unprotect before checks.
+ ALWAYS_INLINE bool ShouldCheckZeroMemory();
// If true, log verbose details of operations.
static constexpr bool kTraceRosAlloc = false;
@@ -485,6 +490,9 @@
// greater than or equal to this value, release pages.
const size_t page_release_size_threshold_;
+ // Whether this allocator is running under Valgrind.
+ bool running_on_valgrind_;
+
// The base address of the memory region that's managed by this allocator.
uint8_t* Begin() { return base_; }
// The end address of the memory region that's managed by this allocator.
@@ -537,6 +545,7 @@
public:
RosAlloc(void* base, size_t capacity, size_t max_capacity,
PageReleaseMode page_release_mode,
+ bool running_on_valgrind,
size_t page_release_size_threshold = kDefaultPageReleaseSizeThreshold);
~RosAlloc();
@@ -551,7 +560,7 @@
LOCKS_EXCLUDED(bulk_free_lock_);
// Returns the size of the allocated slot for a given allocated memory chunk.
- size_t UsableSize(void* ptr);
+ size_t UsableSize(const void* ptr);
// Returns the size of the allocated slot for a given size.
size_t UsableSize(size_t bytes) {
if (UNLIKELY(bytes > kLargeSizeThreshold)) {
diff --git a/runtime/gc/space/dlmalloc_space.cc b/runtime/gc/space/dlmalloc_space.cc
index 3072c23..b8a9dd6 100644
--- a/runtime/gc/space/dlmalloc_space.cc
+++ b/runtime/gc/space/dlmalloc_space.cc
@@ -33,12 +33,9 @@
static constexpr bool kPrefetchDuringDlMallocFreeList = true;
-template class ValgrindMallocSpace<DlMallocSpace, void*>;
-
-DlMallocSpace::DlMallocSpace(const std::string& name, MemMap* mem_map, void* mspace, uint8_t* begin,
- uint8_t* end, uint8_t* limit, size_t growth_limit,
- bool can_move_objects, size_t starting_size,
- size_t initial_size)
+DlMallocSpace::DlMallocSpace(MemMap* mem_map, size_t initial_size, const std::string& name,
+ void* mspace, uint8_t* begin, uint8_t* end, uint8_t* limit,
+ size_t growth_limit, bool can_move_objects, size_t starting_size)
: MallocSpace(name, mem_map, begin, end, limit, growth_limit, true, can_move_objects,
starting_size, initial_size),
mspace_(mspace) {
@@ -65,12 +62,12 @@
// Everything is set so record in immutable structure and leave
uint8_t* begin = mem_map->Begin();
if (Runtime::Current()->RunningOnValgrind()) {
- return new ValgrindMallocSpace<DlMallocSpace, void*>(
- name, mem_map, mspace, begin, end, begin + capacity, growth_limit, initial_size,
+ return new ValgrindMallocSpace<DlMallocSpace, kDefaultValgrindRedZoneBytes, true, false>(
+ mem_map, initial_size, name, mspace, begin, end, begin + capacity, growth_limit,
can_move_objects, starting_size);
} else {
- return new DlMallocSpace(name, mem_map, mspace, begin, end, begin + capacity, growth_limit,
- can_move_objects, starting_size, initial_size);
+ return new DlMallocSpace(mem_map, initial_size, name, mspace, begin, end, begin + capacity,
+ growth_limit, can_move_objects, starting_size);
}
}
@@ -148,12 +145,18 @@
return result;
}
-MallocSpace* DlMallocSpace::CreateInstance(const std::string& name, MemMap* mem_map,
+MallocSpace* DlMallocSpace::CreateInstance(MemMap* mem_map, const std::string& name,
void* allocator, uint8_t* begin, uint8_t* end,
uint8_t* limit, size_t growth_limit,
bool can_move_objects) {
- return new DlMallocSpace(name, mem_map, allocator, begin, end, limit, growth_limit,
- can_move_objects, starting_size_, initial_size_);
+ if (Runtime::Current()->RunningOnValgrind()) {
+ return new ValgrindMallocSpace<DlMallocSpace, kDefaultValgrindRedZoneBytes, true, false>(
+ mem_map, initial_size_, name, allocator, begin, end, limit, growth_limit,
+ can_move_objects, starting_size_);
+ } else {
+ return new DlMallocSpace(mem_map, initial_size_, name, allocator, begin, end, limit,
+ growth_limit, can_move_objects, starting_size_);
+ }
}
size_t DlMallocSpace::Free(Thread* self, mirror::Object* ptr) {
diff --git a/runtime/gc/space/dlmalloc_space.h b/runtime/gc/space/dlmalloc_space.h
index 3b8065e..6ce138c 100644
--- a/runtime/gc/space/dlmalloc_space.h
+++ b/runtime/gc/space/dlmalloc_space.h
@@ -107,7 +107,7 @@
// allocations fail we GC before increasing the footprint limit and allowing the mspace to grow.
void SetFootprintLimit(size_t limit) OVERRIDE;
- MallocSpace* CreateInstance(const std::string& name, MemMap* mem_map, void* allocator,
+ MallocSpace* CreateInstance(MemMap* mem_map, const std::string& name, void* allocator,
uint8_t* begin, uint8_t* end, uint8_t* limit, size_t growth_limit,
bool can_move_objects);
@@ -128,9 +128,9 @@
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
protected:
- DlMallocSpace(const std::string& name, MemMap* mem_map, void* mspace, uint8_t* begin, uint8_t* end,
- uint8_t* limit, size_t growth_limit, bool can_move_objects, size_t starting_size,
- size_t initial_size);
+ DlMallocSpace(MemMap* mem_map, size_t initial_size, const std::string& name, void* mspace,
+ uint8_t* begin, uint8_t* end, uint8_t* limit, size_t growth_limit,
+ bool can_move_objects, size_t starting_size);
private:
mirror::Object* AllocWithoutGrowthLocked(Thread* self, size_t num_bytes, size_t* bytes_allocated,
diff --git a/runtime/gc/space/malloc_space.cc b/runtime/gc/space/malloc_space.cc
index 9d1fbbe..43a2c59 100644
--- a/runtime/gc/space/malloc_space.cc
+++ b/runtime/gc/space/malloc_space.cc
@@ -198,7 +198,7 @@
if (capacity > initial_size_) {
CHECK_MEMORY_CALL(mprotect, (end, capacity - initial_size_, PROT_NONE), alloc_space_name);
}
- *out_malloc_space = CreateInstance(alloc_space_name, mem_map.release(), allocator, End(), end,
+ *out_malloc_space = CreateInstance(mem_map.release(), alloc_space_name, allocator, End(), end,
limit_, growth_limit, CanMoveObjects());
SetLimit(End());
live_bitmap_->SetHeapLimit(reinterpret_cast<uintptr_t>(End()));
diff --git a/runtime/gc/space/malloc_space.h b/runtime/gc/space/malloc_space.h
index cfde460..2fbd5f0 100644
--- a/runtime/gc/space/malloc_space.h
+++ b/runtime/gc/space/malloc_space.h
@@ -114,9 +114,9 @@
void SetGrowthLimit(size_t growth_limit);
- virtual MallocSpace* CreateInstance(const std::string& name, MemMap* mem_map, void* allocator,
- uint8_t* begin, uint8_t* end, uint8_t* limit, size_t growth_limit,
- bool can_move_objects) = 0;
+ virtual MallocSpace* CreateInstance(MemMap* mem_map, const std::string& name, void* allocator,
+ uint8_t* begin, uint8_t* end, uint8_t* limit,
+ size_t growth_limit, bool can_move_objects) = 0;
// Splits ourself into a zygote space and new malloc space which has our unused memory. When true,
// the low memory mode argument specifies that the heap wishes the created space to be more
diff --git a/runtime/gc/space/rosalloc_space-inl.h b/runtime/gc/space/rosalloc_space-inl.h
index fbfef45..5d6642d 100644
--- a/runtime/gc/space/rosalloc_space-inl.h
+++ b/runtime/gc/space/rosalloc_space-inl.h
@@ -18,6 +18,7 @@
#define ART_RUNTIME_GC_SPACE_ROSALLOC_SPACE_INL_H_
#include "gc/allocator/rosalloc-inl.h"
+#include "gc/space/valgrind_settings.h"
#include "rosalloc_space.h"
#include "thread.h"
@@ -26,13 +27,19 @@
namespace space {
inline size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) {
- void* obj_ptr = const_cast<void*>(reinterpret_cast<const void*>(obj));
// obj is a valid object. Use its class in the header to get the size.
// Don't use verification since the object may be dead if we are sweeping.
size_t size = obj->SizeOf<kVerifyNone>();
+ bool running_on_valgrind = RUNNING_ON_VALGRIND != 0;
+ if (running_on_valgrind) {
+ size += 2 * kDefaultValgrindRedZoneBytes;
+ }
size_t size_by_size = rosalloc_->UsableSize(size);
if (kIsDebugBuild) {
- size_t size_by_ptr = rosalloc_->UsableSize(obj_ptr);
+ // On valgrind, the red zone has an impact...
+ const uint8_t* obj_ptr = reinterpret_cast<const uint8_t*>(obj);
+ size_t size_by_ptr = rosalloc_->UsableSize(
+ obj_ptr - (running_on_valgrind ? kDefaultValgrindRedZoneBytes : 0));
if (size_by_size != size_by_ptr) {
LOG(INFO) << "Found a bad sized obj of size " << size
<< " at " << std::hex << reinterpret_cast<intptr_t>(obj_ptr) << std::dec
diff --git a/runtime/gc/space/rosalloc_space.cc b/runtime/gc/space/rosalloc_space.cc
index ff8b570..74d1a2b 100644
--- a/runtime/gc/space/rosalloc_space.cc
+++ b/runtime/gc/space/rosalloc_space.cc
@@ -44,10 +44,10 @@
// TODO: Fix
// template class ValgrindMallocSpace<RosAllocSpace, allocator::RosAlloc*>;
-RosAllocSpace::RosAllocSpace(const std::string& name, MemMap* mem_map,
+RosAllocSpace::RosAllocSpace(MemMap* mem_map, size_t initial_size, const std::string& name,
art::gc::allocator::RosAlloc* rosalloc, uint8_t* begin, uint8_t* end,
uint8_t* limit, size_t growth_limit, bool can_move_objects,
- size_t starting_size, size_t initial_size, bool low_memory_mode)
+ size_t starting_size, bool low_memory_mode)
: MallocSpace(name, mem_map, begin, end, limit, growth_limit, true, can_move_objects,
starting_size, initial_size),
rosalloc_(rosalloc), low_memory_mode_(low_memory_mode) {
@@ -59,8 +59,11 @@
size_t growth_limit, size_t capacity,
bool low_memory_mode, bool can_move_objects) {
DCHECK(mem_map != nullptr);
+
+ bool running_on_valgrind = Runtime::Current()->RunningOnValgrind();
+
allocator::RosAlloc* rosalloc = CreateRosAlloc(mem_map->Begin(), starting_size, initial_size,
- capacity, low_memory_mode);
+ capacity, low_memory_mode, running_on_valgrind);
if (rosalloc == NULL) {
LOG(ERROR) << "Failed to initialize rosalloc for alloc space (" << name << ")";
return NULL;
@@ -76,12 +79,13 @@
uint8_t* begin = mem_map->Begin();
// TODO: Fix RosAllocSpace to support valgrind. There is currently some issues with
// AllocationSize caused by redzones. b/12944686
- if (Runtime::Current()->RunningOnValgrind()) {
- UNIMPLEMENTED(FATAL);
- UNREACHABLE();
+ if (running_on_valgrind) {
+ return new ValgrindMallocSpace<RosAllocSpace, kDefaultValgrindRedZoneBytes, false, true>(
+ mem_map, initial_size, name, rosalloc, begin, end, begin + capacity, growth_limit,
+ can_move_objects, starting_size, low_memory_mode);
} else {
- return new RosAllocSpace(name, mem_map, rosalloc, begin, end, begin + capacity, growth_limit,
- can_move_objects, starting_size, initial_size, low_memory_mode);
+ return new RosAllocSpace(mem_map, initial_size, name, rosalloc, begin, end, begin + capacity,
+ growth_limit, can_move_objects, starting_size, low_memory_mode);
}
}
@@ -128,7 +132,8 @@
allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_start,
size_t initial_size,
- size_t maximum_size, bool low_memory_mode) {
+ size_t maximum_size, bool low_memory_mode,
+ bool running_on_valgrind) {
// clear errno to allow PLOG on error
errno = 0;
// create rosalloc using our backing storage starting at begin and
@@ -138,7 +143,8 @@
begin, morecore_start, maximum_size,
low_memory_mode ?
art::gc::allocator::RosAlloc::kPageReleaseModeAll :
- art::gc::allocator::RosAlloc::kPageReleaseModeSizeAndEnd);
+ art::gc::allocator::RosAlloc::kPageReleaseModeSizeAndEnd,
+ running_on_valgrind);
if (rosalloc != NULL) {
rosalloc->SetFootprintLimit(initial_size);
} else {
@@ -167,12 +173,19 @@
return result;
}
-MallocSpace* RosAllocSpace::CreateInstance(const std::string& name, MemMap* mem_map, void* allocator,
- uint8_t* begin, uint8_t* end, uint8_t* limit, size_t growth_limit,
+MallocSpace* RosAllocSpace::CreateInstance(MemMap* mem_map, const std::string& name,
+ void* allocator, uint8_t* begin, uint8_t* end,
+ uint8_t* limit, size_t growth_limit,
bool can_move_objects) {
- return new RosAllocSpace(name, mem_map, reinterpret_cast<allocator::RosAlloc*>(allocator),
- begin, end, limit, growth_limit, can_move_objects, starting_size_,
- initial_size_, low_memory_mode_);
+ if (Runtime::Current()->RunningOnValgrind()) {
+ return new ValgrindMallocSpace<RosAllocSpace, kDefaultValgrindRedZoneBytes, false, true>(
+ mem_map, initial_size_, name, reinterpret_cast<allocator::RosAlloc*>(allocator), begin, end,
+ limit, growth_limit, can_move_objects, starting_size_, low_memory_mode_);
+ } else {
+ return new RosAllocSpace(mem_map, initial_size_, name,
+ reinterpret_cast<allocator::RosAlloc*>(allocator), begin, end, limit,
+ growth_limit, can_move_objects, starting_size_, low_memory_mode_);
+ }
}
size_t RosAllocSpace::Free(Thread* self, mirror::Object* ptr) {
@@ -353,7 +366,7 @@
SetEnd(begin_ + starting_size_);
delete rosalloc_;
rosalloc_ = CreateRosAlloc(mem_map_->Begin(), starting_size_, initial_size_, Capacity(),
- low_memory_mode_);
+ low_memory_mode_, Runtime::Current()->RunningOnValgrind());
SetFootprintLimit(footprint_limit);
}
diff --git a/runtime/gc/space/rosalloc_space.h b/runtime/gc/space/rosalloc_space.h
index 46fffaa..c856e95 100644
--- a/runtime/gc/space/rosalloc_space.h
+++ b/runtime/gc/space/rosalloc_space.h
@@ -92,7 +92,7 @@
void Clear() OVERRIDE;
- MallocSpace* CreateInstance(const std::string& name, MemMap* mem_map, void* allocator,
+ MallocSpace* CreateInstance(MemMap* mem_map, const std::string& name, void* allocator,
uint8_t* begin, uint8_t* end, uint8_t* limit, size_t growth_limit,
bool can_move_objects) OVERRIDE;
@@ -126,9 +126,10 @@
}
protected:
- RosAllocSpace(const std::string& name, MemMap* mem_map, allocator::RosAlloc* rosalloc,
- uint8_t* begin, uint8_t* end, uint8_t* limit, size_t growth_limit, bool can_move_objects,
- size_t starting_size, size_t initial_size, bool low_memory_mode);
+ RosAllocSpace(MemMap* mem_map, size_t initial_size, const std::string& name,
+ allocator::RosAlloc* rosalloc, uint8_t* begin, uint8_t* end, uint8_t* limit,
+ size_t growth_limit, bool can_move_objects, size_t starting_size,
+ bool low_memory_mode);
private:
template<bool kThreadSafe = true>
@@ -137,10 +138,12 @@
void* CreateAllocator(void* base, size_t morecore_start, size_t initial_size,
size_t maximum_size, bool low_memory_mode) OVERRIDE {
- return CreateRosAlloc(base, morecore_start, initial_size, maximum_size, low_memory_mode);
+ return CreateRosAlloc(base, morecore_start, initial_size, maximum_size, low_memory_mode,
+ RUNNING_ON_VALGRIND != 0);
}
static allocator::RosAlloc* CreateRosAlloc(void* base, size_t morecore_start, size_t initial_size,
- size_t maximum_size, bool low_memory_mode);
+ size_t maximum_size, bool low_memory_mode,
+ bool running_on_valgrind);
void InspectAllRosAlloc(void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
void* arg, bool do_null_callback_at_end)
diff --git a/runtime/gc/space/valgrind_malloc_space-inl.h b/runtime/gc/space/valgrind_malloc_space-inl.h
index a6b837c..793d798 100644
--- a/runtime/gc/space/valgrind_malloc_space-inl.h
+++ b/runtime/gc/space/valgrind_malloc_space-inl.h
@@ -21,68 +21,165 @@
#include <memcheck/memcheck.h>
+#include "valgrind_settings.h"
+
namespace art {
namespace gc {
namespace space {
-// Number of bytes to use as a red zone (rdz). A red zone of this size will be placed before and
-// after each allocation. 8 bytes provides long/double alignment.
-static constexpr size_t kValgrindRedZoneBytes = 8;
+namespace valgrind_details {
-template <typename S, typename A>
-mirror::Object* ValgrindMallocSpace<S, A>::AllocWithGrowth(Thread* self, size_t num_bytes,
- size_t* bytes_allocated,
- size_t* usable_size) {
+template <size_t kValgrindRedZoneBytes, bool kUseObjSizeForUsable>
+inline mirror::Object* AdjustForValgrind(void* obj_with_rdz, size_t num_bytes,
+ size_t bytes_allocated, size_t usable_size,
+ size_t* bytes_allocated_out, size_t* usable_size_out) {
+ if (bytes_allocated_out != nullptr) {
+ *bytes_allocated_out = bytes_allocated;
+ }
+
+ // This cuts over-provision and is a trade-off between testing the over-provisioning code paths
+ // vs checking overflows in the regular paths.
+ if (usable_size_out != nullptr) {
+ if (kUseObjSizeForUsable) {
+ *usable_size_out = num_bytes;
+ } else {
+ *usable_size_out = usable_size - 2 * kValgrindRedZoneBytes;
+ }
+ }
+
+ // Left redzone.
+ VALGRIND_MAKE_MEM_NOACCESS(obj_with_rdz, kValgrindRedZoneBytes);
+
+ // Make requested memory readable.
+ // (If the allocator assumes memory is zeroed out, we might get UNDEFINED warnings, so make
+ // everything DEFINED initially.)
+ mirror::Object* result = reinterpret_cast<mirror::Object*>(
+ reinterpret_cast<uint8_t*>(obj_with_rdz) + kValgrindRedZoneBytes);
+ VALGRIND_MAKE_MEM_DEFINED(result, num_bytes);
+
+ // Right redzone. Assumes that if bytes_allocated > usable_size, then the difference is
+ // management data at the upper end, and for simplicity we will not protect that.
+ // At the moment, this fits RosAlloc (no management data in a slot, usable_size == alloc_size)
+ // and DlMalloc (allocation_size = (usable_size == num_bytes) + 4, 4 is management)
+ VALGRIND_MAKE_MEM_NOACCESS(reinterpret_cast<uint8_t*>(result) + num_bytes,
+ usable_size - (num_bytes + kValgrindRedZoneBytes));
+
+ return result;
+}
+
+inline size_t GetObjSizeNoThreadSafety(mirror::Object* obj) NO_THREAD_SAFETY_ANALYSIS {
+ return obj->SizeOf<kVerifyNone>();
+}
+
+} // namespace valgrind_details
+
+template <typename S,
+ size_t kValgrindRedZoneBytes,
+ bool kAdjustForRedzoneInAllocSize,
+ bool kUseObjSizeForUsable>
+mirror::Object*
+ValgrindMallocSpace<S,
+ kValgrindRedZoneBytes,
+ kAdjustForRedzoneInAllocSize,
+ kUseObjSizeForUsable>::AllocWithGrowth(
+ Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out) {
+ size_t bytes_allocated;
+ size_t usable_size;
void* obj_with_rdz = S::AllocWithGrowth(self, num_bytes + 2 * kValgrindRedZoneBytes,
- bytes_allocated, usable_size);
+ &bytes_allocated, &usable_size);
if (obj_with_rdz == nullptr) {
return nullptr;
}
- mirror::Object* result = reinterpret_cast<mirror::Object*>(
- reinterpret_cast<uint8_t*>(obj_with_rdz) + kValgrindRedZoneBytes);
- // Make redzones as no access.
- VALGRIND_MAKE_MEM_NOACCESS(obj_with_rdz, kValgrindRedZoneBytes);
- VALGRIND_MAKE_MEM_NOACCESS(reinterpret_cast<uint8_t*>(result) + num_bytes, kValgrindRedZoneBytes);
- return result;
+
+ return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes,
+ kUseObjSizeForUsable>(obj_with_rdz, num_bytes,
+ bytes_allocated, usable_size,
+ bytes_allocated_out,
+ usable_size_out);
}
-template <typename S, typename A>
-mirror::Object* ValgrindMallocSpace<S, A>::Alloc(Thread* self, size_t num_bytes,
- size_t* bytes_allocated,
- size_t* usable_size) {
- void* obj_with_rdz = S::Alloc(self, num_bytes + 2 * kValgrindRedZoneBytes, bytes_allocated,
- usable_size);
+template <typename S,
+ size_t kValgrindRedZoneBytes,
+ bool kAdjustForRedzoneInAllocSize,
+ bool kUseObjSizeForUsable>
+mirror::Object* ValgrindMallocSpace<S,
+ kValgrindRedZoneBytes,
+ kAdjustForRedzoneInAllocSize,
+ kUseObjSizeForUsable>::Alloc(
+ Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out) {
+ size_t bytes_allocated;
+ size_t usable_size;
+ void* obj_with_rdz = S::Alloc(self, num_bytes + 2 * kValgrindRedZoneBytes,
+ &bytes_allocated, &usable_size);
if (obj_with_rdz == nullptr) {
return nullptr;
}
- mirror::Object* result = reinterpret_cast<mirror::Object*>(
- reinterpret_cast<uint8_t*>(obj_with_rdz) + kValgrindRedZoneBytes);
- // Make redzones as no access.
- VALGRIND_MAKE_MEM_NOACCESS(obj_with_rdz, kValgrindRedZoneBytes);
- VALGRIND_MAKE_MEM_NOACCESS(reinterpret_cast<uint8_t*>(result) + num_bytes, kValgrindRedZoneBytes);
- return result;
+
+ return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes,
+ kUseObjSizeForUsable>(obj_with_rdz, num_bytes,
+ bytes_allocated, usable_size,
+ bytes_allocated_out,
+ usable_size_out);
}
-template <typename S, typename A>
-size_t ValgrindMallocSpace<S, A>::AllocationSize(mirror::Object* obj, size_t* usable_size) {
+template <typename S,
+ size_t kValgrindRedZoneBytes,
+ bool kAdjustForRedzoneInAllocSize,
+ bool kUseObjSizeForUsable>
+size_t ValgrindMallocSpace<S,
+ kValgrindRedZoneBytes,
+ kAdjustForRedzoneInAllocSize,
+ kUseObjSizeForUsable>::AllocationSize(
+ mirror::Object* obj, size_t* usable_size) {
size_t result = S::AllocationSize(reinterpret_cast<mirror::Object*>(
- reinterpret_cast<uint8_t*>(obj) - kValgrindRedZoneBytes), usable_size);
+ reinterpret_cast<uint8_t*>(obj) - (kAdjustForRedzoneInAllocSize ? kValgrindRedZoneBytes : 0)),
+ usable_size);
+ if (usable_size != nullptr) {
+ if (kUseObjSizeForUsable) {
+ *usable_size = valgrind_details::GetObjSizeNoThreadSafety(obj);
+ } else {
+ *usable_size = *usable_size - 2 * kValgrindRedZoneBytes;
+ }
+ }
return result;
}
-template <typename S, typename A>
-size_t ValgrindMallocSpace<S, A>::Free(Thread* self, mirror::Object* ptr) {
+template <typename S,
+ size_t kValgrindRedZoneBytes,
+ bool kAdjustForRedzoneInAllocSize,
+ bool kUseObjSizeForUsable>
+size_t ValgrindMallocSpace<S,
+ kValgrindRedZoneBytes,
+ kAdjustForRedzoneInAllocSize,
+ kUseObjSizeForUsable>::Free(
+ Thread* self, mirror::Object* ptr) {
void* obj_after_rdz = reinterpret_cast<void*>(ptr);
- void* obj_with_rdz = reinterpret_cast<uint8_t*>(obj_after_rdz) - kValgrindRedZoneBytes;
+ uint8_t* obj_with_rdz = reinterpret_cast<uint8_t*>(obj_after_rdz) - kValgrindRedZoneBytes;
// Make redzones undefined.
- size_t usable_size = 0;
- AllocationSize(ptr, &usable_size);
- VALGRIND_MAKE_MEM_UNDEFINED(obj_with_rdz, usable_size);
+ size_t usable_size;
+ size_t allocation_size = AllocationSize(ptr, &usable_size);
+
+ // Unprotect the allocation.
+ // Use the obj-size-for-usable flag to determine whether usable_size is the more important one,
+ // e.g., whether there's data in the allocation_size (and usable_size can't be trusted).
+ if (kUseObjSizeForUsable) {
+ VALGRIND_MAKE_MEM_UNDEFINED(obj_with_rdz, allocation_size);
+ } else {
+ VALGRIND_MAKE_MEM_UNDEFINED(obj_with_rdz, usable_size + 2 * kValgrindRedZoneBytes);
+ }
+
return S::Free(self, reinterpret_cast<mirror::Object*>(obj_with_rdz));
}
-template <typename S, typename A>
-size_t ValgrindMallocSpace<S, A>::FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) {
+template <typename S,
+ size_t kValgrindRedZoneBytes,
+ bool kAdjustForRedzoneInAllocSize,
+ bool kUseObjSizeForUsable>
+size_t ValgrindMallocSpace<S,
+ kValgrindRedZoneBytes,
+ kAdjustForRedzoneInAllocSize,
+ kUseObjSizeForUsable>::FreeList(
+ Thread* self, size_t num_ptrs, mirror::Object** ptrs) {
size_t freed = 0;
for (size_t i = 0; i < num_ptrs; i++) {
freed += Free(self, ptrs[i]);
@@ -91,15 +188,18 @@
return freed;
}
-template <typename S, typename A>
-ValgrindMallocSpace<S, A>::ValgrindMallocSpace(const std::string& name, MemMap* mem_map,
- A allocator, uint8_t* begin,
- uint8_t* end, uint8_t* limit, size_t growth_limit,
- size_t initial_size,
- bool can_move_objects, size_t starting_size) :
- S(name, mem_map, allocator, begin, end, limit, growth_limit, can_move_objects, starting_size,
- initial_size) {
- VALGRIND_MAKE_MEM_UNDEFINED(mem_map->Begin() + initial_size, mem_map->Size() - initial_size);
+template <typename S,
+ size_t kValgrindRedZoneBytes,
+ bool kAdjustForRedzoneInAllocSize,
+ bool kUseObjSizeForUsable>
+template <typename... Params>
+ValgrindMallocSpace<S,
+ kValgrindRedZoneBytes,
+ kAdjustForRedzoneInAllocSize,
+ kUseObjSizeForUsable>::ValgrindMallocSpace(
+ MemMap* mem_map, size_t initial_size, Params... params) : S(mem_map, initial_size, params...) {
+ VALGRIND_MAKE_MEM_UNDEFINED(mem_map->Begin() + initial_size,
+ mem_map->Size() - initial_size);
}
} // namespace space
diff --git a/runtime/gc/space/valgrind_malloc_space.h b/runtime/gc/space/valgrind_malloc_space.h
index bc870a6..d102f49 100644
--- a/runtime/gc/space/valgrind_malloc_space.h
+++ b/runtime/gc/space/valgrind_malloc_space.h
@@ -27,7 +27,10 @@
// A specialization of DlMallocSpace/RosAllocSpace that places valgrind red zones around
// allocations.
-template <typename BaseMallocSpaceType, typename AllocatorType>
+template <typename BaseMallocSpaceType,
+ size_t kValgrindRedZoneBytes,
+ bool kAdjustForRedzoneInAllocSize,
+ bool kUseObjSizeForUsable>
class ValgrindMallocSpace FINAL : public BaseMallocSpaceType {
public:
mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated,
@@ -47,9 +50,8 @@
UNUSED(ptr);
}
- ValgrindMallocSpace(const std::string& name, MemMap* mem_map, AllocatorType allocator,
- uint8_t* begin, uint8_t* end, uint8_t* limit, size_t growth_limit,
- size_t initial_size, bool can_move_objects, size_t starting_size);
+ template <typename... Params>
+ explicit ValgrindMallocSpace(MemMap* mem_map, size_t initial_size, Params... params);
virtual ~ValgrindMallocSpace() {}
private:
diff --git a/runtime/gc/space/valgrind_settings.h b/runtime/gc/space/valgrind_settings.h
new file mode 100644
index 0000000..73da0fd
--- /dev/null
+++ b/runtime/gc/space/valgrind_settings.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_RUNTIME_GC_SPACE_VALGRIND_SETTINGS_H_
+#define ART_RUNTIME_GC_SPACE_VALGRIND_SETTINGS_H_
+
+namespace art {
+namespace gc {
+namespace space {
+
+// Default number of bytes to use as a red zone (rdz). A red zone of this size will be placed before
+// and after each allocation. 8 bytes provides long/double alignment.
+static constexpr size_t kDefaultValgrindRedZoneBytes = 8;
+
+} // namespace space
+} // namespace gc
+} // namespace art
+
+#endif // ART_RUNTIME_GC_SPACE_VALGRIND_SETTINGS_H_
diff --git a/runtime/utils.cc b/runtime/utils.cc
index 0373708..f2d710d 100644
--- a/runtime/utils.cc
+++ b/runtime/utils.cc
@@ -1121,6 +1121,11 @@
void DumpNativeStack(std::ostream& os, pid_t tid, const char* prefix,
mirror::ArtMethod* current_method) {
#ifdef __linux__
+ // b/18119146
+ if (RUNNING_ON_VALGRIND != 0) {
+ return;
+ }
+
std::unique_ptr<Backtrace> backtrace(Backtrace::Create(BACKTRACE_CURRENT_PROCESS, tid));
if (!backtrace->Unwind(0)) {
os << prefix << "(backtrace::Unwind failed for thread " << tid << ")\n";