ART: Fix valgrind
Allow ValgrindMallocSpace wrapper for RosAlloc.Requires refactoring,
as ValgrindMallocSpace was bound to the signature of DlMallocSpace.
Also turn of native stack dumping when running under Valgrind to
work around b/18119146.
Ritzperf before and after
Mean 3190.725 3082.475
Standard Error 11.68407 10.37911
Mode 3069 2980
Median 3182.5 3051.5
Variance 16382.117 12927.125
Standard Deviation 127.99264 113.69751
Kurtosis 1.1065632 0.3657799
Skewness 0.9013805 0.9117792
Range 644 528
Minimum 2991 2928
Maximum 3635 3456
Count 120 120
Bug: 18119146
Change-Id: I25558ea7cb578406011dede9d3d0bdbfee4ff4d5
diff --git a/runtime/gc/allocator/rosalloc.cc b/runtime/gc/allocator/rosalloc.cc
index f9d6a51..991b956 100644
--- a/runtime/gc/allocator/rosalloc.cc
+++ b/runtime/gc/allocator/rosalloc.cc
@@ -14,13 +14,15 @@
* limitations under the License.
*/
+#include "rosalloc.h"
+
#include "base/mutex-inl.h"
+#include "gc/space/valgrind_settings.h"
#include "mirror/class-inl.h"
#include "mirror/object.h"
#include "mirror/object-inl.h"
#include "thread-inl.h"
#include "thread_list.h"
-#include "rosalloc.h"
#include <map>
#include <list>
@@ -47,13 +49,15 @@
reinterpret_cast<RosAlloc::Run*>(dedicated_full_run_storage_);
RosAlloc::RosAlloc(void* base, size_t capacity, size_t max_capacity,
- PageReleaseMode page_release_mode, size_t page_release_size_threshold)
+ PageReleaseMode page_release_mode, bool running_on_valgrind,
+ size_t page_release_size_threshold)
: base_(reinterpret_cast<uint8_t*>(base)), footprint_(capacity),
capacity_(capacity), max_capacity_(max_capacity),
lock_("rosalloc global lock", kRosAllocGlobalLock),
bulk_free_lock_("rosalloc bulk free lock", kRosAllocBulkFreeLock),
page_release_mode_(page_release_mode),
- page_release_size_threshold_(page_release_size_threshold) {
+ page_release_size_threshold_(page_release_size_threshold),
+ running_on_valgrind_(running_on_valgrind) {
DCHECK_EQ(RoundUp(capacity, kPageSize), capacity);
DCHECK_EQ(RoundUp(max_capacity, kPageSize), max_capacity);
CHECK_LE(capacity, max_capacity);
@@ -317,7 +321,7 @@
}
const size_t byte_size = num_pages * kPageSize;
if (already_zero) {
- if (kCheckZeroMemory) {
+ if (ShouldCheckZeroMemory()) {
const uintptr_t* word_ptr = reinterpret_cast<uintptr_t*>(ptr);
for (size_t i = 0; i < byte_size / sizeof(uintptr_t); ++i) {
CHECK_EQ(word_ptr[i], 0U) << "words don't match at index " << i;
@@ -471,7 +475,7 @@
<< "(" << std::dec << (num_pages * kPageSize) << ")";
}
// Check if the returned memory is really all zero.
- if (kCheckZeroMemory) {
+ if (ShouldCheckZeroMemory()) {
CHECK_EQ(total_bytes % sizeof(uintptr_t), 0U);
const uintptr_t* words = reinterpret_cast<uintptr_t*>(r);
for (size_t i = 0; i < total_bytes / sizeof(uintptr_t); ++i) {
@@ -1433,7 +1437,7 @@
return stream.str();
}
-size_t RosAlloc::UsableSize(void* ptr) {
+size_t RosAlloc::UsableSize(const void* ptr) {
DCHECK_LE(base_, ptr);
DCHECK_LT(ptr, base_ + footprint_);
size_t pm_idx = RoundDownToPageMapIndex(ptr);
@@ -1470,7 +1474,7 @@
Run* run = reinterpret_cast<Run*>(base_ + pm_idx * kPageSize);
DCHECK_EQ(run->magic_num_, kMagicNum);
size_t idx = run->size_bracket_idx_;
- size_t offset_from_slot_base = reinterpret_cast<uint8_t*>(ptr)
+ size_t offset_from_slot_base = reinterpret_cast<const uint8_t*>(ptr)
- (reinterpret_cast<uint8_t*>(run) + headerSizes[idx]);
DCHECK_EQ(offset_from_slot_base % bracketSizes[idx], static_cast<size_t>(0));
return IndexToBracketSize(idx);
@@ -1915,10 +1919,15 @@
num_pages++;
idx++;
}
- void* start = base_ + i * kPageSize;
+ uint8_t* start = base_ + i * kPageSize;
+ if (running_on_valgrind_) {
+ start += ::art::gc::space::kDefaultValgrindRedZoneBytes;
+ }
mirror::Object* obj = reinterpret_cast<mirror::Object*>(start);
size_t obj_size = obj->SizeOf();
- CHECK_GT(obj_size, kLargeSizeThreshold)
+ CHECK_GT(obj_size +
+ (running_on_valgrind_ ? 2 * ::art::gc::space::kDefaultValgrindRedZoneBytes : 0),
+ kLargeSizeThreshold)
<< "A rosalloc large object size must be > " << kLargeSizeThreshold;
CHECK_EQ(num_pages, RoundUp(obj_size, kPageSize) / kPageSize)
<< "A rosalloc large object size " << obj_size
@@ -1986,11 +1995,11 @@
}
// Call Verify() here for the lock order.
for (auto& run : runs) {
- run->Verify(self, this);
+ run->Verify(self, this, running_on_valgrind_);
}
}
-void RosAlloc::Run::Verify(Thread* self, RosAlloc* rosalloc) {
+void RosAlloc::Run::Verify(Thread* self, RosAlloc* rosalloc, bool running_on_valgrind) {
DCHECK_EQ(magic_num_, kMagicNum) << "Bad magic number : " << Dump();
const size_t idx = size_bracket_idx_;
CHECK_LT(idx, kNumOfSizeBrackets) << "Out of range size bracket index : " << Dump();
@@ -2073,6 +2082,9 @@
}
// Check each slot.
size_t slots = 0;
+ size_t valgrind_modifier = running_on_valgrind ?
+ 2 * ::art::gc::space::kDefaultValgrindRedZoneBytes :
+ 0U;
for (size_t v = 0; v < num_vec; v++, slots += 32) {
DCHECK_GE(num_slots, slots) << "Out of bounds";
uint32_t vec = alloc_bit_map_[v];
@@ -2085,14 +2097,17 @@
bool is_thread_local_freed = IsThreadLocal() && ((thread_local_free_vec >> i) & 0x1) != 0;
if (is_allocated && !is_thread_local_freed) {
uint8_t* slot_addr = slot_base + (slots + i) * bracket_size;
+ if (running_on_valgrind) {
+ slot_addr += ::art::gc::space::kDefaultValgrindRedZoneBytes;
+ }
mirror::Object* obj = reinterpret_cast<mirror::Object*>(slot_addr);
size_t obj_size = obj->SizeOf();
- CHECK_LE(obj_size, kLargeSizeThreshold)
+ CHECK_LE(obj_size + valgrind_modifier, kLargeSizeThreshold)
<< "A run slot contains a large object " << Dump();
- CHECK_EQ(SizeToIndex(obj_size), idx)
+ CHECK_EQ(SizeToIndex(obj_size + valgrind_modifier), idx)
<< PrettyTypeOf(obj) << " "
- << "obj_size=" << obj_size << ", idx=" << idx << " "
- << "A run slot contains an object with wrong size " << Dump();
+ << "obj_size=" << obj_size << "(" << obj_size + valgrind_modifier << "), idx=" << idx
+ << " A run slot contains an object with wrong size " << Dump();
}
}
}
@@ -2162,6 +2177,11 @@
// In the debug build, the first page of a free page run
// contains a magic number for debugging. Exclude it.
start += kPageSize;
+
+ // Single pages won't be released.
+ if (start == end) {
+ return 0;
+ }
}
if (!kMadviseZeroes) {
// TODO: Do this when we resurrect the page instead.