Fix zygote live/mark bitmap size.
Fixed some errors with the sizes of mark/live bitmaps after zygote space creation.
This was causing us to occasionally have overlapping mark/live bitmaps.
Added a new verify objects mode called VERIFY_OBJECT_FAST which only checks objects and not their classes.
Refactored/optimized some of the scanning code to use xor to clear bits instead of and+not.
Change-Id: Iec87d9157f69e6a558e300950b51d8781679e3f7
diff --git a/src/space_bitmap.cc b/src/space_bitmap.cc
index 74bc07c..7da8146 100644
--- a/src/space_bitmap.cc
+++ b/src/space_bitmap.cc
@@ -38,8 +38,9 @@
// Clean up any resources associated with the bitmap.
SpaceBitmap::~SpaceBitmap() {}
-void SpaceBitmap::Trim(size_t heap_capacity) {
- size_t new_size = OffsetToIndex(RoundUp(heap_capacity, kAlignment * kBitsPerWord)) * kWordSize;
+void SpaceBitmap::SetHeapLimit(uintptr_t new_end) {
+ DCHECK(IsAligned<kBitsPerWord * kAlignment>(new_end));
+ size_t new_size = OffsetToIndex(new_end - heap_begin_) * kWordSize;
if (new_size < bitmap_size_) {
bitmap_size_ = new_size;
}
@@ -84,13 +85,12 @@
for (uintptr_t i = 0; i <= end; ++i) {
word w = bitmap_begin_[i];
if (UNLIKELY(w != 0)) {
- word high_bit = 1 << (kBitsPerWord - 1);
uintptr_t ptr_base = IndexToOffset(i) + heap_begin_;
while (w != 0) {
- const int shift = CLZ(w);
+ const size_t shift = CLZ(w);
Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
(*callback)(obj, arg);
- w &= ~(high_bit >> shift);
+ w ^= static_cast<size_t>(kWordHighBitMask) >> shift;
}
}
}
@@ -130,14 +130,13 @@
for (size_t i = start; i <= end; i++) {
word w = bitmap_begin_[i];
if (UNLIKELY(w != 0)) {
- word high_bit = 1 << (kBitsPerWord - 1);
uintptr_t ptr_base = IndexToOffset(i) + heap_begin_;
void* finger = reinterpret_cast<void*>(IndexToOffset(i + 1) + heap_begin_);
while (w != 0) {
- const int shift = CLZ(w);
+ const size_t shift = CLZ(w);
Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
(*callback)(obj, finger, arg);
- w &= ~(high_bit >> shift);
+ w ^= static_cast<size_t>(kWordHighBitMask) >> shift;
}
}
}
@@ -146,14 +145,13 @@
for (size_t i = start; i <= end; i++) {
word w = bitmap_begin_[i];
if (UNLIKELY(w != 0)) {
- word high_bit = 1 << (kBitsPerWord - 1);
uintptr_t ptr_base = IndexToOffset(i) + heap_begin_;
void* finger = reinterpret_cast<void*>(IndexToOffset(i + 1) + heap_begin_);
while (w != 0) {
- const int shift = CLZ(w);
+ const size_t shift = CLZ(w);
Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
(*callback)(obj, finger, arg);
- w &= ~(high_bit >> shift);
+ w ^= static_cast<size_t>(kWordHighBitMask) >> shift;
}
}
// update 'end' in case callback modified bitmap
@@ -194,11 +192,10 @@
for (size_t i = start; i <= end; i++) {
word garbage = live[i] & ~mark[i];
if (UNLIKELY(garbage != 0)) {
- word high_bit = 1 << (kBitsPerWord - 1);
uintptr_t ptr_base = IndexToOffset(i) + live_bitmap.heap_begin_;
while (garbage != 0) {
- int shift = CLZ(garbage);
- garbage &= ~(high_bit >> shift);
+ const size_t shift = CLZ(garbage);
+ garbage ^= static_cast<size_t>(kWordHighBitMask) >> shift;
*pb++ = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
}
// Make sure that there are always enough slots available for an
@@ -302,13 +299,12 @@
for (uintptr_t i = 0; i <= end; ++i) {
word w = bitmap_begin_[i];
if (UNLIKELY(w != 0)) {
- word high_bit = 1 << (kBitsPerWord - 1);
uintptr_t ptr_base = IndexToOffset(i) + heap_begin_;
while (w != 0) {
- const int shift = CLZ(w);
+ const size_t shift = CLZ(w);
Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
WalkFieldsInOrder(visited.get(), callback, obj, arg);
- w &= ~(high_bit >> shift);
+ w ^= static_cast<size_t>(kWordHighBitMask) >> shift;
}
}
}