Use the card table to speed up the GSS collector.
Scan only dirty cards, as opposed to the whole space, to find
references from the non-moving spaces to the bump pointer spaces at
bump pointer space only collections.
With this change, the Ritz MemAllocTest speeds up by 8-10% on host and
2-3% on N4. The Ritz EvaluateFibonacci speeds up by 8% and its average
pause time is reduced by 43% on N4.
Bug: 11650816
Change-Id: I1eefe75776bc37e24673b301ffa65a25f9bd4cde
diff --git a/runtime/gc/collector/immune_region.cc b/runtime/gc/collector/immune_region.cc
index 9e65384..70a6213 100644
--- a/runtime/gc/collector/immune_region.cc
+++ b/runtime/gc/collector/immune_region.cc
@@ -56,9 +56,14 @@
}
bool ImmuneRegion::ContainsSpace(const space::ContinuousSpace* space) const {
- return
+ bool contains =
begin_ <= reinterpret_cast<mirror::Object*>(space->Begin()) &&
end_ >= reinterpret_cast<mirror::Object*>(space->Limit());
+ if (kIsDebugBuild && contains) {
+ // A bump pointer space shoult not be in the immune region.
+ DCHECK(space->GetType() != space::kSpaceTypeBumpPointerSpace);
+ }
+ return contains;
}
} // namespace collector
diff --git a/runtime/gc/collector/mark_sweep.cc b/runtime/gc/collector/mark_sweep.cc
index 4f3ad32..fe5a75f 100644
--- a/runtime/gc/collector/mark_sweep.cc
+++ b/runtime/gc/collector/mark_sweep.cc
@@ -201,7 +201,7 @@
Thread* self = Thread::Current();
CHECK(!Locks::mutator_lock_->IsExclusiveHeld(self));
// Process dirty cards and add dirty cards to mod union tables, also ages cards.
- heap_->ProcessCards(timings_);
+ heap_->ProcessCards(timings_, false);
// The checkpoint root marking is required to avoid a race condition which occurs if the
// following happens during a reference write:
// 1. mutator dirties the card (write barrier)
@@ -241,7 +241,7 @@
FindDefaultMarkBitmap();
// Process dirty cards and add dirty cards to mod union tables.
- heap_->ProcessCards(timings_);
+ heap_->ProcessCards(timings_, false);
// Need to do this before the checkpoint since we don't want any threads to add references to
// the live stack during the recursive mark.
diff --git a/runtime/gc/collector/semi_space.cc b/runtime/gc/collector/semi_space.cc
index 23b155c..5b9c397 100644
--- a/runtime/gc/collector/semi_space.cc
+++ b/runtime/gc/collector/semi_space.cc
@@ -27,6 +27,7 @@
#include "base/timing_logger.h"
#include "gc/accounting/heap_bitmap.h"
#include "gc/accounting/mod_union_table.h"
+#include "gc/accounting/remembered_set.h"
#include "gc/accounting/space_bitmap-inl.h"
#include "gc/heap.h"
#include "gc/space/bump_pointer_space.h"
@@ -182,7 +183,7 @@
// Assume the cleared space is already empty.
BindBitmaps();
// Process dirty cards and add dirty cards to mod-union tables.
- heap_->ProcessCards(timings_);
+ heap_->ProcessCards(timings_, kUseRememberedSet && generational_);
// Clear the whole card table since we can not get any additional dirty cards during the
// paused GC. This saves memory but only works for pause the world collectors.
timings_.NewSplit("ClearCardTable");
@@ -214,13 +215,29 @@
"UpdateAndMarkImageModUnionTable",
&timings_);
table->UpdateAndMarkReferences(MarkObjectCallback, this);
+ } else if (heap_->FindRememberedSetFromSpace(space) != nullptr) {
+ DCHECK(kUseRememberedSet);
+ // If a bump pointer space only collection, the non-moving
+ // space is added to the immune space. The non-moving space
+ // doesn't have a mod union table, but has a remembered
+ // set. Its dirty cards will be scanned later in
+ // MarkReachableObjects().
+ DCHECK(generational_ && !whole_heap_collection_ &&
+ (space == heap_->GetNonMovingSpace() || space == heap_->GetPrimaryFreeListSpace()))
+ << "Space " << space->GetName() << " "
+ << "generational_=" << generational_ << " "
+ << "whole_heap_collection_=" << whole_heap_collection_ << " ";
} else {
+ DCHECK(!kUseRememberedSet);
// If a bump pointer space only collection, the non-moving
// space is added to the immune space. But the non-moving
// space doesn't have a mod union table. Instead, its live
// bitmap will be scanned later in MarkReachableObjects().
DCHECK(generational_ && !whole_heap_collection_ &&
- (space == heap_->GetNonMovingSpace() || space == heap_->GetPrimaryFreeListSpace()));
+ (space == heap_->GetNonMovingSpace() || space == heap_->GetPrimaryFreeListSpace()))
+ << "Space " << space->GetName() << " "
+ << "generational_=" << generational_ << " "
+ << "whole_heap_collection_=" << whole_heap_collection_ << " ";
}
}
}
@@ -240,6 +257,42 @@
SemiSpace* const semi_space_;
};
+// Used to verify that there's no references to the from-space.
+class SemiSpaceVerifyNoFromSpaceReferencesVisitor {
+ public:
+ explicit SemiSpaceVerifyNoFromSpaceReferencesVisitor(space::ContinuousMemMapAllocSpace* from_space) :
+ from_space_(from_space) {}
+
+ void operator()(Object* obj, Object* ref, const MemberOffset& offset, bool /* is_static */)
+ const ALWAYS_INLINE {
+ if (from_space_->HasAddress(ref)) {
+ Runtime::Current()->GetHeap()->DumpObject(LOG(INFO), obj);
+ }
+ DCHECK(!from_space_->HasAddress(ref));
+ }
+ private:
+ space::ContinuousMemMapAllocSpace* from_space_;
+};
+
+void SemiSpace::VerifyNoFromSpaceReferences(Object* obj) {
+ DCHECK(obj != NULL);
+ DCHECK(!from_space_->HasAddress(obj)) << "Scanning object " << obj << " in from space";
+ SemiSpaceVerifyNoFromSpaceReferencesVisitor visitor(from_space_);
+ MarkSweep::VisitObjectReferences(obj, visitor, kMovingClasses);
+}
+
+class SemiSpaceVerifyNoFromSpaceReferencesObjectVisitor {
+ public:
+ explicit SemiSpaceVerifyNoFromSpaceReferencesObjectVisitor(SemiSpace* ss) : semi_space_(ss) {}
+ void operator()(Object* obj) const
+ SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_, Locks::mutator_lock_) {
+ DCHECK(obj != nullptr);
+ semi_space_->VerifyNoFromSpaceReferences(obj);
+ }
+ private:
+ SemiSpace* const semi_space_;
+};
+
void SemiSpace::MarkReachableObjects() {
timings_.StartSplit("MarkStackAsLive");
accounting::ObjectStack* live_stack = heap_->GetLiveStack();
@@ -250,18 +303,36 @@
for (auto& space : heap_->GetContinuousSpaces()) {
// If the space is immune and has no mod union table (the
// non-moving space when the bump pointer space only collection is
- // enabled,) then we need to scan its live bitmap as roots
+ // enabled,) then we need to scan its live bitmap or dirty cards as roots
// (including the objects on the live stack which have just marked
// in the live bitmap above in MarkAllocStackAsLive().)
if (immune_region_.ContainsSpace(space) &&
heap_->FindModUnionTableFromSpace(space) == nullptr) {
DCHECK(generational_ && !whole_heap_collection_ &&
(space == GetHeap()->GetNonMovingSpace() || space == GetHeap()->GetPrimaryFreeListSpace()));
- accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap();
- SemiSpaceScanObjectVisitor visitor(this);
- live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()),
- reinterpret_cast<uintptr_t>(space->End()),
- visitor);
+ accounting::RememberedSet* rem_set = heap_->FindRememberedSetFromSpace(space);
+ if (kUseRememberedSet) {
+ DCHECK(rem_set != nullptr);
+ rem_set->UpdateAndMarkReferences(MarkObjectCallback, from_space_, this);
+ if (kIsDebugBuild) {
+ // Verify that there are no from-space references that
+ // remain in the space, that is, the remembered set (and the
+ // card table) didn't miss any from-space references in the
+ // space.
+ accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap();
+ SemiSpaceVerifyNoFromSpaceReferencesObjectVisitor visitor(this);
+ live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()),
+ reinterpret_cast<uintptr_t>(space->End()),
+ visitor);
+ }
+ } else {
+ DCHECK(rem_set == nullptr);
+ accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap();
+ SemiSpaceScanObjectVisitor visitor(this);
+ live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()),
+ reinterpret_cast<uintptr_t>(space->End()),
+ visitor);
+ }
}
}
@@ -447,6 +518,10 @@
} else {
GetHeap()->num_bytes_allocated_.FetchAndAdd(bytes_promoted);
bytes_promoted_ += bytes_promoted;
+ // Dirty the card at the destionation as it may contain
+ // references (including the class pointer) to the bump pointer
+ // space.
+ GetHeap()->WriteBarrierEveryFieldOf(forward_address);
// Handle the bitmaps marking.
accounting::SpaceBitmap* live_bitmap = promo_dest_space->GetLiveBitmap();
DCHECK(live_bitmap != nullptr);
diff --git a/runtime/gc/collector/semi_space.h b/runtime/gc/collector/semi_space.h
index be7ec05..08bfbc4 100644
--- a/runtime/gc/collector/semi_space.h
+++ b/runtime/gc/collector/semi_space.h
@@ -63,6 +63,9 @@
class SemiSpace : public GarbageCollector {
public:
+ // If true, use remembered sets in the generational mode.
+ static constexpr bool kUseRememberedSet = true;
+
explicit SemiSpace(Heap* heap, bool generational = false,
const std::string& name_prefix = "");
@@ -100,6 +103,9 @@
void ScanObject(mirror::Object* obj)
EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_, Locks::mutator_lock_);
+ void VerifyNoFromSpaceReferences(mirror::Object* obj)
+ SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_, Locks::mutator_lock_);
+
// Marks the root set at the start of a garbage collection.
void MarkRoots()
EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_, Locks::mutator_lock_);